cacache 15.0.5 → 15.3.0

package/README.md

@@ -36,6 +36,8 @@ just as easily be used on its own.
     * [`rm.all`](#rm-all)
     * [`rm.entry`](#rm-entry)
     * [`rm.content`](#rm-content)
+    * [`index.compact`](#index-compact)
+    * [`index.insert`](#index-insert)
   * Utilities
     * [`clearMemoized`](#clear-memoized)
     * [`tmp.mkdir`](#tmp-mkdir)
@@ -456,13 +458,17 @@ cacache.rm.all(cachePath).then(() => {
 })
 ```
 
-#### <a name="rm-entry"></a> `> cacache.rm.entry(cache, key) -> Promise`
+#### <a name="rm-entry"></a> `> cacache.rm.entry(cache, key, [opts]) -> Promise`
 
 Alias: `cacache.rm`
 
 Removes the index entry for `key`. Content will still be accessible if
 requested directly by content address ([`get.stream.byDigest`](#get-stream)).
 
+By default, this appends a new entry to the index with an integrity of `null`.
+If `opts.removeFully` is set to `true` then the index file itself will be
+physically deleted rather than appending a `null`.
+
 To remove the content itself (which might still be used by other entries), use
 [`rm.content`](#rm-content). Or, to safely vacuum any unused content, use
 [`verify`](#verify).
@@ -489,6 +495,34 @@ cacache.rm.content(cachePath, 'sha512-SoMeDIGest/IN+BaSE64==').then(() => {
 })
 ```
 
+#### <a name="index-compact"></a> `> cacache.index.compact(cache, key, matchFn, [opts]) -> Promise`
+
+Uses `matchFn`, which must be a synchronous function that accepts two entries
+and returns a boolean indicating whether or not the two entries match, to
+deduplicate all entries in the cache for the given `key`.
+
+If `opts.validateEntry` is provided, it will be called as a function with the
+only parameter being a single index entry. The function must return a Boolean,
+if it returns `true` the entry is considered valid and will be kept in the index,
+if it returns `false` the entry will be removed from the index.
+
+If `opts.validateEntry` is not provided, however, every entry in the index will
+be deduplicated and kept until the first `null` integrity is reached, removing
+all entries that were written before the `null`.
+
+The deduplicated list of entries is both written to the index, replacing the
+existing content, and returned in the Promise.
+
+#### <a name="index-insert"></a> `> cacache.index.insert(cache, key, integrity, opts) -> Promise`
+
+Writes an index entry to the cache for the given `key` without writing content.
+
+It is assumed if you are using this method, you have already stored the content
+some other way and you only wish to add a new index to that content. The `metadata`
+and `size` properties are read from `opts` and used as part of the index entry.
+
+Returns a Promise resolving to the newly added entry.
+
 #### <a name="clear-memoized"></a> `> cacache.clearMemoized()`
 
 Completely resets the in-memory entry cache.

package/get.js

@@ -1,119 +1,112 @@
 'use strict'
 
-const util = require('util')
+const Collect = require('minipass-collect')
+const Minipass = require('minipass')
+const Pipeline = require('minipass-pipeline')
 const fs = require('fs')
+const util = require('util')
+
 const index = require('./lib/entry-index')
 const memo = require('./lib/memoization')
 const read = require('./lib/content/read')
 
-const Minipass = require('minipass')
-const Collect = require('minipass-collect')
-const Pipeline = require('minipass-pipeline')
-
 const writeFile = util.promisify(fs.writeFile)
 
-module.exports = function get (cache, key, opts) {
-  return getData(false, cache, key, opts)
-}
-module.exports.byDigest = function getByDigest (cache, digest, opts) {
-  return getData(true, cache, digest, opts)
-}
-
-function getData (byDigest, cache, key, opts = {}) {
+function getData (cache, key, opts = {}) {
   const { integrity, memoize, size } = opts
-  const memoized = byDigest
-    ? memo.get.byDigest(cache, key, opts)
-    : memo.get(cache, key, opts)
+  const memoized = memo.get(cache, key, opts)
   if (memoized && memoize !== false) {
-    return Promise.resolve(
-      byDigest
-        ? memoized
-        : {
-          metadata: memoized.entry.metadata,
-          data: memoized.data,
-          integrity: memoized.entry.integrity,
-          size: memoized.entry.size
-        }
-    )
+    return Promise.resolve({
+      metadata: memoized.entry.metadata,
+      data: memoized.data,
+      integrity: memoized.entry.integrity,
+      size: memoized.entry.size,
+    })
   }
-  return (byDigest ? Promise.resolve(null) : index.find(cache, key, opts)).then(
-    (entry) => {
-      if (!entry && !byDigest) {
-        throw new index.NotFoundError(cache, key)
+
+  return index.find(cache, key, opts).then((entry) => {
+    if (!entry)
+      throw new index.NotFoundError(cache, key)
+
+    return read(cache, entry.integrity, { integrity, size }).then((data) => {
+      if (memoize)
+        memo.put(cache, entry, data, opts)
+
+      return {
+        data,
+        metadata: entry.metadata,
+        size: entry.size,
+        integrity: entry.integrity,
       }
-      return read(cache, byDigest ? key : entry.integrity, {
-        integrity,
-        size
-      })
-        .then((data) =>
-          byDigest
-            ? data
-            : {
-              data,
-              metadata: entry.metadata,
-              size: entry.size,
-              integrity: entry.integrity
-            }
-        )
-        .then((res) => {
-          if (memoize && byDigest) {
-            memo.put.byDigest(cache, key, res, opts)
-          } else if (memoize) {
-            memo.put(cache, entry, res.data, opts)
-          }
-          return res
-        })
-    }
-  )
+    })
+  })
 }
+module.exports = getData
 
-module.exports.sync = function get (cache, key, opts) {
-  return getDataSync(false, cache, key, opts)
-}
-module.exports.sync.byDigest = function getByDigest (cache, digest, opts) {
-  return getDataSync(true, cache, digest, opts)
+function getDataByDigest (cache, key, opts = {}) {
+  const { integrity, memoize, size } = opts
+  const memoized = memo.get.byDigest(cache, key, opts)
+  if (memoized && memoize !== false)
+    return Promise.resolve(memoized)
+
+  return read(cache, key, { integrity, size }).then((res) => {
+    if (memoize)
+      memo.put.byDigest(cache, key, res, opts)
+    return res
+  })
 }
+module.exports.byDigest = getDataByDigest
 
-function getDataSync (byDigest, cache, key, opts = {}) {
+function getDataSync (cache, key, opts = {}) {
   const { integrity, memoize, size } = opts
-  const memoized = byDigest
-    ? memo.get.byDigest(cache, key, opts)
-    : memo.get(cache, key, opts)
+  const memoized = memo.get(cache, key, opts)
+
   if (memoized && memoize !== false) {
-    return byDigest
-      ? memoized
-      : {
-        metadata: memoized.entry.metadata,
-        data: memoized.data,
-        integrity: memoized.entry.integrity,
-        size: memoized.entry.size
-      }
+    return {
+      metadata: memoized.entry.metadata,
+      data: memoized.data,
+      integrity: memoized.entry.integrity,
+      size: memoized.entry.size,
+    }
   }
-  const entry = !byDigest && index.find.sync(cache, key, opts)
-  if (!entry && !byDigest) {
+  const entry = index.find.sync(cache, key, opts)
+  if (!entry)
     throw new index.NotFoundError(cache, key)
-  }
-  const data = read.sync(cache, byDigest ? key : entry.integrity, {
+  const data = read.sync(cache, entry.integrity, {
     integrity: integrity,
-    size: size
+    size: size,
   })
-  const res = byDigest
-    ? data
-    : {
-      metadata: entry.metadata,
-      data: data,
-      size: entry.size,
-      integrity: entry.integrity
-    }
-  if (memoize && byDigest) {
-    memo.put.byDigest(cache, key, res, opts)
-  } else if (memoize) {
-    memo.put(cache, entry, res.data, opts)
+  const res = {
+    metadata: entry.metadata,
+    data: data,
+    size: entry.size,
+    integrity: entry.integrity,
   }
+  if (memoize)
+    memo.put(cache, entry, res.data, opts)
+
   return res
 }
 
-module.exports.stream = getStream
+module.exports.sync = getDataSync
+
+function getDataByDigestSync (cache, digest, opts = {}) {
+  const { integrity, memoize, size } = opts
+  const memoized = memo.get.byDigest(cache, digest, opts)
+
+  if (memoized && memoize !== false)
+    return memoized
+
+  const res = read.sync(cache, digest, {
+    integrity: integrity,
+    size: size,
+  })
+  if (memoize)
+    memo.put.byDigest(cache, digest, res, opts)
+
+  return res
+}
+module.exports.sync.byDigest = getDataByDigestSync
 
 const getMemoizedStream = (memoized) => {
   const stream = new Minipass()
@@ -129,17 +122,16 @@ const getMemoizedStream = (memoized) => {
 function getStream (cache, key, opts = {}) {
   const { memoize, size } = opts
   const memoized = memo.get(cache, key, opts)
-  if (memoized && memoize !== false) {
+  if (memoized && memoize !== false)
     return getMemoizedStream(memoized)
-  }
 
   const stream = new Pipeline()
   index
     .find(cache, key)
     .then((entry) => {
-      if (!entry) {
+      if (!entry)
         throw new index.NotFoundError(cache, key)
-      }
+
       stream.emit('metadata', entry.metadata)
       stream.emit('integrity', entry.integrity)
       stream.emit('size', entry.size)
@@ -167,7 +159,7 @@ function getStream (cache, key, opts = {}) {
   return stream
 }
 
-module.exports.stream.byDigest = getStreamDigest
+module.exports.stream = getStream
 
 function getStreamDigest (cache, integrity, opts = {}) {
   const { memoize } = opts
@@ -178,9 +170,9 @@ function getStreamDigest (cache, integrity, opts = {}) {
     return stream
   } else {
     const stream = read.readStream(cache, integrity, opts)
-    if (!memoize) {
+    if (!memoize)
       return stream
-    }
+
     const memoStream = new Collect.PassThrough()
     memoStream.on('collect', data => memo.put.byDigest(
       cache,
@@ -192,64 +184,54 @@ function getStreamDigest (cache, integrity, opts = {}) {
   }
 }
 
-module.exports.info = info
+module.exports.stream.byDigest = getStreamDigest
 
 function info (cache, key, opts = {}) {
   const { memoize } = opts
   const memoized = memo.get(cache, key, opts)
-  if (memoized && memoize !== false) {
+  if (memoized && memoize !== false)
     return Promise.resolve(memoized.entry)
-  } else {
+  else
     return index.find(cache, key)
-  }
-}
-
-module.exports.hasContent = read.hasContent
-
-function cp (cache, key, dest, opts) {
-  return copy(false, cache, key, dest, opts)
 }
+module.exports.info = info
 
-module.exports.copy = cp
-
-function cpDigest (cache, digest, dest, opts) {
-  return copy(true, cache, digest, dest, opts)
-}
-
-module.exports.copy.byDigest = cpDigest
-
-function copy (byDigest, cache, key, dest, opts = {}) {
+function copy (cache, key, dest, opts = {}) {
   if (read.copy) {
-    return (byDigest
-      ? Promise.resolve(null)
-      : index.find(cache, key, opts)
-    ).then((entry) => {
-      if (!entry && !byDigest) {
+    return index.find(cache, key, opts).then((entry) => {
+      if (!entry)
         throw new index.NotFoundError(cache, key)
-      }
-      return read
-        .copy(cache, byDigest ? key : entry.integrity, dest, opts)
+      return read.copy(cache, entry.integrity, dest, opts)
         .then(() => {
-          return byDigest
-            ? key
-            : {
-              metadata: entry.metadata,
-              size: entry.size,
-              integrity: entry.integrity
-            }
+          return {
+            metadata: entry.metadata,
+            size: entry.size,
+            integrity: entry.integrity,
+          }
         })
     })
   }
 
-  return getData(byDigest, cache, key, opts).then((res) => {
-    return writeFile(dest, byDigest ? res : res.data).then(() => {
-      return byDigest
-        ? key
-        : {
-          metadata: res.metadata,
-          size: res.size,
-          integrity: res.integrity
-        }
+  return getData(cache, key, opts).then((res) => {
+    return writeFile(dest, res.data).then(() => {
+      return {
+        metadata: res.metadata,
+        size: res.size,
+        integrity: res.integrity,
+      }
     })
   })
 }
+module.exports.copy = copy
+
+function copyByDigest (cache, key, dest, opts = {}) {
+  if (read.copy)
+    return read.copy(cache, key, dest, opts).then(() => key)
+
+  return getDataByDigest(cache, key, opts).then((res) => {
+    return writeFile(dest, res).then(() => key)
+  })
+}
+module.exports.copy.byDigest = copyByDigest
+
+module.exports.hasContent = read.hasContent

package/index.js

@@ -7,6 +7,11 @@ const rm = require('./rm.js')
 const verify = require('./verify.js')
 const { clearMemoized } = require('./lib/memoization.js')
 const tmp = require('./lib/util/tmp.js')
+const index = require('./lib/entry-index.js')
+
+module.exports.index = {}
+module.exports.index.compact = index.compact
+module.exports.index.insert = index.insert
 
 module.exports.ls = ls
 module.exports.ls.stream = ls.stream

package/lib/content/read.js

@@ -20,17 +20,16 @@ function read (cache, integrity, opts = {}) {
     // get size
     return lstat(cpath).then(stat => ({ stat, cpath, sri }))
   }).then(({ stat, cpath, sri }) => {
-    if (typeof size === 'number' && stat.size !== size) {
+    if (typeof size === 'number' && stat.size !== size)
       throw sizeError(size, stat.size)
-    }
-    if (stat.size > MAX_SINGLE_READ_SIZE) {
+
+    if (stat.size > MAX_SINGLE_READ_SIZE)
       return readPipeline(cpath, stat.size, sri, new Pipeline()).concat()
-    }
 
     return readFile(cpath, null).then((data) => {
-      if (!ssri.checkData(data, sri)) {
+      if (!ssri.checkData(data, sri))
         throw integrityError(sri, cpath)
-      }
+
       return data
     })
   })
@@ -40,11 +39,11 @@ const readPipeline = (cpath, size, sri, stream) => {
   stream.push(
     new fsm.ReadStream(cpath, {
       size,
-      readSize: MAX_SINGLE_READ_SIZE
+      readSize: MAX_SINGLE_READ_SIZE,
     }),
     ssri.integrityStream({
       integrity: sri,
-      size
+      size,
     })
   )
   return stream
@@ -56,13 +55,11 @@ function readSync (cache, integrity, opts = {}) {
   const { size } = opts
   return withContentSriSync(cache, integrity, (cpath, sri) => {
     const data = fs.readFileSync(cpath)
-    if (typeof size === 'number' && size !== data.length) {
+    if (typeof size === 'number' && size !== data.length)
       throw sizeError(size, data.length)
-    }
 
-    if (ssri.checkData(data, sri)) {
+    if (ssri.checkData(data, sri))
       return data
-    }
 
     throw integrityError(sri, cpath)
   })
@@ -78,9 +75,9 @@ function readStream (cache, integrity, opts = {}) {
     // just lstat to ensure it exists
     return lstat(cpath).then((stat) => ({ stat, cpath, sri }))
   }).then(({ stat, cpath, sri }) => {
-    if (typeof size === 'number' && size !== stat.size) {
+    if (typeof size === 'number' && size !== stat.size)
       return stream.emit('error', sizeError(size, stat.size))
-    }
+
     readPipeline(cpath, stat.size, sri, stream)
   }, er => stream.emit('error', er))
 
@@ -109,22 +106,21 @@ function copySync (cache, integrity, dest) {
 module.exports.hasContent = hasContent
 
 function hasContent (cache, integrity) {
-  if (!integrity) {
+  if (!integrity)
     return Promise.resolve(false)
-  }
+
   return withContentSri(cache, integrity, (cpath, sri) => {
     return lstat(cpath).then((stat) => ({ size: stat.size, sri, stat }))
   }).catch((err) => {
-    if (err.code === 'ENOENT') {
+    if (err.code === 'ENOENT')
       return false
-    }
+
     if (err.code === 'EPERM') {
       /* istanbul ignore else */
-      if (process.platform !== 'win32') {
+      if (process.platform !== 'win32')
         throw err
-      } else {
+      else
         return false
-      }
     }
   })
 }
@@ -132,24 +128,23 @@ function hasContent (cache, integrity) {
 module.exports.hasContent.sync = hasContentSync
 
 function hasContentSync (cache, integrity) {
-  if (!integrity) {
+  if (!integrity)
     return false
-  }
+
   return withContentSriSync(cache, integrity, (cpath, sri) => {
     try {
       const stat = fs.lstatSync(cpath)
       return { size: stat.size, sri, stat }
     } catch (err) {
-      if (err.code === 'ENOENT') {
+      if (err.code === 'ENOENT')
         return false
-      }
+
       if (err.code === 'EPERM') {
         /* istanbul ignore else */
-        if (process.platform !== 'win32') {
+        if (process.platform !== 'win32')
           throw err
-        } else {
+        else
           return false
-        }
       }
     }
   })
@@ -167,7 +162,8 @@ function withContentSri (cache, integrity, fn) {
       const cpath = contentPath(cache, digests[0])
       return fn(cpath, digests[0])
     } else {
-      // Can't use race here because a generic error can happen before a ENOENT error, and can happen before a valid result
+      // Can't use race here because a generic error can happen before
+      // a ENOENT error, and can happen before a valid result
       return Promise
         .all(digests.map((meta) => {
           return withContentSri(cache, meta, fn)
@@ -184,15 +180,13 @@ function withContentSri (cache, integrity, fn) {
         .then((results) => {
           // Return the first non error if it is found
           const result = results.find((r) => !(r instanceof Error))
-          if (result) {
+          if (result)
             return result
-          }
 
           // Throw the No matching content found error
           const enoentError = results.find((r) => r.code === 'ENOENT')
-          if (enoentError) {
+          if (enoentError)
             throw enoentError
-          }
 
           // Throw generic error
           throw results.find((r) => r instanceof Error)

package/lib/content/rm.js

@@ -11,10 +11,9 @@ module.exports = rm
 function rm (cache, integrity) {
   return hasContent(cache, integrity).then((content) => {
     // ~pretty~ sure we can't end up with a content lacking sri, but be safe
-    if (content && content.sri) {
+    if (content && content.sri)
       return rimraf(contentPath(cache, content.sri)).then(() => true)
-    } else {
+    else
       return false
-    }
   })
 }

package/lib/content/write.js

@@ -22,16 +22,15 @@ module.exports = write
 
 function write (cache, data, opts = {}) {
   const { algorithms, size, integrity } = opts
-  if (algorithms && algorithms.length > 1) {
+  if (algorithms && algorithms.length > 1)
     throw new Error('opts.algorithms only supports a single algorithm for now')
-  }
-  if (typeof size === 'number' && data.length !== size) {
+
+  if (typeof size === 'number' && data.length !== size)
     return Promise.reject(sizeError(size, data.length))
-  }
+
   const sri = ssri.fromData(data, algorithms ? { algorithms } : {})
-  if (integrity && !ssri.checkData(data, integrity, opts)) {
+  if (integrity && !ssri.checkData(data, integrity, opts))
     return Promise.reject(checksumError(integrity, sri))
-  }
 
   return disposer(makeTmp(cache, opts), makeTmpDisposer,
     (tmp) => {
@@ -112,13 +111,17 @@ function pipeToTmp (inputStream, cache, tmpTarget, opts) {
   const hashStream = ssri.integrityStream({
     integrity: opts.integrity,
     algorithms: opts.algorithms,
-    size: opts.size
+    size: opts.size,
+  })
+  hashStream.on('integrity', i => {
+    integrity = i
+  })
+  hashStream.on('size', s => {
+    size = s
   })
-  hashStream.on('integrity', i => { integrity = i })
-  hashStream.on('size', s => { size = s })
 
   const outStream = new fsm.WriteStream(tmpTarget, {
-    flags: 'wx'
+    flags: 'wx',
   })
 
   // NB: this can throw if the hashStream has a problem with
@@ -132,21 +135,23 @@ function pipeToTmp (inputStream, cache, tmpTarget, opts) {
 
   return pipeline.promise()
     .then(() => ({ integrity, size }))
-    .catch(er => rimraf(tmpTarget).then(() => { throw er }))
+    .catch(er => rimraf(tmpTarget).then(() => {
+      throw er
+    }))
 }
 
 function makeTmp (cache, opts) {
   const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
   return fixOwner.mkdirfix(cache, path.dirname(tmpTarget)).then(() => ({
     target: tmpTarget,
-    moved: false
+    moved: false,
   }))
 }
 
 function makeTmpDisposer (tmp) {
-  if (tmp.moved) {
+  if (tmp.moved)
     return Promise.resolve()
-  }
+
   return rimraf(tmp.target)
 }