cacache 16.0.7 → 16.1.0

package/README.md

@@ -413,6 +413,19 @@ with an `EINTEGRITY` error.
 
 `algorithms` has no effect if this option is present.
 
+##### `opts.integrityEmitter`
+
+*Streaming only* If present, uses the provided event emitter as a source of
+truth for both integrity and size. This allows use cases where integrity is
+already being calculated outside of cacache to reuse that data instead of
+calculating it a second time.
+
+The emitter must emit both the `'integrity'` and `'size'` events.
+
+NOTE: If this option is provided, you must verify that you receive the correct
+integrity value yourself and emit an `'error'` event if there is a mismatch.
+[ssri Integrity Streams](https://github.com/npm/ssri#integrity-stream) do this for you when given an expected integrity.
+
 ##### `opts.algorithms`
 
 Default: ['sha512']

package/lib/content/read.js

@@ -1,42 +1,35 @@
 'use strict'
 
-const util = require('util')
-
-const fs = require('fs')
+const fs = require('@npmcli/fs')
 const fsm = require('fs-minipass')
 const ssri = require('ssri')
 const contentPath = require('./path')
 const Pipeline = require('minipass-pipeline')
 
-const lstat = util.promisify(fs.lstat)
-const readFile = util.promisify(fs.readFile)
-const copyFile = util.promisify(fs.copyFile)
-
 module.exports = read
 
 const MAX_SINGLE_READ_SIZE = 64 * 1024 * 1024
-function read (cache, integrity, opts = {}) {
+async function read (cache, integrity, opts = {}) {
   const { size } = opts
-  return withContentSri(cache, integrity, (cpath, sri) => {
+  const { stat, cpath, sri } = await withContentSri(cache, integrity, async (cpath, sri) => {
     // get size
-    return lstat(cpath).then(stat => ({ stat, cpath, sri }))
-  }).then(({ stat, cpath, sri }) => {
-    if (typeof size === 'number' && stat.size !== size) {
-      throw sizeError(size, stat.size)
-    }
+    const stat = await fs.lstat(cpath)
+    return { stat, cpath, sri }
+  })
+  if (typeof size === 'number' && stat.size !== size) {
+    throw sizeError(size, stat.size)
+  }
 
-    if (stat.size > MAX_SINGLE_READ_SIZE) {
-      return readPipeline(cpath, stat.size, sri, new Pipeline()).concat()
-    }
+  if (stat.size > MAX_SINGLE_READ_SIZE) {
+    return readPipeline(cpath, stat.size, sri, new Pipeline()).concat()
+  }
 
-    return readFile(cpath, null).then((data) => {
-      if (!ssri.checkData(data, sri)) {
-        throw integrityError(sri, cpath)
-      }
+  const data = await fs.readFile(cpath, { encoding: null })
+  if (!ssri.checkData(data, sri)) {
+    throw integrityError(sri, cpath)
+  }
 
-      return data
-    })
-  })
+  return data
 }
 
 const readPipeline = (cpath, size, sri, stream) => {
@@ -58,7 +51,7 @@ module.exports.sync = readSync
 function readSync (cache, integrity, opts = {}) {
   const { size } = opts
   return withContentSriSync(cache, integrity, (cpath, sri) => {
-    const data = fs.readFileSync(cpath)
+    const data = fs.readFileSync(cpath, { encoding: null })
     if (typeof size === 'number' && size !== data.length) {
       throw sizeError(size, data.length)
     }
@@ -77,16 +70,19 @@ module.exports.readStream = readStream
 function readStream (cache, integrity, opts = {}) {
   const { size } = opts
   const stream = new Pipeline()
-  withContentSri(cache, integrity, (cpath, sri) => {
-    // just lstat to ensure it exists
-    return lstat(cpath).then((stat) => ({ stat, cpath, sri }))
-  }).then(({ stat, cpath, sri }) => {
+  // Set all this up to run on the stream and then just return the stream
+  Promise.resolve().then(async () => {
+    const { stat, cpath, sri } = await withContentSri(cache, integrity, async (cpath, sri) => {
+      // just lstat to ensure it exists
+      const stat = await fs.lstat(cpath)
+      return { stat, cpath, sri }
+    })
     if (typeof size === 'number' && size !== stat.size) {
       return stream.emit('error', sizeError(size, stat.size))
     }
 
     readPipeline(cpath, stat.size, sri, stream)
-  }, er => stream.emit('error', er))
+  }).catch(err => stream.emit('error', err))
 
   return stream
 }
@@ -96,7 +92,7 @@ module.exports.copy.sync = copySync
 
 function copy (cache, integrity, dest) {
   return withContentSri(cache, integrity, (cpath, sri) => {
-    return copyFile(cpath, dest)
+    return fs.copyFile(cpath, dest)
   })
 }
 
@@ -108,14 +104,17 @@ function copySync (cache, integrity, dest) {
 
 module.exports.hasContent = hasContent
 
-function hasContent (cache, integrity) {
+async function hasContent (cache, integrity) {
   if (!integrity) {
-    return Promise.resolve(false)
+    return false
   }
 
-  return withContentSri(cache, integrity, (cpath, sri) => {
-    return lstat(cpath).then((stat) => ({ size: stat.size, sri, stat }))
-  }).catch((err) => {
+  try {
+    return await withContentSri(cache, integrity, async (cpath, sri) => {
+      const stat = await fs.lstat(cpath)
+      return { size: stat.size, sri, stat }
+    })
+  } catch (err) {
     if (err.code === 'ENOENT') {
       return false
     }
@@ -128,7 +127,7 @@ function hasContent (cache, integrity) {
         return false
       }
     }
-  })
+  }
 }
 
 module.exports.hasContent.sync = hasContentSync
@@ -159,61 +158,47 @@ function hasContentSync (cache, integrity) {
   })
 }
 
-function withContentSri (cache, integrity, fn) {
-  const tryFn = () => {
-    const sri = ssri.parse(integrity)
-    // If `integrity` has multiple entries, pick the first digest
-    // with available local data.
-    const algo = sri.pickAlgorithm()
-    const digests = sri[algo]
-
-    if (digests.length <= 1) {
-      const cpath = contentPath(cache, digests[0])
-      return fn(cpath, digests[0])
-    } else {
-      // Can't use race here because a generic error can happen before
-      // a ENOENT error, and can happen before a valid result
-      return Promise
-        .all(digests.map((meta) => {
-          return withContentSri(cache, meta, fn)
-            .catch((err) => {
-              if (err.code === 'ENOENT') {
-                return Object.assign(
-                  new Error('No matching content found for ' + sri.toString()),
-                  { code: 'ENOENT' }
-                )
-              }
-              return err
-            })
-        }))
-        .then((results) => {
-          // Return the first non error if it is found
-          const result = results.find((r) => !(r instanceof Error))
-          if (result) {
-            return result
-          }
-
-          // Throw the No matching content found error
-          const enoentError = results.find((r) => r.code === 'ENOENT')
-          if (enoentError) {
-            throw enoentError
-          }
-
-          // Throw generic error
-          throw results.find((r) => r instanceof Error)
-        })
+async function withContentSri (cache, integrity, fn) {
+  const sri = ssri.parse(integrity)
+  // If `integrity` has multiple entries, pick the first digest
+  // with available local data.
+  const algo = sri.pickAlgorithm()
+  const digests = sri[algo]
+
+  if (digests.length <= 1) {
+    const cpath = contentPath(cache, digests[0])
+    return fn(cpath, digests[0])
+  } else {
+    // Can't use race here because a generic error can happen before
+    // a ENOENT error, and can happen before a valid result
+    const results = await Promise.all(digests.map(async (meta) => {
+      try {
+        return await withContentSri(cache, meta, fn)
+      } catch (err) {
+        if (err.code === 'ENOENT') {
+          return Object.assign(
+            new Error('No matching content found for ' + sri.toString()),
+            { code: 'ENOENT' }
+          )
+        }
+        return err
+      }
+    }))
+    // Return the first non error if it is found
+    const result = results.find((r) => !(r instanceof Error))
+    if (result) {
+      return result
     }
-  }
 
-  return new Promise((resolve, reject) => {
-    try {
-      tryFn()
-        .then(resolve)
-        .catch(reject)
-    } catch (err) {
-      reject(err)
+    // Throw the No matching content found error
+    const enoentError = results.find((r) => r.code === 'ENOENT')
+    if (enoentError) {
+      throw enoentError
     }
-  })
+
+    // Throw generic error
+    throw results.find((r) => r instanceof Error)
+  }
 }
 
 function withContentSriSync (cache, integrity, fn) {

package/lib/content/rm.js

@@ -8,13 +8,13 @@ const rimraf = util.promisify(require('rimraf'))
 
 module.exports = rm
 
-function rm (cache, integrity) {
-  return hasContent(cache, integrity).then((content) => {
-    // ~pretty~ sure we can't end up with a content lacking sri, but be safe
-    if (content && content.sri) {
-      return rimraf(contentPath(cache, content.sri)).then(() => true)
-    } else {
-      return false
-    }
-  })
+async function rm (cache, integrity) {
+  const content = await hasContent(cache, integrity)
+  // ~pretty~ sure we can't end up with a content lacking sri, but be safe
+  if (content && content.sri) {
+    await rimraf(contentPath(cache, content.sri))
+    return true
+  } else {
+    return false
+  }
 }

package/lib/content/write.js

@@ -1,10 +1,11 @@
 'use strict'
 
+const events = require('events')
 const util = require('util')
 
 const contentPath = require('./path')
 const fixOwner = require('../util/fix-owner')
-const fs = require('fs')
+const fs = require('@npmcli/fs')
 const moveFile = require('../util/move-file')
 const Minipass = require('minipass')
 const Pipeline = require('minipass-pipeline')
@@ -15,8 +16,6 @@ const ssri = require('ssri')
 const uniqueFilename = require('unique-filename')
 const fsm = require('fs-minipass')
 
-const writeFile = util.promisify(fs.writeFile)
-
 module.exports = write
 
 async function write (cache, data, opts = {}) {
@@ -36,7 +35,7 @@ async function write (cache, data, opts = {}) {
 
   const tmp = await makeTmp(cache, opts)
   try {
-    await writeFile(tmp.target, data, { flag: 'wx' })
+    await fs.writeFile(tmp.target, data, { flag: 'wx' })
     await moveToDestination(tmp, cache, sri, opts)
     return { integrity: sri, size: data.length }
   } finally {
@@ -115,7 +114,21 @@ async function handleContent (inputStream, cache, opts) {
   }
 }
 
-function pipeToTmp (inputStream, cache, tmpTarget, opts) {
+async function pipeToTmp (inputStream, cache, tmpTarget, opts) {
+  const outStream = new fsm.WriteStream(tmpTarget, {
+    flags: 'wx',
+  })
+
+  if (opts.integrityEmitter) {
+    // we need to create these all simultaneously since they can fire in any order
+    const [integrity, size] = await Promise.all([
+      events.once(opts.integrityEmitter, 'integrity').then(res => res[0]),
+      events.once(opts.integrityEmitter, 'size').then(res => res[0]),
+      new Pipeline(inputStream, outStream).promise(),
+    ])
+    return { integrity, size }
+  }
+
   let integrity
   let size
   const hashStream = ssri.integrityStream({
@@ -130,43 +143,28 @@ function pipeToTmp (inputStream, cache, tmpTarget, opts) {
     size = s
   })
 
-  const outStream = new fsm.WriteStream(tmpTarget, {
-    flags: 'wx',
-  })
-
-  // NB: this can throw if the hashStream has a problem with
-  // it, and the data is fully written.  but pipeToTmp is only
-  // called in promisory contexts where that is handled.
-  const pipeline = new Pipeline(
-    inputStream,
-    hashStream,
-    outStream
-  )
-
-  return pipeline.promise().then(() => ({ integrity, size }))
+  const pipeline = new Pipeline(inputStream, hashStream, outStream)
+  await pipeline.promise()
+  return { integrity, size }
 }
 
-function makeTmp (cache, opts) {
+async function makeTmp (cache, opts) {
   const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
-  return fixOwner.mkdirfix(cache, path.dirname(tmpTarget)).then(() => ({
+  await fixOwner.mkdirfix(cache, path.dirname(tmpTarget))
+  return {
     target: tmpTarget,
     moved: false,
-  }))
+  }
 }
 
-function moveToDestination (tmp, cache, sri, opts) {
+async function moveToDestination (tmp, cache, sri, opts) {
   const destination = contentPath(cache, sri)
   const destDir = path.dirname(destination)
 
-  return fixOwner
-    .mkdirfix(cache, destDir)
-    .then(() => {
-      return moveFile(tmp.target, destination)
-    })
-    .then(() => {
-      tmp.moved = true
-      return fixOwner.chownr(cache, destination)
-    })
+  await fixOwner.mkdirfix(cache, destDir)
+  await moveFile(tmp.target, destination)
+  tmp.moved = true
+  await fixOwner.chownr(cache, destination)
 }
 
 function sizeError (expected, found) {

package/lib/entry-index.js

@@ -2,7 +2,7 @@
 
 const util = require('util')
 const crypto = require('crypto')
-const fs = require('fs')
+const fs = require('@npmcli/fs')
 const Minipass = require('minipass')
 const path = require('path')
 const ssri = require('ssri')
@@ -17,11 +17,6 @@ const _rimraf = require('rimraf')
 const rimraf = util.promisify(_rimraf)
 rimraf.sync = _rimraf.sync
 
-const appendFile = util.promisify(fs.appendFile)
-const readFile = util.promisify(fs.readFile)
-const readdir = util.promisify(fs.readdir)
-const writeFile = util.promisify(fs.writeFile)
-
 module.exports.NotFoundError = class NotFoundError extends Error {
   constructor (cache, key) {
     super(`No cache entry for ${key} found in ${cache}`)
@@ -85,7 +80,7 @@ async function compact (cache, key, matchFn, opts = {}) {
   }
 
   const write = async (tmp) => {
-    await writeFile(tmp.target, newIndex, { flag: 'wx' })
+    await fs.writeFile(tmp.target, newIndex, { flag: 'wx' })
     await fixOwner.mkdirfix(cache, path.dirname(bucket))
     // we use @npmcli/move-file directly here because we
     // want to overwrite the existing file
@@ -118,7 +113,7 @@ async function compact (cache, key, matchFn, opts = {}) {
 
 module.exports.insert = insert
 
-function insert (cache, key, integrity, opts = {}) {
+async function insert (cache, key, integrity, opts = {}) {
   const { metadata, size } = opts
   const bucket = bucketPath(cache, key)
   const entry = {
@@ -128,36 +123,32 @@ function insert (cache, key, integrity, opts = {}) {
     size,
     metadata,
   }
-  return fixOwner
-    .mkdirfix(cache, path.dirname(bucket))
-    .then(() => {
-      const stringified = JSON.stringify(entry)
-      // NOTE - Cleverness ahoy!
-      //
-      // This works because it's tremendously unlikely for an entry to corrupt
-      // another while still preserving the string length of the JSON in
-      // question. So, we just slap the length in there and verify it on read.
-      //
-      // Thanks to @isaacs for the whiteboarding session that ended up with
-      // this.
-      return appendFile(bucket, `\n${hashEntry(stringified)}\t${stringified}`)
-    })
-    .then(() => fixOwner.chownr(cache, bucket))
-    .catch((err) => {
-      if (err.code === 'ENOENT') {
-        return undefined
-      }
+  try {
+    await fixOwner.mkdirfix(cache, path.dirname(bucket))
+    const stringified = JSON.stringify(entry)
+    // NOTE - Cleverness ahoy!
+    //
+    // This works because it's tremendously unlikely for an entry to corrupt
+    // another while still preserving the string length of the JSON in
+    // question. So, we just slap the length in there and verify it on read.
+    //
+    // Thanks to @isaacs for the whiteboarding session that ended up with
+    // this.
+    await fs.appendFile(bucket, `\n${hashEntry(stringified)}\t${stringified}`)
+    await fixOwner.chownr(cache, bucket)
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return undefined
+    }
 
-      throw err
-      // There's a class of race conditions that happen when things get deleted
-      // during fixOwner, or between the two mkdirfix/chownr calls.
-      //
-      // It's perfectly fine to just not bother in those cases and lie
-      // that the index entry was written. Because it's a cache.
-    })
-    .then(() => {
-      return formatEntry(cache, entry)
-    })
+    throw err
+    // There's a class of race conditions that happen when things get deleted
+    // during fixOwner, or between the two mkdirfix/chownr calls.
+    //
+    // It's perfectly fine to just not bother in those cases and lie
+    // that the index entry was written. Because it's a cache.
+  }
+  return formatEntry(cache, entry)
 }
 
 module.exports.insert.sync = insertSync
@@ -187,25 +178,24 @@ function insertSync (cache, key, integrity, opts = {}) {
 
 module.exports.find = find
 
-function find (cache, key) {
+async function find (cache, key) {
   const bucket = bucketPath(cache, key)
-  return bucketEntries(bucket)
-    .then((entries) => {
-      return entries.reduce((latest, next) => {
-        if (next && next.key === key) {
-          return formatEntry(cache, next)
-        } else {
-          return latest
-        }
-      }, null)
-    })
-    .catch((err) => {
-      if (err.code === 'ENOENT') {
-        return null
+  try {
+    const entries = await bucketEntries(bucket)
+    return entries.reduce((latest, next) => {
+      if (next && next.key === key) {
+        return formatEntry(cache, next)
       } else {
-        throw err
+        return latest
       }
-    })
+    }, null)
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return null
+    } else {
+      throw err
+    }
+  }
 }
 
 module.exports.find.sync = findSync
@@ -257,67 +247,64 @@ function lsStream (cache) {
   const indexDir = bucketDir(cache)
   const stream = new Minipass({ objectMode: true })
 
-  readdirOrEmpty(indexDir).then(buckets => Promise.all(
-    buckets.map(bucket => {
+  // Set all this up to run on the stream and then just return the stream
+  Promise.resolve().then(async () => {
+    const buckets = await readdirOrEmpty(indexDir)
+    await Promise.all(buckets.map(async (bucket) => {
       const bucketPath = path.join(indexDir, bucket)
-      return readdirOrEmpty(bucketPath).then(subbuckets => Promise.all(
-        subbuckets.map(subbucket => {
-          const subbucketPath = path.join(bucketPath, subbucket)
-
-          // "/cachename/<bucket 0xFF>/<bucket 0xFF>./*"
-          return readdirOrEmpty(subbucketPath).then(entries => Promise.all(
-            entries.map(entry => {
-              const entryPath = path.join(subbucketPath, entry)
-              return bucketEntries(entryPath).then(entries =>
-                // using a Map here prevents duplicate keys from
-                // showing up twice, I guess?
-                entries.reduce((acc, entry) => {
-                  acc.set(entry.key, entry)
-                  return acc
-                }, new Map())
-              ).then(reduced => {
-                // reduced is a map of key => entry
-                for (const entry of reduced.values()) {
-                  const formatted = formatEntry(cache, entry)
-                  if (formatted) {
-                    stream.write(formatted)
-                  }
-                }
-              }).catch(err => {
-                if (err.code === 'ENOENT') {
-                  return undefined
-                }
-                throw err
-              })
-            })
-          ))
-        })
-      ))
-    })
-  ))
-    .then(
-      () => stream.end(),
-      err => stream.emit('error', err)
-    )
+      const subbuckets = await readdirOrEmpty(bucketPath)
+      await Promise.all(subbuckets.map(async (subbucket) => {
+        const subbucketPath = path.join(bucketPath, subbucket)
+
+        // "/cachename/<bucket 0xFF>/<bucket 0xFF>./*"
+        const subbucketEntries = await readdirOrEmpty(subbucketPath)
+        await Promise.all(subbucketEntries.map(async (entry) => {
+          const entryPath = path.join(subbucketPath, entry)
+          try {
+            const entries = await bucketEntries(entryPath)
+            // using a Map here prevents duplicate keys from showing up
+            // twice, I guess?
+            const reduced = entries.reduce((acc, entry) => {
+              acc.set(entry.key, entry)
+              return acc
+            }, new Map())
+            // reduced is a map of key => entry
+            for (const entry of reduced.values()) {
+              const formatted = formatEntry(cache, entry)
+              if (formatted) {
+                stream.write(formatted)
+              }
+            }
+          } catch (err) {
+            if (err.code === 'ENOENT') {
+              return undefined
+            }
+            throw err
+          }
+        }))
+      }))
+    }))
+    stream.end()
+  }).catch(err => stream.emit('error', err))
 
   return stream
 }
 
 module.exports.ls = ls
 
-function ls (cache) {
-  return lsStream(cache).collect().then(entries =>
-    entries.reduce((acc, xs) => {
-      acc[xs.key] = xs
-      return acc
-    }, {})
-  )
+async function ls (cache) {
+  const entries = await lsStream(cache).collect()
+  return entries.reduce((acc, xs) => {
+    acc[xs.key] = xs
+    return acc
+  }, {})
 }
 
 module.exports.bucketEntries = bucketEntries
 
-function bucketEntries (bucket, filter) {
-  return readFile(bucket, 'utf8').then((data) => _bucketEntries(data, filter))
+async function bucketEntries (bucket, filter) {
+  const data = await fs.readFile(bucket, 'utf8')
+  return _bucketEntries(data, filter)
 }
 
 module.exports.bucketEntries.sync = bucketEntriesSync
@@ -406,7 +393,7 @@ function formatEntry (cache, entry, keepAll) {
 }
 
 function readdirOrEmpty (dir) {
-  return readdir(dir).catch((err) => {
+  return fs.readdir(dir).catch((err) => {
     if (err.code === 'ENOENT' || err.code === 'ENOTDIR') {
       return []
     }