cacache 15.0.5 → 15.3.0

package/lib/entry-index.js

@@ -1,20 +1,27 @@
 'use strict'
 
 const util = require('util')
-
 const crypto = require('crypto')
 const fs = require('fs')
 const Minipass = require('minipass')
 const path = require('path')
 const ssri = require('ssri')
+const uniqueFilename = require('unique-filename')
+
+const { disposer } = require('./util/disposer')
 const contentPath = require('./content/path')
 const fixOwner = require('./util/fix-owner')
 const hashToSegments = require('./util/hash-to-segments')
 const indexV = require('../package.json')['cache-version'].index
+const moveFile = require('@npmcli/move-file')
+const _rimraf = require('rimraf')
+const rimraf = util.promisify(_rimraf)
+rimraf.sync = _rimraf.sync
 
 const appendFile = util.promisify(fs.appendFile)
 const readFile = util.promisify(fs.readFile)
 const readdir = util.promisify(fs.readdir)
+const writeFile = util.promisify(fs.writeFile)
 
 module.exports.NotFoundError = class NotFoundError extends Error {
   constructor (cache, key) {
@@ -25,6 +32,82 @@ module.exports.NotFoundError = class NotFoundError extends Error {
   }
 }
 
+module.exports.compact = compact
+
+async function compact (cache, key, matchFn, opts = {}) {
+  const bucket = bucketPath(cache, key)
+  const entries = await bucketEntries(bucket)
+  const newEntries = []
+  // we loop backwards because the bottom-most result is the newest
+  // since we add new entries with appendFile
+  for (let i = entries.length - 1; i >= 0; --i) {
+    const entry = entries[i]
+    // a null integrity could mean either a delete was appended
+    // or the user has simply stored an index that does not map
+    // to any content. we determine if the user wants to keep the
+    // null integrity based on the validateEntry function passed in options.
+    // if the integrity is null and no validateEntry is provided, we break
+    // as we consider the null integrity to be a deletion of everything
+    // that came before it.
+    if (entry.integrity === null && !opts.validateEntry)
+      break
+
+    // if this entry is valid, and it is either the first entry or
+    // the newEntries array doesn't already include an entry that
+    // matches this one based on the provided matchFn, then we add
+    // it to the beginning of our list
+    if ((!opts.validateEntry || opts.validateEntry(entry) === true) &&
+      (newEntries.length === 0 ||
+        !newEntries.find((oldEntry) => matchFn(oldEntry, entry))))
+      newEntries.unshift(entry)
+  }
+
+  const newIndex = '\n' + newEntries.map((entry) => {
+    const stringified = JSON.stringify(entry)
+    const hash = hashEntry(stringified)
+    return `${hash}\t${stringified}`
+  }).join('\n')
+
+  const setup = async () => {
+    const target = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
+    await fixOwner.mkdirfix(cache, path.dirname(target))
+    return {
+      target,
+      moved: false,
+    }
+  }
+
+  const teardown = async (tmp) => {
+    if (!tmp.moved)
+      return rimraf(tmp.target)
+  }
+
+  const write = async (tmp) => {
+    await writeFile(tmp.target, newIndex, { flag: 'wx' })
+    await fixOwner.mkdirfix(cache, path.dirname(bucket))
+    // we use @npmcli/move-file directly here because we
+    // want to overwrite the existing file
+    await moveFile(tmp.target, bucket)
+    tmp.moved = true
+    try {
+      await fixOwner.chownr(cache, bucket)
+    } catch (err) {
+      if (err.code !== 'ENOENT')
+        throw err
+    }
+  }
+
+  // write the file atomically
+  await disposer(setup(), teardown, write)
+
+  // we reverse the list we generated such that the newest
+  // entries come first in order to make looping through them easier
+  // the true passed to formatEntry tells it to keep null
+  // integrity values, if they made it this far it's because
+  // validateEntry returned true, and as such we should return it
+  return newEntries.reverse().map((entry) => formatEntry(cache, entry, true))
+}
+
 module.exports.insert = insert
 
 function insert (cache, key, integrity, opts = {}) {
@@ -35,7 +118,7 @@ function insert (cache, key, integrity, opts = {}) {
     integrity: integrity && ssri.stringify(integrity),
     time: Date.now(),
     size,
-    metadata
+    metadata,
   }
   return fixOwner
     .mkdirfix(cache, path.dirname(bucket))
@@ -47,14 +130,15 @@ function insert (cache, key, integrity, opts = {}) {
       // another while still preserving the string length of the JSON in
       // question. So, we just slap the length in there and verify it on read.
       //
-      // Thanks to @isaacs for the whiteboarding session that ended up with this.
+      // Thanks to @isaacs for the whiteboarding session that ended up with
+      // this.
       return appendFile(bucket, `\n${hashEntry(stringified)}\t${stringified}`)
     })
     .then(() => fixOwner.chownr(cache, bucket))
     .catch((err) => {
-      if (err.code === 'ENOENT') {
+      if (err.code === 'ENOENT')
         return undefined
-      }
+
       throw err
       // There's a class of race conditions that happen when things get deleted
       // during fixOwner, or between the two mkdirfix/chownr calls.
@@ -77,7 +161,7 @@ function insertSync (cache, key, integrity, opts = {}) {
     integrity: integrity && ssri.stringify(integrity),
     time: Date.now(),
     size,
-    metadata
+    metadata,
   }
   fixOwner.mkdirfix.sync(cache, path.dirname(bucket))
   const stringified = JSON.stringify(entry)
@@ -85,9 +169,8 @@ function insertSync (cache, key, integrity, opts = {}) {
   try {
     fixOwner.chownr.sync(cache, bucket)
   } catch (err) {
-    if (err.code !== 'ENOENT') {
+    if (err.code !== 'ENOENT')
       throw err
-    }
   }
   return formatEntry(cache, entry)
 }
@@ -99,19 +182,17 @@ function find (cache, key) {
   return bucketEntries(bucket)
     .then((entries) => {
       return entries.reduce((latest, next) => {
-        if (next && next.key === key) {
+        if (next && next.key === key)
           return formatEntry(cache, next)
-        } else {
+        else
           return latest
-        }
       }, null)
     })
     .catch((err) => {
-      if (err.code === 'ENOENT') {
+      if (err.code === 'ENOENT')
         return null
-      } else {
+      else
         throw err
-      }
     })
 }
 
@@ -121,31 +202,37 @@ function findSync (cache, key) {
   const bucket = bucketPath(cache, key)
   try {
     return bucketEntriesSync(bucket).reduce((latest, next) => {
-      if (next && next.key === key) {
+      if (next && next.key === key)
         return formatEntry(cache, next)
-      } else {
+      else
         return latest
-      }
     }, null)
   } catch (err) {
-    if (err.code === 'ENOENT') {
+    if (err.code === 'ENOENT')
       return null
-    } else {
+    else
       throw err
-    }
   }
 }
 
 module.exports.delete = del
 
-function del (cache, key, opts) {
-  return insert(cache, key, null, opts)
+function del (cache, key, opts = {}) {
+  if (!opts.removeFully)
+    return insert(cache, key, null, opts)
+
+  const bucket = bucketPath(cache, key)
+  return rimraf(bucket)
 }
 
 module.exports.delete.sync = delSync
 
-function delSync (cache, key, opts) {
-  return insertSync(cache, key, null, opts)
+function delSync (cache, key, opts = {}) {
+  if (!opts.removeFully)
+    return insertSync(cache, key, null, opts)
+
+  const bucket = bucketPath(cache, key)
+  return rimraf.sync(bucket)
 }
 
 module.exports.lsStream = lsStream
@@ -176,12 +263,12 @@ function lsStream (cache) {
                 // reduced is a map of key => entry
                 for (const entry of reduced.values()) {
                   const formatted = formatEntry(cache, entry)
-                  if (formatted) {
+                  if (formatted)
                     stream.write(formatted)
-                  }
                 }
               }).catch(err => {
-                if (err.code === 'ENOENT') { return undefined }
+                if (err.code === 'ENOENT')
+                  return undefined
                 throw err
               })
             })
@@ -209,10 +296,14 @@ function ls (cache) {
   )
 }
 
+module.exports.bucketEntries = bucketEntries
+
 function bucketEntries (bucket, filter) {
   return readFile(bucket, 'utf8').then((data) => _bucketEntries(data, filter))
 }
 
+module.exports.bucketEntries.sync = bucketEntriesSync
+
 function bucketEntriesSync (bucket, filter) {
   const data = fs.readFileSync(bucket, 'utf8')
   return _bucketEntries(data, filter)
@@ -221,9 +312,9 @@ function bucketEntriesSync (bucket, filter) {
 function _bucketEntries (data, filter) {
   const entries = []
   data.split('\n').forEach((entry) => {
-    if (!entry) {
+    if (!entry)
       return
-    }
+
     const pieces = entry.split('\t')
     if (!pieces[1] || hashEntry(pieces[1]) !== pieces[0]) {
       // Hash is no good! Corruption or malice? Doesn't matter!
@@ -237,9 +328,8 @@ function _bucketEntries (data, filter) {
       // Entry is corrupted!
       return
     }
-    if (obj) {
+    if (obj)
       entries.push(obj)
-    }
   })
   return entries
 }
@@ -279,26 +369,25 @@ function hash (str, digest) {
     .digest('hex')
 }
 
-function formatEntry (cache, entry) {
+function formatEntry (cache, entry, keepAll) {
   // Treat null digests as deletions. They'll shadow any previous entries.
-  if (!entry.integrity) {
+  if (!entry.integrity && !keepAll)
     return null
-  }
+
   return {
     key: entry.key,
     integrity: entry.integrity,
-    path: contentPath(cache, entry.integrity),
+    path: entry.integrity ? contentPath(cache, entry.integrity) : undefined,
     size: entry.size,
     time: entry.time,
-    metadata: entry.metadata
+    metadata: entry.metadata,
   }
 }
 
 function readdirOrEmpty (dir) {
   return readdir(dir).catch((err) => {
-    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') {
+    if (err.code === 'ENOENT' || err.code === 'ENOTDIR')
       return []
-    }
 
     throw err
   })

package/lib/memoization.js

@@ -8,7 +8,7 @@ const MAX_AGE = 3 * 60 * 1000
 const MEMOIZED = new LRU({
   max: MAX_SIZE,
   maxAge: MAX_AGE,
-  length: (entry, key) => key.startsWith('key:') ? entry.data.length : entry.length
+  length: (entry, key) => key.startsWith('key:') ? entry.data.length : entry.length,
 })
 
 module.exports.clearMemoized = clearMemoized
@@ -62,13 +62,12 @@ class ObjProxy {
 }
 
 function pickMem (opts) {
-  if (!opts || !opts.memoize) {
+  if (!opts || !opts.memoize)
     return MEMOIZED
-  } else if (opts.memoize.get && opts.memoize.set) {
+  else if (opts.memoize.get && opts.memoize.set)
     return opts.memoize
-  } else if (typeof opts.memoize === 'object') {
+  else if (typeof opts.memoize === 'object')
     return new ObjProxy(opts.memoize)
-  } else {
+  else
     return MEMOIZED
-  }
 }

package/lib/util/disposer.js

@@ -8,9 +8,9 @@ function disposer (creatorFn, disposerFn, fn) {
       .then(
         // disposer resolved, do something with original fn's promise
         () => {
-          if (shouldThrow) {
+          if (shouldThrow)
             throw result
-          }
+
           return result
         },
         // Disposer fn failed, crash process

package/lib/util/fix-owner.js

@@ -49,9 +49,8 @@ function fixOwner (cache, filepath) {
     const { uid, gid } = owner
 
     // No need to override if it's already what we used.
-    if (self.uid === uid && self.gid === gid) {
+    if (self.uid === uid && self.gid === gid)
       return
-    }
 
     return inflight('fixOwner: fixing ownership on ' + filepath, () =>
       chownr(
@@ -59,9 +58,9 @@ function fixOwner (cache, filepath) {
         typeof uid === 'number' ? uid : self.uid,
         typeof gid === 'number' ? gid : self.gid
       ).catch((err) => {
-        if (err.code === 'ENOENT') {
+        if (err.code === 'ENOENT')
           return null
-        }
+
         throw err
       })
     )
@@ -94,9 +93,9 @@ function fixOwnerSync (cache, filepath) {
     )
   } catch (err) {
     // only catch ENOENT, any other error is a problem.
-    if (err.code === 'ENOENT') {
+    if (err.code === 'ENOENT')
       return null
-    }
+
     throw err
   }
 }
@@ -111,14 +110,13 @@ function mkdirfix (cache, p, cb) {
   return Promise.resolve(inferOwner(cache)).then(() => {
     return mkdirp(p)
       .then((made) => {
-        if (made) {
+        if (made)
           return fixOwner(cache, made).then(() => made)
-        }
       })
       .catch((err) => {
-        if (err.code === 'EEXIST') {
+        if (err.code === 'EEXIST')
           return fixOwner(cache, p).then(() => null)
-        }
+
         throw err
       })
   })
@@ -138,8 +136,7 @@ function mkdirfixSync (cache, p) {
     if (err.code === 'EEXIST') {
       fixOwnerSync(cache, p)
       return null
-    } else {
+    } else
       throw err
-    }
   }
 }

package/lib/util/move-file.js

@@ -38,19 +38,17 @@ function moveFile (src, dest) {
         } else if (err.code === 'EEXIST' || err.code === 'EBUSY') {
           // file already exists, so whatever
           return resolve()
-        } else {
+        } else
           return reject(err)
-        }
-      } else {
+      } else
         return resolve()
-      }
     })
   })
     .then(() => {
       // content should never change for any reason, so make it read-only
       return Promise.all([
         unlink(src),
-        !isWindows && chmod(dest, '0444')
+        !isWindows && chmod(dest, '0444'),
       ])
     })
     .catch(() => {

package/lib/util/tmp.js

@@ -1,21 +1,21 @@
 'use strict'
 
-const util = require('util')
+const fs = require('@npmcli/fs')
 
 const fixOwner = require('./fix-owner')
 const path = require('path')
-const rimraf = util.promisify(require('rimraf'))
-const uniqueFilename = require('unique-filename')
-const { disposer } = require('./disposer')
 
 module.exports.mkdir = mktmpdir
 
 function mktmpdir (cache, opts = {}) {
   const { tmpPrefix } = opts
-  const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), tmpPrefix)
-  return fixOwner.mkdirfix(cache, tmpTarget).then(() => {
-    return tmpTarget
-  })
+  const tmpDir = path.join(cache, 'tmp')
+  return fs.mkdir(tmpDir, { recursive: true, owner: 'inherit' })
+    .then(() => {
+      // do not use path.join(), it drops the trailing / if tmpPrefix is unset
+      const target = `${tmpDir}${path.sep}${tmpPrefix || ''}`
+      return fs.mkdtemp(target, { owner: 'inherit' })
+    })
 }
 
 module.exports.withTmp = withTmp
@@ -25,7 +25,7 @@ function withTmp (cache, opts, cb) {
     cb = opts
     opts = {}
   }
-  return disposer(mktmpdir(cache, opts), rimraf, cb)
+  return fs.withTempDir(path.join(cache, 'tmp'), cb, opts)
 }
 
 module.exports.fix = fixtmpdir

package/lib/verify.js

@@ -24,7 +24,7 @@ const readFile = util.promisify(fs.readFile)
 const verifyOpts = (opts) => ({
   concurrency: 20,
   log: { silly () {} },
-  ...opts
+  ...opts,
 })
 
 module.exports = verify
@@ -40,7 +40,7 @@ function verify (cache, opts) {
     rebuildIndex,
     cleanTmp,
     writeVerifile,
-    markEndTime
+    markEndTime,
   ]
 
   return steps
@@ -54,9 +54,9 @@ function verify (cache, opts) {
               stats[k] = s[k]
             })
           const end = new Date()
-          if (!stats.runTime) {
+          if (!stats.runTime)
             stats.runTime = {}
-          }
+
           stats.runTime[label] = end - start
           return Promise.resolve(stats)
         })
@@ -108,9 +108,9 @@ function garbageCollect (cache, opts) {
   const indexStream = index.lsStream(cache)
   const liveContent = new Set()
   indexStream.on('data', (entry) => {
-    if (opts.filter && !opts.filter(entry)) {
+    if (opts.filter && !opts.filter(entry))
       return
-    }
+
     liveContent.add(entry.integrity.toString())
   })
   return new Promise((resolve, reject) => {
@@ -120,14 +120,14 @@ function garbageCollect (cache, opts) {
     return glob(path.join(contentDir, '**'), {
       follow: false,
       nodir: true,
-      nosort: true
+      nosort: true,
     }).then((files) => {
       return Promise.resolve({
         verifiedContent: 0,
         reclaimedCount: 0,
         reclaimedSize: 0,
         badContentCount: 0,
-        keptSize: 0
+        keptSize: 0,
       }).then((stats) =>
         pMap(
           files,
@@ -171,14 +171,14 @@ function verifyContent (filepath, sri) {
     .then((s) => {
       const contentInfo = {
         size: s.size,
-        valid: true
+        valid: true,
       }
       return ssri
         .checkStream(new fsm.ReadStream(filepath), sri)
         .catch((err) => {
-          if (err.code !== 'EINTEGRITY') {
+          if (err.code !== 'EINTEGRITY')
             throw err
-          }
+
           return rimraf(filepath).then(() => {
             contentInfo.valid = false
           })
@@ -186,9 +186,9 @@ function verifyContent (filepath, sri) {
         .then(() => contentInfo)
     })
     .catch((err) => {
-      if (err.code === 'ENOENT') {
+      if (err.code === 'ENOENT')
         return { size: 0, valid: false }
-      }
+
       throw err
     })
 }
@@ -199,7 +199,7 @@ function rebuildIndex (cache, opts) {
     const stats = {
       missingContent: 0,
       rejectedEntries: 0,
-      totalEntries: 0
+      totalEntries: 0,
     }
     const buckets = {}
     for (const k in entries) {
@@ -209,9 +209,9 @@ function rebuildIndex (cache, opts) {
         const entry = entries[k]
         const excluded = opts.filter && !opts.filter(entry)
         excluded && stats.rejectedEntries++
-        if (buckets[hashed] && !excluded) {
+        if (buckets[hashed] && !excluded)
           buckets[hashed].push(entry)
-        } else if (buckets[hashed] && excluded) {
+        else if (buckets[hashed] && excluded) {
           // skip
         } else if (excluded) {
           buckets[hashed] = []
@@ -244,7 +244,7 @@ function rebuildBucket (cache, bucket, stats, opts) {
             return index
               .insert(cache, entry.key, entry.integrity, {
                 metadata: entry.metadata,
-                size: entry.size
+                size: entry.size,
               })
               .then(() => {
                 stats.totalEntries++

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cacache",
-  "version": "15.0.5",
+  "version": "15.3.0",
   "cache-version": {
     "content": "2",
     "index": "5"
@@ -13,15 +13,17 @@
   ],
   "scripts": {
     "benchmarks": "node test/benchmarks",
-    "lint": "standard",
-    "postrelease": "npm publish",
-    "posttest": "npm run lint",
-    "prepublishOnly": "git push --follow-tags",
-    "prerelease": "npm t",
-    "release": "standard-version -s",
+    "preversion": "npm test",
+    "postversion": "npm publish",
+    "prepublishOnly": "git push origin --follow-tags",
     "test": "tap",
+    "snap": "tap",
     "coverage": "tap",
-    "test-docker": "docker run -it --rm --name pacotest -v \"$PWD\":/tmp -w /tmp node:latest npm test"
+    "test-docker": "docker run -it --rm --name pacotest -v \"$PWD\":/tmp -w /tmp node:latest npm test",
+    "lint": "npm run npmclilint -- \"*.*js\" \"lib/**/*.*js\" \"test/**/*.*js\"",
+    "npmclilint": "npmcli-lint",
+    "lintfix": "npm run lint -- --fix",
+    "postsnap": "npm run lintfix --"
   },
   "repository": "https://github.com/npm/cacache",
   "keywords": [
@@ -39,25 +41,9 @@
     "disk cache",
     "disk storage"
   ],
-  "author": {
-    "name": "Kat Marchán",
-    "email": "kzm@sykosomatic.org",
-    "twitter": "maybekatz"
-  },
-  "contributors": [
-    {
-      "name": "Charlotte Spencer",
-      "email": "charlottelaspencer@gmail.com",
-      "twitter": "charlotteis"
-    },
-    {
-      "name": "Rebecca Turner",
-      "email": "me@re-becca.org",
-      "twitter": "ReBeccaOrg"
-    }
-  ],
   "license": "ISC",
   "dependencies": {
+    "@npmcli/fs": "^1.0.0",
     "@npmcli/move-file": "^1.0.1",
     "chownr": "^2.0.0",
     "fs-minipass": "^2.0.0",
@@ -72,18 +58,17 @@
     "p-map": "^4.0.0",
     "promise-inflight": "^1.0.1",
     "rimraf": "^3.0.2",
-    "ssri": "^8.0.0",
+    "ssri": "^8.0.1",
     "tar": "^6.0.2",
     "unique-filename": "^1.1.1"
   },
   "devDependencies": {
+    "@npmcli/lint": "^1.0.1",
     "benchmark": "^2.1.4",
     "chalk": "^4.0.0",
     "require-inject": "^1.4.4",
-    "standard": "^14.3.1",
-    "standard-version": "^7.1.0",
     "tacks": "^1.3.0",
-    "tap": "^14.10.6"
+    "tap": "^15.0.9"
   },
   "tap": {
     "100": true,