pacote 11.2.7 → 11.3.3

package/README.md

@@ -168,6 +168,18 @@ resolved, and other properties, as they are determined.
   times (even just to validate the cache) for a given packument, since it
   is unlikely to change in the span of a single command.
 
+
+### Advanced API
+
+Each different type of fetcher is exposed for more advanced usage such as
+using helper methods from this classes:
+
+* `DirFetcher`
+* `FileFetcher`
+* `GitFetcher`
+* `RegistryFetcher`
+* `RemoteFetcher`
+
 ## Extracted File Modes
 
 Files are extracted with a mode matching the following formula:

package/lib/dir.js

@@ -4,11 +4,10 @@ const cacache = require('cacache')
 const Minipass = require('minipass')
 const { promisify } = require('util')
 const readPackageJson = require('read-package-json-fast')
-const isPackageBin = require('./util/is-package-bin.js')
+const tarCreateOptions = require('./util/tar-create-options.js')
 const packlist = require('npm-packlist')
 const tar = require('tar')
 const _prepareDir = Symbol('_prepareDir')
-const _tarcOpts = Symbol('_tarcOpts')
 const { resolve } = require('path')
 
 const runScript = require('@npmcli/run-script')
@@ -21,6 +20,11 @@ class DirFetcher extends Fetcher {
     this.resolved = this.spec.fetchSpec
   }
 
+  // exposes tarCreateOptions as public API
+  static tarCreateOptions (manifest) {
+    return tarCreateOptions(manifest)
+  }
+
   get types () {
     return ['directory']
   }
@@ -65,35 +69,12 @@ class DirFetcher extends Fetcher {
     // pipe to the stream, and proxy errors the chain.
     this[_prepareDir]()
       .then(() => packlist({ path: this.resolved }))
-      .then(files => tar.c(this[_tarcOpts](), files)
+      .then(files => tar.c(tarCreateOptions(this.package), files)
         .on('error', er => stream.emit('error', er)).pipe(stream))
       .catch(er => stream.emit('error', er))
     return stream
   }
 
-  [_tarcOpts] () {
-    return {
-      cwd: this.resolved,
-      prefix: 'package/',
-      portable: true,
-      gzip: true,
-
-      // ensure that package bins are always executable
-      // Note that npm-packlist is already filtering out
-      // anything that is not a regular file, ignored by
-      // .npmignore or package.json "files", etc.
-      filter: (path, stat) => {
-        if (isPackageBin(this.package, path))
-          stat.mode |= 0o111
-        return true
-      },
-
-      // Provide a specific date in the 1980s for the benefit of zip,
-      // which is confounded by files dated at the Unix epoch 0.
-      mtime: new Date('1985-10-26T08:15:00.000Z'),
-    }
-  }
-
   manifest () {
     if (this.package)
       return Promise.resolve(this.package)

package/lib/fetcher.js

@@ -40,6 +40,7 @@ const _istream = Symbol('_istream')
 const _assertType = Symbol('_assertType')
 const _tarballFromCache = Symbol('_tarballFromCache')
 const _tarballFromResolved = Symbol.for('pacote.Fetcher._tarballFromResolved')
+const _cacheFetches = Symbol.for('pacote.Fetcher._cacheFetches')
 
 class FetcherBase {
   constructor (spec, opts) {
@@ -166,25 +167,19 @@ class FetcherBase {
   }
 
   // private, should be overridden.
-  // Note that they should *not* calculate or check integrity, but *just*
-  // return the raw tarball data stream.
+  // Note that they should *not* calculate or check integrity or cache,
+  // but *just*  return the raw tarball data stream.
   [_tarballFromResolved] () {
     throw this.notImplementedError
   }
 
   // public, should not be overridden
   tarball () {
-    return this.tarballStream(stream => new Promise((res, rej) => {
-      const buf = []
-      stream.on('error', er => rej(er))
-      stream.on('end', () => {
-        const data = Buffer.concat(buf)
-        data.integrity = this.integrity && String(this.integrity)
-        data.resolved = this.resolved
-        data.from = this.from
-        return res(data)
-      })
-      stream.on('data', d => buf.push(d))
+    return this.tarballStream(stream => stream.concat().then(data => {
+      data.integrity = this.integrity && String(this.integrity)
+      data.resolved = this.resolved
+      data.from = this.from
+      return data
     }))
   }
 
@@ -194,6 +189,10 @@ class FetcherBase {
     return cacache.get.stream.byDigest(this.cache, this.integrity, this.opts)
   }
 
+  get [_cacheFetches] () {
+    return true
+  }
+
   [_istream] (stream) {
     // everyone will need one of these, either for verifying or calculating
     // We always set it, because we have might only have a weak legacy hex
@@ -203,7 +202,31 @@ class FetcherBase {
     // gets to the point of re-setting the integrity.
     const istream = ssri.integrityStream(this.opts)
     istream.on('integrity', i => this.integrity = i)
-    return stream.on('error', er => istream.emit('error', er)).pipe(istream)
+    stream.on('error', er => istream.emit('error', er))
+
+    // if not caching this, just pipe through to the istream and return it
+    if (!this.opts.cache || !this[_cacheFetches])
+      return stream.pipe(istream)
+
+    // we have to return a stream that gets ALL the data, and proxies errors,
+    // but then pipe from the original tarball stream into the cache as well.
+    // To do this without losing any data, and since the cacache put stream
+    // is not a passthrough, we have to pipe from the original stream into
+    // the cache AFTER we pipe into the istream.  Since the cache stream
+    // has an asynchronous flush to write its contents to disk, we need to
+    // defer the istream end until the cache stream ends.
+    stream.pipe(istream, { end: false })
+    const cstream = cacache.put.stream(
+      this.opts.cache,
+      `pacote:tarball:${this.from}`,
+      this.opts
+    )
+    stream.pipe(cstream)
+    // defer istream end until after cstream
+    // cache write errors should not crash the fetch, this is best-effort.
+    cstream.promise().catch(() => {}).then(() => istream.end())
+
+    return istream
   }
 
   pickIntegrityAlgorithm () {
@@ -232,7 +255,9 @@ class FetcherBase {
   // An ENOENT trying to read a tgz file, for example, is Right Out.
   isRetriableError (er) {
     // TODO: check error class, once those are rolled out to our deps
-    return this.isDataCorruptionError(er) || er.code === 'ENOENT'
+    return this.isDataCorruptionError(er) ||
+      er.code === 'ENOENT' ||
+      er.code === 'EISDIR'
   }
 
   // Mostly internal, but has some uses

package/lib/index.js

@@ -1,5 +1,16 @@
 const { get } = require('./fetcher.js')
+const GitFetcher = require('./git.js')
+const RegistryFetcher = require('./registry.js')
+const FileFetcher = require('./file.js')
+const DirFetcher = require('./dir.js')
+const RemoteFetcher = require('./remote.js')
+
 module.exports = {
+  GitFetcher,
+  RegistryFetcher,
+  FileFetcher,
+  DirFetcher,
+  RemoteFetcher,
   resolve: (spec, opts) => get(spec, opts).resolve(),
   extract: (spec, dest, opts) => get(spec, opts).extract(dest),
   manifest: (spec, opts) => get(spec, opts).manifest(),

package/lib/registry.js

@@ -3,6 +3,7 @@ const RemoteFetcher = require('./remote.js')
 const _tarballFromResolved = Symbol.for('pacote.Fetcher._tarballFromResolved')
 const pacoteVersion = require('../package.json').version
 const npa = require('npm-package-arg')
+const rpj = require('read-package-json-fast')
 const pickManifest = require('npm-pick-manifest')
 const ssri = require('ssri')
 const Minipass = require('minipass')
@@ -156,7 +157,8 @@ class RegistryFetcher extends Fetcher {
         }
         if (this.integrity)
           mani._integrity = String(this.integrity)
-        return this.package = mani
+        this.package = rpj.normalize(mani)
+        return this.package
       })
   }
 

package/lib/remote.js

@@ -8,6 +8,7 @@ const Minipass = require('minipass')
 // The default registry URL is a string of great magic.
 const magic = /^https?:\/\/registry\.npmjs\.org\//
 
+const _cacheFetches = Symbol.for('pacote.Fetcher._cacheFetches')
 const _headers = Symbol('_headers')
 class RemoteFetcher extends Fetcher {
   constructor (spec, opts) {
@@ -21,6 +22,12 @@ class RemoteFetcher extends Fetcher {
     this.pkgid = opts.pkgid ? opts.pkgid : `remote:${nameat}${this.resolved}`
   }
 
+  // Don't need to cache tarball fetches in pacote, because make-fetch-happen
+  // will write into cacache anyway.
+  get [_cacheFetches] () {
+    return false
+  }
+
   [_tarballFromResolved] () {
     const stream = new Minipass()
     const fetchOpts = {

package/lib/util/tar-create-options.js

@@ -0,0 +1,30 @@
+const isPackageBin = require('./is-package-bin.js')
+
+const tarCreateOptions = manifest => ({
+  cwd: manifest._resolved,
+  prefix: 'package/',
+  portable: true,
+  gzip: {
+    // forcing the level to 9 seems to avoid some
+    // platform specific optimizations that cause
+    // integrity mismatch errors due to differing
+    // end results after compression
+    level: 9
+  },
+
+  // ensure that package bins are always executable
+  // Note that npm-packlist is already filtering out
+  // anything that is not a regular file, ignored by
+  // .npmignore or package.json "files", etc.
+  filter: (path, stat) => {
+    if (isPackageBin(manifest, path))
+      stat.mode |= 0o111
+    return true
+  },
+
+  // Provide a specific date in the 1980s for the benefit of zip,
+  // which is confounded by files dated at the Unix epoch 0.
+  mtime: new Date('1985-10-26T08:15:00.000Z'),
+})
+
+module.exports = tarCreateOptions

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pacote",
-  "version": "11.2.7",
+  "version": "11.3.3",
   "description": "JavaScript package downloader",
   "author": "Isaac Z. Schlueter <i@izs.me> (https://izs.me)",
   "bin": {
@@ -17,15 +17,12 @@
   },
   "tap": {
     "timeout": 300,
-    "check-coverage": true,
-    "coverage-map": "map.js",
-    "esm": false
+    "coverage-map": "map.js"
   },
   "devDependencies": {
     "mutate-fs": "^2.1.1",
     "npm-registry-mock": "^1.3.1",
-    "require-inject": "^1.4.4",
-    "tap": "^14.11.0"
+    "tap": "^15.0.4"
   },
   "files": [
     "lib/**/*.js"
@@ -49,7 +46,7 @@
     "npm-package-arg": "^8.0.1",
     "npm-packlist": "^2.1.4",
     "npm-pick-manifest": "^6.0.0",
-    "npm-registry-fetch": "^9.0.0",
+    "npm-registry-fetch": "^10.0.0",
     "promise-retry": "^2.0.1",
     "read-package-json-fast": "^2.0.1",
     "rimraf": "^3.0.2",