@preventive/upstream 1.2.0 → 1.3.0

package/github.d.ts

@@ -127,7 +127,7 @@ export interface Client {
   // getRepoTreeTarball's for the tree GitHub names for that commit, asked
   // on every call, cached or not. So its top directory is named for the
   // tree, and files marked `export-subst` are as committed, not rewritten.
-  getRepoTarball(options: { repo: RepoName; sha: string }): Promise<Uint8Array>
+  getRepoTarball(options: { repo: RepoName; sha: string; symlinksAbove?: number }): Promise<Uint8Array>
   // The id of the tree at `directory`, `/`-separated as npm's is, in the
   // full commit `sha`, or of its root without one: from the tree GitHub
   // names for the commit, down GitHub's listings a directory at a time,
@@ -153,7 +153,13 @@ export interface Client {
   // a repo set to include Git LFS objects in archives, which replace the
   // pointers git has; that repo still gets a tree already cached, pointers
   // and all.
-  getRepoTreeTarball(options: { repo: RepoName; tree: string }): Promise<Uint8Array>
+  // It is read as @preventive/archive reads a tarball, and refused for
+  // anything it refuses: a pax record repeated, a name or a symlink that
+  // climbs out of it, an absolute symlink. `symlinksAbove` lets a symlink
+  // climb that many levels above the tarball's root, none unless given: a
+  // subdirectory's, its links reaching the repo's root as a monorepo
+  // package's LICENSE does, needs as many as it is deep, less one.
+  getRepoTreeTarball(options: { repo: RepoName; tree: string; symlinksAbove?: number }): Promise<Uint8Array>
   // The repository's published security advisories, as GitHub's
   // repository advisory objects: every page of 100, each found by the
   // cursor in the one before's Link header. More than 100 pages is an

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@preventive/upstream",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Minimal npm, crates.io, Soldeer, GitHub and git checkout clients",
   "type": "module",
   "exports": {
@@ -124,6 +124,9 @@
   "publishConfig": {
     "access": "public"
   },
+  "dependencies": {
+    "@preventive/archive": "^1.2.0"
+  },
   "peerDependencies": {
     "semver": "^7.5.2"
   },

package/src/args.js

@@ -68,6 +68,7 @@ export function assertNames(method, what, names, options, assertName) {
 
 export const assertBoolean = assertion('a boolean', (value) => typeof value === 'boolean')
 export const assertNumber = assertion('a positive integer', (value) => Number.isSafeInteger(value) && value > 0)
+export const assertLevels = assertion('a whole number of levels', (value) => Number.isSafeInteger(value) && value >= 0)
 export const assertLine = assertion('a non-empty single line', (value) => isString(value, 1024) && value.trim() !== '')
 // 65,536 is GitHub's own limit on a body.
 export const assertText = assertion('text with no control characters', (value) => isString(value, 65_536, '\t\n\r'))

package/src/download.js

@@ -20,10 +20,11 @@ const DIGESTS = {
 // whether set or not (read, never written; a mismatch is passed over),
 // else the cache set, or a `cache` store in its place (readBytes; a
 // mismatch throws), else what `locate` answers, checked and then kept as
-// writeBytes keeps it.
-export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], ours = false, cache, locate, options = {}, list }) {
+// writeBytes keeps it. A tree's `list` and `symlinksAbove` are
+// gitTreeOfTarball's.
+export async function verifiedDownload({ method, dir, what, ext, algorithm, expected, local = [], ours = false, cache, locate, options = {}, list, symlinksAbove }) {
   assert.ok(Object.hasOwn(DIGESTS, algorithm) && typeof expected === 'string' && expected !== '', `${method}: nothing to check ${what} against`)
-  const digest = (bytes) => DIGESTS[algorithm](bytes, { expected, list })
+  const digest = (bytes) => DIGESTS[algorithm](bytes, { expected, list, symlinksAbove })
   const key = `${what}.${ext}`
   for (const path of ours ? [...local, ...ourCachePaths(dir, key)] : local) {
     const bytes = await readRegularFile(path)

package/src/github/read.js

@@ -1,7 +1,7 @@
 import assert from 'node:assert/strict'
 import { Buffer } from 'node:buffer'
 
-import { assertArgs, assertBoolean, assertGhsa, assertLogin, assertNumber, assertPath, assertRef, assertRepo, assertSha, assertTagName, assertTreeId, isRepo, isSha, isSha1, isTagName, optional, sameName, show } from '../args.js'
+import { assertArgs, assertBoolean, assertGhsa, assertLevels, assertLogin, assertNumber, assertPath, assertRef, assertRepo, assertSha, assertTagName, assertTreeId, isRepo, isSha, isSha1, isTagName, optional, sameName, show } from '../args.js'
 import { verifiedDownload } from '../download.js'
 import { HttpError, decode, encodeSegment, requestWithHeaders } from '../http.js'
 import { FORMATS, lockfileOf, projectFormats } from '../lockfiles.js'
@@ -137,14 +137,15 @@ async function getRepoFile(headers, options) {
 // with nothing in it or the LF blob of a file marked `eol=crlf`, comes
 // from GitHub's listings of the trees, which the id checks as well: a
 // directory at a time, as a recursive listing of a large tree is cut short.
-async function treeTarball(method, headers, repo, tree) {
+// Its symlinks climb no more than `symlinksAbove` levels above its root.
+async function treeTarball(method, headers, repo, tree, symlinksAbove) {
   const listings = new Map()
   const list = (sha) => {
     if (!listings.has(sha)) listings.set(sha, call(headers, repoApi(repo, ['git', 'trees', sha])).then((listing) => (Array.isArray(listing?.tree) ? listing.tree : [])))
     return listings.get(sha)
   }
   const locate = () => repoApi(repo, ['tarball', tree])
-  return await verifiedDownload({ method, dir: 'github/trees', what: tree, ext: 'tgz', algorithm: 'tree', expected: tree, locate, options: { headers, redirect: 'follow' }, list })
+  return await verifiedDownload({ method, dir: 'github/trees', what: tree, ext: 'tgz', algorithm: 'tree', expected: tree, locate, options: { headers, redirect: 'follow' }, list, symlinksAbove })
 }
 
 async function commitTree(method, headers, repo, sha) {
@@ -157,15 +158,15 @@ async function commitTree(method, headers, repo, sha) {
 // now. Its tree's tarball, not its own, in which `git archive` rewrites the
 // files marked `export-subst`.
 async function getRepoTarball(headers, options) {
-  assertArgs('getRepoTarball', options, { repo: assertRepo, sha: assertSha })
-  const { repo, sha } = options
+  assertArgs('getRepoTarball', options, { repo: assertRepo, sha: assertSha, symlinksAbove: optional(assertLevels) })
+  const { repo, sha, symlinksAbove } = options
   const tree = await commitTree('getRepoTarball', headers, repo, sha)
-  return await treeTarball('getRepoTarball', headers, repo, tree)
+  return await treeTarball('getRepoTarball', headers, repo, tree, symlinksAbove)
 }
 
 async function getRepoTreeTarball(headers, options) {
-  assertArgs('getRepoTreeTarball', options, { repo: assertRepo, tree: assertTreeId })
-  return await treeTarball('getRepoTreeTarball', headers, options.repo, options.tree)
+  assertArgs('getRepoTreeTarball', options, { repo: assertRepo, tree: assertTreeId, symlinksAbove: optional(assertLevels) })
+  return await treeTarball('getRepoTreeTarball', headers, options.repo, options.tree, options.symlinksAbove)
 }
 
 // Not recursive: GitHub cuts short a recursive listing of a large tree.

package/src/tree.js

@@ -1,56 +1,34 @@
 import { Buffer } from 'node:buffer'
 import { createHash } from 'node:crypto'
-import { gunzipSync } from 'node:zlib'
+
+import { CompressionError, decompress } from '@preventive/archive/compression.js'
+import { ArchiveError, unpackStream } from '@preventive/archive/tar.js'
 
 import { isSha1, matches } from './args.js'
 import { withAttributes, writtenWithCrlf } from './attributes.js'
 
-const BLOCK = 512
 const MAX_UNPACKED_BYTES = 2 ** 30
-// Only what git writes is read: anything else, a pax `size` or a mode of
-// 0000, would be extracted as it is not hashed. Modes are as its umask,
-// 002 or 022, leaves them: a file 0664 or 0644, 0775 or 0755 executable;
-// a directory 0775 or 0755; a symlink 0777.
+// Read by @preventive/archive, which holds a tarball to its checks -- each
+// header's checksum and fields, the padding and the end, pax records of
+// which none repeats, names and link targets that stay within it, no name
+// twice as another entry -- and past those to only what git writes:
+// anything else, a pax `size` or a mode of 0000, would be extracted as it
+// is not hashed. POSIX ustar headers, as under GNU's magic a reader that
+// knows none takes GNU's own bytes for a name's prefix, their numbers as
+// octal digits; no extended header but an entry's own pax one; type flags
+// and modes as stored. Modes are as
+// its umask, 002 or 022, leaves them: a file 0664 or 0644, 0775 or 0755
+// executable; a directory 0775 or 0755; a symlink 0777.
 const PAX_KEYS = new Set(['path', 'linkpath'])
 const MODES = { 0: [0o664, 0o644, 0o775, 0o755], 2: [0o777], 5: [0o775, 0o755] }
+// The extended headers git does not write, by the type a tar names them by.
+const FLAGS = { global: 'g', longname: 'L', longlink: 'K' }
 // The top directory is not hashed, so it is held to GitHub's name for it,
 // owner-repo-id: another, `..\..`, `C:` or a Windows device such as
 // `con.txt`, could take an extractor elsewhere.
 const isTop = matches(/^(?!(?:con|prn|aux|nul|com\d|lpt\d)(?:\.|$))[\da-z][\w.-]*$/iu)
 
-// Names are kept as latin1 strings, a char per byte, so they sort and hash
-// as the bytes git has.
-const field = (header, start, end) => {
-  const bytes = header.subarray(start, end)
-  const nul = bytes.indexOf(0)
-  return Buffer.from(nul === -1 ? bytes : bytes.subarray(0, nul)).toString('latin1')
-}
-const octal = (header, start, end) => {
-  const text = field(header, start, end).trim()
-  return /^[0-7]{1,12}$/u.test(text) ? Number.parseInt(text, 8) : Number.NaN
-}
 export const objectId = (type, content) => createHash('sha1').update(`${type} ${content.length}\0`).update(content).digest()
-const checksum = (header) => header.reduce((sum, byte, i) => sum + (i >= 148 && i < 156 ? 32 : byte), 0)
-const isZero = (bytes) => bytes.every((byte) => byte === 0)
-
-// `<length> <key>=<value>\n`, the length counting the whole record. A NUL
-// would end a path for an extractor, and a name in a tree for git.
-function paxRecords(body) {
-  const text = Buffer.from(body).toString('latin1')
-  const records = new Map()
-  for (let at = 0; at < text.length;) {
-    const space = text.indexOf(' ', at)
-    const length = text.slice(at, space)
-    const end = at + Number(length)
-    if (!/^[1-9]\d*$/u.test(length) || end > text.length || text[end - 1] !== '\n') return null
-    const record = text.slice(space + 1, end - 1)
-    const equals = record.indexOf('=')
-    if (equals < 1 || record.includes('\0')) return null
-    records.set(record.slice(0, equals), record.slice(equals + 1))
-    at = end
-  }
-  return records
-}
 
 // Git keeps no empty directory: in a tarball, one is a submodule. Each as
 // [the directory holding it, its name, its path].
@@ -73,6 +51,8 @@ function treeId(dir) {
 const EMPTY_TREE = objectId('tree', Buffer.alloc(0)).toString('hex')
 const isName = matches(/^[^\0/]+$/u)
 const isEntry = (entry) => isName(entry?.path) && isSha1(entry.sha)
+// Names are kept as latin1 strings of their UTF-8, a char per byte, so they
+// sort and hash as the bytes git has.
 const nameOf = (path) => Buffer.from(path).toString('latin1')
 const subtree = (sha) => ({ mode: '40000', id: Buffer.from(sha, 'hex') })
 
@@ -139,6 +119,35 @@ async function mend(dir, sha, listed, base = '', above = []) {
   }
 }
 
+// One entry of a tarball into `root`, the tree being built, whose top
+// directory is `top`, the first entry's: a reason where git writes no such
+// entry there, else nothing.
+function take(root, { storedName, storedLinkname, typeflag, storedMode, uid, gid, uname, gname, data, pax, extended, magic, octal }, top) {
+  const path = nameOf(storedName)
+  const flag = extended.find(({ kind }) => kind !== 'pax')
+  if (flag !== undefined) return `no tree: an entry of type ${JSON.stringify(FLAGS[flag.kind])}`
+  if (magic !== 'ustar' || extended.some((header) => header.magic !== 'ustar')) return 'no tree: a header that is not POSIX ustar'
+  const other = [...pax.keys()].find((key) => !PAX_KEYS.has(key))
+  if (other !== undefined) return `no tree: a pax record git does not write, ${JSON.stringify(other)}`
+  if (!Object.hasOwn(MODES, typeflag)) return `no tree: an entry of type ${JSON.stringify(typeflag)}`
+  const asGitWrites = MODES[typeflag].includes(storedMode) && path.endsWith('/') === (typeflag === '5')
+    && uid === 0 && gid === 0 && uname === 'root' && gname === 'root'
+    && octal && extended.every((header) => header.octal)
+  if (!asGitWrites) return `no tree: a header git does not write, ${JSON.stringify(path)}`
+  const [first, ...parts] = path.replace(/\/$/u, '').split('/')
+  const name = typeflag === '5' ? null : parts.pop()
+  if (!isTop(first) || first !== top || name === undefined || [...parts, name].some((part) => ['', '.', '..'].includes(part))) return `no tree: an entry outside one top directory, ${JSON.stringify(path)}`
+  let dir = root
+  for (const part of parts) {
+    if (!dir.has(part)) dir.set(part, new Map())
+    dir = dir.get(part)
+    if (!(dir instanceof Map)) return `no tree: a file where a directory is, ${JSON.stringify(path)}`
+  }
+  if (name === null) return
+  if (dir.has(name)) return `no tree: ${JSON.stringify(path)} twice`
+  dir.set(name, typeflag === '0' ? { mode: storedMode & 0o100 ? '100755' : '100644', id: objectId('blob', data), body: data } : { mode: '120000', id: objectId('blob', Buffer.from(storedLinkname)) })
+}
+
 // The id of the git tree a gzipped tarball holds, as `git archive` writes
 // one, under a single top directory: a file's mode is its exec bit, a
 // symlink's blob its target. What a tarball cannot show comes from `list`,
@@ -147,62 +156,31 @@ async function mend(dir, sha, listed, base = '', above = []) {
 // commit for an empty directory, and, where the id comes out otherwise, the
 // subtrees with nothing in them it leaves out and the blobs of the files it
 // wrote with CRLF for `eol=crlf`. The id is `expected` only if those are
-// right. Where there is no such tree, a reason, never an id.
-export async function gitTreeOfTarball(gzipped, { expected, list } = {}) {
-  let bytes
+// right. Where there is no such tree, a reason, never an id. A symlink may
+// climb `symlinksAbove` levels above the tarball's root, none unless given,
+// as @preventive/archive lets it: a subtree's, linking to the tree's root
+// as a monorepo package's LICENSE does, as many as it is deep, less one.
+export async function gitTreeOfTarball(gzipped, { expected, list, symlinksAbove = 0 } = {}) {
+  let tar
   try {
-    bytes = gunzipSync(gzipped, { maxOutputLength: MAX_UNPACKED_BYTES })
-  } catch {
-    return 'no tree: not gzip, or larger than 1 GiB unpacked'
+    tar = await decompress(gzipped, 'gzip', { limit: MAX_UNPACKED_BYTES })
+  } catch (error) {
+    if (error instanceof CompressionError) return 'no tree: not gzip, or larger than 1 GiB unpacked'
+    throw error
   }
   const root = new Map()
   let top
-  let pax = null
-  for (let at = 0; ;) {
-    const header = bytes.subarray(at, at + BLOCK)
-    if (header.length < BLOCK) return 'no tree: the tarball is cut short'
-    if (isZero(header)) {
-      if (!isZero(bytes.subarray(at))) return 'no tree: data after the end of the tarball'
-      break
-    }
-    if (Buffer.from(header.subarray(257, 265)).toString('latin1') !== 'ustar\u000000') return 'no tree: a header that is not POSIX ustar'
-    if (octal(header, 148, 156) !== checksum(header)) return 'no tree: a header that fails its checksum'
-    const size = octal(header, 124, 136)
-    const body = bytes.subarray(at + BLOCK, at + BLOCK + size)
-    if (body.length !== size) return 'no tree: the tarball is cut short'
-    at += BLOCK + Math.ceil(size / BLOCK) * BLOCK
-    const type = String.fromCodePoint(header[156])
-    if (type === 'x') {
-      const records = paxRecords(body)
-      if (records === null) return 'no tree: a malformed pax header'
-      const other = [...records.keys()].find((key) => !PAX_KEYS.has(key))
-      if (other !== undefined) return `no tree: a pax record git does not write, ${JSON.stringify(other)}`
-      if (pax !== null) return 'no tree: two pax headers for one entry'
-      pax = records
-      continue
-    }
-    const prefix = field(header, 345, 500)
-    const path = pax?.get('path') ?? (prefix ? `${prefix}/${field(header, 0, 100)}` : field(header, 0, 100))
-    const target = pax?.get('linkpath') ?? field(header, 157, 257)
-    pax = null
-    if (!Object.hasOwn(MODES, type)) return `no tree: an entry of type ${JSON.stringify(type)}`
-    const mode = octal(header, 100, 108)
-    const asGitWrites = MODES[type].includes(mode) && (type === '0' || size === 0) && path.endsWith('/') === (type === '5')
-      && octal(header, 108, 116) === 0 && octal(header, 116, 124) === 0 && field(header, 265, 297) === 'root' && field(header, 297, 329) === 'root'
-    if (!asGitWrites) return `no tree: a header git does not write, ${JSON.stringify(path)}`
-    const [first, ...parts] = path.replace(/\/$/u, '').split('/')
-    top ??= first
-    const name = type === '5' ? null : parts.pop()
-    if (!isTop(first) || first !== top || name === undefined || [...parts, name].some((part) => ['', '.', '..'].includes(part))) return `no tree: an entry outside one top directory, ${JSON.stringify(path)}`
-    let dir = root
-    for (const part of parts) {
-      if (!dir.has(part)) dir.set(part, new Map())
-      dir = dir.get(part)
-      if (!(dir instanceof Map)) return `no tree: a file where a directory is, ${JSON.stringify(path)}`
+  // An entry at a time, each taken into the tree as it is read, its data a
+  // view of the tar: a header that repeats an earlier one, which git never
+  // writes, is refused where it stands, so no run of them is held.
+  try {
+    for await (const entry of unpackStream([tar], { symlinksAbove })) {
+      const reason = take(root, entry, top ??= nameOf(entry.storedName).split('/')[0])
+      if (reason !== undefined) return reason
     }
-    if (name === null) continue
-    if (dir.has(name)) return `no tree: ${JSON.stringify(path)} twice`
-    dir.set(name, type === '0' ? { mode: mode & 0o100 ? '100755' : '100644', id: objectId('blob', body), body } : { mode: '120000', id: objectId('blob', Buffer.from(target, 'latin1')) })
+  } catch (error) {
+    if (error instanceof ArchiveError) return `no tree: ${error.message}`
+    throw error
   }
   if (top === undefined) return 'no tree: an empty tarball'
   const listed = async (sha) => (list && isSha1(sha) ? await list(sha) : [])