@preventive/triage 1.0.0-alpha.37 → 1.0.0-alpha.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/common/bundle-compare-diff.d.ts +3 -0
- package/common/bundle-compare-diff.js +522 -0
- package/common/bundle-compare-inputs.d.ts +2 -0
- package/common/bundle-compare-inputs.js +65 -0
- package/common/bundle-metadata.d.ts +6 -2
- package/common/bundle-metadata.js +63 -20
- package/common/bundle-pkg-of.d.ts +1 -0
- package/common/bundle-pkg-of.js +90 -0
- package/common/bundle-source-links.js +7 -0
- package/common/bundle-sourcemap.d.ts +10 -0
- package/common/bundle-sourcemap.js +100 -0
- package/common/bundle-sources.d.ts +3 -1
- package/common/bundle-sources.js +47 -19
- package/common/code-language.js +16 -2
- package/common/default-scan-models.ts +2 -1
- package/common/dependency-reviews.d.ts +23 -0
- package/common/dependency-reviews.js +81 -0
- package/common/finding-duplicates.js +58 -0
- package/common/finding-filters.js +274 -0
- package/common/github-pr.ts +1 -1
- package/common/gzip.js +59 -0
- package/common/line-diff.d.ts +8 -0
- package/common/line-diff.js +56 -0
- package/common/linked-workspace-groups.js +63 -0
- package/common/managed/npm-packages.d.ts +5 -0
- package/common/managed/npm-packages.js +34 -0
- package/common/managed/permissions.ts +1 -5
- package/common/managed/routes.js +123 -12
- package/common/managed/team-app.d.ts +2 -0
- package/common/managed/team-app.js +31 -0
- package/common/managed/triage.ts +1 -1
- package/common/managed/upload-seal.ts +126 -0
- package/common/pretty-print.d.ts +5 -0
- package/common/pretty-print.js +24 -0
- package/common/revalidation-conflicts.js +48 -0
- package/common/revalidation-input-groups.js +56 -0
- package/common/workspace-app.js +23 -0
- package/common/workspace-groups.js +176 -0
- package/out/brotli-fallback.js +3 -3
- package/out/chunk-2KBF5TZ7.js +2 -0
- package/out/chunk-2XUEZHIP.js +28 -0
- package/out/chunk-35VE6CDH.js +2 -0
- package/out/chunk-3BZ7JERB.js +1 -0
- package/out/chunk-4XCDFGDP.js +10 -0
- package/out/chunk-5FKRIJGF.js +1 -0
- package/out/chunk-6Z5XLSPJ.js +1 -0
- package/out/chunk-ABW5FVSN.js +1 -0
- package/out/chunk-BJKRVROW.js +1 -0
- package/out/chunk-BYUEDJOM.js +7 -0
- package/out/chunk-C4Z6ZBXV.js +10 -0
- package/out/chunk-CMQA4DM6.js +1 -0
- package/out/chunk-CVVJAXVY.js +2 -0
- package/out/chunk-DCQX2J3C.js +223 -0
- package/out/chunk-DIJJNTRH.js +10 -0
- package/out/chunk-F4OMDNEY.js +1 -0
- package/out/chunk-FU4MTW7F.js +1 -0
- package/out/chunk-GJ4IUCNS.js +1 -0
- package/out/chunk-GMFXFEUA.js +2 -0
- package/out/chunk-GNXOKIGP.js +80 -0
- package/out/chunk-GZ4MS3TE.js +13 -0
- package/out/chunk-HIV7SVI7.js +1 -0
- package/out/chunk-IU3CCINR.js +3 -0
- package/out/chunk-JAKWIZWL.js +2 -0
- package/out/chunk-K3X5EJBQ.js +2 -0
- package/out/chunk-KEX2ATEW.js +2 -0
- package/out/chunk-KYCNJVB7.js +1 -0
- package/out/chunk-NRPC74AM.js +1 -0
- package/out/chunk-OJ5DGP5E.js +33 -0
- package/out/chunk-OOJM4CTU.js +1 -0
- package/out/chunk-PNHBZGDD.js +1 -0
- package/out/chunk-R5O6FAW4.js +2 -0
- package/out/chunk-RFZQQXSG.js +17 -0
- package/out/chunk-SD2WJAXN.js +48 -0
- package/out/chunk-TUII34K2.js +1 -0
- package/out/chunk-TZ5KB2Y5.js +10 -0
- package/out/chunk-UNTRGE26.js +2 -0
- package/out/chunk-UYAXVRVL.js +1 -0
- package/out/chunk-V3ME6FI5.js +2 -0
- package/out/chunk-V5O7M2YT.js +1 -0
- package/out/chunk-VIHGO77Y.js +1 -0
- package/out/chunk-W4J3MGLB.js +1 -0
- package/out/chunk-WGG7KQUH.js +1 -0
- package/out/chunk-XYFXXB2G.js +10 -0
- package/out/chunk-YVZH5SHP.js +1 -0
- package/out/chunk-ZPD5R5RP.js +3 -0
- package/out/client-managed-import.js +3 -87
- package/out/client-managed.js +26 -338
- package/out/client-sync.js +8 -14
- package/out/graph.js +17 -30
- package/out/index.html +34 -0
- package/out/prism.js +6 -2
- package/out/terminal.js +241 -257
- package/out/view.css +1 -1
- package/out/view.js +76 -520
- package/package.json +88 -7
- package/risk/files/characters.js +196 -0
- package/risk/files/hosts.js +205 -0
- package/risk/files/readability.js +164 -0
- package/risk/files/runs-code.js +329 -0
- package/risk/files/script.js +112 -0
- package/risk/files/stamps.js +46 -0
- package/risk/index.d.ts +142 -0
- package/risk/index.js +7 -0
- package/risk/npm/assess.js +219 -0
- package/risk/npm/combined.js +65 -0
- package/risk/npm/compare.js +222 -0
- package/risk/npm/dependencies.js +153 -0
- package/risk/npm/diff.js +43 -0
- package/risk/npm/downloads.js +84 -0
- package/risk/npm/files.js +34 -0
- package/risk/npm/held.js +60 -0
- package/risk/npm/install-scripts.js +33 -0
- package/risk/npm/malware.js +36 -0
- package/risk/npm/manifest.js +65 -0
- package/risk/npm/new-dependency.js +30 -0
- package/risk/npm/old-version.js +108 -0
- package/risk/npm/registry.js +282 -0
- package/risk/npm/release.js +85 -0
- package/risk/npm/specs.js +124 -0
- package/risk/npm/tarball.js +113 -0
- package/server-e2e/http.ts +7 -7
- package/server-e2e/npm-proxy.ts +1 -1
- package/server-e2e/objstore/fetch-mint-guard.ts +2 -2
- package/server-e2e/objstore/handlers.ts +1 -1
- package/server-e2e/objstore/tokens.ts +1 -1
- package/server-e2e/sign.ts +1 -1
- package/server-managed/bundle-advisories.ts +68 -11
- package/server-managed/bundle-cache.ts +37 -4
- package/server-managed/bundle-commits.ts +140 -0
- package/server-managed/bundle-summary-cache.ts +24 -10
- package/server-managed/config.ts +4 -0
- package/server-managed/db-methods.ts +98 -11
- package/server-managed/db-neon.ts +39 -4
- package/server-managed/db-schema.ts +14 -2
- package/server-managed/db-table-names.ts +1 -1
- package/server-managed/db.ts +7 -3
- package/server-managed/dependency-reviews.ts +248 -0
- package/server-managed/github-issue-oauth.ts +5 -6
- package/server-managed/github-metadata.ts +115 -0
- package/server-managed/github-oauth.ts +38 -2
- package/server-managed/http-response.ts +26 -0
- package/server-managed/http.ts +943 -52
- package/server-managed/index.ts +11 -5
- package/server-managed/kept.ts +28 -0
- package/server-managed/locked-packages.ts +85 -0
- package/server-managed/lru.ts +30 -0
- package/server-managed/npm-insights.ts +440 -0
- package/server-managed/npm-kept.ts +65 -0
- package/server-managed/npm-loads.ts +167 -0
- package/server-managed/npm-manifest.ts +115 -0
- package/server-managed/npm-packages.ts +378 -0
- package/server-managed/npm-provenance.ts +123 -0
- package/server-managed/npm-risk.ts +177 -0
- package/server-managed/pretty-print.ts +352 -0
- package/server-managed/public-workspace.ts +78 -7
- package/server-managed/rename-diff-worker.js +26 -0
- package/server-managed/rename-diff.ts +270 -0
- package/server-managed/report-sources.ts +13 -2
- package/server-managed/repository-browser.ts +19 -5
- package/server-managed/repository-risk.ts +413 -0
- package/server-managed/session.ts +16 -2
- package/server-managed/sql.ts +5 -2
- package/server-managed/storage-encryption.ts +5 -3
- package/server-managed/storage-payload.ts +4 -0
- package/server-managed/storage-stores.ts +2 -0
- package/server-managed/storage.ts +6 -3
- package/server-managed/team-app.ts +94 -0
- package/server-managed/team-feed.ts +1 -1
- package/server-managed/team-npm-scopes.ts +77 -0
- package/server-managed/type-only-worker.js +19 -0
- package/server-managed/type-only.ts +103 -0
- package/server-managed/uploads.ts +25 -4
- package/server-managed/upstream-cache.ts +147 -5
- package/server-managed/view-as.ts +58 -0
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
// The managed npm package viewer: a published version's files, read from the
|
|
2
|
+
// registry's tarball, for the Overview and Code tabs the bundle view shows.
|
|
3
|
+
//
|
|
4
|
+
// Anyone with workspace access may read a public package. A private one also
|
|
5
|
+
// needs NPM_TOKEN on the server, and a reader with access to it: an admin or
|
|
6
|
+
// manager, or a member of a visible team listing its scope (team-npm-scopes.ts).
|
|
7
|
+
// For everyone else, a version is public only when the registry answers for
|
|
8
|
+
// it without credentials, asked on every request and never from a cache:
|
|
9
|
+
// upstream's caches, which bundle builds fill using the token, hold no
|
|
10
|
+
// answer here. Its tarball may come from them (npm-loads.ts), as only bytes
|
|
11
|
+
// matching the integrity that anonymous answer gives are served; else it is
|
|
12
|
+
// read without credentials too, and held to that integrity.
|
|
13
|
+
import { Buffer } from 'node:buffer'
|
|
14
|
+
import { createHash } from 'node:crypto'
|
|
15
|
+
import { rm } from 'node:fs/promises'
|
|
16
|
+
import { resolve } from 'node:path'
|
|
17
|
+
import { isExactVersion } from '@preventive/upstream/semver.js'
|
|
18
|
+
import { isNpmPackageName, isNpmPackageSpec, npmPackageScope } from '../common/managed/npm-packages.js'
|
|
19
|
+
import type { Role } from '../common/managed/roles.ts'
|
|
20
|
+
import { npmManifest, npmVersionFields, plainObject, stringRecord } from './npm-manifest.ts'
|
|
21
|
+
import { keepJson, readKeptJson, upstreamCachePath } from './upstream-cache.ts'
|
|
22
|
+
|
|
23
|
+
export { npmManifest, plainObject }
|
|
24
|
+
|
|
25
|
+
export const NPM_REGISTRY = 'https://registry.npmjs.org'
|
|
26
|
+
const VERSION_DOCUMENT_BYTES = 8 * 1024 * 1024
|
|
27
|
+
// Abbreviated packuments of packages with thousands of versions run to tens of MiB.
|
|
28
|
+
const PACKUMENT_BYTES = 64 * 1024 * 1024
|
|
29
|
+
// What registry documents being read at once may hold, each counted at its
|
|
30
|
+
// limit until it is read and parsed: four packuments, or 32 version documents.
|
|
31
|
+
const MAX_DOCUMENT_BYTES = 4 * PACKUMENT_BYTES
|
|
32
|
+
// What a package may unpack to: the files' bytes and their count; the tar
|
|
33
|
+
// stream holding them is bounded in npm-loads.ts.
|
|
34
|
+
export const MAX_NPM_PACKAGE_BYTES = 64 * 1024 * 1024
|
|
35
|
+
export const MAX_NPM_PACKAGE_FILES = 20_000
|
|
36
|
+
// What the files may take as the viewer's JSON. Escaping grows a text up to
|
|
37
|
+
// six times (a control character becomes `\u0001`), so it is counted before
|
|
38
|
+
// anything is serialized.
|
|
39
|
+
export const MAX_NPM_JSON_LENGTH = 96 * 1024 * 1024
|
|
40
|
+
export class NpmPackageError extends Error {
|
|
41
|
+
status: number
|
|
42
|
+
constructor(status: number, code: string) { super(code); this.status = status }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Who reads, as the session has it now: the role, and the scopes of the
|
|
46
|
+
// visible teams they are a member of.
|
|
47
|
+
export interface NpmReader { role: Role; scopes: ReadonlySet<string>; userId: string }
|
|
48
|
+
|
|
49
|
+
// Whether a reader may have a package read with the server's token.
|
|
50
|
+
export function canReadPrivateNpm(reader: NpmReader, name: string): boolean {
|
|
51
|
+
if (reader.role === 'admin' || reader.role === 'manage') return true
|
|
52
|
+
const scope = npmPackageScope(name)
|
|
53
|
+
return scope !== null && reader.scopes.has(scope)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export const npmToken = () => process.env['NPM_TOKEN'] || null
|
|
57
|
+
|
|
58
|
+
function registryUrl(name: string, spec?: string): string {
|
|
59
|
+
return `${NPM_REGISTRY}/${name}${spec === undefined ? '' : `/${encodeURIComponent(spec)}`}`
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// What an upstream (npm's registry and APIs, Socket, GitHub) answers, read
|
|
63
|
+
// whole: null where it says it has none (`absent`, 404 unless said). Asked
|
|
64
|
+
// without redirects, and given up on after `ms`, 30 seconds unless said, or
|
|
65
|
+
// once `signal` aborts, its abort rethrown as it is; a 502 where it can't be
|
|
66
|
+
// asked, answers otherwise or stops, and `upstream-too-large` past `limit`
|
|
67
|
+
// bytes (readLimited). `answered` sees the response first, its status and
|
|
68
|
+
// headers.
|
|
69
|
+
export async function askUpstream(url: string, { limit, ms = 30_000, signal, absent = [404], answered, ...init }: Omit<RequestInit, 'signal'>
|
|
70
|
+
& { limit: number; ms?: number; signal?: AbortSignal; absent?: number[]; answered?: (res: Response) => void }): Promise<Buffer | null> {
|
|
71
|
+
try {
|
|
72
|
+
const res = await fetch(url, { ...init, redirect: 'error', signal: AbortSignal.any([AbortSignal.timeout(ms), ...signal ? [signal] : []]) })
|
|
73
|
+
answered?.(res)
|
|
74
|
+
if (absent.includes(res.status)) { await res.body?.cancel(); return null }
|
|
75
|
+
if (!res.ok) { await res.body?.cancel(); throw new NpmPackageError(502, 'upstream-unavailable') }
|
|
76
|
+
return await readLimited(res, limit)
|
|
77
|
+
} catch (err) {
|
|
78
|
+
if (signal?.aborted || err instanceof NpmPackageError) throw err
|
|
79
|
+
throw new NpmPackageError(502, 'upstream-unavailable')
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// The same, read as JSON (by `parse`, as one line of ndjson): a 502
|
|
84
|
+
// `upstream-invalid` where it isn't.
|
|
85
|
+
export async function askUpstreamJson(url: string, options: Parameters<typeof askUpstream>[1], parse = (text: string): unknown => JSON.parse(text)): Promise<unknown> {
|
|
86
|
+
const body = await askUpstream(url, options)
|
|
87
|
+
if (body === null) return null
|
|
88
|
+
try { return parse(body.toString('utf8')) } catch { throw new NpmPackageError(502, 'upstream-invalid') }
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export async function readLimited(res: Response, limit: number): Promise<Buffer> {
|
|
92
|
+
const declared = Number(res.headers.get('content-length'))
|
|
93
|
+
if (declared > limit) { await res.body?.cancel(); throw new NpmPackageError(502, 'upstream-too-large') }
|
|
94
|
+
const chunks: Uint8Array[] = []
|
|
95
|
+
let size = 0
|
|
96
|
+
for await (const chunk of res.body ?? []) {
|
|
97
|
+
size += chunk.byteLength
|
|
98
|
+
if (size > limit) throw new NpmPackageError(502, 'upstream-too-large')
|
|
99
|
+
chunks.push(chunk)
|
|
100
|
+
}
|
|
101
|
+
return Buffer.concat(chunks)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// A registry document, or null where the registry has none for this caller:
|
|
105
|
+
// 404, or 401/403 as it answers some private packages asked anonymously.
|
|
106
|
+
// Credentials go only with `token`; nothing is cached on either side, but for
|
|
107
|
+
// public whole documents (publicWholeDocument).
|
|
108
|
+
async function registryDocument(url: string, { token, accept, limit, signal }: { token: string | null; accept: string; limit: number; signal: AbortSignal }): Promise<Record<string, unknown> | null> {
|
|
109
|
+
const json = await askUpstreamJson(url, { headers: { accept, ...(token ? { authorization: `Bearer ${token}` } : {}) }, limit, signal, absent: [401, 403, 404] })
|
|
110
|
+
if (json === null || plainObject(json)) return json
|
|
111
|
+
throw new NpmPackageError(502, 'upstream-invalid')
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// Where public packages' whole documents are kept, off Vercel: the upstream
|
|
115
|
+
// cache dir, set at startup (index.ts); null keeps none.
|
|
116
|
+
let documentCache: string | null = null
|
|
117
|
+
export const setNpmDocumentCache = (dir: string | null) => { documentCache = dir === null ? null : resolve(dir) }
|
|
118
|
+
|
|
119
|
+
// A package's whole document as npm answers anyone (`url` its own), kept on
|
|
120
|
+
// disk where the server keeps them (setNpmDocumentCache) with npm's ETag, and
|
|
121
|
+
// asked for again each time with it: npm answers 304, and nothing more, while
|
|
122
|
+
// it is unchanged. Null where npm has none for anyone, nothing then kept.
|
|
123
|
+
async function publicWholeDocument(name: string, url: string, { limit, signal }: { limit: number; signal: AbortSignal }): Promise<Record<string, unknown> | null> {
|
|
124
|
+
const path = documentCache === null ? null : upstreamCachePath(documentCache, 'npm/packuments', `${name}.json.gz`)
|
|
125
|
+
const kept = path === null ? null : await readKeptJson(path, limit)
|
|
126
|
+
let etag: string | null = null, status = 0
|
|
127
|
+
const body = await askUpstream(url, { headers: { accept: 'application/json', ...kept && { 'if-none-match': kept.etag } }, limit, signal, absent: [304, 401, 403, 404],
|
|
128
|
+
answered: res => { status = res.status; etag = res.headers.get('etag') } })
|
|
129
|
+
if (status === 304 && kept) return kept.json
|
|
130
|
+
if (body === null) {
|
|
131
|
+
if (path !== null && kept) await rm(path, { force: true }).catch(() => {})
|
|
132
|
+
return null
|
|
133
|
+
}
|
|
134
|
+
let json: unknown
|
|
135
|
+
try { json = JSON.parse(body.toString('utf8')) } catch { throw new NpmPackageError(502, 'upstream-invalid') }
|
|
136
|
+
if (!plainObject(json)) throw new NpmPackageError(502, 'upstream-invalid')
|
|
137
|
+
if (path !== null && etag !== null) await keepJson(path, etag, body)
|
|
138
|
+
return json
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// Asked anonymously first: a reader without private access stops there. One
|
|
142
|
+
// with it asks again with the token, where there is one and the package is
|
|
143
|
+
// scoped (npm's private packages always are), and the answer is private. A
|
|
144
|
+
// whole document (`whole`) anyone may read is kept (publicWholeDocument).
|
|
145
|
+
async function readDocument(name: string, url: string, privileged: boolean, { whole = false, ...options }: { accept: string; limit: number; signal: AbortSignal; whole?: boolean }) {
|
|
146
|
+
const open = whole ? await publicWholeDocument(name, url, options) : await registryDocument(url, { ...options, token: null })
|
|
147
|
+
if (open) return { json: open, private: false }
|
|
148
|
+
const token = npmToken()
|
|
149
|
+
if (!privileged || !token || npmPackageScope(name) === null) return null
|
|
150
|
+
const json = await registryDocument(url, { ...options, token })
|
|
151
|
+
return json ? { json, private: true } : null
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
let documentBytes = 0
|
|
155
|
+
|
|
156
|
+
// A `spare` read, one no reader waits on, is made only where it leaves room
|
|
157
|
+
// for two of the largest documents more: readers' own reads come first.
|
|
158
|
+
async function withDocumentBudget<T>(limit: number, read: () => Promise<T>, spare = false): Promise<T> {
|
|
159
|
+
if (documentBytes + limit + (spare ? 2 * PACKUMENT_BYTES : 0) > MAX_DOCUMENT_BYTES) throw new NpmPackageError(429, 'npm-busy')
|
|
160
|
+
documentBytes += limit
|
|
161
|
+
try { return await read() } finally { documentBytes -= limit }
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export interface NpmVersionDocument {
|
|
165
|
+
name: string
|
|
166
|
+
version: string
|
|
167
|
+
private: boolean
|
|
168
|
+
dist: { tarball: string; integrity: string; unpackedSize: number | null; fileCount: number | null }
|
|
169
|
+
manifest: Record<string, unknown>
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const count = (value: unknown) => Number.isSafeInteger(value) && (value as number) >= 0 ? value as number : null
|
|
173
|
+
|
|
174
|
+
// The version `json`, a document of package `name`, tells of.
|
|
175
|
+
function versionDocument(name: string, json: Record<string, unknown>, isPrivate: boolean): NpmVersionDocument {
|
|
176
|
+
const dist = json['dist']
|
|
177
|
+
if (json['name'] !== name || typeof json['version'] !== 'string' || !isNpmPackageSpec(json['version']) || !plainObject(dist)
|
|
178
|
+
|| typeof dist['tarball'] !== 'string' || typeof dist['integrity'] !== 'string') throw new NpmPackageError(502, 'upstream-invalid')
|
|
179
|
+
return {
|
|
180
|
+
name, version: json['version'], private: isPrivate,
|
|
181
|
+
dist: { tarball: dist['tarball'], integrity: dist['integrity'], unpackedSize: count(dist['unpackedSize']), fileCount: count(dist['fileCount']) },
|
|
182
|
+
manifest: npmManifest(json),
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Where public versions' documents are kept, of the fields read of them
|
|
187
|
+
// (npm-manifest.ts npmVersionFields), as the database keeps them (npm-kept.ts).
|
|
188
|
+
export interface NpmVersionStore {
|
|
189
|
+
read(name: string, version: string): Promise<{ at: number; json: Record<string, unknown> } | null>
|
|
190
|
+
write(name: string, version: string, json: Record<string, unknown>): Promise<void>
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// The version a spec names, a dist-tag resolved by the registry, with its
|
|
194
|
+
// dist. Null where the registry has none for this reader. With `store`, an
|
|
195
|
+
// exact version kept there for less than a day (VERSION_KEPT_MS) is read from
|
|
196
|
+
// it, not asked of npm, and a public one npm answers is kept. npm never
|
|
197
|
+
// changes a version's files, but its document can change: a deprecation, the
|
|
198
|
+
// package made private (a scoped one can be), the version removed (which whoever
|
|
199
|
+
// reads it from a store learns elsewhere too: risk does). A day bounds each.
|
|
200
|
+
export const VERSION_KEPT_MS = 24 * 60 * 60_000
|
|
201
|
+
export function readNpmVersion(name: string, spec: string, privileged: boolean, signal: AbortSignal, store?: NpmVersionStore): Promise<NpmVersionDocument | null> {
|
|
202
|
+
if (!isNpmPackageName(name) || !isNpmPackageSpec(spec)) return Promise.reject(new NpmPackageError(400, 'bad-package'))
|
|
203
|
+
// A kept one is read within the same budget as npm's answer.
|
|
204
|
+
return withDocumentBudget(VERSION_DOCUMENT_BYTES, async () => {
|
|
205
|
+
const kept = store && isExactVersion(spec) ? await store.read(name, spec) : null
|
|
206
|
+
if (kept && Date.now() - kept.at < VERSION_KEPT_MS) try { return versionDocument(name, kept.json, false) } catch {}
|
|
207
|
+
const found = await readDocument(name, registryUrl(name, spec), privileged, { accept: 'application/json', limit: VERSION_DOCUMENT_BYTES, signal })
|
|
208
|
+
if (!found) return null
|
|
209
|
+
const doc = versionDocument(name, found.json, found.private)
|
|
210
|
+
if (store && !found.private) await store.write(name, doc.version, npmVersionFields(found.json))
|
|
211
|
+
return doc
|
|
212
|
+
})
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// The package's dist-tags and versions, newest published first, from its
|
|
216
|
+
// abbreviated document; with `times`, from its whole one, which alone has
|
|
217
|
+
// when each was published, `times` by version (ISO dates), and `created`,
|
|
218
|
+
// when its first version was, one since unpublished among them: the earliest
|
|
219
|
+
// time it lists, which npm's own `created` can postdate, as a placeholder's
|
|
220
|
+
// does for a package npm took down. With `spare`, as a read no reader waits
|
|
221
|
+
// on (withDocumentBudget). Null where the registry has none for this reader.
|
|
222
|
+
export function readNpmVersions(name: string, privileged: boolean, signal: AbortSignal, { times = false, spare = false } = {}) {
|
|
223
|
+
if (!isNpmPackageName(name)) return Promise.reject(new NpmPackageError(400, 'bad-package'))
|
|
224
|
+
return withDocumentBudget(PACKUMENT_BYTES, async () => {
|
|
225
|
+
const accept = times ? 'application/json' : 'application/vnd.npm.install-v1+json'
|
|
226
|
+
const found = await readDocument(name, registryUrl(name), privileged, { accept, limit: PACKUMENT_BYTES, signal, whole: times })
|
|
227
|
+
if (!found) return null
|
|
228
|
+
const { json } = found
|
|
229
|
+
if (json['name'] !== name || !plainObject(json['versions'])) throw new NpmPackageError(502, 'upstream-invalid')
|
|
230
|
+
const versions = Object.keys(json['versions']).filter(isNpmPackageSpec).toReversed()
|
|
231
|
+
const distTags = Object.fromEntries(Object.entries(stringRecord(json['dist-tags']) ?? {}).filter(([, version]) => versions.includes(version)))
|
|
232
|
+
const listed = { name, private: found.private, distTags, versions }
|
|
233
|
+
if (!times) return listed
|
|
234
|
+
const time = stringRecord(json['time']) ?? {}
|
|
235
|
+
const published = Object.fromEntries(versions.filter(version => isPublishTime(time[version])).map(version => [version, time[version]!]))
|
|
236
|
+
for (const [version, at] of Object.entries(published)) publishTimes.set(`${name}@${version}`, at)
|
|
237
|
+
while (publishTimes.size > PUBLISH_TIMES) publishTimes.delete(publishTimes.keys().next().value!)
|
|
238
|
+
const listedAt = Object.entries(time).filter(([key, at]) => key !== 'modified' && isPublishTime(at)).map(([, at]) => at)
|
|
239
|
+
const created = listedAt.reduce<string | null>((first, at) => first === null || Date.parse(at) < Date.parse(first) ? at : first, null)
|
|
240
|
+
return { ...listed, times: published, created }
|
|
241
|
+
}, spare)
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// A package's whole document as npm has it for this reader, for risk
|
|
245
|
+
// (npm-risk.ts), within the bounds and budget every document read here is
|
|
246
|
+
// held to; null where npm has none for this reader, or the name is none npm
|
|
247
|
+
// takes.
|
|
248
|
+
export function readNpmPackument(name: string, privileged: boolean, signal: AbortSignal): Promise<Record<string, unknown> | null> {
|
|
249
|
+
if (!isNpmPackageName(name)) return Promise.resolve(null)
|
|
250
|
+
return withDocumentBudget(PACKUMENT_BYTES, async () => (await readDocument(name, registryUrl(name), privileged, { accept: 'application/json', limit: PACKUMENT_BYTES, signal, whole: true }))?.json ?? null)
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const isPublishTime = (value: string | undefined) => typeof value === 'string' && value.length <= 40 && !Number.isNaN(Date.parse(value))
|
|
254
|
+
|
|
255
|
+
// When each version was published, by `name@version`, as whole documents read
|
|
256
|
+
// with `times` have said: a version keeps its time once published, so it is
|
|
257
|
+
// kept for as long as the server runs, the first noted going first past a
|
|
258
|
+
// bound, and answered only with the version, once its reader's access is.
|
|
259
|
+
const PUBLISH_TIMES = 100_000
|
|
260
|
+
const publishTimes = new Map<string, string>()
|
|
261
|
+
|
|
262
|
+
export const npmPublishTime = (name: string, version: string): string | null => publishTimes.get(`${name}@${version}`) ?? null
|
|
263
|
+
|
|
264
|
+
export interface NpmPackageFile { path: string; bytes: Uint8Array }
|
|
265
|
+
|
|
266
|
+
const textOf = (bytes: Uint8Array, start: number, length: number) => {
|
|
267
|
+
const field = bytes.subarray(start, start + length)
|
|
268
|
+
const end = field.indexOf(0)
|
|
269
|
+
return Buffer.from(end === -1 ? field : field.subarray(0, end)).toString('utf8')
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function sizeOf(bytes: Uint8Array, offset: number): number {
|
|
273
|
+
// GNU base-256 for sizes past the 11 octal digits.
|
|
274
|
+
if (bytes[offset]! & 0x80) {
|
|
275
|
+
let size = bytes[offset]! & 0x7f
|
|
276
|
+
for (let i = 1; i < 12; i++) size = size * 256 + bytes[offset + i]!
|
|
277
|
+
return size
|
|
278
|
+
}
|
|
279
|
+
const text = textOf(bytes, offset, 12).trim()
|
|
280
|
+
return /^[0-7]+$/u.test(text) ? Number.parseInt(text, 8) : Number.NaN
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// The path a package's file is installed at: npm drops the first component
|
|
284
|
+
// (`package/`, or whatever the archive used), and a path that would escape the
|
|
285
|
+
// package, or names nothing, is not extracted.
|
|
286
|
+
function packagePath(raw: string): string | null {
|
|
287
|
+
const parts = raw.split('/').slice(1).filter(part => part !== '' && part !== '.')
|
|
288
|
+
return parts.length === 0 || parts.includes('..') ? null : parts.join('/')
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// A pax header's path. Each record is `<length> <key>=<value>\n`, its length
|
|
292
|
+
// in bytes, so records are walked as bytes and only their values decoded.
|
|
293
|
+
function paxPath(data: Uint8Array): string | null {
|
|
294
|
+
let path = null
|
|
295
|
+
let at = 0
|
|
296
|
+
while (at < data.length) {
|
|
297
|
+
const space = data.indexOf(0x20, at)
|
|
298
|
+
const length = Number.parseInt(Buffer.from(data.subarray(at, space)).toString('latin1'), 10)
|
|
299
|
+
if (space === -1 || !Number.isSafeInteger(length) || length <= 0) break
|
|
300
|
+
const record = Buffer.from(data.subarray(space + 1, at + length - 1)).toString('utf8')
|
|
301
|
+
if (record.startsWith('path=')) path = record.slice(5)
|
|
302
|
+
at += length
|
|
303
|
+
}
|
|
304
|
+
return path
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// The regular files of a tar stream, by package path, a later entry
|
|
308
|
+
// replacing an earlier one as extraction would. Directories, links and
|
|
309
|
+
// devices hold no file content and are skipped.
|
|
310
|
+
export function readNpmTar(tar: Uint8Array): NpmPackageFile[] {
|
|
311
|
+
const files = new Map<string, Uint8Array>()
|
|
312
|
+
let total = 0
|
|
313
|
+
let longName: string | null = null
|
|
314
|
+
let offset = 0
|
|
315
|
+
while (offset + 512 <= tar.length) {
|
|
316
|
+
const header = tar.subarray(offset, offset + 512)
|
|
317
|
+
if (header.every(byte => byte === 0)) break
|
|
318
|
+
const size = sizeOf(header, 124)
|
|
319
|
+
const type = String.fromCodePoint(header[156]!)
|
|
320
|
+
const start = offset + 512
|
|
321
|
+
if (!Number.isSafeInteger(size) || size < 0 || start + size > tar.length) throw new NpmPackageError(422, 'bad-tarball')
|
|
322
|
+
const data = tar.subarray(start, start + size)
|
|
323
|
+
offset = start + Math.ceil(size / 512) * 512
|
|
324
|
+
if (type === 'x' || type === 'L') { longName = type === 'x' ? paxPath(data) : textOf(data, 0, data.length); continue }
|
|
325
|
+
if (type === 'g' || type === 'K') continue
|
|
326
|
+
const prefix = textOf(header, 257, 6) === 'ustar' ? textOf(header, 345, 155) : ''
|
|
327
|
+
const name = longName ?? (prefix ? `${prefix}/${textOf(header, 0, 100)}` : textOf(header, 0, 100))
|
|
328
|
+
longName = null
|
|
329
|
+
if (!['0', '\0', '7'].includes(type)) continue
|
|
330
|
+
const path = packagePath(name)
|
|
331
|
+
if (path === null) continue
|
|
332
|
+
total += size - (files.get(path)?.length ?? 0)
|
|
333
|
+
files.set(path, data)
|
|
334
|
+
if (files.size > MAX_NPM_PACKAGE_FILES || total > MAX_NPM_PACKAGE_BYTES) throw new NpmPackageError(413, 'package-too-large')
|
|
335
|
+
}
|
|
336
|
+
return [...files].map(([path, bytes]) => ({ path, bytes })).toSorted((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true })
|
|
340
|
+
|
|
341
|
+
// A file's text, or null for bytes that are not UTF-8 text.
|
|
342
|
+
export function npmFileText(bytes: Uint8Array): string | null {
|
|
343
|
+
if (bytes.includes(0)) return null
|
|
344
|
+
try { return utf8.decode(bytes) } catch { return null }
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// A string's length in JSON: quotes, backslashes and the control characters
|
|
348
|
+
// with a short escape take two characters, the other control characters six.
|
|
349
|
+
function jsonLength(text: string): number {
|
|
350
|
+
let length = text.length + 2
|
|
351
|
+
for (let i = 0; i < text.length; i++) {
|
|
352
|
+
const code = text.codePointAt(i)!
|
|
353
|
+
if (code === 0x22 || code === 0x5c) length += 1
|
|
354
|
+
else if (code < 0x20) length += code >= 0x08 && code <= 0x0d && code !== 0x0b ? 1 : 5
|
|
355
|
+
}
|
|
356
|
+
return length
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// A row's brackets, size, null and digest, past its strings.
|
|
360
|
+
const ROW_LENGTH = 80
|
|
361
|
+
|
|
362
|
+
// The files as the viewer gets them: `[path, bytes, text]`, text null for a
|
|
363
|
+
// file that is not UTF-8, which instead carries its bytes' sha256, so a
|
|
364
|
+
// comparison of two versions tells a changed one from an unchanged one.
|
|
365
|
+
export function npmFileRows(files: NpmPackageFile[]): ([string, number, string] | [string, number, null, string])[] {
|
|
366
|
+
let length = 0
|
|
367
|
+
return files.map(({ path, bytes }) => {
|
|
368
|
+
const text = npmFileText(bytes)
|
|
369
|
+
length += ROW_LENGTH + jsonLength(path) + (text === null ? 0 : jsonLength(text))
|
|
370
|
+
if (length > MAX_NPM_JSON_LENGTH) throw new NpmPackageError(413, 'package-too-large')
|
|
371
|
+
return text === null ? [path, bytes.byteLength, null, `sha256-${createHash('sha256').update(bytes).digest('base64')}`] : [path, bytes.byteLength, text]
|
|
372
|
+
})
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// The tarball's filename as `npm pack` writes it.
|
|
376
|
+
export function npmTarballFilename(name: string, version: string): string {
|
|
377
|
+
return `${name.replace(/^@/u, '').replace('/', '-')}-${version}.tgz`
|
|
378
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// npm's provenance for a public version (npm-packages.ts npmManifest marks
|
|
2
|
+
// one that has it): the SLSA statement npm's registry keeps beside it,
|
|
3
|
+
// saying where the version was built: from which repository and commit, by
|
|
4
|
+
// which workflow, on which builder, in which run, with its entry in
|
|
5
|
+
// Sigstore's transparency log. Only a statement naming this version's
|
|
6
|
+
// tarball (its name, version and sha512 digest) is read; its signature isn't
|
|
7
|
+
// checked here, as npm checks it when the version is published. npm is asked
|
|
8
|
+
// without the server's token, and only about a version it answers for
|
|
9
|
+
// without one (http.ts), each answer kept as npm-insights.ts keeps its own.
|
|
10
|
+
import { Buffer } from 'node:buffer'
|
|
11
|
+
import { kept } from './kept.ts'
|
|
12
|
+
import { NPM_REGISTRY, askUpstreamJson, plainObject } from './npm-packages.ts'
|
|
13
|
+
|
|
14
|
+
const ATTESTATIONS_API = `${NPM_REGISTRY}/-/npm/v1/attestations`
|
|
15
|
+
const ATTESTATIONS_BYTES = 1024 * 1024
|
|
16
|
+
// SLSA's v1 statement, as npm writes it now, and v0.2, as it first did.
|
|
17
|
+
const SLSA = /^https:\/\/slsa\.dev\/provenance\/v(?:1|0\.2)$/u
|
|
18
|
+
const COMMIT = /^[0-9a-f]{40}$/u
|
|
19
|
+
const GITHUB_REPOSITORY = /^https:\/\/github\.com\/([\w.-]+\/[\w.-]+)$/u
|
|
20
|
+
|
|
21
|
+
// Where a version was built: its source `repository` (an https URL) and
|
|
22
|
+
// `commit`, the `workflow` file and the `ref` it ran on, the `builder` and
|
|
23
|
+
// the `run` (an https URL), its transparency log entry (`logIndex`), and
|
|
24
|
+
// whose runner built it (`runner`: `github-hosted`, `gitlab-hosted` or
|
|
25
|
+
// `self-hosted`, as its signing certificate says); each null where they
|
|
26
|
+
// don't say.
|
|
27
|
+
export interface NpmProvenance {
|
|
28
|
+
repository: string; commit: string | null; workflow: string | null; ref: string | null
|
|
29
|
+
builder: string | null; run: string | null; logIndex: number | null; runner: string | null
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const text = (value: unknown) => typeof value === 'string' && value !== '' && value.length <= 2048 ? value : null
|
|
33
|
+
const https = (value: unknown) => {
|
|
34
|
+
const url = text(value)
|
|
35
|
+
try { return url && new URL(url).protocol === 'https:' ? url : null } catch { return null }
|
|
36
|
+
}
|
|
37
|
+
// `git+https://github.com/org/repo@refs/heads/main` → the repository and
|
|
38
|
+
// ref; GitLab's names no ref (`git+https://gitlab.com/group/project`).
|
|
39
|
+
function sourceOf(uri: unknown) {
|
|
40
|
+
const match = /^git\+(https:\/\/[^@\s]+?)(?:\.git)?(?:@(.+))?$/u.exec(text(uri) ?? '')
|
|
41
|
+
return match ? { repository: https(match[1]), ref: match[2] ?? null } : { repository: null, ref: null }
|
|
42
|
+
}
|
|
43
|
+
// What `keys` lead to in `value`, through objects and arrays.
|
|
44
|
+
const field = (value: unknown, ...keys: string[]): unknown =>
|
|
45
|
+
keys.reduce<unknown>((at, key) => plainObject(at) || Array.isArray(at) ? (at as Record<string, unknown>)[key] : undefined, value)
|
|
46
|
+
|
|
47
|
+
// What a statement says of the build, as v1 or v0.2 says it; null where it
|
|
48
|
+
// names no source repository.
|
|
49
|
+
function readStatement(predicateType: string, predicate: unknown): NpmProvenance | null {
|
|
50
|
+
if (predicateType.endsWith('v1')) {
|
|
51
|
+
const workflow = field(predicate, 'buildDefinition', 'externalParameters', 'workflow')
|
|
52
|
+
const source = field(predicate, 'buildDefinition', 'resolvedDependencies', '0')
|
|
53
|
+
const repository = https(field(workflow, 'repository')) ?? sourceOf(field(source, 'uri')).repository
|
|
54
|
+
if (!repository) return null
|
|
55
|
+
const commit = text(field(source, 'digest', 'gitCommit'))
|
|
56
|
+
return {
|
|
57
|
+
repository, commit: commit && COMMIT.test(commit) ? commit : null, workflow: text(field(workflow, 'path')),
|
|
58
|
+
ref: text(field(workflow, 'ref')) ?? sourceOf(field(source, 'uri')).ref, builder: https(field(predicate, 'runDetails', 'builder', 'id')),
|
|
59
|
+
run: https(field(predicate, 'runDetails', 'metadata', 'invocationId')), logIndex: null, runner: null,
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const config = field(predicate, 'invocation', 'configSource')
|
|
63
|
+
const { repository, ref } = sourceOf(field(config, 'uri'))
|
|
64
|
+
if (!repository) return null
|
|
65
|
+
const commit = text(field(config, 'digest', 'sha1'))
|
|
66
|
+
// Its run: GitLab's job URL, or GitHub's as `<run id>-<attempt>`.
|
|
67
|
+
const invocation = field(predicate, 'metadata', 'buildInvocationId')
|
|
68
|
+
const run = /^(\d+)-(\d+)$/u.exec(text(invocation) ?? '')
|
|
69
|
+
const github = GITHUB_REPOSITORY.exec(repository)?.[1]
|
|
70
|
+
return {
|
|
71
|
+
repository, commit: commit && COMMIT.test(commit) ? commit : null, workflow: text(field(config, 'entryPoint')), ref,
|
|
72
|
+
builder: https(field(predicate, 'builder', 'id')),
|
|
73
|
+
run: https(invocation) ?? (run && github ? `https://github.com/${github}/actions/runs/${run[1]}/attempts/${run[2]}` : null), logIndex: null, runner: null,
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Fulcio's Runner Environment extension (1.3.6.1.4.1.57264.1.11) in the
|
|
78
|
+
// certificate that signed it: its OID, then its value, a short UTF8String
|
|
79
|
+
// in an OCTET STRING, as the CI's token gave it. A builder's id doesn't
|
|
80
|
+
// tell GitLab's own runners from a project's.
|
|
81
|
+
const RUNNER_ENVIRONMENT = Buffer.from('060a2b0601040183bf30010b', 'hex')
|
|
82
|
+
function runnerOf(bundle: unknown): string | null {
|
|
83
|
+
const raw = field(bundle, 'verificationMaterial', 'certificate', 'rawBytes') ?? field(bundle, 'verificationMaterial', 'x509CertificateChain', 'certificates', '0', 'rawBytes')
|
|
84
|
+
if (typeof raw !== 'string') return null
|
|
85
|
+
const der = Buffer.from(raw, 'base64')
|
|
86
|
+
const at = der.indexOf(RUNNER_ENVIRONMENT) + RUNNER_ENVIRONMENT.length
|
|
87
|
+
if (at < RUNNER_ENVIRONMENT.length || der[at] !== 0x04 || der[at + 2] !== 0x0c || der[at + 1] !== der[at + 3]! + 2) return null
|
|
88
|
+
return text(der.subarray(at + 4, at + 4 + der[at + 3]!).toString('utf8'))
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function askProvenance(name: string, version: string, integrity: string): Promise<NpmProvenance | null> {
|
|
92
|
+
const json = await askUpstreamJson(`${ATTESTATIONS_API}/${name.replace('/', '%2f')}@${encodeURIComponent(version)}`,
|
|
93
|
+
{ headers: { accept: 'application/json' }, limit: ATTESTATIONS_BYTES })
|
|
94
|
+
if (json === null) return null
|
|
95
|
+
const attestations: unknown[] = Array.isArray(field(json, 'attestations')) ? field(json, 'attestations') as unknown[] : []
|
|
96
|
+
const slsa = attestations.find(item => SLSA.test(text(field(item, 'predicateType')) ?? ''))
|
|
97
|
+
if (!slsa) return null
|
|
98
|
+
// The statement, whatever its length (the answer is bounded as a whole),
|
|
99
|
+
// unlike the fields read out of it.
|
|
100
|
+
const payload = field(slsa, 'bundle', 'dsseEnvelope', 'payload')
|
|
101
|
+
let statement: unknown
|
|
102
|
+
try { statement = JSON.parse(Buffer.from(typeof payload === 'string' ? payload : '', 'base64').toString('utf8')) } catch { return null }
|
|
103
|
+
// Only one naming this version's tarball: its purl, and its digest as the
|
|
104
|
+
// version's integrity has it.
|
|
105
|
+
const purl = `pkg:npm/${name.replace(/^@/u, '%40')}@${version}`
|
|
106
|
+
const digest = /^sha512-(.+)$/u.exec(integrity)?.[1]
|
|
107
|
+
const subjects: unknown[] = Array.isArray(field(statement, 'subject')) ? field(statement, 'subject') as unknown[] : []
|
|
108
|
+
if (!digest || !subjects.some(subject => field(subject, 'name') === purl && field(subject, 'digest', 'sha512') === Buffer.from(digest, 'base64').toString('hex'))) return null
|
|
109
|
+
const provenance = readStatement(text(field(slsa, 'predicateType'))!, field(statement, 'predicate'))
|
|
110
|
+
const logIndex = Number(field(slsa, 'bundle', 'verificationMaterial', 'tlogEntries', '0', 'logIndex'))
|
|
111
|
+
return provenance && { ...provenance, logIndex: Number.isSafeInteger(logIndex) && logIndex >= 0 ? logIndex : null, runner: runnerOf(field(slsa, 'bundle')) }
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const provenances = kept<NpmProvenance | null>(1000)
|
|
115
|
+
|
|
116
|
+
// npm's provenance for a version of a public package, its tarball's
|
|
117
|
+
// `integrity` as the registry has it; null where npm keeps none for it.
|
|
118
|
+
export function npmProvenance(name: string, version: string, integrity: string): Promise<NpmProvenance | null> {
|
|
119
|
+
return provenances(`${name}@${version}`, () => askProvenance(name, version, integrity))
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// One kept, asking for nothing; undefined where none is.
|
|
123
|
+
export const keptNpmProvenance = (name: string, version: string) => provenances.peek(`${name}@${version}`)
|