@preventive/triage 1.0.0-alpha.37 → 1.0.0-alpha.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/README.md +16 -0
  2. package/common/bundle-compare-diff.d.ts +3 -0
  3. package/common/bundle-compare-diff.js +522 -0
  4. package/common/bundle-compare-inputs.d.ts +2 -0
  5. package/common/bundle-compare-inputs.js +65 -0
  6. package/common/bundle-metadata.d.ts +6 -2
  7. package/common/bundle-metadata.js +63 -20
  8. package/common/bundle-pkg-of.d.ts +1 -0
  9. package/common/bundle-pkg-of.js +90 -0
  10. package/common/bundle-source-links.js +7 -0
  11. package/common/bundle-sourcemap.d.ts +10 -0
  12. package/common/bundle-sourcemap.js +100 -0
  13. package/common/bundle-sources.d.ts +3 -1
  14. package/common/bundle-sources.js +47 -19
  15. package/common/code-language.js +16 -2
  16. package/common/default-scan-models.ts +2 -1
  17. package/common/dependency-reviews.d.ts +23 -0
  18. package/common/dependency-reviews.js +81 -0
  19. package/common/finding-duplicates.js +58 -0
  20. package/common/finding-filters.js +274 -0
  21. package/common/github-pr.ts +1 -1
  22. package/common/gzip.js +59 -0
  23. package/common/line-diff.d.ts +8 -0
  24. package/common/line-diff.js +56 -0
  25. package/common/linked-workspace-groups.js +63 -0
  26. package/common/managed/npm-packages.d.ts +5 -0
  27. package/common/managed/npm-packages.js +34 -0
  28. package/common/managed/permissions.ts +1 -5
  29. package/common/managed/routes.js +123 -12
  30. package/common/managed/team-app.d.ts +2 -0
  31. package/common/managed/team-app.js +31 -0
  32. package/common/managed/triage.ts +1 -1
  33. package/common/managed/upload-seal.ts +126 -0
  34. package/common/pretty-print.d.ts +5 -0
  35. package/common/pretty-print.js +24 -0
  36. package/common/revalidation-conflicts.js +48 -0
  37. package/common/revalidation-input-groups.js +56 -0
  38. package/common/workspace-app.js +23 -0
  39. package/common/workspace-groups.js +176 -0
  40. package/out/brotli-fallback.js +3 -3
  41. package/out/chunk-2KBF5TZ7.js +2 -0
  42. package/out/chunk-2XUEZHIP.js +28 -0
  43. package/out/chunk-35VE6CDH.js +2 -0
  44. package/out/chunk-3BZ7JERB.js +1 -0
  45. package/out/chunk-45WIN3SW.js +1 -0
  46. package/out/chunk-4XCDFGDP.js +10 -0
  47. package/out/chunk-6C76GAXK.js +1 -0
  48. package/out/chunk-7ZKDKGAH.js +1 -0
  49. package/out/chunk-ABW5FVSN.js +1 -0
  50. package/out/chunk-C4Z6ZBXV.js +10 -0
  51. package/out/chunk-CMQA4DM6.js +1 -0
  52. package/out/chunk-CVVJAXVY.js +2 -0
  53. package/out/chunk-DIJJNTRH.js +10 -0
  54. package/out/chunk-F4OMDNEY.js +1 -0
  55. package/out/chunk-GJ4IUCNS.js +1 -0
  56. package/out/chunk-GMFXFEUA.js +2 -0
  57. package/out/chunk-GNXOKIGP.js +80 -0
  58. package/out/chunk-GZ4MS3TE.js +13 -0
  59. package/out/chunk-HIV7SVI7.js +1 -0
  60. package/out/chunk-IU3CCINR.js +3 -0
  61. package/out/chunk-JVBB4LZX.js +1 -0
  62. package/out/chunk-K3X5EJBQ.js +2 -0
  63. package/out/chunk-KEX2ATEW.js +2 -0
  64. package/out/chunk-KYCNJVB7.js +1 -0
  65. package/out/chunk-NRPC74AM.js +1 -0
  66. package/out/chunk-OOJM4CTU.js +1 -0
  67. package/out/chunk-PNHBZGDD.js +1 -0
  68. package/out/chunk-QSLWGJBQ.js +223 -0
  69. package/out/chunk-R5O6FAW4.js +2 -0
  70. package/out/chunk-RFZQQXSG.js +17 -0
  71. package/out/chunk-TINZZKP5.js +33 -0
  72. package/out/chunk-TUII34K2.js +1 -0
  73. package/out/chunk-TZ5KB2Y5.js +10 -0
  74. package/out/chunk-UNTRGE26.js +2 -0
  75. package/out/chunk-UYAXVRVL.js +1 -0
  76. package/out/chunk-V3ME6FI5.js +2 -0
  77. package/out/chunk-V5O7M2YT.js +1 -0
  78. package/out/chunk-VIHGO77Y.js +1 -0
  79. package/out/chunk-W4J3MGLB.js +1 -0
  80. package/out/chunk-WGG7KQUH.js +1 -0
  81. package/out/chunk-XNGEQO7Y.js +2 -0
  82. package/out/chunk-XRWAV3DX.js +7 -0
  83. package/out/chunk-XYDGU6BM.js +51 -0
  84. package/out/chunk-XYFXXB2G.js +10 -0
  85. package/out/chunk-YVZH5SHP.js +1 -0
  86. package/out/chunk-ZPD5R5RP.js +3 -0
  87. package/out/client-managed-import.js +3 -87
  88. package/out/client-managed.js +26 -338
  89. package/out/client-sync.js +8 -14
  90. package/out/graph.js +17 -30
  91. package/out/index.html +34 -0
  92. package/out/prism.js +6 -2
  93. package/out/terminal.js +241 -257
  94. package/out/view.css +1 -1
  95. package/out/view.js +76 -520
  96. package/package.json +89 -7
  97. package/risk/files/characters.js +196 -0
  98. package/risk/files/hosts.js +205 -0
  99. package/risk/files/readability.js +171 -0
  100. package/risk/files/runs-code.js +358 -0
  101. package/risk/files/script.js +114 -0
  102. package/risk/files/stamps.js +46 -0
  103. package/risk/index.d.ts +142 -0
  104. package/risk/index.js +7 -0
  105. package/risk/npm/assess.js +219 -0
  106. package/risk/npm/combined.js +65 -0
  107. package/risk/npm/compare.js +228 -0
  108. package/risk/npm/dependencies.js +153 -0
  109. package/risk/npm/diff.js +43 -0
  110. package/risk/npm/downloads.js +84 -0
  111. package/risk/npm/files.js +34 -0
  112. package/risk/npm/held.js +60 -0
  113. package/risk/npm/install-scripts.js +34 -0
  114. package/risk/npm/malware.js +36 -0
  115. package/risk/npm/manifest.js +65 -0
  116. package/risk/npm/new-dependency.js +30 -0
  117. package/risk/npm/old-version.js +108 -0
  118. package/risk/npm/registry.js +282 -0
  119. package/risk/npm/release.js +85 -0
  120. package/risk/npm/specs.js +124 -0
  121. package/risk/npm/tarball.js +113 -0
  122. package/server-e2e/http.ts +7 -7
  123. package/server-e2e/npm-proxy.ts +1 -1
  124. package/server-e2e/objstore/fetch-mint-guard.ts +2 -2
  125. package/server-e2e/objstore/handlers.ts +1 -1
  126. package/server-e2e/objstore/tokens.ts +1 -1
  127. package/server-e2e/sign.ts +1 -1
  128. package/server-managed/bundle-advisories.ts +74 -12
  129. package/server-managed/bundle-cache.ts +37 -4
  130. package/server-managed/bundle-commits.ts +140 -0
  131. package/server-managed/bundle-summary-cache.ts +24 -10
  132. package/server-managed/config.ts +4 -0
  133. package/server-managed/db-methods.ts +98 -11
  134. package/server-managed/db-neon.ts +39 -4
  135. package/server-managed/db-schema.ts +14 -2
  136. package/server-managed/db-table-names.ts +1 -1
  137. package/server-managed/db.ts +7 -3
  138. package/server-managed/dependency-reviews.ts +248 -0
  139. package/server-managed/github-app.ts +9 -3
  140. package/server-managed/github-errors.ts +21 -2
  141. package/server-managed/github-issue-oauth.ts +5 -6
  142. package/server-managed/github-metadata.ts +115 -0
  143. package/server-managed/github-oauth.ts +38 -2
  144. package/server-managed/http-response.ts +26 -0
  145. package/server-managed/http.ts +952 -53
  146. package/server-managed/index.ts +11 -5
  147. package/server-managed/kept.ts +28 -0
  148. package/server-managed/locked-packages.ts +85 -0
  149. package/server-managed/lru.ts +30 -0
  150. package/server-managed/npm-insights.ts +440 -0
  151. package/server-managed/npm-kept.ts +65 -0
  152. package/server-managed/npm-loads.ts +167 -0
  153. package/server-managed/npm-manifest.ts +115 -0
  154. package/server-managed/npm-packages.ts +378 -0
  155. package/server-managed/npm-provenance.ts +123 -0
  156. package/server-managed/npm-risk.ts +177 -0
  157. package/server-managed/pretty-print.ts +352 -0
  158. package/server-managed/public-workspace.ts +78 -7
  159. package/server-managed/rename-diff-worker.js +26 -0
  160. package/server-managed/rename-diff.ts +270 -0
  161. package/server-managed/report-sources.ts +13 -2
  162. package/server-managed/repository-browser.ts +19 -5
  163. package/server-managed/repository-risk.ts +418 -0
  164. package/server-managed/session.ts +16 -2
  165. package/server-managed/sql.ts +5 -2
  166. package/server-managed/storage-encryption.ts +5 -3
  167. package/server-managed/storage-payload.ts +4 -0
  168. package/server-managed/storage-stores.ts +2 -0
  169. package/server-managed/storage.ts +6 -3
  170. package/server-managed/team-app.ts +94 -0
  171. package/server-managed/team-feed.ts +1 -1
  172. package/server-managed/team-npm-scopes.ts +77 -0
  173. package/server-managed/type-only-worker.js +19 -0
  174. package/server-managed/type-only.ts +103 -0
  175. package/server-managed/uploads.ts +25 -4
  176. package/server-managed/upstream-cache.ts +147 -5
  177. package/server-managed/view-as.ts +58 -0
@@ -0,0 +1,378 @@
1
+ // The managed npm package viewer: a published version's files, read from the
2
+ // registry's tarball, for the Overview and Code tabs the bundle view shows.
3
+ //
4
+ // Anyone with workspace access may read a public package. A private one also
5
+ // needs NPM_TOKEN on the server, and a reader with access to it: an admin or
6
+ // manager, or a member of a visible team listing its scope (team-npm-scopes.ts).
7
+ // For everyone else, a version is public only when the registry answers for
8
+ // it without credentials, asked on every request and never from a cache:
9
+ // upstream's caches, which bundle builds fill using the token, hold no
10
+ // answer here. Its tarball may come from them (npm-loads.ts), as only bytes
11
+ // matching the integrity that anonymous answer gives are served; else it is
12
+ // read without credentials too, and held to that integrity.
13
+ import { Buffer } from 'node:buffer'
14
+ import { createHash } from 'node:crypto'
15
+ import { rm } from 'node:fs/promises'
16
+ import { resolve } from 'node:path'
17
+ import { isExactVersion } from '@preventive/upstream/semver.js'
18
+ import { isNpmPackageName, isNpmPackageSpec, npmPackageScope } from '../common/managed/npm-packages.js'
19
+ import type { Role } from '../common/managed/roles.ts'
20
+ import { npmManifest, npmVersionFields, plainObject, stringRecord } from './npm-manifest.ts'
21
+ import { keepJson, readKeptJson, upstreamCachePath } from './upstream-cache.ts'
22
+
23
+ export { npmManifest, plainObject }
24
+
25
+ export const NPM_REGISTRY = 'https://registry.npmjs.org'
26
+ const VERSION_DOCUMENT_BYTES = 8 * 1024 * 1024
27
+ // Abbreviated packuments of packages with thousands of versions run to tens of MiB.
28
+ const PACKUMENT_BYTES = 64 * 1024 * 1024
29
+ // What registry documents being read at once may hold, each counted at its
30
+ // limit until it is read and parsed: four packuments, or 32 version documents.
31
+ const MAX_DOCUMENT_BYTES = 4 * PACKUMENT_BYTES
32
+ // What a package may unpack to: the files' bytes and their count; the tar
33
+ // stream holding them is bounded in npm-loads.ts.
34
+ export const MAX_NPM_PACKAGE_BYTES = 64 * 1024 * 1024
35
+ export const MAX_NPM_PACKAGE_FILES = 20_000
36
+ // What the files may take as the viewer's JSON. Escaping grows a text up to
37
+ // six times (a control character becomes `\u0001`), so it is counted before
38
+ // anything is serialized.
39
+ export const MAX_NPM_JSON_LENGTH = 96 * 1024 * 1024
40
+ export class NpmPackageError extends Error {
41
+ status: number
42
+ constructor(status: number, code: string) { super(code); this.status = status }
43
+ }
44
+
45
+ // Who reads, as the session has it now: the role, and the scopes of the
46
+ // visible teams they are a member of.
47
+ export interface NpmReader { role: Role; scopes: ReadonlySet<string>; userId: string }
48
+
49
+ // Whether a reader may have a package read with the server's token.
50
+ export function canReadPrivateNpm(reader: NpmReader, name: string): boolean {
51
+ if (reader.role === 'admin' || reader.role === 'manage') return true
52
+ const scope = npmPackageScope(name)
53
+ return scope !== null && reader.scopes.has(scope)
54
+ }
55
+
56
+ export const npmToken = () => process.env['NPM_TOKEN'] || null
57
+
58
+ function registryUrl(name: string, spec?: string): string {
59
+ return `${NPM_REGISTRY}/${name}${spec === undefined ? '' : `/${encodeURIComponent(spec)}`}`
60
+ }
61
+
62
+ // What an upstream (npm's registry and APIs, Socket, GitHub) answers, read
63
+ // whole: null where it says it has none (`absent`, 404 unless said). Asked
64
+ // without redirects, and given up on after `ms`, 30 seconds unless said, or
65
+ // once `signal` aborts, its abort rethrown as it is; a 502 where it can't be
66
+ // asked, answers otherwise or stops, and `upstream-too-large` past `limit`
67
+ // bytes (readLimited). `answered` sees the response first, its status and
68
+ // headers.
69
+ export async function askUpstream(url: string, { limit, ms = 30_000, signal, absent = [404], answered, ...init }: Omit<RequestInit, 'signal'>
70
+ & { limit: number; ms?: number; signal?: AbortSignal; absent?: number[]; answered?: (res: Response) => void }): Promise<Buffer | null> {
71
+ try {
72
+ const res = await fetch(url, { ...init, redirect: 'error', signal: AbortSignal.any([AbortSignal.timeout(ms), ...signal ? [signal] : []]) })
73
+ answered?.(res)
74
+ if (absent.includes(res.status)) { await res.body?.cancel(); return null }
75
+ if (!res.ok) { await res.body?.cancel(); throw new NpmPackageError(502, 'upstream-unavailable') }
76
+ return await readLimited(res, limit)
77
+ } catch (err) {
78
+ if (signal?.aborted || err instanceof NpmPackageError) throw err
79
+ throw new NpmPackageError(502, 'upstream-unavailable')
80
+ }
81
+ }
82
+
83
+ // The same, read as JSON (by `parse`, as one line of ndjson): a 502
84
+ // `upstream-invalid` where it isn't.
85
+ export async function askUpstreamJson(url: string, options: Parameters<typeof askUpstream>[1], parse = (text: string): unknown => JSON.parse(text)): Promise<unknown> {
86
+ const body = await askUpstream(url, options)
87
+ if (body === null) return null
88
+ try { return parse(body.toString('utf8')) } catch { throw new NpmPackageError(502, 'upstream-invalid') }
89
+ }
90
+
91
+ export async function readLimited(res: Response, limit: number): Promise<Buffer> {
92
+ const declared = Number(res.headers.get('content-length'))
93
+ if (declared > limit) { await res.body?.cancel(); throw new NpmPackageError(502, 'upstream-too-large') }
94
+ const chunks: Uint8Array[] = []
95
+ let size = 0
96
+ for await (const chunk of res.body ?? []) {
97
+ size += chunk.byteLength
98
+ if (size > limit) throw new NpmPackageError(502, 'upstream-too-large')
99
+ chunks.push(chunk)
100
+ }
101
+ return Buffer.concat(chunks)
102
+ }
103
+
104
+ // A registry document, or null where the registry has none for this caller:
105
+ // 404, or 401/403 as it answers some private packages asked anonymously.
106
+ // Credentials go only with `token`; nothing is cached on either side, but for
107
+ // public whole documents (publicWholeDocument).
108
+ async function registryDocument(url: string, { token, accept, limit, signal }: { token: string | null; accept: string; limit: number; signal: AbortSignal }): Promise<Record<string, unknown> | null> {
109
+ const json = await askUpstreamJson(url, { headers: { accept, ...(token ? { authorization: `Bearer ${token}` } : {}) }, limit, signal, absent: [401, 403, 404] })
110
+ if (json === null || plainObject(json)) return json
111
+ throw new NpmPackageError(502, 'upstream-invalid')
112
+ }
113
+
114
+ // Where public packages' whole documents are kept, off Vercel: the upstream
115
+ // cache dir, set at startup (index.ts); null keeps none.
116
+ let documentCache: string | null = null
117
+ export const setNpmDocumentCache = (dir: string | null) => { documentCache = dir === null ? null : resolve(dir) }
118
+
119
+ // A package's whole document as npm answers anyone (`url` its own), kept on
120
+ // disk where the server keeps them (setNpmDocumentCache) with npm's ETag, and
121
+ // asked for again each time with it: npm answers 304, and nothing more, while
122
+ // it is unchanged. Null where npm has none for anyone, nothing then kept.
123
+ async function publicWholeDocument(name: string, url: string, { limit, signal }: { limit: number; signal: AbortSignal }): Promise<Record<string, unknown> | null> {
124
+ const path = documentCache === null ? null : upstreamCachePath(documentCache, 'npm/packuments', `${name}.json.gz`)
125
+ const kept = path === null ? null : await readKeptJson(path, limit)
126
+ let etag: string | null = null, status = 0
127
+ const body = await askUpstream(url, { headers: { accept: 'application/json', ...kept && { 'if-none-match': kept.etag } }, limit, signal, absent: [304, 401, 403, 404],
128
+ answered: res => { status = res.status; etag = res.headers.get('etag') } })
129
+ if (status === 304 && kept) return kept.json
130
+ if (body === null) {
131
+ if (path !== null && kept) await rm(path, { force: true }).catch(() => {})
132
+ return null
133
+ }
134
+ let json: unknown
135
+ try { json = JSON.parse(body.toString('utf8')) } catch { throw new NpmPackageError(502, 'upstream-invalid') }
136
+ if (!plainObject(json)) throw new NpmPackageError(502, 'upstream-invalid')
137
+ if (path !== null && etag !== null) await keepJson(path, etag, body)
138
+ return json
139
+ }
140
+
141
+ // Asked anonymously first: a reader without private access stops there. One
142
+ // with it asks again with the token, where there is one and the package is
143
+ // scoped (npm's private packages always are), and the answer is private. A
144
+ // whole document (`whole`) anyone may read is kept (publicWholeDocument).
145
+ async function readDocument(name: string, url: string, privileged: boolean, { whole = false, ...options }: { accept: string; limit: number; signal: AbortSignal; whole?: boolean }) {
146
+ const open = whole ? await publicWholeDocument(name, url, options) : await registryDocument(url, { ...options, token: null })
147
+ if (open) return { json: open, private: false }
148
+ const token = npmToken()
149
+ if (!privileged || !token || npmPackageScope(name) === null) return null
150
+ const json = await registryDocument(url, { ...options, token })
151
+ return json ? { json, private: true } : null
152
+ }
153
+
154
+ let documentBytes = 0
155
+
156
+ // A `spare` read, one no reader waits on, is made only where it leaves room
157
+ // for two of the largest documents more: readers' own reads come first.
158
+ async function withDocumentBudget<T>(limit: number, read: () => Promise<T>, spare = false): Promise<T> {
159
+ if (documentBytes + limit + (spare ? 2 * PACKUMENT_BYTES : 0) > MAX_DOCUMENT_BYTES) throw new NpmPackageError(429, 'npm-busy')
160
+ documentBytes += limit
161
+ try { return await read() } finally { documentBytes -= limit }
162
+ }
163
+
164
+ export interface NpmVersionDocument {
165
+ name: string
166
+ version: string
167
+ private: boolean
168
+ dist: { tarball: string; integrity: string; unpackedSize: number | null; fileCount: number | null }
169
+ manifest: Record<string, unknown>
170
+ }
171
+
172
+ const count = (value: unknown) => Number.isSafeInteger(value) && (value as number) >= 0 ? value as number : null
173
+
174
+ // The version `json`, a document of package `name`, tells of.
175
+ function versionDocument(name: string, json: Record<string, unknown>, isPrivate: boolean): NpmVersionDocument {
176
+ const dist = json['dist']
177
+ if (json['name'] !== name || typeof json['version'] !== 'string' || !isNpmPackageSpec(json['version']) || !plainObject(dist)
178
+ || typeof dist['tarball'] !== 'string' || typeof dist['integrity'] !== 'string') throw new NpmPackageError(502, 'upstream-invalid')
179
+ return {
180
+ name, version: json['version'], private: isPrivate,
181
+ dist: { tarball: dist['tarball'], integrity: dist['integrity'], unpackedSize: count(dist['unpackedSize']), fileCount: count(dist['fileCount']) },
182
+ manifest: npmManifest(json),
183
+ }
184
+ }
185
+
186
+ // Where public versions' documents are kept, of the fields read of them
187
+ // (npm-manifest.ts npmVersionFields), as the database keeps them (npm-kept.ts).
188
+ export interface NpmVersionStore {
189
+ read(name: string, version: string): Promise<{ at: number; json: Record<string, unknown> } | null>
190
+ write(name: string, version: string, json: Record<string, unknown>): Promise<void>
191
+ }
192
+
193
+ // The version a spec names, a dist-tag resolved by the registry, with its
194
+ // dist. Null where the registry has none for this reader. With `store`, an
195
+ // exact version kept there for less than a day (VERSION_KEPT_MS) is read from
196
+ // it, not asked of npm, and a public one npm answers is kept. npm never
197
+ // changes a version's files, but its document can change: a deprecation, the
198
+ // package made private (a scoped one can be), the version removed (which whoever
199
+ // reads it from a store learns elsewhere too: risk does). A day bounds each.
200
+ export const VERSION_KEPT_MS = 24 * 60 * 60_000
201
+ export function readNpmVersion(name: string, spec: string, privileged: boolean, signal: AbortSignal, store?: NpmVersionStore): Promise<NpmVersionDocument | null> {
202
+ if (!isNpmPackageName(name) || !isNpmPackageSpec(spec)) return Promise.reject(new NpmPackageError(400, 'bad-package'))
203
+ // A kept one is read within the same budget as npm's answer.
204
+ return withDocumentBudget(VERSION_DOCUMENT_BYTES, async () => {
205
+ const kept = store && isExactVersion(spec) ? await store.read(name, spec) : null
206
+ if (kept && Date.now() - kept.at < VERSION_KEPT_MS) try { return versionDocument(name, kept.json, false) } catch {}
207
+ const found = await readDocument(name, registryUrl(name, spec), privileged, { accept: 'application/json', limit: VERSION_DOCUMENT_BYTES, signal })
208
+ if (!found) return null
209
+ const doc = versionDocument(name, found.json, found.private)
210
+ if (store && !found.private) await store.write(name, doc.version, npmVersionFields(found.json))
211
+ return doc
212
+ })
213
+ }
214
+
215
+ // The package's dist-tags and versions, newest published first, from its
216
+ // abbreviated document; with `times`, from its whole one, which alone has
217
+ // when each was published, `times` by version (ISO dates), and `created`,
218
+ // when its first version was, one since unpublished among them: the earliest
219
+ // time it lists, which npm's own `created` can postdate, as a placeholder's
220
+ // does for a package npm took down. With `spare`, as a read no reader waits
221
+ // on (withDocumentBudget). Null where the registry has none for this reader.
222
+ export function readNpmVersions(name: string, privileged: boolean, signal: AbortSignal, { times = false, spare = false } = {}) {
223
+ if (!isNpmPackageName(name)) return Promise.reject(new NpmPackageError(400, 'bad-package'))
224
+ return withDocumentBudget(PACKUMENT_BYTES, async () => {
225
+ const accept = times ? 'application/json' : 'application/vnd.npm.install-v1+json'
226
+ const found = await readDocument(name, registryUrl(name), privileged, { accept, limit: PACKUMENT_BYTES, signal, whole: times })
227
+ if (!found) return null
228
+ const { json } = found
229
+ if (json['name'] !== name || !plainObject(json['versions'])) throw new NpmPackageError(502, 'upstream-invalid')
230
+ const versions = Object.keys(json['versions']).filter(isNpmPackageSpec).toReversed()
231
+ const distTags = Object.fromEntries(Object.entries(stringRecord(json['dist-tags']) ?? {}).filter(([, version]) => versions.includes(version)))
232
+ const listed = { name, private: found.private, distTags, versions }
233
+ if (!times) return listed
234
+ const time = stringRecord(json['time']) ?? {}
235
+ const published = Object.fromEntries(versions.filter(version => isPublishTime(time[version])).map(version => [version, time[version]!]))
236
+ for (const [version, at] of Object.entries(published)) publishTimes.set(`${name}@${version}`, at)
237
+ while (publishTimes.size > PUBLISH_TIMES) publishTimes.delete(publishTimes.keys().next().value!)
238
+ const listedAt = Object.entries(time).filter(([key, at]) => key !== 'modified' && isPublishTime(at)).map(([, at]) => at)
239
+ const created = listedAt.reduce<string | null>((first, at) => first === null || Date.parse(at) < Date.parse(first) ? at : first, null)
240
+ return { ...listed, times: published, created }
241
+ }, spare)
242
+ }
243
+
244
+ // A package's whole document as npm has it for this reader, for risk
245
+ // (npm-risk.ts), within the bounds and budget every document read here is
246
+ // held to; null where npm has none for this reader, or the name is none npm
247
+ // takes.
248
+ export function readNpmPackument(name: string, privileged: boolean, signal: AbortSignal): Promise<Record<string, unknown> | null> {
249
+ if (!isNpmPackageName(name)) return Promise.resolve(null)
250
+ return withDocumentBudget(PACKUMENT_BYTES, async () => (await readDocument(name, registryUrl(name), privileged, { accept: 'application/json', limit: PACKUMENT_BYTES, signal, whole: true }))?.json ?? null)
251
+ }
252
+
253
+ const isPublishTime = (value: string | undefined) => typeof value === 'string' && value.length <= 40 && !Number.isNaN(Date.parse(value))
254
+
255
+ // When each version was published, by `name@version`, as whole documents read
256
+ // with `times` have said: a version keeps its time once published, so it is
257
+ // kept for as long as the server runs, the first noted going first past a
258
+ // bound, and answered only with the version, once its reader's access is.
259
+ const PUBLISH_TIMES = 100_000
260
+ const publishTimes = new Map<string, string>()
261
+
262
+ export const npmPublishTime = (name: string, version: string): string | null => publishTimes.get(`${name}@${version}`) ?? null
263
+
264
+ export interface NpmPackageFile { path: string; bytes: Uint8Array }
265
+
266
+ const textOf = (bytes: Uint8Array, start: number, length: number) => {
267
+ const field = bytes.subarray(start, start + length)
268
+ const end = field.indexOf(0)
269
+ return Buffer.from(end === -1 ? field : field.subarray(0, end)).toString('utf8')
270
+ }
271
+
272
+ function sizeOf(bytes: Uint8Array, offset: number): number {
273
+ // GNU base-256 for sizes past the 11 octal digits.
274
+ if (bytes[offset]! & 0x80) {
275
+ let size = bytes[offset]! & 0x7f
276
+ for (let i = 1; i < 12; i++) size = size * 256 + bytes[offset + i]!
277
+ return size
278
+ }
279
+ const text = textOf(bytes, offset, 12).trim()
280
+ return /^[0-7]+$/u.test(text) ? Number.parseInt(text, 8) : Number.NaN
281
+ }
282
+
283
+ // The path a package's file is installed at: npm drops the first component
284
+ // (`package/`, or whatever the archive used), and a path that would escape the
285
+ // package, or names nothing, is not extracted.
286
+ function packagePath(raw: string): string | null {
287
+ const parts = raw.split('/').slice(1).filter(part => part !== '' && part !== '.')
288
+ return parts.length === 0 || parts.includes('..') ? null : parts.join('/')
289
+ }
290
+
291
+ // A pax header's path. Each record is `<length> <key>=<value>\n`, its length
292
+ // in bytes, so records are walked as bytes and only their values decoded.
293
+ function paxPath(data: Uint8Array): string | null {
294
+ let path = null
295
+ let at = 0
296
+ while (at < data.length) {
297
+ const space = data.indexOf(0x20, at)
298
+ const length = Number.parseInt(Buffer.from(data.subarray(at, space)).toString('latin1'), 10)
299
+ if (space === -1 || !Number.isSafeInteger(length) || length <= 0) break
300
+ const record = Buffer.from(data.subarray(space + 1, at + length - 1)).toString('utf8')
301
+ if (record.startsWith('path=')) path = record.slice(5)
302
+ at += length
303
+ }
304
+ return path
305
+ }
306
+
307
+ // The regular files of a tar stream, by package path, a later entry
308
+ // replacing an earlier one as extraction would. Directories, links and
309
+ // devices hold no file content and are skipped.
310
+ export function readNpmTar(tar: Uint8Array): NpmPackageFile[] {
311
+ const files = new Map<string, Uint8Array>()
312
+ let total = 0
313
+ let longName: string | null = null
314
+ let offset = 0
315
+ while (offset + 512 <= tar.length) {
316
+ const header = tar.subarray(offset, offset + 512)
317
+ if (header.every(byte => byte === 0)) break
318
+ const size = sizeOf(header, 124)
319
+ const type = String.fromCodePoint(header[156]!)
320
+ const start = offset + 512
321
+ if (!Number.isSafeInteger(size) || size < 0 || start + size > tar.length) throw new NpmPackageError(422, 'bad-tarball')
322
+ const data = tar.subarray(start, start + size)
323
+ offset = start + Math.ceil(size / 512) * 512
324
+ if (type === 'x' || type === 'L') { longName = type === 'x' ? paxPath(data) : textOf(data, 0, data.length); continue }
325
+ if (type === 'g' || type === 'K') continue
326
+ const prefix = textOf(header, 257, 6) === 'ustar' ? textOf(header, 345, 155) : ''
327
+ const name = longName ?? (prefix ? `${prefix}/${textOf(header, 0, 100)}` : textOf(header, 0, 100))
328
+ longName = null
329
+ if (!['0', '\0', '7'].includes(type)) continue
330
+ const path = packagePath(name)
331
+ if (path === null) continue
332
+ total += size - (files.get(path)?.length ?? 0)
333
+ files.set(path, data)
334
+ if (files.size > MAX_NPM_PACKAGE_FILES || total > MAX_NPM_PACKAGE_BYTES) throw new NpmPackageError(413, 'package-too-large')
335
+ }
336
+ return [...files].map(([path, bytes]) => ({ path, bytes })).toSorted((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)
337
+ }
338
+
339
+ const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true })
340
+
341
+ // A file's text, or null for bytes that are not UTF-8 text.
342
+ export function npmFileText(bytes: Uint8Array): string | null {
343
+ if (bytes.includes(0)) return null
344
+ try { return utf8.decode(bytes) } catch { return null }
345
+ }
346
+
347
+ // A string's length in JSON: quotes, backslashes and the control characters
348
+ // with a short escape take two characters, the other control characters six.
349
+ function jsonLength(text: string): number {
350
+ let length = text.length + 2
351
+ for (let i = 0; i < text.length; i++) {
352
+ const code = text.codePointAt(i)!
353
+ if (code === 0x22 || code === 0x5c) length += 1
354
+ else if (code < 0x20) length += code >= 0x08 && code <= 0x0d && code !== 0x0b ? 1 : 5
355
+ }
356
+ return length
357
+ }
358
+
359
+ // A row's brackets, size, null and digest, past its strings.
360
+ const ROW_LENGTH = 80
361
+
362
+ // The files as the viewer gets them: `[path, bytes, text]`, text null for a
363
+ // file that is not UTF-8, which instead carries its bytes' sha256, so a
364
+ // comparison of two versions tells a changed one from an unchanged one.
365
+ export function npmFileRows(files: NpmPackageFile[]): ([string, number, string] | [string, number, null, string])[] {
366
+ let length = 0
367
+ return files.map(({ path, bytes }) => {
368
+ const text = npmFileText(bytes)
369
+ length += ROW_LENGTH + jsonLength(path) + (text === null ? 0 : jsonLength(text))
370
+ if (length > MAX_NPM_JSON_LENGTH) throw new NpmPackageError(413, 'package-too-large')
371
+ return text === null ? [path, bytes.byteLength, null, `sha256-${createHash('sha256').update(bytes).digest('base64')}`] : [path, bytes.byteLength, text]
372
+ })
373
+ }
374
+
375
+ // The tarball's filename as `npm pack` writes it.
376
+ export function npmTarballFilename(name: string, version: string): string {
377
+ return `${name.replace(/^@/u, '').replace('/', '-')}-${version}.tgz`
378
+ }
@@ -0,0 +1,123 @@
1
+ // npm's provenance for a public version (npm-packages.ts npmManifest marks
2
+ // one that has it): the SLSA statement npm's registry keeps beside it,
3
+ // saying where the version was built: from which repository and commit, by
4
+ // which workflow, on which builder, in which run, with its entry in
5
+ // Sigstore's transparency log. Only a statement naming this version's
6
+ // tarball (its name, version and sha512 digest) is read; its signature isn't
7
+ // checked here, as npm checks it when the version is published. npm is asked
8
+ // without the server's token, and only about a version it answers for
9
+ // without one (http.ts), each answer kept as npm-insights.ts keeps its own.
10
+ import { Buffer } from 'node:buffer'
11
+ import { kept } from './kept.ts'
12
+ import { NPM_REGISTRY, askUpstreamJson, plainObject } from './npm-packages.ts'
13
+
14
+ const ATTESTATIONS_API = `${NPM_REGISTRY}/-/npm/v1/attestations`
15
+ const ATTESTATIONS_BYTES = 1024 * 1024
16
+ // SLSA's v1 statement, as npm writes it now, and v0.2, as it first did.
17
+ const SLSA = /^https:\/\/slsa\.dev\/provenance\/v(?:1|0\.2)$/u
18
+ const COMMIT = /^[0-9a-f]{40}$/u
19
+ const GITHUB_REPOSITORY = /^https:\/\/github\.com\/([\w.-]+\/[\w.-]+)$/u
20
+
21
+ // Where a version was built: its source `repository` (an https URL) and
22
+ // `commit`, the `workflow` file and the `ref` it ran on, the `builder` and
23
+ // the `run` (an https URL), its transparency log entry (`logIndex`), and
24
+ // whose runner built it (`runner`: `github-hosted`, `gitlab-hosted` or
25
+ // `self-hosted`, as its signing certificate says); each null where they
26
+ // don't say.
27
+ export interface NpmProvenance {
28
+ repository: string; commit: string | null; workflow: string | null; ref: string | null
29
+ builder: string | null; run: string | null; logIndex: number | null; runner: string | null
30
+ }
31
+
32
+ const text = (value: unknown) => typeof value === 'string' && value !== '' && value.length <= 2048 ? value : null
33
+ const https = (value: unknown) => {
34
+ const url = text(value)
35
+ try { return url && new URL(url).protocol === 'https:' ? url : null } catch { return null }
36
+ }
37
+ // `git+https://github.com/org/repo@refs/heads/main` → the repository and
38
+ // ref; GitLab's names no ref (`git+https://gitlab.com/group/project`).
39
+ function sourceOf(uri: unknown) {
40
+ const match = /^git\+(https:\/\/[^@\s]+?)(?:\.git)?(?:@(.+))?$/u.exec(text(uri) ?? '')
41
+ return match ? { repository: https(match[1]), ref: match[2] ?? null } : { repository: null, ref: null }
42
+ }
43
+ // What `keys` lead to in `value`, through objects and arrays.
44
+ const field = (value: unknown, ...keys: string[]): unknown =>
45
+ keys.reduce<unknown>((at, key) => plainObject(at) || Array.isArray(at) ? (at as Record<string, unknown>)[key] : undefined, value)
46
+
47
+ // What a statement says of the build, as v1 or v0.2 says it; null where it
48
+ // names no source repository.
49
+ function readStatement(predicateType: string, predicate: unknown): NpmProvenance | null {
50
+ if (predicateType.endsWith('v1')) {
51
+ const workflow = field(predicate, 'buildDefinition', 'externalParameters', 'workflow')
52
+ const source = field(predicate, 'buildDefinition', 'resolvedDependencies', '0')
53
+ const repository = https(field(workflow, 'repository')) ?? sourceOf(field(source, 'uri')).repository
54
+ if (!repository) return null
55
+ const commit = text(field(source, 'digest', 'gitCommit'))
56
+ return {
57
+ repository, commit: commit && COMMIT.test(commit) ? commit : null, workflow: text(field(workflow, 'path')),
58
+ ref: text(field(workflow, 'ref')) ?? sourceOf(field(source, 'uri')).ref, builder: https(field(predicate, 'runDetails', 'builder', 'id')),
59
+ run: https(field(predicate, 'runDetails', 'metadata', 'invocationId')), logIndex: null, runner: null,
60
+ }
61
+ }
62
+ const config = field(predicate, 'invocation', 'configSource')
63
+ const { repository, ref } = sourceOf(field(config, 'uri'))
64
+ if (!repository) return null
65
+ const commit = text(field(config, 'digest', 'sha1'))
66
+ // Its run: GitLab's job URL, or GitHub's as `<run id>-<attempt>`.
67
+ const invocation = field(predicate, 'metadata', 'buildInvocationId')
68
+ const run = /^(\d+)-(\d+)$/u.exec(text(invocation) ?? '')
69
+ const github = GITHUB_REPOSITORY.exec(repository)?.[1]
70
+ return {
71
+ repository, commit: commit && COMMIT.test(commit) ? commit : null, workflow: text(field(config, 'entryPoint')), ref,
72
+ builder: https(field(predicate, 'builder', 'id')),
73
+ run: https(invocation) ?? (run && github ? `https://github.com/${github}/actions/runs/${run[1]}/attempts/${run[2]}` : null), logIndex: null, runner: null,
74
+ }
75
+ }
76
+
77
+ // Fulcio's Runner Environment extension (1.3.6.1.4.1.57264.1.11) in the
78
+ // certificate that signed it: its OID, then its value, a short UTF8String
79
+ // in an OCTET STRING, as the CI's token gave it. A builder's id doesn't
80
+ // tell GitLab's own runners from a project's.
81
+ const RUNNER_ENVIRONMENT = Buffer.from('060a2b0601040183bf30010b', 'hex')
82
+ function runnerOf(bundle: unknown): string | null {
83
+ const raw = field(bundle, 'verificationMaterial', 'certificate', 'rawBytes') ?? field(bundle, 'verificationMaterial', 'x509CertificateChain', 'certificates', '0', 'rawBytes')
84
+ if (typeof raw !== 'string') return null
85
+ const der = Buffer.from(raw, 'base64')
86
+ const at = der.indexOf(RUNNER_ENVIRONMENT) + RUNNER_ENVIRONMENT.length
87
+ if (at < RUNNER_ENVIRONMENT.length || der[at] !== 0x04 || der[at + 2] !== 0x0c || der[at + 1] !== der[at + 3]! + 2) return null
88
+ return text(der.subarray(at + 4, at + 4 + der[at + 3]!).toString('utf8'))
89
+ }
90
+
91
+ async function askProvenance(name: string, version: string, integrity: string): Promise<NpmProvenance | null> {
92
+ const json = await askUpstreamJson(`${ATTESTATIONS_API}/${name.replace('/', '%2f')}@${encodeURIComponent(version)}`,
93
+ { headers: { accept: 'application/json' }, limit: ATTESTATIONS_BYTES })
94
+ if (json === null) return null
95
+ const attestations: unknown[] = Array.isArray(field(json, 'attestations')) ? field(json, 'attestations') as unknown[] : []
96
+ const slsa = attestations.find(item => SLSA.test(text(field(item, 'predicateType')) ?? ''))
97
+ if (!slsa) return null
98
+ // The statement, whatever its length (the answer is bounded as a whole),
99
+ // unlike the fields read out of it.
100
+ const payload = field(slsa, 'bundle', 'dsseEnvelope', 'payload')
101
+ let statement: unknown
102
+ try { statement = JSON.parse(Buffer.from(typeof payload === 'string' ? payload : '', 'base64').toString('utf8')) } catch { return null }
103
+ // Only one naming this version's tarball: its purl, and its digest as the
104
+ // version's integrity has it.
105
+ const purl = `pkg:npm/${name.replace(/^@/u, '%40')}@${version}`
106
+ const digest = /^sha512-(.+)$/u.exec(integrity)?.[1]
107
+ const subjects: unknown[] = Array.isArray(field(statement, 'subject')) ? field(statement, 'subject') as unknown[] : []
108
+ if (!digest || !subjects.some(subject => field(subject, 'name') === purl && field(subject, 'digest', 'sha512') === Buffer.from(digest, 'base64').toString('hex'))) return null
109
+ const provenance = readStatement(text(field(slsa, 'predicateType'))!, field(statement, 'predicate'))
110
+ const logIndex = Number(field(slsa, 'bundle', 'verificationMaterial', 'tlogEntries', '0', 'logIndex'))
111
+ return provenance && { ...provenance, logIndex: Number.isSafeInteger(logIndex) && logIndex >= 0 ? logIndex : null, runner: runnerOf(field(slsa, 'bundle')) }
112
+ }
113
+
114
+ const provenances = kept<NpmProvenance | null>(1000)
115
+
116
+ // npm's provenance for a version of a public package, its tarball's
117
+ // `integrity` as the registry has it; null where npm keeps none for it.
118
+ export function npmProvenance(name: string, version: string, integrity: string): Promise<NpmProvenance | null> {
119
+ return provenances(`${name}@${version}`, () => askProvenance(name, version, integrity))
120
+ }
121
+
122
+ // One kept, asking for nothing; undefined where none is.
123
+ export const keptNpmProvenance = (name: string, version: string) => provenances.peek(`${name}@${version}`)