@preventive/triage 1.0.0-alpha.4 → 1.0.0-alpha.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/README.md +16 -0
- package/api/reap.ts +17 -0
- package/cli.js +6 -0
- package/client/dependency-paths.d.ts +2 -0
- package/client/dependency-paths.js +24 -0
- package/client/finding-link.js +305 -0
- package/client/linked-findings.d.ts +1 -0
- package/client/linked-findings.js +112 -0
- package/common/bundle-commit.d.ts +1 -0
- package/common/bundle-commit.js +4 -0
- package/common/bundle-compare-diff.d.ts +3 -0
- package/common/bundle-compare-diff.js +522 -0
- package/common/bundle-compare-inputs.d.ts +2 -0
- package/common/bundle-compare-inputs.js +65 -0
- package/common/bundle-metadata.d.ts +14 -0
- package/common/bundle-metadata.js +226 -0
- package/common/bundle-pkg-of.d.ts +1 -0
- package/common/bundle-pkg-of.js +90 -0
- package/common/bundle-reasons.d.ts +2 -0
- package/common/bundle-reasons.js +21 -0
- package/common/bundle-source-links.d.ts +6 -0
- package/common/bundle-source-links.js +216 -0
- package/common/bundle-sourcemap.d.ts +10 -0
- package/common/bundle-sourcemap.js +100 -0
- package/common/bundle-sources.d.ts +7 -0
- package/common/bundle-sources.js +310 -0
- package/common/bundle-stats.js +41 -0
- package/common/bundle-tabs.js +1 -0
- package/common/code-language.js +65 -0
- package/common/default-scan-models.ts +31 -0
- package/common/dependency-reviews.d.ts +23 -0
- package/common/dependency-reviews.js +81 -0
- package/common/finding-duplicates.js +58 -0
- package/common/finding-filters.js +274 -0
- package/common/finding-id.js +47 -0
- package/common/github-issue-labels.d.ts +1 -0
- package/common/github-issue-labels.js +11 -0
- package/common/github-pr.ts +56 -0
- package/common/gzip.js +59 -0
- package/common/line-diff.d.ts +8 -0
- package/common/line-diff.js +56 -0
- package/common/linked-workspace-groups.js +63 -0
- package/common/managed/comments.ts +34 -0
- package/common/managed/npm-packages.d.ts +5 -0
- package/common/managed/npm-packages.js +34 -0
- package/common/managed/permissions.ts +31 -0
- package/common/managed/report-content.ts +72 -0
- package/common/managed/report-filter.ts +108 -0
- package/common/managed/repository-alias.ts +65 -0
- package/common/managed/roles.ts +28 -0
- package/common/managed/routes.d.ts +2 -0
- package/common/managed/routes.js +254 -0
- package/common/managed/scan-models.ts +6 -0
- package/common/managed/team-app.d.ts +2 -0
- package/common/managed/team-app.js +31 -0
- package/common/managed/triage.ts +83 -0
- package/common/managed/upload-seal.ts +126 -0
- package/common/pretty-print.d.ts +5 -0
- package/common/pretty-print.js +24 -0
- package/common/revalidation-conflicts.js +48 -0
- package/common/revalidation-input-groups.js +56 -0
- package/common/save-error-reason.ts +20 -7
- package/common/scan-server.ts +13 -0
- package/common/server-info.ts +35 -0
- package/common/utf8.d.ts +3 -0
- package/common/utf8.js +45 -0
- package/common/workspace-app.js +23 -0
- package/common/workspace-groups.js +176 -0
- package/out/brotli-fallback.js +3 -3
- package/out/chunk-2AFQKRAM.js +51 -0
- package/out/chunk-2KBF5TZ7.js +2 -0
- package/out/chunk-2XUEZHIP.js +28 -0
- package/out/chunk-35VE6CDH.js +2 -0
- package/out/chunk-3BZ7JERB.js +1 -0
- package/out/chunk-4F6JSF57.js +3 -0
- package/out/chunk-4XCDFGDP.js +10 -0
- package/out/chunk-C4Z6ZBXV.js +10 -0
- package/out/chunk-CMQA4DM6.js +1 -0
- package/out/chunk-CVVJAXVY.js +2 -0
- package/out/chunk-DIJJNTRH.js +10 -0
- package/out/chunk-GJ4IUCNS.js +1 -0
- package/out/chunk-GNXOKIGP.js +80 -0
- package/out/chunk-GZ4MS3TE.js +13 -0
- package/out/chunk-IU3CCINR.js +3 -0
- package/out/chunk-K3X5EJBQ.js +2 -0
- package/out/chunk-KEX2ATEW.js +2 -0
- package/out/chunk-KVEUXIEU.js +1 -0
- package/out/chunk-KYCNJVB7.js +1 -0
- package/out/chunk-NRPC74AM.js +1 -0
- package/out/chunk-OOJM4CTU.js +1 -0
- package/out/chunk-PNHBZGDD.js +1 -0
- package/out/chunk-QZ3A376W.js +1 -0
- package/out/chunk-R5O6FAW4.js +2 -0
- package/out/chunk-RFZQQXSG.js +17 -0
- package/out/chunk-RSHMVYNS.js +1 -0
- package/out/chunk-TUII34K2.js +1 -0
- package/out/chunk-TY4RZD3Z.js +226 -0
- package/out/chunk-TZ5KB2Y5.js +10 -0
- package/out/chunk-U6OPEXIA.js +1 -0
- package/out/chunk-UNTRGE26.js +2 -0
- package/out/chunk-UXNUAIIC.js +33 -0
- package/out/chunk-UYAXVRVL.js +1 -0
- package/out/chunk-UZAZM2SX.js +2 -0
- package/out/chunk-V3ME6FI5.js +2 -0
- package/out/chunk-V5O7M2YT.js +1 -0
- package/out/chunk-VIHGO77Y.js +1 -0
- package/out/chunk-VQJCXGIY.js +7 -0
- package/out/chunk-W4J3MGLB.js +1 -0
- package/out/chunk-WGG7KQUH.js +1 -0
- package/out/chunk-WZDTZE64.js +1 -0
- package/out/chunk-XL35ZV6R.js +1 -0
- package/out/chunk-XYFXXB2G.js +10 -0
- package/out/chunk-YVZH5SHP.js +1 -0
- package/out/chunk-ZBBOBQ43.js +1 -0
- package/out/chunk-ZMQQCGF6.js +2 -0
- package/out/client-managed-import.js +4 -0
- package/out/client-managed.js +37 -0
- package/out/client-sync.js +10 -13
- package/out/github-issue-authorized.html +3 -0
- package/out/github-issue-authorized.js +1 -0
- package/out/graph.js +17 -4
- package/out/index.html +90 -8
- package/out/prism.js +6 -2
- package/out/stasis.svg +20 -0
- package/out/terminal.js +365 -39
- package/out/view.css +1 -1
- package/out/view.js +83 -117
- package/package.json +298 -59
- package/risk/files/characters.js +196 -0
- package/risk/files/hosts.js +205 -0
- package/risk/files/readability.js +171 -0
- package/risk/files/runs-code.js +595 -0
- package/risk/files/script.js +114 -0
- package/risk/files/stamps.js +46 -0
- package/risk/index.d.ts +156 -0
- package/risk/index.js +7 -0
- package/risk/npm/assess.js +223 -0
- package/risk/npm/combined.js +65 -0
- package/risk/npm/compare.js +228 -0
- package/risk/npm/dependencies.js +153 -0
- package/risk/npm/diff.js +43 -0
- package/risk/npm/downloads.js +84 -0
- package/risk/npm/files.js +34 -0
- package/risk/npm/held.js +60 -0
- package/risk/npm/install-scripts.js +34 -0
- package/risk/npm/malware.js +36 -0
- package/risk/npm/manifest.js +65 -0
- package/risk/npm/new-dependency.js +30 -0
- package/risk/npm/old-version.js +108 -0
- package/risk/npm/registry.js +282 -0
- package/risk/npm/release.js +85 -0
- package/risk/npm/score.js +59 -0
- package/risk/npm/specs.js +124 -0
- package/risk/npm/tarball.js +113 -0
- package/server-common/database-config.ts +16 -0
- package/server-common/initialize.ts +18 -0
- package/server-common/npm-advisories.ts +101 -0
- package/{server → server-common}/origin.ts +5 -5
- package/server-common/reap.ts +48 -0
- package/server-common/scan-config.ts +19 -0
- package/server-common/standalone.ts +29 -0
- package/server-common/storage-crypto.ts +182 -0
- package/server-common/storage-log.ts +38 -0
- package/server-common/vercel-blob.ts +103 -0
- package/server-e2e/app.ts +486 -0
- package/{server → server-e2e}/auth.ts +5 -1
- package/{server → server-e2e}/bus-receiver.ts +2 -1
- package/{server → server-e2e}/cli.js +9 -4
- package/{server → server-e2e}/config.ts +56 -35
- package/{server → server-e2e}/db-neon.ts +2 -2
- package/{server → server-e2e}/db-revision-sql.ts +1 -1
- package/{server → server-e2e}/db-stmt.ts +2 -2
- package/{server → server-e2e}/db.ts +7 -7
- package/{server → server-e2e}/http.ts +116 -18
- package/{server → server-e2e}/hub.ts +40 -9
- package/server-e2e/index.ts +17 -0
- package/server-e2e/lifecycle.ts +95 -0
- package/{server → server-e2e}/neon-driver.ts +2 -2
- package/{server → server-e2e}/npm-proxy.ts +12 -145
- package/{server → server-e2e}/objstore/blob-vercel.ts +10 -114
- package/{server → server-e2e}/objstore/blob.ts +1 -1
- package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
- package/{server → server-e2e}/objstore/handlers.ts +9 -8
- package/{server → server-e2e}/objstore/init.ts +35 -18
- package/{server → server-e2e}/objstore/reaper.ts +1 -1
- package/server-e2e/objstore/rest-deny.ts +28 -0
- package/server-e2e/objstore/rest-mint.ts +224 -0
- package/{server → server-e2e}/objstore/rest.ts +27 -26
- package/{server → server-e2e}/objstore/sign.ts +105 -0
- package/{server → server-e2e}/objstore/store-neon.ts +6 -10
- package/{server → server-e2e}/objstore/store.ts +7 -7
- package/{server → server-e2e}/objstore/tokens.ts +2 -2
- package/{server → server-e2e}/pubsub.ts +12 -9
- package/{server → server-e2e}/sign.ts +4 -4
- package/{server → server-e2e}/sse-server.ts +102 -68
- package/{server → server-e2e}/sse-session.ts +28 -8
- package/{server → server-e2e}/static.ts +31 -21
- package/server-e2e/sync-handlers.ts +408 -0
- package/{server → server-e2e}/ws-server.ts +20 -9
- package/server-managed/activity.ts +245 -0
- package/server-managed/avatar-store.ts +12 -0
- package/server-managed/blob-store.ts +17 -0
- package/server-managed/brotli.ts +10 -0
- package/server-managed/bundle-advisories.ts +120 -0
- package/server-managed/bundle-advisory-inventory.ts +77 -0
- package/server-managed/bundle-build-diagnostics.d.ts +10 -0
- package/server-managed/bundle-build-diagnostics.js +23 -0
- package/server-managed/bundle-build-leases.ts +41 -0
- package/server-managed/bundle-build-worker.js +111 -0
- package/server-managed/bundle-build.ts +198 -0
- package/server-managed/bundle-cache.ts +199 -0
- package/server-managed/bundle-catalog.ts +29 -0
- package/server-managed/bundle-commits.ts +140 -0
- package/server-managed/bundle-store.ts +24 -0
- package/server-managed/bundle-summary-cache.ts +130 -0
- package/server-managed/bundle.ts +120 -0
- package/server-managed/cache-storage.ts +19 -0
- package/server-managed/cli.js +21 -0
- package/server-managed/combined.ts +46 -0
- package/server-managed/comments.ts +151 -0
- package/server-managed/config.ts +158 -0
- package/server-managed/content-access.ts +15 -0
- package/server-managed/crypto.ts +25 -0
- package/server-managed/db-methods.ts +1725 -0
- package/server-managed/db-neon.ts +367 -0
- package/server-managed/db-schema.ts +227 -0
- package/server-managed/db-table-names.ts +22 -0
- package/server-managed/db.ts +132 -0
- package/server-managed/dependency-reviews.ts +248 -0
- package/server-managed/github-app.ts +391 -0
- package/server-managed/github-errors.ts +68 -0
- package/server-managed/github-issue-links.ts +127 -0
- package/server-managed/github-issue-oauth.ts +47 -0
- package/server-managed/github-issues.ts +152 -0
- package/server-managed/github-metadata.ts +213 -0
- package/server-managed/github-oauth.ts +235 -0
- package/server-managed/github-pulls.ts +227 -0
- package/server-managed/http-response.ts +50 -0
- package/server-managed/http.ts +3472 -0
- package/server-managed/import-triage.ts +51 -0
- package/server-managed/index.ts +187 -0
- package/server-managed/kept.ts +28 -0
- package/server-managed/link-reports.ts +158 -0
- package/server-managed/locked-packages.ts +85 -0
- package/server-managed/lru.ts +30 -0
- package/server-managed/managed-issues.ts +75 -0
- package/server-managed/management-catalog.ts +45 -0
- package/server-managed/management.ts +99 -0
- package/server-managed/npm-insights.ts +440 -0
- package/server-managed/npm-kept.ts +65 -0
- package/server-managed/npm-loads.ts +167 -0
- package/server-managed/npm-manifest.ts +115 -0
- package/server-managed/npm-packages.ts +378 -0
- package/server-managed/npm-provenance.ts +123 -0
- package/server-managed/npm-risk.ts +177 -0
- package/server-managed/object-storage-disk.ts +148 -0
- package/server-managed/object-storage-vercel.ts +67 -0
- package/server-managed/object-storage.ts +74 -0
- package/server-managed/package-entry-points.ts +51 -0
- package/server-managed/pretty-print.ts +352 -0
- package/server-managed/public-workspace.ts +220 -0
- package/server-managed/rename-diff-worker.js +26 -0
- package/server-managed/rename-diff.ts +270 -0
- package/server-managed/repo-path.ts +21 -0
- package/server-managed/report-migration.ts +35 -0
- package/server-managed/report-query.ts +10 -0
- package/server-managed/report-response.ts +16 -0
- package/server-managed/report-sources.ts +178 -0
- package/server-managed/report-store.ts +47 -0
- package/server-managed/repository-aliases.ts +91 -0
- package/server-managed/repository-browser.ts +153 -0
- package/server-managed/repository-discovery.ts +82 -0
- package/server-managed/repository-policy.ts +27 -0
- package/server-managed/repository-risk.ts +420 -0
- package/server-managed/revisions.ts +35 -0
- package/server-managed/session.ts +92 -0
- package/server-managed/slugs.ts +38 -0
- package/server-managed/solidity-entry-points.ts +69 -0
- package/server-managed/sql-postgres.ts +31 -0
- package/server-managed/sql.ts +76 -0
- package/server-managed/static.ts +28 -0
- package/server-managed/storage-command.ts +16 -0
- package/server-managed/storage-db.ts +175 -0
- package/server-managed/storage-encryption.ts +279 -0
- package/server-managed/storage-maintenance.ts +256 -0
- package/server-managed/storage-payload.ts +103 -0
- package/server-managed/storage-stores.ts +85 -0
- package/server-managed/storage.ts +78 -0
- package/server-managed/team-app.ts +94 -0
- package/server-managed/team-catalog.ts +7 -0
- package/server-managed/team-feed.ts +173 -0
- package/server-managed/team-npm-scopes.ts +77 -0
- package/server-managed/team-reports.ts +228 -0
- package/server-managed/triage-response.ts +16 -0
- package/server-managed/type-only-worker.js +19 -0
- package/server-managed/type-only.ts +103 -0
- package/server-managed/uploads.ts +86 -0
- package/server-managed/upstream-cache.ts +215 -0
- package/server-managed/view-as.ts +58 -0
- package/server-managed/workspace-shares.ts +166 -0
- package/server.ts +50 -0
- package/server/index.ts +0 -477
- package/server/lifecycle.ts +0 -204
- package/server/sync-handlers.ts +0 -320
- /package/{LICENSE → LICENSE-MIT} +0 -0
- /package/{server → server-e2e}/config.example.json +0 -0
- /package/{server → server-e2e}/objstore/blob-fs.ts +0 -0
- /package/{server → server-e2e}/objstore/fs.ts +0 -0
- /package/{server → server-e2e}/peer.ts +0 -0
- /package/{server → server-e2e}/util.ts +0 -0
- /package/{server → server-e2e}/validation.ts +0 -0
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
// What calls for a look at an npm version, as risk tells it (risk/), for the
|
|
2
|
+
// managed viewer: risk is asked on the server alone, its records kept where
|
|
3
|
+
// each belongs by its type (risk/npm/registry.js, assess.js):
|
|
4
|
+
//
|
|
5
|
+
// - `risk/npm/assessments`, what of an assessment holds whatever the time:
|
|
6
|
+
// a public package's in the public npm cache, as its pretty-printed files
|
|
7
|
+
// are (storage-payload.ts PUBLIC_CACHE), by a hash of its key; a private
|
|
8
|
+
// one's in this instance's memory alone.
|
|
9
|
+
// - `risk/npm/versions`, what a version's document says, which npm never
|
|
10
|
+
// changes once published: a public package's in the database's upstream
|
|
11
|
+
// cache table, shared by every instance; a private one's in memory.
|
|
12
|
+
// - `risk/npm/packages`, `risk/npm/public`, `risk/npm/advisories` and
|
|
13
|
+
// `risk/npm/downloads`, kept an hour or a day by risk itself: in memory.
|
|
14
|
+
// - `npm/tarballs`: none asked, as tarballs are read as the viewer's loads
|
|
15
|
+
// read them, kept where they keep them (npm-loads.ts loadNpmTarball), and
|
|
16
|
+
// an assessment kept needs none.
|
|
17
|
+
//
|
|
18
|
+
// What memory keeps is held to 2,000 records and 64 MiB as JSON, the first
|
|
19
|
+
// kept going first.
|
|
20
|
+
//
|
|
21
|
+
// A cache failure is a miss, never a failed assessment. Packages' documents
|
|
22
|
+
// are read as every other npm document here is, within their bounds and
|
|
23
|
+
// shared budget (npm-packages.ts readNpmPackument), and tarballs as the
|
|
24
|
+
// viewer's loads read them, bounded as they arrive and among the loads made
|
|
25
|
+
// at once. Assessments made at once are bounded as those loads are, each
|
|
26
|
+
// unpacking up to two tarballs: one past them is refused as busy.
|
|
27
|
+
import { Buffer } from 'node:buffer'
|
|
28
|
+
import { createHash } from 'node:crypto'
|
|
29
|
+
import { buffer } from 'node:stream/consumers'
|
|
30
|
+
import type { CacheStore } from '@preventive/upstream/npm.js'
|
|
31
|
+
import { DEFAULT_MINIMUM_RELEASE_AGE, type NpmAssessment, assessNpm } from '../risk/index.js'
|
|
32
|
+
import { CacheMissError, type CacheStorage } from './cache-storage.ts'
|
|
33
|
+
import { loadNpmTarball } from './npm-loads.ts'
|
|
34
|
+
import { NpmPackageError, type NpmVersionDocument, readNpmPackument, readNpmVersion } from './npm-packages.ts'
|
|
35
|
+
import { npmPackageScope } from '../common/managed/npm-packages.js'
|
|
36
|
+
import type { UpstreamCacheStore } from './upstream-cache.ts'
|
|
37
|
+
|
|
38
|
+
const MEMORY_ENTRIES = 2000
|
|
39
|
+
// As npm-loads.ts MAX_ACTIVE_LOADS.
|
|
40
|
+
const MAX_ACTIVE_ASSESSMENTS = 4
|
|
41
|
+
// Of them, those no reader waits on (`spare`), such as a scan's: the rest are
|
|
42
|
+
// left for readers.
|
|
43
|
+
export const MAX_SPARE_ASSESSMENTS = 2
|
|
44
|
+
let assessing = 0
|
|
45
|
+
// What memory holds at most, by its records' length as JSON: the first kept
|
|
46
|
+
// go first past it, and a record past an eighth of it isn't kept at all.
|
|
47
|
+
const MEMORY_LENGTH = 64 * 1024 * 1024
|
|
48
|
+
let memoryLength = 0
|
|
49
|
+
const memory = new Map<string, { value: unknown; length: number }>()
|
|
50
|
+
function forget(key: string) {
|
|
51
|
+
const kept = memory.get(key)
|
|
52
|
+
if (!kept) return
|
|
53
|
+
memory.delete(key)
|
|
54
|
+
memoryLength -= kept.length
|
|
55
|
+
}
|
|
56
|
+
function remember(key: string, value: unknown) {
|
|
57
|
+
forget(key)
|
|
58
|
+
const length = JSON.stringify(value)?.length ?? 0
|
|
59
|
+
if (length > MEMORY_LENGTH / 8) return
|
|
60
|
+
memory.set(key, { value, length })
|
|
61
|
+
memoryLength += length
|
|
62
|
+
while (memory.size > MEMORY_ENTRIES || memoryLength > MEMORY_LENGTH) forget(memory.keys().next().value!)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const hashed = (type: string, key: string) => `risk/${createHash('sha256').update(`${type}\0${key}`).digest('hex')}`
|
|
66
|
+
|
|
67
|
+
export interface NpmRiskStores {
|
|
68
|
+
db: UpstreamCacheStore
|
|
69
|
+
npmCache?: CacheStorage | undefined
|
|
70
|
+
debug?: boolean | undefined
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Risk's cache for one package: `isPublic` whether anyone can read it, so
|
|
74
|
+
// that what is kept of it can be shared.
|
|
75
|
+
export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: boolean): CacheStore {
|
|
76
|
+
const warn = (what: string, error: unknown) => { if (debug) console.warn(`managed: risk cache ${what} failed:`, error) }
|
|
77
|
+
return {
|
|
78
|
+
async read(type, key) {
|
|
79
|
+
if (type === 'npm/tarballs') return null
|
|
80
|
+
try {
|
|
81
|
+
if (isPublic && type === 'risk/npm/assessments' && npmCache) return JSON.parse((await buffer((await npmCache.open(hashed(type, key))).stream)).toString('utf8'))
|
|
82
|
+
if (isPublic && type === 'risk/npm/versions') { const value = await db.getUpstreamCacheEntry(`${type}/${key}`); return value === null ? null : JSON.parse(value) }
|
|
83
|
+
} catch (error) { if (!(error instanceof CacheMissError)) warn('read', error); return null }
|
|
84
|
+
return memory.get(`${isPublic ? '' : 'private\0'}${type}\0${key}`)?.value ?? null
|
|
85
|
+
},
|
|
86
|
+
async write(type, key, value) {
|
|
87
|
+
if (type === 'npm/tarballs') return
|
|
88
|
+
try {
|
|
89
|
+
if (isPublic && type === 'risk/npm/assessments' && npmCache) { await npmCache.put(hashed(type, key), Buffer.from(JSON.stringify(value))); return }
|
|
90
|
+
if (isPublic && type === 'risk/npm/versions') { await db.setUpstreamCacheEntry(`${type}/${key}`, JSON.stringify(value), Date.now()); return }
|
|
91
|
+
} catch (error) { warn('write', error); return }
|
|
92
|
+
remember(`${isPublic ? '' : 'private\0'}${type}\0${key}`, value)
|
|
93
|
+
},
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// A version's tarball as the viewer's loads read it (npm-loads.ts), its bytes
|
|
98
|
+
// held to the integrity its reader's document names: given up on, and the
|
|
99
|
+
// load released, once `signal` aborts, the load going on for its other
|
|
100
|
+
// readers, so that a reader gone holds no assessment's place.
|
|
101
|
+
async function tarballOf(doc: NpmVersionDocument | undefined, signal: AbortSignal): Promise<Uint8Array> {
|
|
102
|
+
if (!doc) throw new NpmPackageError(502, 'upstream-invalid')
|
|
103
|
+
signal.throwIfAborted()
|
|
104
|
+
const load = loadNpmTarball(doc)
|
|
105
|
+
let onAbort = () => {}
|
|
106
|
+
const aborted = new Promise<never>((_, reject) => { onAbort = () => reject(signal.reason); signal.addEventListener('abort', onAbort, { once: true }) })
|
|
107
|
+
try { return await Promise.race([load.result, aborted]) } finally {
|
|
108
|
+
signal.removeEventListener('abort', onAbort)
|
|
109
|
+
load.release()
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// The version `doc` names assessed against `old`'s, where given, each as the
|
|
114
|
+
// reader's readNpmVersion read it, or against the one risk picks of its
|
|
115
|
+
// versions (`'pick'`, assessNpm `oldVersion: true`), its document read as the
|
|
116
|
+
// open version's is where its tarball is wanted: trusted however new and however few
|
|
117
|
+
// download it where a team lists its scope (`teamScoped`), its own packages;
|
|
118
|
+
// else held to risk's minimum release age and flagged as unpopular. A new
|
|
119
|
+
// dependency on a package of a scope a team lists (`ownScope`, asked once a
|
|
120
|
+
// scope) is told no tier nor age, as an own package. Its
|
|
121
|
+
// documents are read with npm's token where the reader may have it
|
|
122
|
+
// (`privileged`, as readNpmVersion), its dependencies' without; what is kept
|
|
123
|
+
// of it is shared only where neither version is private, and OSV is asked of
|
|
124
|
+
// its advisories only where it isn't, a private package's name never sent to
|
|
125
|
+
// OSV, as it never is to Socket (http.ts). A `spare` one, no
|
|
126
|
+
// reader waiting on it, is made only within MAX_SPARE_ASSESSMENTS, and tells
|
|
127
|
+
// its dependencies without asking npm of their packages (assessNpm
|
|
128
|
+
// `dependencies`), nor its advisories, which a scan asks of a lockfile's
|
|
129
|
+
// packages at once (repository-risk.ts).
|
|
130
|
+
export function npmRisk(doc: NpmVersionDocument, old: NpmVersionDocument | 'pick' | null,
|
|
131
|
+
{ teamScoped, ownScope = () => Promise.resolve(false), fromEmpty = false, privileged, signal, spare = false, ...stores }: NpmRiskStores & { teamScoped: boolean;
|
|
132
|
+
ownScope?: (scope: string) => Promise<boolean>; fromEmpty?: boolean; privileged: boolean; signal: AbortSignal; spare?: boolean }): Promise<NpmAssessment> {
|
|
133
|
+
const scopes = new Map<string, Promise<boolean>>()
|
|
134
|
+
const owned = (name: string) => {
|
|
135
|
+
const scope = npmPackageScope(name)
|
|
136
|
+
if (scope === null) return false
|
|
137
|
+
if (!scopes.has(scope)) scopes.set(scope, ownScope(scope))
|
|
138
|
+
return scopes.get(scope)!
|
|
139
|
+
}
|
|
140
|
+
const given = old === 'pick' ? null : old
|
|
141
|
+
const documentOf = async (version: string) => [doc, given].find(read => read?.version === version) ?? await readNpmVersion(doc.name, version, privileged, signal) ?? undefined
|
|
142
|
+
return bounded(spare, () => assessNpm(doc.name, doc.version, { oldVersion: old === 'pick' ? true : old?.version ?? null, fromEmpty,
|
|
143
|
+
minimumReleaseAge: teamScoped ? 0 : DEFAULT_MINIMUM_RELEASE_AGE,
|
|
144
|
+
popularity: !teamScoped, cache: npmRiskCache(stores, !doc.private && !given?.private), signal, dependencies: !spare, advisories: !spare, osv: !doc.private, owned,
|
|
145
|
+
packument: (name, { authorized }) => readNpmPackument(name, privileged && authorized, signal),
|
|
146
|
+
tarball: async (_name, version) => tarballOf(await documentOf(version), signal) }))
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// An assessment made within MAX_ACTIVE_ASSESSMENTS, a `spare` one within
|
|
150
|
+
// MAX_SPARE_ASSESSMENTS, all across the server; one past them refused as
|
|
151
|
+
// busy (`npm-busy`).
|
|
152
|
+
async function bounded(spare: boolean, assess: () => Promise<NpmAssessment>): Promise<NpmAssessment> {
|
|
153
|
+
if (assessing >= (spare ? MAX_SPARE_ASSESSMENTS : MAX_ACTIVE_ASSESSMENTS)) throw new NpmPackageError(429, 'npm-busy')
|
|
154
|
+
assessing++
|
|
155
|
+
try { return await assess() } finally { assessing-- }
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// A version npm has no document of, as risk tells one npm removed, as it
|
|
159
|
+
// removes every one it finds malware in (assessNpm): its package taken down
|
|
160
|
+
// by npm, as a scan's (`spare`), and its advisories of malware, npm's and,
|
|
161
|
+
// where anyone can read its package, OSV's, which alone has some of those
|
|
162
|
+
// npm removed (nx's), a private package's name never sent to OSV. Its
|
|
163
|
+
// package's document read anonymously, else with npm's token where the
|
|
164
|
+
// reader may have it (`privileged`), and kept as a private package's is; no
|
|
165
|
+
// tarball read, nothing of the version being left. Spare, within
|
|
166
|
+
// MAX_SPARE_ASSESSMENTS as a scan's others are. An NpmRiskError where
|
|
167
|
+
// nothing tells of it.
|
|
168
|
+
export function npmRiskRemoved(name: string, version: string,
|
|
169
|
+
{ teamScoped, privileged, signal, ...stores }: NpmRiskStores & { teamScoped: boolean; privileged: boolean; signal: AbortSignal }): Promise<NpmAssessment> {
|
|
170
|
+
return bounded(true, async () => {
|
|
171
|
+
const open = await readNpmPackument(name, false, signal)
|
|
172
|
+
return assessNpm(name, version, { minimumReleaseAge: teamScoped ? 0 : DEFAULT_MINIMUM_RELEASE_AGE, popularity: false, downloads: false, dependencies: false,
|
|
173
|
+
osv: open !== null, cache: npmRiskCache(stores, false), signal,
|
|
174
|
+
packument: (read, { authorized }) => read === name && open ? Promise.resolve(open) : readNpmPackument(read, privileged && authorized, signal),
|
|
175
|
+
tarball: () => Promise.reject(new NpmPackageError(502, 'upstream-invalid')) })
|
|
176
|
+
})
|
|
177
|
+
}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto'
|
|
2
|
+
import { link, lstat, mkdir, open, readdir, rename, rm, rmdir, stat } from 'node:fs/promises'
|
|
3
|
+
import { dirname, join } from 'node:path'
|
|
4
|
+
import { type ObjectPage, type RawObjectStorage, objectPath } from './object-storage.ts'
|
|
5
|
+
|
|
6
|
+
function missing(err: unknown): boolean { return (err as NodeJS.ErrnoException).code === 'ENOENT' }
|
|
7
|
+
function version(info: { dev: number; ino: number; size: number; mtimeMs: number; ctimeMs: number }): string {
|
|
8
|
+
return JSON.stringify([info.dev, info.ino, info.size, info.mtimeMs, info.ctimeMs])
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
async function syncDirectory(dir: string): Promise<void> {
|
|
12
|
+
try {
|
|
13
|
+
const directory = await open(dir, 'r')
|
|
14
|
+
try { await directory.sync() } finally { await directory.close() }
|
|
15
|
+
} catch (err) {
|
|
16
|
+
// Windows and some mounted filesystems cannot open/fsync directories.
|
|
17
|
+
// Preserve real I/O failures; file fsync remains mandatory on every write.
|
|
18
|
+
if (!['EINVAL', 'ENOTSUP', 'EOPNOTSUPP', 'ENOSYS', 'EISDIR', 'EBADF', 'EPERM'].includes((err as NodeJS.ErrnoException).code ?? '')) throw err
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// Keyset pagination survives concurrent file creation/deletion. A final sweep
|
|
23
|
+
// catches objects inserted before the cursor during migration.
|
|
24
|
+
async function listFiles(dir: string, prefix: string, cursor: string | null, limit: number, signal?: AbortSignal): Promise<ObjectPage> {
|
|
25
|
+
const objects: ObjectPage['objects'] = []
|
|
26
|
+
async function visit(key: string): Promise<void> {
|
|
27
|
+
signal?.throwIfAborted()
|
|
28
|
+
let entries
|
|
29
|
+
try {
|
|
30
|
+
if (!(await lstat(join(dir, key))).isDirectory()) {
|
|
31
|
+
console.warn('managed-storage: skipping unsupported directory', JSON.stringify(key))
|
|
32
|
+
return
|
|
33
|
+
}
|
|
34
|
+
entries = await readdir(join(dir, key), { withFileTypes: true })
|
|
35
|
+
}
|
|
36
|
+
catch (err) { if (missing(err)) return; throw err }
|
|
37
|
+
const order = (entry: typeof entries[number]) => `${entry.name}${entry.isDirectory() ? '/' : ''}`
|
|
38
|
+
entries.sort((a, b) => order(a) < order(b) ? -1 : order(a) > order(b) ? 1 : 0)
|
|
39
|
+
for (const entry of entries) {
|
|
40
|
+
signal?.throwIfAborted()
|
|
41
|
+
const child = `${key}${entry.name}`
|
|
42
|
+
if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) {
|
|
43
|
+
console.warn('managed-storage: skipping unsupported entry', JSON.stringify(child))
|
|
44
|
+
continue
|
|
45
|
+
}
|
|
46
|
+
try { objectPath(child) }
|
|
47
|
+
catch { console.warn('managed-storage: skipping invalid entry', JSON.stringify(child)); continue }
|
|
48
|
+
if (entry.isDirectory()) {
|
|
49
|
+
const folder = `${child}/`
|
|
50
|
+
if (cursor && folder < cursor && !cursor.startsWith(folder)) continue
|
|
51
|
+
await visit(folder)
|
|
52
|
+
if (objects.length > limit) return
|
|
53
|
+
}
|
|
54
|
+
else if (entry.isFile() && (!cursor || child > cursor)) {
|
|
55
|
+
try { objects.push({ key: objectPath(child), modifiedAt: (await stat(join(dir, child))).mtimeMs }) }
|
|
56
|
+
catch (err) { if (!missing(err)) throw err }
|
|
57
|
+
if (objects.length > limit) return
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
// Another migration worker can observe an unlink before its writer has
|
|
61
|
+
// synced the directory. Persist those observed removals before an empty
|
|
62
|
+
// inventory can mark the shared migration complete.
|
|
63
|
+
try { await syncDirectory(join(dir, key)) } catch (err) { if (!missing(err)) throw err }
|
|
64
|
+
}
|
|
65
|
+
if (!prefix.endsWith('/') || !Number.isInteger(limit) || limit < 1) throw new Error('Invalid storage page')
|
|
66
|
+
objectPath(`${prefix}validate`)
|
|
67
|
+
await visit(prefix)
|
|
68
|
+
return { objects: objects.slice(0, limit), cursor: objects.length > limit ? objects[limit - 1]!.key : null }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function createDiskObjectStorage(dir: string): RawObjectStorage {
|
|
72
|
+
const path = (key: string) => join(dir, objectPath(key))
|
|
73
|
+
return {
|
|
74
|
+
async head(key, signal) {
|
|
75
|
+
signal?.throwIfAborted()
|
|
76
|
+
try {
|
|
77
|
+
const info = await stat(path(key))
|
|
78
|
+
return { size: info.size, version: version(info), modifiedAt: info.mtimeMs }
|
|
79
|
+
} catch (err) { if (missing(err)) return null; throw err }
|
|
80
|
+
},
|
|
81
|
+
async open(key, signal) {
|
|
82
|
+
signal?.throwIfAborted()
|
|
83
|
+
let file
|
|
84
|
+
try { file = await open(path(key), 'r') }
|
|
85
|
+
catch (err) { if (missing(err)) return null; throw err }
|
|
86
|
+
try {
|
|
87
|
+
const info = await file.stat()
|
|
88
|
+
return { size: info.size, version: version(info), modifiedAt: info.mtimeMs,
|
|
89
|
+
stream: file.createReadStream(signal ? { signal } : {}) }
|
|
90
|
+
} catch (err) { await file.close(); throw err }
|
|
91
|
+
},
|
|
92
|
+
async put(key, bytes, signal, expected) {
|
|
93
|
+
const target = path(key)
|
|
94
|
+
// Temp-file + rename makes replacement atomic to readers. Concurrent
|
|
95
|
+
// migrations use the same persisted data key and immutable plaintext.
|
|
96
|
+
const temp = `${target}.${randomUUID()}.tmp`
|
|
97
|
+
await mkdir(dirname(target), { recursive: true })
|
|
98
|
+
const file = await open(temp, 'wx', 0o600)
|
|
99
|
+
try {
|
|
100
|
+
await file.writeFile(bytes, signal ? { signal } : {})
|
|
101
|
+
await file.sync(); await file.close()
|
|
102
|
+
signal?.throwIfAborted()
|
|
103
|
+
if (expected === null) {
|
|
104
|
+
try { await link(temp, target) }
|
|
105
|
+
catch (err) { if ((err as NodeJS.ErrnoException).code === 'EEXIST') return false; throw err }
|
|
106
|
+
} else if (expected === undefined) await rename(temp, target)
|
|
107
|
+
else {
|
|
108
|
+
try { if (version(await stat(target)) !== expected) return false }
|
|
109
|
+
catch (err) { if (missing(err)) return false; throw err }
|
|
110
|
+
await rename(temp, target)
|
|
111
|
+
}
|
|
112
|
+
await syncDirectory(dirname(target))
|
|
113
|
+
return true
|
|
114
|
+
} finally { await file.close(); await rm(temp, { force: true }) }
|
|
115
|
+
},
|
|
116
|
+
async delete(key, expected, signal) {
|
|
117
|
+
signal?.throwIfAborted()
|
|
118
|
+
try {
|
|
119
|
+
if (expected !== undefined && version(await stat(path(key))) !== expected) return false
|
|
120
|
+
await rm(path(key), { force: true })
|
|
121
|
+
// Persist plaintext removal before migration checkpoints completion.
|
|
122
|
+
// Otherwise a crash could restore the directory entry after SQL has
|
|
123
|
+
// durably disabled further plaintext inventory sweeps.
|
|
124
|
+
await syncDirectory(dirname(path(key)))
|
|
125
|
+
return true
|
|
126
|
+
} catch (err) { if (missing(err)) return true; throw err }
|
|
127
|
+
},
|
|
128
|
+
async sync(key, signal) { signal?.throwIfAborted(); await syncDirectory(dirname(path(key))) },
|
|
129
|
+
async prune(prefix, signal) {
|
|
130
|
+
if (!prefix.endsWith('/')) throw new Error('Invalid storage prefix')
|
|
131
|
+
objectPath(`${prefix}validate`)
|
|
132
|
+
async function visit(folder: string): Promise<void> {
|
|
133
|
+
signal?.throwIfAborted()
|
|
134
|
+
let entries
|
|
135
|
+
try {
|
|
136
|
+
if (!(await lstat(folder)).isDirectory()) return
|
|
137
|
+
entries = await readdir(folder, { withFileTypes: true })
|
|
138
|
+
}
|
|
139
|
+
catch (err) { if (missing(err)) return; throw err }
|
|
140
|
+
for (const entry of entries) if (entry.isDirectory()) await visit(join(folder, entry.name))
|
|
141
|
+
try { await rmdir(folder); await syncDirectory(dirname(folder)) }
|
|
142
|
+
catch (err) { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes((err as NodeJS.ErrnoException).code ?? '')) throw err }
|
|
143
|
+
}
|
|
144
|
+
await visit(join(dir, prefix))
|
|
145
|
+
},
|
|
146
|
+
list: (prefix, cursor, limit, signal) => listFiles(dir, prefix, cursor, limit, signal),
|
|
147
|
+
}
|
|
148
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { Readable } from 'node:stream'
|
|
2
|
+
import { Buffer } from 'node:buffer'
|
|
3
|
+
import { type VercelBlobSdk, isNotFound, loadVercelBlobSdk } from '../server-common/vercel-blob.ts'
|
|
4
|
+
import { type RawObjectStorage, objectPath } from './object-storage.ts'
|
|
5
|
+
|
|
6
|
+
export async function openVercelObjectStorage(token: string, sdk?: VercelBlobSdk): Promise<RawObjectStorage> {
|
|
7
|
+
const blobs = sdk ?? await loadVercelBlobSdk()
|
|
8
|
+
const path = (key: string) => `.managed/${objectPath(key)}`
|
|
9
|
+
const storage: RawObjectStorage = {
|
|
10
|
+
async head(key, signal) {
|
|
11
|
+
try {
|
|
12
|
+
const meta = await blobs.head(path(key), { token, ...(signal ? { abortSignal: signal } : {}) })
|
|
13
|
+
if (!meta.etag) throw new Error('Missing object version')
|
|
14
|
+
return { size: meta.size, version: meta.etag, modifiedAt: new Date(meta.uploadedAt ?? Date.now()).getTime() }
|
|
15
|
+
} catch (err) { if (isNotFound(err, blobs)) return null; throw err }
|
|
16
|
+
},
|
|
17
|
+
async open(key, signal) {
|
|
18
|
+
try {
|
|
19
|
+
// Read the stored representation: transport compression can weaken its
|
|
20
|
+
// ETag, which cannot satisfy ifMatch, and changes the advertised size.
|
|
21
|
+
const result = await blobs.get(path(key), { token, access: 'private', useCache: false,
|
|
22
|
+
headers: { 'accept-encoding': 'identity' }, ...(signal ? { abortSignal: signal } : {}) })
|
|
23
|
+
if (!result) return null
|
|
24
|
+
if (result.statusCode !== 200 || !result.stream || !result.blob.etag) {
|
|
25
|
+
await result.stream?.cancel()
|
|
26
|
+
throw new Error('Unexpected blob response or missing object version')
|
|
27
|
+
}
|
|
28
|
+
return { size: result.blob.size === 0 ? null : result.blob.size, version: result.blob.etag,
|
|
29
|
+
modifiedAt: new Date(result.blob.uploadedAt ?? Date.now()).getTime(),
|
|
30
|
+
stream: Readable.fromWeb(result.stream as Parameters<typeof Readable.fromWeb>[0], signal ? { signal } : {}) }
|
|
31
|
+
} catch (err) { if (isNotFound(err, blobs)) return null; throw err }
|
|
32
|
+
},
|
|
33
|
+
async put(key, bytes, signal, expected, sizeHint) {
|
|
34
|
+
const multipart = (sizeHint ?? (Buffer.isBuffer(bytes) ? bytes.length : Infinity)) >= 5 * 1024 * 1024
|
|
35
|
+
try { await blobs.put(path(key), bytes, { token, access: 'private', addRandomSuffix: false, allowOverwrite: expected !== null,
|
|
36
|
+
...(expected ? { ifMatch: expected } : {}), multipart, contentType: 'application/octet-stream', cacheControlMaxAge: 60, ...(signal ? { abortSignal: signal } : {}) })
|
|
37
|
+
return true
|
|
38
|
+
} catch (err) {
|
|
39
|
+
if (blobs.BlobPreconditionFailedError && err instanceof blobs.BlobPreconditionFailedError) return false
|
|
40
|
+
// The SDK uses a generic error for an occupied pathname. Reconcile a
|
|
41
|
+
// create-only failure against storage; the caller verifies the winner.
|
|
42
|
+
if (expected === null && await storage.head(key, signal)) return false
|
|
43
|
+
throw err
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
async delete(key, version, signal) {
|
|
47
|
+
signal?.throwIfAborted()
|
|
48
|
+
try { await blobs.del(path(key), { token, ...(version ? { ifMatch: version } : {}), ...(signal ? { abortSignal: signal } : {}) }); return true }
|
|
49
|
+
catch (err) {
|
|
50
|
+
if (isNotFound(err, blobs)) return true
|
|
51
|
+
if (blobs.BlobPreconditionFailedError && err instanceof blobs.BlobPreconditionFailedError) return false
|
|
52
|
+
throw err
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
async list(prefix, cursor, limit, signal) {
|
|
56
|
+
signal?.throwIfAborted()
|
|
57
|
+
if (!prefix.endsWith('/') || !Number.isInteger(limit) || limit < 1) throw new Error('Invalid storage page')
|
|
58
|
+
objectPath(`${prefix}validate`)
|
|
59
|
+
const page = await blobs.list({ prefix: `.managed/${prefix}`, token, limit, ...(cursor ? { cursor } : {}), ...(signal ? { abortSignal: signal } : {}) })
|
|
60
|
+
if (page.hasMore && (!page.cursor || page.cursor === cursor)) throw new Error('Invalid blob pagination')
|
|
61
|
+
return { objects: page.blobs.filter(blob => blob.pathname.startsWith(`.managed/${prefix}`)).map(blob => ({ key: blob.pathname.slice('.managed/'.length),
|
|
62
|
+
modifiedAt: new Date(blob.uploadedAt ?? Date.now()).getTime() })),
|
|
63
|
+
cursor: page.hasMore ? page.cursor! : null }
|
|
64
|
+
},
|
|
65
|
+
}
|
|
66
|
+
return storage
|
|
67
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type { Buffer } from 'node:buffer'
|
|
2
|
+
import type { Readable } from 'node:stream'
|
|
3
|
+
import type { OpenedBlob } from './blob-store.ts'
|
|
4
|
+
import { validateCacheKey } from './cache-storage.ts'
|
|
5
|
+
|
|
6
|
+
export const ENCRYPTED_CACHE_PREFIX = 'cache-encrypted-v1/'
|
|
7
|
+
const STORAGE_PREFIXES = ['reports/', 'bundles/', 'avatars/', 'uploads/', 'cache/', ENCRYPTED_CACHE_PREFIX]
|
|
8
|
+
export interface RawObject extends OpenedBlob { version: string; modifiedAt: number }
|
|
9
|
+
type ObjectMetadata = Omit<RawObject, 'stream'>
|
|
10
|
+
interface ListedObject { key: string; modifiedAt: number }
|
|
11
|
+
export interface ObjectPage { objects: ListedObject[]; cursor: string | null }
|
|
12
|
+
export interface RawObjectStorage {
|
|
13
|
+
open(key: string, signal?: AbortSignal): Promise<RawObject | null>
|
|
14
|
+
head(key: string, signal?: AbortSignal): Promise<ObjectMetadata | null>
|
|
15
|
+
prune?(prefix: string, signal?: AbortSignal): Promise<void>
|
|
16
|
+
// Persist an observed disk replacement before its SQL migration checkpoint.
|
|
17
|
+
sync?(key: string, signal?: AbortSignal): Promise<void>
|
|
18
|
+
// A plaintext size hint selects multipart uploads without buffering streams.
|
|
19
|
+
// Publish atomically only after consuming the body through clean EOF; a body
|
|
20
|
+
// error must preserve the existing object (migration verifies while streaming).
|
|
21
|
+
// A null expected version creates only if absent, without replacing a winner.
|
|
22
|
+
put(key: string, bytes: Buffer | Readable, signal?: AbortSignal, expected?: string | null, sizeHint?: number): Promise<boolean>
|
|
23
|
+
// Return false if the original version changed; do not delete its replacement.
|
|
24
|
+
delete(key: string, version?: string, signal?: AbortSignal): Promise<boolean>
|
|
25
|
+
list(prefix: string, cursor: string | null, limit: number, signal?: AbortSignal): Promise<ObjectPage>
|
|
26
|
+
}
|
|
27
|
+
export interface ObjectStorage {
|
|
28
|
+
open(key: string): Promise<OpenedBlob | null>
|
|
29
|
+
get(key: string): Promise<Buffer | null>
|
|
30
|
+
exists(key: string): Promise<boolean>
|
|
31
|
+
put(key: string, bytes: Buffer, options?: { replaceReport?: boolean }): Promise<string | null>
|
|
32
|
+
delete(key: string): Promise<void>
|
|
33
|
+
deletePrefix(prefix: string): Promise<void>
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function isBlobId(value: string): boolean {
|
|
37
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u.test(value)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function objectPath(key: string): string {
|
|
41
|
+
validateCacheKey(key)
|
|
42
|
+
if (!STORAGE_PREFIXES.some(prefix => key.startsWith(prefix))) throw new Error('Invalid managed storage namespace')
|
|
43
|
+
return key
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Hash verification and migration may consume a stream before it can be used.
|
|
47
|
+
// Reopen only the same object version, closing any concurrent replacement.
|
|
48
|
+
export async function openObjectVersion(raw: RawObjectStorage, key: string, version: string, signal?: AbortSignal): Promise<RawObject | null> {
|
|
49
|
+
const current = await raw.open(key, signal)
|
|
50
|
+
if (current?.version === version) return current
|
|
51
|
+
current?.stream.destroy()
|
|
52
|
+
return null
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export async function deleteObjects(raw: RawObjectStorage, prefix: string, signal: AbortSignal, before?: number): Promise<number> {
|
|
56
|
+
const eligible = (modifiedAt: number) => before === undefined || modifiedAt < before
|
|
57
|
+
// Finish listing before deleting: provider cursors may be offset-based.
|
|
58
|
+
const cursors = new Set<string>(), keys = new Set<string>()
|
|
59
|
+
let cursor: string | null = null
|
|
60
|
+
do {
|
|
61
|
+
const page = await raw.list(prefix, cursor, 100, signal)
|
|
62
|
+
for (const object of page.objects) if (eligible(object.modifiedAt)) keys.add(object.key)
|
|
63
|
+
cursor = page.cursor
|
|
64
|
+
if (cursor !== null && cursors.has(cursor)) throw new Error('Invalid blob pagination')
|
|
65
|
+
if (cursor !== null) cursors.add(cursor)
|
|
66
|
+
} while (cursor !== null)
|
|
67
|
+
let removed = 0
|
|
68
|
+
for (const key of keys) {
|
|
69
|
+
const stored = await raw.head(key, signal)
|
|
70
|
+
// Ignore stale listings and preserve concurrent replacements.
|
|
71
|
+
if (stored && eligible(stored.modifiedAt) && await raw.delete(key, stored.version, signal)) removed++
|
|
72
|
+
}
|
|
73
|
+
return removed
|
|
74
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
2
|
+
import { normalizeTeamPath } from './repo-path.ts'
|
|
3
|
+
|
|
4
|
+
export const MAX_PACKAGE_BYTES = 256 * 1024
|
|
5
|
+
const MAX_ENTRY_POINTS = 100
|
|
6
|
+
|
|
7
|
+
// Declared paths, not a Node resolver: never guess defaults, expand export
|
|
8
|
+
// patterns, or execute package code. Keep every suggestion in its package.
|
|
9
|
+
export function packageEntryPoints(manifest: unknown, directory: string): string[] {
|
|
10
|
+
if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) return []
|
|
11
|
+
const pkg = manifest as { main?: unknown; exports?: unknown; bin?: unknown }
|
|
12
|
+
const paths = new Set<string>()
|
|
13
|
+
const add = (value: unknown) => {
|
|
14
|
+
if (typeof value !== 'string' || value.startsWith('/') || value.endsWith('/') || /[*?]|^[a-z][a-z\d+.-]*:/iu.test(value)) return
|
|
15
|
+
const normalized = normalizeTeamPath(value)
|
|
16
|
+
if (!normalized.ok || !normalized.path || normalized.path === 'package.json') return
|
|
17
|
+
const path = directory ? `${directory}/${normalized.path}` : normalized.path
|
|
18
|
+
if (path.length <= 500 && paths.size < MAX_ENTRY_POINTS) paths.add(path)
|
|
19
|
+
}
|
|
20
|
+
add(pkg.main)
|
|
21
|
+
const pending = [pkg.exports]
|
|
22
|
+
for (let visited = 0; pending.length > 0 && visited < 4096; visited++) {
|
|
23
|
+
const value = pending.pop()
|
|
24
|
+
if (typeof value === 'string') { if (value.startsWith('./')) add(value); continue }
|
|
25
|
+
if (Array.isArray(value)) pending.push(...value.slice(0, 4096 - visited).toReversed())
|
|
26
|
+
else if (value && typeof value === 'object') {
|
|
27
|
+
const entries = Object.entries(value).filter(([key]) => key !== 'types' && !key.startsWith('types@'))
|
|
28
|
+
pending.push(...entries.slice(0, 4096 - visited).toReversed().map(([, target]) => target))
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (typeof pkg.bin === 'string') add(pkg.bin)
|
|
32
|
+
else if (pkg.bin && typeof pkg.bin === 'object' && !Array.isArray(pkg.bin)) Object.values(pkg.bin).forEach(add)
|
|
33
|
+
return [...paths]
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// Use the immutable blob from the authorized listing, never download_url or a
|
|
37
|
+
// resolved symlink target. Malformed or oversized manifests are optional.
|
|
38
|
+
export async function readPackageEntryPoints(sha: string, directory: string, read: (suffix: string) => Promise<unknown>): Promise<string[]> {
|
|
39
|
+
try {
|
|
40
|
+
const blob = await read(`/git/blobs/${sha}`) as { encoding?: unknown; content?: unknown; size?: unknown }
|
|
41
|
+
if (blob?.encoding !== 'base64' || typeof blob.content !== 'string' || typeof blob.size !== 'number'
|
|
42
|
+
|| blob.size < 0 || blob.size > MAX_PACKAGE_BYTES || blob.content.length > MAX_PACKAGE_BYTES * 2) return []
|
|
43
|
+
const bytes = Buffer.from(blob.content, 'base64')
|
|
44
|
+
if (bytes.length !== blob.size || bytes.length > MAX_PACKAGE_BYTES) return []
|
|
45
|
+
return packageEntryPoints(JSON.parse(bytes.toString('utf8')), directory)
|
|
46
|
+
} catch {
|
|
47
|
+
// Suggestions must not prevent browsing. The handler still rechecks
|
|
48
|
+
// GitHub and managed access after this optional upstream read.
|
|
49
|
+
return []
|
|
50
|
+
}
|
|
51
|
+
}
|