@preventive/triage 1.0.0-alpha.5 → 1.0.0-alpha.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/README.md +16 -0
- package/api/reap.ts +17 -0
- package/cli.js +6 -0
- package/client/dependency-paths.d.ts +3 -0
- package/client/dependency-paths.js +36 -0
- package/client/finding-link.js +305 -0
- package/client/linked-findings.d.ts +1 -0
- package/client/linked-findings.js +115 -0
- package/common/bundle-commit.d.ts +1 -0
- package/common/bundle-commit.js +4 -0
- package/common/bundle-compare-diff.d.ts +3 -0
- package/common/bundle-compare-diff.js +357 -0
- package/common/bundle-compare-inputs.d.ts +2 -0
- package/common/bundle-compare-inputs.js +65 -0
- package/common/bundle-compare-versions.js +172 -0
- package/common/bundle-comparison.d.ts +8 -0
- package/common/bundle-comparison.js +62 -0
- package/common/bundle-metadata.d.ts +15 -0
- package/common/bundle-metadata.js +226 -0
- package/common/bundle-pkg-of.d.ts +1 -0
- package/common/bundle-pkg-of.js +90 -0
- package/common/bundle-reasons.d.ts +2 -0
- package/common/bundle-reasons.js +21 -0
- package/common/bundle-source-links.d.ts +6 -0
- package/common/bundle-source-links.js +216 -0
- package/common/bundle-sourcemap.d.ts +10 -0
- package/common/bundle-sourcemap.js +100 -0
- package/common/bundle-sources.d.ts +7 -0
- package/common/bundle-sources.js +333 -0
- package/common/bundle-stats.js +41 -0
- package/common/bundle-tabs.js +1 -0
- package/common/code-language.js +65 -0
- package/common/default-scan-models.ts +31 -0
- package/common/dependency-reviews.d.ts +18 -0
- package/common/dependency-reviews.js +64 -0
- package/common/finding-duplicates.js +58 -0
- package/common/finding-filters.js +274 -0
- package/common/finding-id.js +47 -0
- package/common/github-issue-labels.d.ts +1 -0
- package/common/github-issue-labels.js +11 -0
- package/common/github-pr.ts +56 -0
- package/common/gzip.js +59 -0
- package/common/line-diff.d.ts +8 -0
- package/common/line-diff.js +57 -0
- package/common/linked-workspace-groups.js +63 -0
- package/common/managed/agent-tasks.ts +292 -0
- package/common/managed/comments.ts +34 -0
- package/common/managed/finding-detail.d.ts +2 -0
- package/common/managed/finding-detail.js +32 -0
- package/common/managed/github-agents.ts +114 -0
- package/common/managed/github-logins.ts +49 -0
- package/common/managed/npm-packages.d.ts +5 -0
- package/common/managed/npm-packages.js +34 -0
- package/common/managed/permissions.ts +31 -0
- package/common/managed/report-content.ts +72 -0
- package/common/managed/report-filter.ts +109 -0
- package/common/managed/repository-alias.ts +65 -0
- package/common/managed/roles.ts +35 -0
- package/common/managed/routes.d.ts +2 -0
- package/common/managed/routes.js +347 -0
- package/common/managed/scan-models.ts +6 -0
- package/common/managed/team-app.d.ts +5 -0
- package/common/managed/team-app.js +31 -0
- package/common/managed/triage.ts +83 -0
- package/common/managed/upload-seal.ts +126 -0
- package/common/pretty-print.d.ts +5 -0
- package/common/pretty-print.js +24 -0
- package/common/revalidation-conflicts.js +48 -0
- package/common/revalidation-input-groups.js +56 -0
- package/common/save-error-reason.ts +20 -7
- package/common/scan-server.ts +13 -0
- package/common/security-links.d.ts +5 -0
- package/common/security-links.js +33 -0
- package/common/server-info.ts +46 -0
- package/common/utf8.d.ts +3 -0
- package/common/utf8.js +45 -0
- package/common/workspace-app.js +39 -0
- package/common/workspace-groups.js +176 -0
- package/out/activity.js +4 -0
- package/out/brotli-fallback.js +3 -3
- package/out/chunk-2WDZPXVL.js +3 -0
- package/out/chunk-2XUEZHIP.js +28 -0
- package/out/chunk-2ZDZWLFY.js +1 -0
- package/out/chunk-4HI2MSAV.js +17 -0
- package/out/chunk-4XMP7YVH.js +36 -0
- package/out/chunk-6LNQTKAV.js +52 -0
- package/out/chunk-6S2U2UYG.js +232 -0
- package/out/chunk-75YLHBCM.js +3 -0
- package/out/chunk-7BKKYVWR.js +7 -0
- package/out/chunk-7C7DWL7I.js +1 -0
- package/out/chunk-7VAWINQF.js +6 -0
- package/out/chunk-C4Z6ZBXV.js +10 -0
- package/out/chunk-FC65TPUB.js +1 -0
- package/out/chunk-FKBXNUW4.js +1 -0
- package/out/chunk-GDPCLSOS.js +2 -0
- package/out/chunk-GO7FT4OD.js +1 -0
- package/out/chunk-HTBYHSPG.js +1 -0
- package/out/chunk-IYPY47MC.js +1 -0
- package/out/chunk-IZ7POWI3.js +18 -0
- package/out/chunk-JCIIBLKR.js +1 -0
- package/out/chunk-NRPC74AM.js +1 -0
- package/out/chunk-OOJM4CTU.js +1 -0
- package/out/chunk-PBZPYIG7.js +1 -0
- package/out/chunk-PNHBZGDD.js +1 -0
- package/out/chunk-PWFUOLZK.js +17 -0
- package/out/chunk-PXK7EU4V.js +1 -0
- package/out/chunk-QB4CBOX2.js +1 -0
- package/out/chunk-R3NZFXVR.js +1 -0
- package/out/chunk-R5O6FAW4.js +2 -0
- package/out/chunk-SXV2T5C4.js +80 -0
- package/out/chunk-TQJDYHVE.js +2 -0
- package/out/chunk-TUII34K2.js +1 -0
- package/out/chunk-UK7RGJT6.js +3 -0
- package/out/chunk-VFWMQ3LG.js +2 -0
- package/out/chunk-VH4XMZIW.js +2 -0
- package/out/chunk-WCPJDAYA.js +10 -0
- package/out/chunk-WFYLMF6Z.js +10 -0
- package/out/chunk-WYLQWYLA.js +1 -0
- package/out/client-managed-import.js +4 -0
- package/out/client-managed.js +33 -0
- package/out/client-sync.js +11 -17
- package/out/compare.js +35 -0
- package/out/dialogs.js +55 -0
- package/out/github-issue-authorized.html +3 -0
- package/out/github-issue-authorized.js +1 -0
- package/out/graph.js +17 -4
- package/out/index.html +90 -8
- package/out/prism.js +14 -10
- package/out/scan.js +2 -0
- package/out/stasis.svg +20 -0
- package/out/terminal.js +366 -39
- package/out/view.css +1 -1
- package/out/view.js +19 -144
- package/package.json +362 -59
- package/risk/actions/findings.js +161 -0
- package/risk/actions/resolve.js +219 -0
- package/risk/actions/scripts/commands.js +182 -0
- package/risk/actions/scripts/grade.js +113 -0
- package/risk/actions/scripts/packages.js +123 -0
- package/risk/actions/scripts/shell.js +169 -0
- package/risk/actions/uses.js +76 -0
- package/risk/actions/workflow.js +110 -0
- package/risk/files/characters.js +196 -0
- package/risk/files/hosts.js +205 -0
- package/risk/files/readability.js +177 -0
- package/risk/files/runs-code.js +615 -0
- package/risk/files/script.js +118 -0
- package/risk/files/stamps.js +47 -0
- package/risk/index.d.ts +254 -0
- package/risk/index.js +11 -0
- package/risk/npm/assess.js +285 -0
- package/risk/npm/combined.js +68 -0
- package/risk/npm/compare.js +257 -0
- package/risk/npm/dependencies.js +172 -0
- package/risk/npm/diff.js +79 -0
- package/risk/npm/downloads.js +84 -0
- package/risk/npm/files.js +445 -0
- package/risk/npm/held.js +63 -0
- package/risk/npm/install-scripts.js +34 -0
- package/risk/npm/malware.js +36 -0
- package/risk/npm/manifest.js +65 -0
- package/risk/npm/new-dependency.js +60 -0
- package/risk/npm/old-version.js +108 -0
- package/risk/npm/registry.js +388 -0
- package/risk/npm/release.js +85 -0
- package/risk/npm/score.js +59 -0
- package/risk/npm/specs.js +124 -0
- package/risk/npm/tarball.js +96 -0
- package/server-common/database-config.ts +16 -0
- package/server-common/initialize.ts +18 -0
- package/server-common/npm-advisories.ts +101 -0
- package/{server → server-common}/origin.ts +5 -5
- package/server-common/reap.ts +48 -0
- package/server-common/scan-config.ts +19 -0
- package/server-common/standalone.ts +29 -0
- package/server-common/storage-crypto.ts +182 -0
- package/server-common/storage-log.ts +38 -0
- package/server-common/vercel-blob.ts +103 -0
- package/server-e2e/app.ts +487 -0
- package/{server → server-e2e}/bus-receiver.ts +2 -1
- package/{server → server-e2e}/cli.js +9 -4
- package/{server → server-e2e}/config.ts +50 -42
- package/{server → server-e2e}/db-neon.ts +2 -2
- package/{server → server-e2e}/db-revision-sql.ts +1 -1
- package/{server → server-e2e}/db-stmt.ts +2 -2
- package/{server → server-e2e}/db.ts +7 -7
- package/{server → server-e2e}/http.ts +44 -13
- package/{server → server-e2e}/hub.ts +40 -9
- package/server-e2e/index.ts +17 -0
- package/server-e2e/lifecycle.ts +95 -0
- package/{server → server-e2e}/neon-driver.ts +2 -2
- package/{server → server-e2e}/npm-proxy.ts +12 -145
- package/{server → server-e2e}/objstore/blob-vercel.ts +10 -114
- package/{server → server-e2e}/objstore/blob.ts +1 -1
- package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
- package/{server → server-e2e}/objstore/handlers.ts +9 -8
- package/{server → server-e2e}/objstore/init.ts +19 -27
- package/{server → server-e2e}/objstore/reaper.ts +1 -1
- package/server-e2e/objstore/rest-deny.ts +28 -0
- package/server-e2e/objstore/rest-mint.ts +224 -0
- package/{server → server-e2e}/objstore/rest.ts +3 -3
- package/{server → server-e2e}/objstore/store-neon.ts +6 -10
- package/{server → server-e2e}/objstore/store.ts +7 -7
- package/{server → server-e2e}/objstore/tokens.ts +2 -2
- package/{server → server-e2e}/pubsub.ts +12 -9
- package/{server → server-e2e}/sign.ts +4 -4
- package/{server → server-e2e}/sse-server.ts +60 -33
- package/{server → server-e2e}/static.ts +31 -21
- package/{server → server-e2e}/sync-handlers.ts +64 -30
- package/{server → server-e2e}/ws-server.ts +20 -9
- package/server-managed/activity.ts +258 -0
- package/server-managed/agent-merges.ts +109 -0
- package/server-managed/app-agents.ts +21 -0
- package/server-managed/avatar-store.ts +13 -0
- package/server-managed/blob-store.ts +17 -0
- package/server-managed/brotli.ts +10 -0
- package/server-managed/bundle-advisories.ts +120 -0
- package/server-managed/bundle-advisory-inventory.ts +77 -0
- package/server-managed/bundle-build-diagnostics.d.ts +10 -0
- package/server-managed/bundle-build-diagnostics.js +23 -0
- package/server-managed/bundle-build-leases.ts +41 -0
- package/server-managed/bundle-build-worker.js +115 -0
- package/server-managed/bundle-build.ts +200 -0
- package/server-managed/bundle-cache.ts +211 -0
- package/server-managed/bundle-catalog.ts +29 -0
- package/server-managed/bundle-commits.ts +140 -0
- package/server-managed/bundle-store.ts +24 -0
- package/server-managed/bundle-summary-cache.ts +130 -0
- package/server-managed/bundle.ts +119 -0
- package/server-managed/cache-storage.ts +20 -0
- package/server-managed/cli.js +21 -0
- package/server-managed/combined.ts +47 -0
- package/server-managed/comments.ts +151 -0
- package/server-managed/commit-coauthors.ts +106 -0
- package/server-managed/commit-lockfiles.ts +73 -0
- package/server-managed/comparison.ts +41 -0
- package/server-managed/config.ts +164 -0
- package/server-managed/content-access.ts +15 -0
- package/server-managed/crypto.ts +25 -0
- package/server-managed/db-methods.ts +1923 -0
- package/server-managed/db-neon.ts +458 -0
- package/server-managed/db-schema.ts +238 -0
- package/server-managed/db-table-names.ts +22 -0
- package/server-managed/db.ts +142 -0
- package/server-managed/dependabot-access.ts +60 -0
- package/server-managed/dependabot-alerts.ts +185 -0
- package/server-managed/dependency-review-comments.ts +100 -0
- package/server-managed/dependency-reviews.ts +346 -0
- package/server-managed/github-accounts.ts +60 -0
- package/server-managed/github-actions.ts +71 -0
- package/server-managed/github-app.ts +415 -0
- package/server-managed/github-errors.ts +70 -0
- package/server-managed/github-graphql.ts +39 -0
- package/server-managed/github-issue-links.ts +140 -0
- package/server-managed/github-issue-oauth.ts +47 -0
- package/server-managed/github-issues.ts +152 -0
- package/server-managed/github-link-oauth.ts +69 -0
- package/server-managed/github-links.ts +204 -0
- package/server-managed/github-metadata.ts +213 -0
- package/server-managed/github-oauth.ts +238 -0
- package/server-managed/github-pulls.ts +227 -0
- package/server-managed/github-requests.ts +39 -0
- package/server-managed/github-users.ts +30 -0
- package/server-managed/http-response.ts +76 -0
- package/server-managed/http.ts +5329 -0
- package/server-managed/import-triage.ts +51 -0
- package/server-managed/index.ts +192 -0
- package/server-managed/kept-briefly.ts +37 -0
- package/server-managed/kept.ts +28 -0
- package/server-managed/link-reports.ts +162 -0
- package/server-managed/locked-packages.ts +129 -0
- package/server-managed/lockfile-scope.ts +49 -0
- package/server-managed/lru.ts +30 -0
- package/server-managed/managed-issues.ts +78 -0
- package/server-managed/management-catalog.ts +45 -0
- package/server-managed/management.ts +99 -0
- package/server-managed/npm-delta-worker.js +23 -0
- package/server-managed/npm-delta.ts +194 -0
- package/server-managed/npm-insights.ts +449 -0
- package/server-managed/npm-kept.ts +215 -0
- package/server-managed/npm-loads.ts +264 -0
- package/server-managed/npm-manifest.ts +115 -0
- package/server-managed/npm-packages.ts +454 -0
- package/server-managed/npm-provenance.ts +123 -0
- package/server-managed/npm-risk.ts +285 -0
- package/server-managed/object-storage-disk.ts +148 -0
- package/server-managed/object-storage-vercel.ts +67 -0
- package/server-managed/object-storage.ts +75 -0
- package/server-managed/package-entry-points.ts +51 -0
- package/server-managed/pretty-print.ts +434 -0
- package/server-managed/public-workspace.ts +238 -0
- package/server-managed/pull-comment-reviews.ts +161 -0
- package/server-managed/pull-dependencies.ts +348 -0
- package/server-managed/pull-dependency-reviews.ts +328 -0
- package/server-managed/pull-reviews.ts +205 -0
- package/server-managed/queue.ts +39 -0
- package/server-managed/release-commits.ts +84 -0
- package/server-managed/rename-diff-worker.js +26 -0
- package/server-managed/rename-diff.ts +279 -0
- package/server-managed/repo-path.ts +21 -0
- package/server-managed/report-migration.ts +35 -0
- package/server-managed/report-query.ts +10 -0
- package/server-managed/report-response.ts +16 -0
- package/server-managed/report-sources.ts +178 -0
- package/server-managed/report-store.ts +85 -0
- package/server-managed/repository-actions.ts +218 -0
- package/server-managed/repository-agents.ts +355 -0
- package/server-managed/repository-aliases.ts +91 -0
- package/server-managed/repository-browser.ts +294 -0
- package/server-managed/repository-changes.ts +284 -0
- package/server-managed/repository-code.ts +215 -0
- package/server-managed/repository-dependency-reviews.ts +165 -0
- package/server-managed/repository-discovery.ts +82 -0
- package/server-managed/repository-history.ts +432 -0
- package/server-managed/repository-issues.ts +195 -0
- package/server-managed/repository-policy.ts +27 -0
- package/server-managed/repository-pulls.ts +156 -0
- package/server-managed/repository-pulse.ts +460 -0
- package/server-managed/repository-risk-assess.ts +105 -0
- package/server-managed/repository-risk-audit.ts +79 -0
- package/server-managed/repository-risk-read.ts +170 -0
- package/server-managed/repository-risk.ts +491 -0
- package/server-managed/repository-squashes.ts +168 -0
- package/server-managed/request-timing.ts +63 -0
- package/server-managed/reviewed-dependencies.ts +150 -0
- package/server-managed/revisions.ts +35 -0
- package/server-managed/scan-downloads.ts +87 -0
- package/server-managed/session.ts +132 -0
- package/server-managed/slugs.ts +38 -0
- package/server-managed/solidity-entry-points.ts +71 -0
- package/server-managed/sql-postgres.ts +34 -0
- package/server-managed/sql.ts +88 -0
- package/server-managed/static.ts +28 -0
- package/server-managed/storage-command.ts +16 -0
- package/server-managed/storage-db.ts +183 -0
- package/server-managed/storage-encryption.ts +290 -0
- package/server-managed/storage-maintenance.ts +259 -0
- package/server-managed/storage-payload.ts +121 -0
- package/server-managed/storage-stores.ts +91 -0
- package/server-managed/storage.ts +82 -0
- package/server-managed/team-app.ts +134 -0
- package/server-managed/team-catalog.ts +7 -0
- package/server-managed/team-feed.ts +180 -0
- package/server-managed/team-npm-scopes.ts +106 -0
- package/server-managed/team-reports.ts +307 -0
- package/server-managed/triage-response.ts +16 -0
- package/server-managed/type-only-worker.js +19 -0
- package/server-managed/type-only.ts +105 -0
- package/server-managed/uploads.ts +106 -0
- package/server-managed/upstream-cache.ts +243 -0
- package/server-managed/view-as.ts +58 -0
- package/server-managed/workspace-shares.ts +169 -0
- package/server.ts +70 -0
- package/server/index.ts +0 -480
- package/server/lifecycle.ts +0 -208
- /package/{LICENSE → LICENSE-MIT} +0 -0
- /package/{server → server-e2e}/auth.ts +0 -0
- /package/{server → server-e2e}/config.example.json +0 -0
- /package/{server → server-e2e}/objstore/blob-fs.ts +0 -0
- /package/{server → server-e2e}/objstore/fs.ts +0 -0
- /package/{server → server-e2e}/objstore/sign.ts +0 -0
- /package/{server → server-e2e}/peer.ts +0 -0
- /package/{server → server-e2e}/sse-session.ts +0 -0
- /package/{server → server-e2e}/util.ts +0 -0
- /package/{server → server-e2e}/validation.ts +0 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Whitelist diagnostic fields; never inspect/serialize an entire upstream
|
|
2
|
+
// error, request, response, or workerData (which contains the GitHub token).
|
|
3
|
+
export function bundleBuildDiagnostic(error, token, depth = 0) {
|
|
4
|
+
const redact = value => {
|
|
5
|
+
let text = String(value)
|
|
6
|
+
if (token) text = text.replaceAll(token, '[redacted]')
|
|
7
|
+
return text.replaceAll(/\b(?:github_pat_|gh[pousr]_)[\w]+/gu, '[redacted]')
|
|
8
|
+
.replaceAll(/(https?:\/\/)[^\s/@]+:[^\s/@]+@/gu, '$1[redacted]@')
|
|
9
|
+
.replaceAll(/([?&](?:access_token|token|key)=)[^\s&#]*/giu, '$1[redacted]')
|
|
10
|
+
}
|
|
11
|
+
if (!error || typeof error !== 'object') return { message: redact(error).slice(0, 2048) }
|
|
12
|
+
// Upstream HttpError messages append the response body after the status.
|
|
13
|
+
// Keep the method, URL and status, but omit that body from message and stack.
|
|
14
|
+
const raw = typeof error.message === 'string' ? error.message : 'Unknown error'
|
|
15
|
+
const message = error.name === 'HttpError' ? raw.replace(/(\s\d{3}):[\s\S]*$/u, '$1') : raw
|
|
16
|
+
const result = { message: redact(message).slice(0, 2048) }
|
|
17
|
+
if (typeof error.name === 'string') result.name = redact(error.name).slice(0, 128)
|
|
18
|
+
if (typeof error.code === 'string') result.code = redact(error.code).slice(0, 128)
|
|
19
|
+
if (typeof error.status === 'number') result.status = error.status
|
|
20
|
+
if (typeof error.stack === 'string') result.stack = redact(error.stack.replace(raw, message)).slice(0, 8192)
|
|
21
|
+
if (error.cause && depth < 3) result.cause = bundleBuildDiagnostic(error.cause, token, depth + 1)
|
|
22
|
+
return result
|
|
23
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { ManagedSql } from './sql.ts'
|
|
2
|
+
|
|
3
|
+
export const BUNDLE_BUILD_TIMEOUT_MS = 180_000
|
|
4
|
+
// Leave time to terminate a timed-out worker before a crashed owner's slot
|
|
5
|
+
// becomes reusable. Expiration uses the database clock, not an instance's clock.
|
|
6
|
+
const LEASE_MS = BUNDLE_BUILD_TIMEOUT_MS + 60_000
|
|
7
|
+
export const BUNDLE_BUILD_LEASE_SCHEMA = `
|
|
8
|
+
CREATE TABLE IF NOT EXISTS managed_bundle_build_lease (
|
|
9
|
+
slot INTEGER PRIMARY KEY CHECK (slot IN (1, 2)),
|
|
10
|
+
owner TEXT NOT NULL UNIQUE,
|
|
11
|
+
user_id TEXT NOT NULL UNIQUE,
|
|
12
|
+
expires_at INTEGER NOT NULL
|
|
13
|
+
) STRICT;
|
|
14
|
+
`
|
|
15
|
+
|
|
16
|
+
export interface BundleBuildLeaseStore {
|
|
17
|
+
claimBundleBuildLease(userId: string, owner: string): Promise<boolean>
|
|
18
|
+
releaseBundleBuildLease(owner: string): Promise<void>
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function bundleBuildLeaseMethods(db: ManagedSql): BundleBuildLeaseStore {
|
|
22
|
+
return {
|
|
23
|
+
// scopeManagedMethods holds SQLite's write transaction / PostgreSQL's
|
|
24
|
+
// advisory transaction lock across cleanup, inspection, and admission.
|
|
25
|
+
async claimBundleBuildLease(userId, owner) {
|
|
26
|
+
const { now } = await db.prepare("SELECT CAST(unixepoch('subsec') * 1000 AS INTEGER) AS now").get() as { now: number }
|
|
27
|
+
await db.prepare('DELETE FROM managed_bundle_build_lease WHERE expires_at <= ?').run(now)
|
|
28
|
+
const leases = await db.prepare('SELECT slot, user_id AS userId FROM managed_bundle_build_lease').all() as { slot: number; userId: string }[]
|
|
29
|
+
if (leases.length >= 2 || leases.some(lease => lease.userId === userId)) return false
|
|
30
|
+
const slot = leases.some(lease => lease.slot === 1) ? 2 : 1
|
|
31
|
+
await db.prepare('INSERT INTO managed_bundle_build_lease (slot, owner, user_id, expires_at) VALUES (?, ?, ?, ?)')
|
|
32
|
+
.run(slot, owner, userId, now + LEASE_MS)
|
|
33
|
+
return true
|
|
34
|
+
},
|
|
35
|
+
// No user FK: deleting an account must not free a still-running worker's
|
|
36
|
+
// slot. A delayed/crashed owner's cleanup cannot release its successor.
|
|
37
|
+
async releaseBundleBuildLease(owner) {
|
|
38
|
+
await db.prepare('DELETE FROM managed_bundle_build_lease WHERE owner = ?').run(owner)
|
|
39
|
+
},
|
|
40
|
+
}
|
|
41
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { parentPort, workerData } from 'node:worker_threads'
|
|
2
|
+
import { Buffer } from 'node:buffer'
|
|
3
|
+
import { posix } from 'node:path'
|
|
4
|
+
import { brotliCompressSync } from 'node:zlib'
|
|
5
|
+
import { buildGitHubBundle, setCacheDir } from '@exodus/stasis/vfs-bundle'
|
|
6
|
+
import { brotliOptions } from '@exodus/stasis-core/brotli'
|
|
7
|
+
import { HttpError, createClient } from '@preventive/upstream/github.js'
|
|
8
|
+
import { githubBundleFilename } from './bundle-build.ts'
|
|
9
|
+
import { bundleBuildDiagnostic } from './bundle-build-diagnostics.js'
|
|
10
|
+
|
|
11
|
+
const regularFile = entry => entry.type === 'blob' && ['100644', '100755'].includes(entry.mode)
|
|
12
|
+
|
|
13
|
+
async function projectDirectory(input, github, client) {
|
|
14
|
+
// Input validation permits either JS/TS entries or Solidity entries, never a mix.
|
|
15
|
+
const manifests = input.entries[0].endsWith('.sol') ? ['foundry.toml', 'soldeer.toml', 'soldeer.lock'] : ['package.json']
|
|
16
|
+
if (!input.directory) return ''
|
|
17
|
+
for (let directory = input.directory; ; directory = posix.dirname(directory).replace(/^\.$/u, '')) {
|
|
18
|
+
const entries = await client.listRepoDir({ repo: github, sha: input.commit, directory: directory || undefined })
|
|
19
|
+
if (entries.some(entry => manifests.includes(entry.path) && regularFile(entry))) return directory
|
|
20
|
+
if (!directory) return input.directory
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// The name Stasis read from the package.json in the project directory we
|
|
25
|
+
// provide, where its module holds bundled files. JS builds only: a Solidity
|
|
26
|
+
// build records `solidity-bundle` for a package.json without name and version.
|
|
27
|
+
async function packageName(input, github, project, bundle, client) {
|
|
28
|
+
if (!project || input.entries[0].endsWith('.sol')) return null
|
|
29
|
+
const entries = await client.listRepoDir({ repo: github, sha: input.commit, directory: project })
|
|
30
|
+
if (!entries.some(entry => entry.path === 'package.json' && regularFile(entry))) return null
|
|
31
|
+
const name = bundle.modules.get(posix.relative(bundle.repo?.directory ?? '', project) || '.')?.name
|
|
32
|
+
return typeof name === 'string' ? name : null
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Prisma Client is what `prisma generate` writes, not what the repo holds: for
|
|
36
|
+
// a JS project whose package.json depends on a @prisma/ package, Stasis
|
|
37
|
+
// generates it before the scan (generate: ['prisma']).
|
|
38
|
+
async function prismaGenerate(input, github, project, client) {
|
|
39
|
+
if (input.entries[0].endsWith('.sol')) return []
|
|
40
|
+
const listing = await client.listRepoDir({ repo: github, sha: input.commit, directory: project || undefined })
|
|
41
|
+
if (!listing.some(entry => entry.path === 'package.json' && regularFile(entry))) return []
|
|
42
|
+
const text = await client.getRepoFile({ repo: github, path: posix.join(project, 'package.json'), ref: input.commit })
|
|
43
|
+
let manifest
|
|
44
|
+
try { manifest = JSON.parse(text) } catch { return [] }
|
|
45
|
+
const named = ['dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies']
|
|
46
|
+
.flatMap(field => manifest?.[field] !== null && typeof manifest?.[field] === 'object' ? Object.keys(manifest[field]) : [])
|
|
47
|
+
return named.some(name => name.startsWith('@prisma/')) ? ['prisma'] : []
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export async function buildStasisBundle({ input, github, token, maxBytes, scopes, cacheDir = null }, client = createClient({ token }), progress = () => {}) {
|
|
51
|
+
const allowed = path => scopes.some(scope => !scope || path === scope || path.startsWith(scope + '/'))
|
|
52
|
+
// Reuse these immutable listings when Stasis discovers the lockfile root.
|
|
53
|
+
const listings = new Map()
|
|
54
|
+
const buildClient = { ...client, listRepoDir(options) {
|
|
55
|
+
const key = JSON.stringify([options.repo, options.sha, options.directory ?? ''])
|
|
56
|
+
if (!listings.has(key)) listings.set(key, client.listRepoDir(options))
|
|
57
|
+
return listings.get(key)
|
|
58
|
+
} }
|
|
59
|
+
// Upstream's disk cache keeps the npm tarballs and version documents, and
|
|
60
|
+
// the repo's tree, in `cacheDir`; without one (Vercel), nothing is kept on
|
|
61
|
+
// disk. This fresh worker inherits no application credentials but NPM_TOKEN.
|
|
62
|
+
setCacheDir(cacheDir ?? false)
|
|
63
|
+
progress('build')
|
|
64
|
+
const project = await projectDirectory(input, github, buildClient)
|
|
65
|
+
if (!allowed(project)) throw new Error('build-scope')
|
|
66
|
+
const generate = await prismaGenerate(input, github, project, buildClient)
|
|
67
|
+
const { bundle } = await buildGitHubBundle({
|
|
68
|
+
github, sha: input.commit, directory: project || undefined,
|
|
69
|
+
entries: input.entries.map(entry => posix.relative(project || '.', entry)),
|
|
70
|
+
...input.options, client: buildClient, ...(cacheDir === null && { cache: false }), ...(generate.length > 0 && { generate }),
|
|
71
|
+
})
|
|
72
|
+
const directory = bundle.repo?.directory ?? ''
|
|
73
|
+
progress('scope')
|
|
74
|
+
// Imports outside the selected package may widen the root. Never publish it under a
|
|
75
|
+
// narrower team grant: the storage location must cover the actual bundle.
|
|
76
|
+
if (!allowed(directory)) throw new Error('build-scope')
|
|
77
|
+
progress('serialize')
|
|
78
|
+
const serialized = bundle.serialize()
|
|
79
|
+
if (Buffer.byteLength(serialized) > 200 * 1024 * 1024) throw new Error('too-large')
|
|
80
|
+
progress('compress')
|
|
81
|
+
const bytes = brotliCompressSync(serialized, brotliOptions())
|
|
82
|
+
if (bytes.length > maxBytes) throw new Error('too-large')
|
|
83
|
+
const name = await packageName(input, github, project, bundle, buildClient)
|
|
84
|
+
return { bytes, directory, filename: githubBundleFilename(github, project, input.commit, name) }
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// The code a failed build answers with. Never upstream bodies or paths outside
|
|
88
|
+
// the caller's grants: those stay in the diagnostic. A missing or ambiguous
|
|
89
|
+
// lockfile, or the named one missing, is the lockfile's.
|
|
90
|
+
export function buildErrorCode(error, message) {
|
|
91
|
+
if (['build-scope', 'too-large'].includes(message)) return message
|
|
92
|
+
if (error instanceof HttpError) return error.status === 429 ? 'github-rate-limited' : 'github-build-failed'
|
|
93
|
+
return /no packageManager given|lockfile installs|no (?:pnpm-lock\.yaml|yarn\.lock|package-lock\.json|soldeer\.lock) found in /u.test(message) ? 'build-lockfile' : 'build-failed'
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (parentPort && workerData?.type === 'managed-bundle-build') {
|
|
97
|
+
// Send diagnostics to the request's parent thread, which writes them before
|
|
98
|
+
// returning the HTTP response. Worker stdout can be lost on termination.
|
|
99
|
+
const send = (message, transfer = []) => {
|
|
100
|
+
// Node worker messages have no browser target origin.
|
|
101
|
+
// oxlint-disable-next-line unicorn/require-post-message-target-origin
|
|
102
|
+
parentPort.postMessage(message, transfer)
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
const built = await buildStasisBundle(workerData, undefined, stage => send({ type: 'progress', stage }))
|
|
106
|
+
// The bytes handed over rather than copied, where they alone fill their
|
|
107
|
+
// buffer (a small one may share Node's pool, which can't be moved).
|
|
108
|
+
const { bytes } = built
|
|
109
|
+
send(built, bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength ? [bytes.buffer] : [])
|
|
110
|
+
} catch (error) {
|
|
111
|
+
const diagnostic = bundleBuildDiagnostic(error, workerData.token)
|
|
112
|
+
const code = buildErrorCode(error, diagnostic.message)
|
|
113
|
+
send({ error: code, status: code === 'too-large' ? 413 : code === 'build-scope' ? 403 : 422, diagnostic })
|
|
114
|
+
}
|
|
115
|
+
}
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
3
|
+
import { posix } from 'node:path'
|
|
4
|
+
import { Worker } from 'node:worker_threads'
|
|
5
|
+
import { BUNDLE_BUILD_TIMEOUT_MS, type BundleBuildLeaseStore } from './bundle-build-leases.ts'
|
|
6
|
+
import { type BundleBuildDiagnostic, type BundleBuildStage, bundleBuildDiagnostic } from './bundle-build-diagnostics.js'
|
|
7
|
+
import type { BundleBuildConditions } from './db-methods.ts'
|
|
8
|
+
|
|
9
|
+
export class BundleBuildError extends Error {
|
|
10
|
+
status: number
|
|
11
|
+
code: string
|
|
12
|
+
constructor(status: number, code: string) { super(code); this.status = status; this.code = code }
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface BundleBuildInput {
|
|
16
|
+
repoId: number
|
|
17
|
+
commit: string
|
|
18
|
+
entries: string[]
|
|
19
|
+
directory: string
|
|
20
|
+
options: { conditions?: string[]; mainFields?: string[]; metro?: boolean; platforms?: string[]; jsx?: boolean; packageManager?: 'soldeer' }
|
|
21
|
+
// What the stored bundle records; null for Solidity, which takes no conditions.
|
|
22
|
+
conditions: BundleBuildConditions | null
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function path(value: unknown): value is string {
|
|
26
|
+
return typeof value === 'string' && value.length > 0 && value.length <= 1024 && !/[\\\p{Cc}]/u.test(value)
|
|
27
|
+
&& value.split('/').every(part => part !== '' && part !== '.' && part !== '..')
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function parseBundleBuild(value: unknown): BundleBuildInput {
|
|
31
|
+
const fail = (code = 'bad-request'): never => { throw new BundleBuildError(400, code) }
|
|
32
|
+
if (value == null || typeof value !== 'object' || Array.isArray(value)) return fail()
|
|
33
|
+
const { repoId, commit, entries, conditions } = value as Record<string, unknown>
|
|
34
|
+
if (typeof repoId !== 'number' || !Number.isSafeInteger(repoId) || repoId <= 0
|
|
35
|
+
|| typeof commit !== 'string' || !/^[a-f\d]{40}$/u.test(commit)
|
|
36
|
+
|| !Array.isArray(entries) || entries.length === 0 || entries.length > 100 || !entries.every(path)) return fail()
|
|
37
|
+
const selected = [...new Set(entries as string[])]
|
|
38
|
+
const scripts = selected.every(entry => /\.[cm]?[tj]sx?$/iu.test(entry))
|
|
39
|
+
if (!scripts && !selected.every(entry => entry.endsWith('.sol'))) return fail('unsupported-entries')
|
|
40
|
+
const parts = posix.dirname(selected[0]!).split('/').filter(part => part !== '.')
|
|
41
|
+
for (const entry of selected) {
|
|
42
|
+
const parent = posix.dirname(entry).split('/')
|
|
43
|
+
while (parts.some((part, i) => parent[i] !== part)) parts.pop()
|
|
44
|
+
}
|
|
45
|
+
// Stasis builds Solidity with Soldeer alone. Named, a JS lockfile beside or
|
|
46
|
+
// above the project no longer leaves it unable to tell which one installs it.
|
|
47
|
+
const options: BundleBuildInput['options'] = scripts ? {} : { packageManager: 'soldeer' }
|
|
48
|
+
let recorded: BundleBuildConditions | null = null
|
|
49
|
+
if (scripts) {
|
|
50
|
+
if (conditions == null || typeof conditions !== 'object' || Array.isArray(conditions)) return fail('bad-conditions')
|
|
51
|
+
const { preset, conditions: names, platforms } = conditions as Record<string, unknown>
|
|
52
|
+
if (typeof preset !== 'string' || !['node', 'browser', 'metro'].includes(preset) || !Array.isArray(names) || names.length > 16
|
|
53
|
+
|| !names.every(name => typeof name === 'string' && name.length > 0 && name.length <= 64
|
|
54
|
+
&& !/^\.|^\d+$|[\s,/\\\p{Cc}]/u.test(name) && !['default', 'import', 'require'].includes(name))) return fail('bad-conditions')
|
|
55
|
+
if (preset === 'metro') {
|
|
56
|
+
if (names.length !== 1 || names[0] !== 'react-native') return fail('metro-conditions')
|
|
57
|
+
if (!Array.isArray(platforms) || platforms.length === 0 || platforms.length > 2
|
|
58
|
+
|| !platforms.every(platform => ['ios', 'android'].includes(platform))) return fail('bad-conditions')
|
|
59
|
+
// React Native's convention puts JSX in .js files, which Stasis parses as
|
|
60
|
+
// JSX only on request. TypeScript is left to Stasis to detect.
|
|
61
|
+
Object.assign(options, { metro: true, platforms: [...new Set(platforms)], jsx: true })
|
|
62
|
+
} else {
|
|
63
|
+
options.conditions = [...new Set(names)]
|
|
64
|
+
if (preset === 'browser') options.mainFields = ['browser', 'module', 'main']
|
|
65
|
+
}
|
|
66
|
+
recorded = { preset: preset as BundleBuildConditions['preset'], conditions: [...new Set(names as string[])],
|
|
67
|
+
platforms: options.platforms ?? [] }
|
|
68
|
+
}
|
|
69
|
+
return { repoId, commit, entries: selected, directory: parts.join('/'), options, conditions: recorded }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// A package name as a filename base: `@scope/name` as `scope-name`, or as
|
|
73
|
+
// `name` alone where it is `scope` or already starts with `scope-`. Null for
|
|
74
|
+
// a name npm could not publish by its length or leading dot.
|
|
75
|
+
function packageBase(name: string): string | null {
|
|
76
|
+
const match = name.length <= 214 ? /^(?:@([^/.][^/]*)\/)?([^/.][^/]*)$/u.exec(name) : null
|
|
77
|
+
if (!match) return null
|
|
78
|
+
const base = match[2]!, scope = match[1]
|
|
79
|
+
return scope === undefined || base === scope || base.startsWith(`${scope}-`) ? base : `${scope}-${base}`
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Same portable name and 255-character limit as Stasis's githubBundleFile
|
|
83
|
+
// (src/cmd/github-bundle.js), which is not a public package export, except
|
|
84
|
+
// that a usable `packageName` replaces both the repo and the directory.
|
|
85
|
+
export function githubBundleFilename(github: string, directory: string, commit: string, packageName: string | null = null): string {
|
|
86
|
+
const portable = (value: string) => value.replaceAll(/[^\w.-]/gu, '_')
|
|
87
|
+
const tail = `.${portable(commit.slice(0, 7))}.stasis.code.br`
|
|
88
|
+
const named = packageName === null ? null : packageBase(packageName)
|
|
89
|
+
if (named !== null) return `${portable(named)}${tail}`
|
|
90
|
+
const repo = portable(github.replace('/', '-'))
|
|
91
|
+
if (!directory) return `${repo}${tail}`
|
|
92
|
+
const room = 255 - `${repo}.${tail}`.length
|
|
93
|
+
let dir = portable(directory.replaceAll('/', '-'))
|
|
94
|
+
if (dir.length > room) dir = `${dir.slice(0, room - 9)}_${createHash('sha256').update(directory).digest('hex').slice(0, 8)}`
|
|
95
|
+
return `${repo}.${dir}${tail}`
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export interface BuiltBundle { bytes: Buffer; directory: string; filename: string }
|
|
99
|
+
export interface BuildRequest {
|
|
100
|
+
input: BundleBuildInput; github: string; token: string | null; maxBytes: number; scopes: (string | null)[]
|
|
101
|
+
// Upstream's disk cache for what a build fetches (setCacheDir), or null for
|
|
102
|
+
// none, as on Vercel. Resolved here: the worker's env holds NPM_TOKEN alone.
|
|
103
|
+
cacheDir: string | null
|
|
104
|
+
}
|
|
105
|
+
// An extra per-process ceiling; admission must also hold the shared DB lease.
|
|
106
|
+
const active = new Set<string>()
|
|
107
|
+
|
|
108
|
+
export async function withBundleBuildLease<T>(db: BundleBuildLeaseStore, userId: string, signal: AbortSignal,
|
|
109
|
+
work: (signal: AbortSignal) => Promise<T>): Promise<T> {
|
|
110
|
+
signal.throwIfAborted()
|
|
111
|
+
const owner = randomUUID(), started = performance.now(), timeout = new AbortController()
|
|
112
|
+
const expired = () => timeout.abort(new BundleBuildError(504, 'build-timeout'))
|
|
113
|
+
const timer = setTimeout(expired, BUNDLE_BUILD_TIMEOUT_MS)
|
|
114
|
+
const combined = AbortSignal.any([signal, timeout.signal])
|
|
115
|
+
let claimed = false
|
|
116
|
+
try {
|
|
117
|
+
claimed = await db.claimBundleBuildLease(userId, owner)
|
|
118
|
+
if (!claimed) throw new BundleBuildError(429, 'build-busy')
|
|
119
|
+
// Account for a slow/lost claim response or a suspended invocation before
|
|
120
|
+
// starting work. The three-minute budget starts before lease acquisition.
|
|
121
|
+
if (performance.now() - started >= BUNDLE_BUILD_TIMEOUT_MS) expired()
|
|
122
|
+
combined.throwIfAborted()
|
|
123
|
+
return await work(combined)
|
|
124
|
+
} catch (error) {
|
|
125
|
+
if (timeout.signal.aborted && !signal.aborted) throw timeout.signal.reason
|
|
126
|
+
throw error
|
|
127
|
+
} finally {
|
|
128
|
+
clearTimeout(timer)
|
|
129
|
+
// buildRepositoryBundle waits for worker termination before settling.
|
|
130
|
+
// Unknown claim/release outcomes expire safely without starting more work.
|
|
131
|
+
if (claimed) await db.releaseBundleBuildLease(owner)
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Parsing repositories and compressing bundles stays off the HTTP event loop.
|
|
136
|
+
// Cap concurrent builds, terminate disconnected/timed-out work, and share no
|
|
137
|
+
// credentials or mutable Stasis state between requests.
|
|
138
|
+
export async function buildRepositoryBundle(userId: string, request: BuildRequest, signal: AbortSignal): Promise<BuiltBundle> {
|
|
139
|
+
if (active.has(userId) || active.size >= 2) throw new BundleBuildError(429, 'build-busy')
|
|
140
|
+
signal.throwIfAborted()
|
|
141
|
+
active.add(userId)
|
|
142
|
+
let worker: Worker | undefined
|
|
143
|
+
let workerUrl: URL | undefined
|
|
144
|
+
let stage: BundleBuildStage = 'worker-start'
|
|
145
|
+
const buildId = randomUUID(), started = performance.now()
|
|
146
|
+
const context = { buildId, repoId: request.input.repoId, github: request.github, commit: request.input.commit,
|
|
147
|
+
directory: request.input.directory, entryCount: request.input.entries.length, node: process.version }
|
|
148
|
+
const log = (event: string, details: Record<string, unknown> = {}) => {
|
|
149
|
+
const write = event === 'failed' ? console.error : console.info
|
|
150
|
+
write('managed-bundle-build:', JSON.stringify({ ...context, workerUrl: workerUrl?.href, event, stage, elapsedMs: Math.round(performance.now() - started), ...details }))
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
workerUrl = new URL('./bundle-build-worker.js', import.meta.url)
|
|
154
|
+
log('started')
|
|
155
|
+
// Upstream reads NPM_TOKEN from the env for scoped packages, which npm
|
|
156
|
+
// answers with 404 when private and asked without it. Nothing else passes.
|
|
157
|
+
const npmToken = process.env['NPM_TOKEN']
|
|
158
|
+
const env = npmToken ? { NPM_TOKEN: npmToken } : {}
|
|
159
|
+
worker = new Worker(workerUrl, {
|
|
160
|
+
workerData: { ...request, type: 'managed-bundle-build' }, env, execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 512 },
|
|
161
|
+
})
|
|
162
|
+
const running = worker
|
|
163
|
+
return await new Promise((resolve, reject) => {
|
|
164
|
+
let settled = false
|
|
165
|
+
const cleanup = () => { clearTimeout(timer); signal.removeEventListener('abort', abort) }
|
|
166
|
+
const fail = (status: number, code: string, details: Record<string, unknown> = {}) => {
|
|
167
|
+
if (settled) return
|
|
168
|
+
settled = true
|
|
169
|
+
cleanup()
|
|
170
|
+
log(code === 'build-cancelled' ? 'cancelled' : 'failed', { code, ...details })
|
|
171
|
+
reject(new BundleBuildError(status, code))
|
|
172
|
+
}
|
|
173
|
+
const timer = setTimeout(() => fail(504, 'build-timeout'), BUNDLE_BUILD_TIMEOUT_MS)
|
|
174
|
+
const abort = () => fail(signal.reason instanceof BundleBuildError ? signal.reason.status : 499,
|
|
175
|
+
signal.reason instanceof BundleBuildError ? signal.reason.code : 'build-cancelled')
|
|
176
|
+
signal.addEventListener('abort', abort, { once: true })
|
|
177
|
+
running.on('message', (message: Omit<BuiltBundle, 'bytes'> & { bytes: Uint8Array; type?: string; stage?: BundleBuildStage; error?: string; status?: number; diagnostic?: BundleBuildDiagnostic }) => {
|
|
178
|
+
if (settled) return
|
|
179
|
+
if (message.type === 'progress' && message.stage) { stage = message.stage; log('progress'); return }
|
|
180
|
+
if (message.error) { fail(message.status ?? 422, message.error, { diagnostic: message.diagnostic }); return }
|
|
181
|
+
settled = true
|
|
182
|
+
cleanup()
|
|
183
|
+
log('completed', { byteSize: message.bytes.length })
|
|
184
|
+
// A view of the bytes the worker handed over, not a copy of them.
|
|
185
|
+
const { bytes } = message
|
|
186
|
+
resolve({ bytes: Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength), directory: message.directory, filename: message.filename })
|
|
187
|
+
})
|
|
188
|
+
running.once('error', error => fail(422, 'build-failed', { diagnostic: bundleBuildDiagnostic(error, request.token) }))
|
|
189
|
+
running.once('exit', exitCode => fail(422, 'build-failed', { exitCode }))
|
|
190
|
+
if (signal.aborted) abort()
|
|
191
|
+
})
|
|
192
|
+
} catch (error) {
|
|
193
|
+
if (!(error instanceof BundleBuildError)) {
|
|
194
|
+
log('failed', { code: 'build-failed', diagnostic: bundleBuildDiagnostic(error, request.token) })
|
|
195
|
+
}
|
|
196
|
+
throw error
|
|
197
|
+
} finally {
|
|
198
|
+
try { await worker?.terminate() } finally { active.delete(userId) }
|
|
199
|
+
}
|
|
200
|
+
}
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
// Metadata is cached as Brotli. Contents use the stored Brotli bytes directly:
|
|
2
|
+
// unchanged Stasis uploads or sourcemaps compressed once at upload.
|
|
3
|
+
import { Buffer } from 'node:buffer'
|
|
4
|
+
import { brotliDecompress } from 'node:zlib'
|
|
5
|
+
import { promisify } from 'node:util'
|
|
6
|
+
import { bundleEdges } from '@preventive/sourcemap/edges.js'
|
|
7
|
+
import { BUNDLE_METADATA_VERSION, type BundleDetails, createBundleMetadata, createBundleSummary, parseBundleContents, parseBundleMetadata } from '../common/bundle-metadata.js'
|
|
8
|
+
import { bundleReasons } from '../common/bundle-reasons.js'
|
|
9
|
+
import { sourcemapEdges } from '../common/bundle-sourcemap.js'
|
|
10
|
+
import { type BundleAdvisoryInventory, bundleAdvisoryInventory } from './bundle-advisory-inventory.ts'
|
|
11
|
+
import { decodeUtf8 } from '../common/utf8.js'
|
|
12
|
+
import type { OpenedBlob } from './blob-store.ts'
|
|
13
|
+
import type { BundleStore } from './bundle-store.ts'
|
|
14
|
+
import { encodeBrotli } from './brotli.ts'
|
|
15
|
+
import type { ManagedBundle, ManagedDb } from './db.ts'
|
|
16
|
+
import { CacheMissError } from './cache-storage.ts'
|
|
17
|
+
import { SUMMARY_FILENAME, createBundleSummaryCache } from './bundle-summary-cache.ts'
|
|
18
|
+
|
|
19
|
+
const decompress = promisify(brotliDecompress)
|
|
20
|
+
const MAX_DECODED_BYTES = 512 * 1024 * 1024
|
|
21
|
+
export const MAX_PACKAGE_INVENTORY_BYTES = 1024 * 1024
|
|
22
|
+
export type BundleCachePart = 'metadata' | 'contents'
|
|
23
|
+
export type BundleCacheRecord = Pick<ManagedBundle, 'id' | 'integrity' | 'filename' | 'kind' | 'byteSize'>
|
|
24
|
+
export interface BundleSummary { files: number; codeFiles: number; lines: number; commit?: string; stasisVersion?: number; versionedPackages?: number }
|
|
25
|
+
|
|
26
|
+
export async function readBundleDetails(record: BundleCacheRecord, store: BundleStore) {
|
|
27
|
+
return (await readSizedBundleDetails(record, store))?.details ?? null
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// As readBundleDetails, with the bytes the bundle decoded to: about what
|
|
31
|
+
// holding its details takes.
|
|
32
|
+
export async function readSizedBundleDetails(record: BundleCacheRecord, store: BundleStore) {
|
|
33
|
+
const bytes = await store.get(record.id, record.kind)
|
|
34
|
+
if (!bytes) return null
|
|
35
|
+
const decoded = record.kind === 'stasis' || record.kind === 'sourcemap' ? await decompress(bytes, { maxOutputLength: MAX_DECODED_BYTES }) : bytes
|
|
36
|
+
if (decoded.length > MAX_DECODED_BYTES) throw new Error('Decoded bundle too large')
|
|
37
|
+
return { details: parseBundleContents(decodeUtf8(decoded), { integrity: record.integrity, kind: record.kind, size: record.byteSize }), decodedBytes: decoded.length }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface BundleCacheStorage {
|
|
41
|
+
exists(id: string, file: string): Promise<boolean>
|
|
42
|
+
put(id: string, file: string, bytes: Buffer): Promise<void>
|
|
43
|
+
open(id: string, file: string): Promise<OpenedBlob>
|
|
44
|
+
// Remove all cached versions for this bundle, including legacy derivatives.
|
|
45
|
+
delete(id: string): Promise<void>
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const filename = `v${BUNDLE_METADATA_VERSION}-metadata.json.br`
|
|
49
|
+
const packagesFilename = 'v6-advisory-inventory.json'
|
|
50
|
+
|
|
51
|
+
// All scopes share one bounded derivative. Never decode full bundle metadata
|
|
52
|
+
// on advisory requests, including when selecting a reason. Persist null when
|
|
53
|
+
// the combined inventory exceeds the budget, without disabling metadata.
|
|
54
|
+
function encodePackageInventory(details: BundleDetails): Buffer {
|
|
55
|
+
const parts: string[] = []
|
|
56
|
+
let size = 0
|
|
57
|
+
function append(part: string) {
|
|
58
|
+
size += Buffer.byteLength(part)
|
|
59
|
+
if (size > MAX_PACKAGE_INVENTORY_BYTES) return false
|
|
60
|
+
parts.push(part)
|
|
61
|
+
return true
|
|
62
|
+
}
|
|
63
|
+
function entries(rows: unknown[]) {
|
|
64
|
+
if (!append('[')) return false
|
|
65
|
+
let first = true
|
|
66
|
+
for (const row of rows) {
|
|
67
|
+
if (!append(`${first ? '' : ','}${JSON.stringify(row)}`)) return false
|
|
68
|
+
first = false
|
|
69
|
+
}
|
|
70
|
+
return append(']')
|
|
71
|
+
}
|
|
72
|
+
function inventory(paths: Set<string> | null) {
|
|
73
|
+
const { packages, skipped } = bundleAdvisoryInventory(details, paths)
|
|
74
|
+
return append('{"packages":') && entries(packages) && append(',"skipped":') && entries(skipped) && append('}')
|
|
75
|
+
}
|
|
76
|
+
append('{"all":')
|
|
77
|
+
if (!inventory(null) || !append(',"reasons":{')) return Buffer.from('null')
|
|
78
|
+
let first = true
|
|
79
|
+
for (const [reason, paths] of bundleReasons(details)) {
|
|
80
|
+
if (!append(`${first ? '' : ','}${JSON.stringify(reason)}:`) || !inventory(paths)) return Buffer.from('null')
|
|
81
|
+
first = false
|
|
82
|
+
}
|
|
83
|
+
if (!append('}}')) return Buffer.from('null')
|
|
84
|
+
return Buffer.from(parts.join(''))
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// What a catalog sends of a bundle before it opens. A Stasis bundle's
|
|
88
|
+
// `versionedPackages` counts the name@version pairs its unscoped advisory
|
|
89
|
+
// audit covers, audited or skipped: with none, Advisories has nothing to show.
|
|
90
|
+
export function bundleSummary(details: BundleDetails, metadata?: Parameters<typeof createBundleSummary>[1]): BundleSummary {
|
|
91
|
+
const summary = createBundleSummary(details, metadata)
|
|
92
|
+
if (details.kind !== 'stasis') return summary
|
|
93
|
+
const { packages, skipped } = bundleAdvisoryInventory(details)
|
|
94
|
+
return { ...summary, versionedPackages: packages.reduce((count, pkg) => count + pkg.versions.length, 0) + skipped.length }
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// A sourcemap's edges, read with the parser (edges.js) that the client goes
|
|
98
|
+
// without: every map's, where the client reads Metro's alone. Without it,
|
|
99
|
+
// the metadata keeps what the client would read. A map with no
|
|
100
|
+
// sourcesContent has no source to read them from.
|
|
101
|
+
function readSourcemapEdges(details: BundleDetails) {
|
|
102
|
+
if (details.kind !== 'sourcemap' || !details.map?.files.some(file => file.content !== null)) return
|
|
103
|
+
try { details.edges = sourcemapEdges(details.map, bundleEdges) }
|
|
104
|
+
catch (err) { console.warn('managed: sourcemap edges failed:', err) }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async function build(record: BundleCacheRecord, storage: BundleCacheStorage, db: ManagedDb, store: BundleStore) {
|
|
108
|
+
const details = await readBundleDetails(record, store)
|
|
109
|
+
if (!details) throw new Error('Bundle bytes unavailable')
|
|
110
|
+
readSourcemapEdges(details)
|
|
111
|
+
const metadata = { ...await createBundleMetadata(details), id: record.id, filename: record.filename }
|
|
112
|
+
const body = await encodeBrotli(Buffer.from(JSON.stringify(metadata)))
|
|
113
|
+
if (!(await db.getBundle(record.id))) throw new Error('Bundle deleted')
|
|
114
|
+
const summary = bundleSummary(details, metadata)
|
|
115
|
+
// Written together, each settled before the row is looked at again: one
|
|
116
|
+
// failing leaves the others, which a later open builds over.
|
|
117
|
+
const written = await Promise.allSettled([
|
|
118
|
+
storage.put(record.id, filename, body),
|
|
119
|
+
...record.kind === 'stasis' ? [storage.put(record.id, packagesFilename, encodePackageInventory(details))] : [],
|
|
120
|
+
storage.put(record.id, SUMMARY_FILENAME, Buffer.from(JSON.stringify(summary))),
|
|
121
|
+
])
|
|
122
|
+
// A different instance may have deleted the row while these writes ran.
|
|
123
|
+
// Reconcile after publishing so its cleanup cannot be undone by us.
|
|
124
|
+
if (!(await db.getBundle(record.id))) {
|
|
125
|
+
await storage.delete(record.id)
|
|
126
|
+
throw new Error('Bundle deleted')
|
|
127
|
+
}
|
|
128
|
+
const failed = written.find(result => result.status === 'rejected')
|
|
129
|
+
if (failed) throw failed.reason
|
|
130
|
+
return summary
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export function createBundleCache(storage: BundleCacheStorage, db: ManagedDb, store: BundleStore) {
|
|
134
|
+
const pending = new Map<string, Promise<void>>()
|
|
135
|
+
// The database deduplicates by integrity, so each hash has one persistent
|
|
136
|
+
// bundle/cache directory, shared across teams and repeated uploads.
|
|
137
|
+
const summaries = createBundleSummaryCache(storage, async record => {
|
|
138
|
+
const details = await readBundleDetails(record, store)
|
|
139
|
+
if (!details) throw new Error('Bundle bytes unavailable')
|
|
140
|
+
return bundleSummary(details)
|
|
141
|
+
}, id => db.getBundle(id))
|
|
142
|
+
let queue = Promise.resolve()
|
|
143
|
+
async function ensure(record: BundleCacheRecord): Promise<void> {
|
|
144
|
+
const existing = pending.get(record.id)
|
|
145
|
+
if (existing) return existing
|
|
146
|
+
const job = (async () => {
|
|
147
|
+
if (await storage.exists(record.id, filename) && (record.kind !== 'stasis' || await storage.exists(record.id, packagesFilename))) return
|
|
148
|
+
const work = queue.then(async () => summaries.remember(record, await build(record, storage, db, store)))
|
|
149
|
+
queue = work.catch(() => {})
|
|
150
|
+
await work
|
|
151
|
+
})()
|
|
152
|
+
pending.set(record.id, job)
|
|
153
|
+
try { await job } finally { if (pending.get(record.id) === job) pending.delete(record.id) }
|
|
154
|
+
}
|
|
155
|
+
async function openCached(record: BundleCacheRecord, file: string) {
|
|
156
|
+
try { return await storage.open(record.id, file) }
|
|
157
|
+
catch (error) { if (!(error instanceof CacheMissError)) throw error }
|
|
158
|
+
await ensure(record)
|
|
159
|
+
return storage.open(record.id, file)
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
162
|
+
prebuild: ensure,
|
|
163
|
+
summary: summaries.summary,
|
|
164
|
+
summaryStatus: summaries.summaryStatus,
|
|
165
|
+
backfillSummaries: summaries.backfill,
|
|
166
|
+
async open(record: ManagedBundle, part: BundleCachePart) {
|
|
167
|
+
if (part === 'contents') {
|
|
168
|
+
if (record.kind !== 'stasis' && record.kind !== 'sourcemap') throw new Error('Unsupported bundle')
|
|
169
|
+
const stored = await store.open(record.id, record.kind)
|
|
170
|
+
if (!stored) throw new Error('Bundle bytes unavailable')
|
|
171
|
+
return stored
|
|
172
|
+
}
|
|
173
|
+
return openCached(record, filename)
|
|
174
|
+
},
|
|
175
|
+
// A sourcemap's edges as its metadata keeps them, read with the parser
|
|
176
|
+
// once, for a derivative that reads the bundle itself (report previews).
|
|
177
|
+
async sourcemapEdges(record: ManagedBundle): Promise<BundleDetails['edges']> {
|
|
178
|
+
const cached = await openCached(record, filename)
|
|
179
|
+
const chunks: Buffer[] = []
|
|
180
|
+
try { for await (const chunk of cached.stream) chunks.push(Buffer.from(chunk)) }
|
|
181
|
+
finally { cached.stream.destroy() }
|
|
182
|
+
const decoded = await decompress(Buffer.concat(chunks), { maxOutputLength: MAX_DECODED_BYTES })
|
|
183
|
+
return parseBundleMetadata(JSON.parse(decodeUtf8(decoded)), record.integrity).edges
|
|
184
|
+
},
|
|
185
|
+
async advisoryInventory(record: ManagedBundle, reason = ''): Promise<BundleAdvisoryInventory | null | undefined> {
|
|
186
|
+
if (record.kind !== 'stasis') return { packages: [], skipped: [] }
|
|
187
|
+
const cached = await openCached(record, packagesFilename)
|
|
188
|
+
try {
|
|
189
|
+
if (cached.size != null && cached.size > MAX_PACKAGE_INVENTORY_BYTES) return null
|
|
190
|
+
const chunks: Buffer[] = []
|
|
191
|
+
let size = 0
|
|
192
|
+
for await (const chunk of cached.stream) {
|
|
193
|
+
size += chunk.length
|
|
194
|
+
if (size > MAX_PACKAGE_INVENTORY_BYTES) return null
|
|
195
|
+
chunks.push(Buffer.from(chunk))
|
|
196
|
+
}
|
|
197
|
+
const inventory = JSON.parse(decodeUtf8(Buffer.concat(chunks)))
|
|
198
|
+
if (inventory === null) return null
|
|
199
|
+
return reason ? (Object.hasOwn(inventory.reasons, reason) ? inventory.reasons[reason] : undefined) : inventory.all
|
|
200
|
+
} finally { cached.stream.destroy() }
|
|
201
|
+
},
|
|
202
|
+
async delete(id: string) {
|
|
203
|
+
// Call after deleting the row. Waiting prevents an in-flight builder
|
|
204
|
+
// from recreating its files after deletion has completed.
|
|
205
|
+
await pending.get(id)?.catch(() => {})
|
|
206
|
+
await summaries.forget(id)
|
|
207
|
+
await storage.delete(id)
|
|
208
|
+
},
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
export type BundleCache = ReturnType<typeof createBundleCache>
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { BundleCache, BundleSummary } from './bundle-cache.ts'
|
|
2
|
+
import type { AdminBundle, UserTeamBundle } from './db.ts'
|
|
3
|
+
|
|
4
|
+
const SUMMARY_READ_CONCURRENCY = 8
|
|
5
|
+
|
|
6
|
+
// Read only existing summaries: cold/malformed bundles must never trigger
|
|
7
|
+
// parsing or wait for metadata builds before the catalog response is sent.
|
|
8
|
+
export async function bundleSummaries(bundles: readonly (AdminBundle | UserTeamBundle)[], cache?: BundleCache) {
|
|
9
|
+
const summaries = new Map<string, { summary: BundleSummary | null; summaryRetryAt: number | null }>()
|
|
10
|
+
if (cache) {
|
|
11
|
+
const unique = [...new Map(bundles.map(bundle => [bundle.integrity, bundle])).values()]
|
|
12
|
+
let next = 0
|
|
13
|
+
// Cold reads may each access remote storage. Bound those requests as well
|
|
14
|
+
// as the subsequent backfill, without dropping entries from large catalogs.
|
|
15
|
+
await Promise.all(Array.from({ length: Math.min(SUMMARY_READ_CONCURRENCY, unique.length) }, async () => {
|
|
16
|
+
while (next < unique.length) {
|
|
17
|
+
const bundle = unique[next++]!
|
|
18
|
+
summaries.set(bundle.integrity, await cache.summaryStatus(bundle).catch(() => ({ summary: null, summaryRetryAt: null })))
|
|
19
|
+
}
|
|
20
|
+
}))
|
|
21
|
+
}
|
|
22
|
+
return summaries
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Called after sending the response. Each cache permits one small batch at a
|
|
26
|
+
// time, deduplicated by content hash, and skips persisted retry-backoff markers.
|
|
27
|
+
export async function backfillBundleSummaries(bundles: readonly (AdminBundle | UserTeamBundle)[], cache?: BundleCache) {
|
|
28
|
+
await cache?.backfillSummaries([...new Map(bundles.map(bundle => [bundle.integrity, bundle])).values()])
|
|
29
|
+
}
|