@preventive/triage 1.0.0-alpha.5 → 1.0.0-alpha.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/README.md +16 -0
- package/api/reap.ts +17 -0
- package/cli.js +6 -0
- package/client/dependency-paths.d.ts +3 -0
- package/client/dependency-paths.js +36 -0
- package/client/finding-link.js +305 -0
- package/client/linked-findings.d.ts +1 -0
- package/client/linked-findings.js +115 -0
- package/common/bundle-commit.d.ts +1 -0
- package/common/bundle-commit.js +4 -0
- package/common/bundle-compare-diff.d.ts +3 -0
- package/common/bundle-compare-diff.js +357 -0
- package/common/bundle-compare-inputs.d.ts +2 -0
- package/common/bundle-compare-inputs.js +65 -0
- package/common/bundle-compare-versions.js +172 -0
- package/common/bundle-comparison.d.ts +8 -0
- package/common/bundle-comparison.js +62 -0
- package/common/bundle-metadata.d.ts +15 -0
- package/common/bundle-metadata.js +226 -0
- package/common/bundle-pkg-of.d.ts +1 -0
- package/common/bundle-pkg-of.js +90 -0
- package/common/bundle-reasons.d.ts +2 -0
- package/common/bundle-reasons.js +21 -0
- package/common/bundle-source-links.d.ts +6 -0
- package/common/bundle-source-links.js +216 -0
- package/common/bundle-sourcemap.d.ts +10 -0
- package/common/bundle-sourcemap.js +100 -0
- package/common/bundle-sources.d.ts +7 -0
- package/common/bundle-sources.js +333 -0
- package/common/bundle-stats.js +41 -0
- package/common/bundle-tabs.js +1 -0
- package/common/code-language.js +65 -0
- package/common/default-scan-models.ts +31 -0
- package/common/dependency-reviews.d.ts +18 -0
- package/common/dependency-reviews.js +64 -0
- package/common/finding-duplicates.js +58 -0
- package/common/finding-filters.js +274 -0
- package/common/finding-id.js +47 -0
- package/common/github-issue-labels.d.ts +1 -0
- package/common/github-issue-labels.js +11 -0
- package/common/github-pr.ts +56 -0
- package/common/gzip.js +59 -0
- package/common/line-diff.d.ts +8 -0
- package/common/line-diff.js +57 -0
- package/common/linked-workspace-groups.js +63 -0
- package/common/managed/agent-tasks.ts +292 -0
- package/common/managed/comments.ts +34 -0
- package/common/managed/finding-detail.d.ts +2 -0
- package/common/managed/finding-detail.js +32 -0
- package/common/managed/github-agents.ts +114 -0
- package/common/managed/github-logins.ts +49 -0
- package/common/managed/npm-packages.d.ts +5 -0
- package/common/managed/npm-packages.js +34 -0
- package/common/managed/permissions.ts +31 -0
- package/common/managed/report-content.ts +72 -0
- package/common/managed/report-filter.ts +109 -0
- package/common/managed/repository-alias.ts +65 -0
- package/common/managed/roles.ts +35 -0
- package/common/managed/routes.d.ts +2 -0
- package/common/managed/routes.js +347 -0
- package/common/managed/scan-models.ts +6 -0
- package/common/managed/team-app.d.ts +5 -0
- package/common/managed/team-app.js +31 -0
- package/common/managed/triage.ts +83 -0
- package/common/managed/upload-seal.ts +126 -0
- package/common/pretty-print.d.ts +5 -0
- package/common/pretty-print.js +24 -0
- package/common/revalidation-conflicts.js +48 -0
- package/common/revalidation-input-groups.js +56 -0
- package/common/save-error-reason.ts +20 -7
- package/common/scan-server.ts +13 -0
- package/common/security-links.d.ts +5 -0
- package/common/security-links.js +33 -0
- package/common/server-info.ts +46 -0
- package/common/utf8.d.ts +3 -0
- package/common/utf8.js +45 -0
- package/common/workspace-app.js +39 -0
- package/common/workspace-groups.js +176 -0
- package/out/activity.js +4 -0
- package/out/brotli-fallback.js +3 -3
- package/out/chunk-2WDZPXVL.js +3 -0
- package/out/chunk-2XUEZHIP.js +28 -0
- package/out/chunk-2ZDZWLFY.js +1 -0
- package/out/chunk-4HI2MSAV.js +17 -0
- package/out/chunk-4XMP7YVH.js +36 -0
- package/out/chunk-6LNQTKAV.js +52 -0
- package/out/chunk-6S2U2UYG.js +232 -0
- package/out/chunk-75YLHBCM.js +3 -0
- package/out/chunk-7BKKYVWR.js +7 -0
- package/out/chunk-7C7DWL7I.js +1 -0
- package/out/chunk-7VAWINQF.js +6 -0
- package/out/chunk-C4Z6ZBXV.js +10 -0
- package/out/chunk-FC65TPUB.js +1 -0
- package/out/chunk-FKBXNUW4.js +1 -0
- package/out/chunk-GDPCLSOS.js +2 -0
- package/out/chunk-GO7FT4OD.js +1 -0
- package/out/chunk-HTBYHSPG.js +1 -0
- package/out/chunk-IYPY47MC.js +1 -0
- package/out/chunk-IZ7POWI3.js +18 -0
- package/out/chunk-JCIIBLKR.js +1 -0
- package/out/chunk-NRPC74AM.js +1 -0
- package/out/chunk-OOJM4CTU.js +1 -0
- package/out/chunk-PBZPYIG7.js +1 -0
- package/out/chunk-PNHBZGDD.js +1 -0
- package/out/chunk-PWFUOLZK.js +17 -0
- package/out/chunk-PXK7EU4V.js +1 -0
- package/out/chunk-QB4CBOX2.js +1 -0
- package/out/chunk-R3NZFXVR.js +1 -0
- package/out/chunk-R5O6FAW4.js +2 -0
- package/out/chunk-SXV2T5C4.js +80 -0
- package/out/chunk-TQJDYHVE.js +2 -0
- package/out/chunk-TUII34K2.js +1 -0
- package/out/chunk-UK7RGJT6.js +3 -0
- package/out/chunk-VFWMQ3LG.js +2 -0
- package/out/chunk-VH4XMZIW.js +2 -0
- package/out/chunk-WCPJDAYA.js +10 -0
- package/out/chunk-WFYLMF6Z.js +10 -0
- package/out/chunk-WYLQWYLA.js +1 -0
- package/out/client-managed-import.js +4 -0
- package/out/client-managed.js +33 -0
- package/out/client-sync.js +11 -17
- package/out/compare.js +35 -0
- package/out/dialogs.js +55 -0
- package/out/github-issue-authorized.html +3 -0
- package/out/github-issue-authorized.js +1 -0
- package/out/graph.js +17 -4
- package/out/index.html +90 -8
- package/out/prism.js +14 -10
- package/out/scan.js +2 -0
- package/out/stasis.svg +20 -0
- package/out/terminal.js +366 -39
- package/out/view.css +1 -1
- package/out/view.js +19 -144
- package/package.json +362 -59
- package/risk/actions/findings.js +161 -0
- package/risk/actions/resolve.js +219 -0
- package/risk/actions/scripts/commands.js +182 -0
- package/risk/actions/scripts/grade.js +113 -0
- package/risk/actions/scripts/packages.js +123 -0
- package/risk/actions/scripts/shell.js +169 -0
- package/risk/actions/uses.js +76 -0
- package/risk/actions/workflow.js +110 -0
- package/risk/files/characters.js +196 -0
- package/risk/files/hosts.js +205 -0
- package/risk/files/readability.js +177 -0
- package/risk/files/runs-code.js +615 -0
- package/risk/files/script.js +118 -0
- package/risk/files/stamps.js +47 -0
- package/risk/index.d.ts +254 -0
- package/risk/index.js +11 -0
- package/risk/npm/assess.js +285 -0
- package/risk/npm/combined.js +68 -0
- package/risk/npm/compare.js +257 -0
- package/risk/npm/dependencies.js +172 -0
- package/risk/npm/diff.js +79 -0
- package/risk/npm/downloads.js +84 -0
- package/risk/npm/files.js +445 -0
- package/risk/npm/held.js +63 -0
- package/risk/npm/install-scripts.js +34 -0
- package/risk/npm/malware.js +36 -0
- package/risk/npm/manifest.js +65 -0
- package/risk/npm/new-dependency.js +60 -0
- package/risk/npm/old-version.js +108 -0
- package/risk/npm/registry.js +388 -0
- package/risk/npm/release.js +85 -0
- package/risk/npm/score.js +59 -0
- package/risk/npm/specs.js +124 -0
- package/risk/npm/tarball.js +96 -0
- package/server-common/database-config.ts +16 -0
- package/server-common/initialize.ts +18 -0
- package/server-common/npm-advisories.ts +101 -0
- package/{server → server-common}/origin.ts +5 -5
- package/server-common/reap.ts +48 -0
- package/server-common/scan-config.ts +19 -0
- package/server-common/standalone.ts +29 -0
- package/server-common/storage-crypto.ts +182 -0
- package/server-common/storage-log.ts +38 -0
- package/server-common/vercel-blob.ts +103 -0
- package/server-e2e/app.ts +487 -0
- package/{server → server-e2e}/bus-receiver.ts +2 -1
- package/{server → server-e2e}/cli.js +9 -4
- package/{server → server-e2e}/config.ts +50 -42
- package/{server → server-e2e}/db-neon.ts +2 -2
- package/{server → server-e2e}/db-revision-sql.ts +1 -1
- package/{server → server-e2e}/db-stmt.ts +2 -2
- package/{server → server-e2e}/db.ts +7 -7
- package/{server → server-e2e}/http.ts +44 -13
- package/{server → server-e2e}/hub.ts +40 -9
- package/server-e2e/index.ts +17 -0
- package/server-e2e/lifecycle.ts +95 -0
- package/{server → server-e2e}/neon-driver.ts +2 -2
- package/{server → server-e2e}/npm-proxy.ts +12 -145
- package/{server → server-e2e}/objstore/blob-vercel.ts +10 -114
- package/{server → server-e2e}/objstore/blob.ts +1 -1
- package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
- package/{server → server-e2e}/objstore/handlers.ts +9 -8
- package/{server → server-e2e}/objstore/init.ts +19 -27
- package/{server → server-e2e}/objstore/reaper.ts +1 -1
- package/server-e2e/objstore/rest-deny.ts +28 -0
- package/server-e2e/objstore/rest-mint.ts +224 -0
- package/{server → server-e2e}/objstore/rest.ts +3 -3
- package/{server → server-e2e}/objstore/store-neon.ts +6 -10
- package/{server → server-e2e}/objstore/store.ts +7 -7
- package/{server → server-e2e}/objstore/tokens.ts +2 -2
- package/{server → server-e2e}/pubsub.ts +12 -9
- package/{server → server-e2e}/sign.ts +4 -4
- package/{server → server-e2e}/sse-server.ts +60 -33
- package/{server → server-e2e}/static.ts +31 -21
- package/{server → server-e2e}/sync-handlers.ts +64 -30
- package/{server → server-e2e}/ws-server.ts +20 -9
- package/server-managed/activity.ts +258 -0
- package/server-managed/agent-merges.ts +109 -0
- package/server-managed/app-agents.ts +21 -0
- package/server-managed/avatar-store.ts +13 -0
- package/server-managed/blob-store.ts +17 -0
- package/server-managed/brotli.ts +10 -0
- package/server-managed/bundle-advisories.ts +120 -0
- package/server-managed/bundle-advisory-inventory.ts +77 -0
- package/server-managed/bundle-build-diagnostics.d.ts +10 -0
- package/server-managed/bundle-build-diagnostics.js +23 -0
- package/server-managed/bundle-build-leases.ts +41 -0
- package/server-managed/bundle-build-worker.js +115 -0
- package/server-managed/bundle-build.ts +200 -0
- package/server-managed/bundle-cache.ts +211 -0
- package/server-managed/bundle-catalog.ts +29 -0
- package/server-managed/bundle-commits.ts +140 -0
- package/server-managed/bundle-store.ts +24 -0
- package/server-managed/bundle-summary-cache.ts +130 -0
- package/server-managed/bundle.ts +119 -0
- package/server-managed/cache-storage.ts +20 -0
- package/server-managed/cli.js +21 -0
- package/server-managed/combined.ts +47 -0
- package/server-managed/comments.ts +151 -0
- package/server-managed/commit-coauthors.ts +106 -0
- package/server-managed/commit-lockfiles.ts +73 -0
- package/server-managed/comparison.ts +41 -0
- package/server-managed/config.ts +164 -0
- package/server-managed/content-access.ts +15 -0
- package/server-managed/crypto.ts +25 -0
- package/server-managed/db-methods.ts +1923 -0
- package/server-managed/db-neon.ts +458 -0
- package/server-managed/db-schema.ts +238 -0
- package/server-managed/db-table-names.ts +22 -0
- package/server-managed/db.ts +142 -0
- package/server-managed/dependabot-access.ts +60 -0
- package/server-managed/dependabot-alerts.ts +185 -0
- package/server-managed/dependency-review-comments.ts +100 -0
- package/server-managed/dependency-reviews.ts +346 -0
- package/server-managed/github-accounts.ts +60 -0
- package/server-managed/github-actions.ts +71 -0
- package/server-managed/github-app.ts +415 -0
- package/server-managed/github-errors.ts +70 -0
- package/server-managed/github-graphql.ts +39 -0
- package/server-managed/github-issue-links.ts +140 -0
- package/server-managed/github-issue-oauth.ts +47 -0
- package/server-managed/github-issues.ts +152 -0
- package/server-managed/github-link-oauth.ts +69 -0
- package/server-managed/github-links.ts +204 -0
- package/server-managed/github-metadata.ts +213 -0
- package/server-managed/github-oauth.ts +238 -0
- package/server-managed/github-pulls.ts +227 -0
- package/server-managed/github-requests.ts +39 -0
- package/server-managed/github-users.ts +30 -0
- package/server-managed/http-response.ts +76 -0
- package/server-managed/http.ts +5329 -0
- package/server-managed/import-triage.ts +51 -0
- package/server-managed/index.ts +192 -0
- package/server-managed/kept-briefly.ts +37 -0
- package/server-managed/kept.ts +28 -0
- package/server-managed/link-reports.ts +162 -0
- package/server-managed/locked-packages.ts +129 -0
- package/server-managed/lockfile-scope.ts +49 -0
- package/server-managed/lru.ts +30 -0
- package/server-managed/managed-issues.ts +78 -0
- package/server-managed/management-catalog.ts +45 -0
- package/server-managed/management.ts +99 -0
- package/server-managed/npm-delta-worker.js +23 -0
- package/server-managed/npm-delta.ts +194 -0
- package/server-managed/npm-insights.ts +449 -0
- package/server-managed/npm-kept.ts +215 -0
- package/server-managed/npm-loads.ts +264 -0
- package/server-managed/npm-manifest.ts +115 -0
- package/server-managed/npm-packages.ts +454 -0
- package/server-managed/npm-provenance.ts +123 -0
- package/server-managed/npm-risk.ts +285 -0
- package/server-managed/object-storage-disk.ts +148 -0
- package/server-managed/object-storage-vercel.ts +67 -0
- package/server-managed/object-storage.ts +75 -0
- package/server-managed/package-entry-points.ts +51 -0
- package/server-managed/pretty-print.ts +434 -0
- package/server-managed/public-workspace.ts +238 -0
- package/server-managed/pull-comment-reviews.ts +161 -0
- package/server-managed/pull-dependencies.ts +348 -0
- package/server-managed/pull-dependency-reviews.ts +328 -0
- package/server-managed/pull-reviews.ts +205 -0
- package/server-managed/queue.ts +39 -0
- package/server-managed/release-commits.ts +84 -0
- package/server-managed/rename-diff-worker.js +26 -0
- package/server-managed/rename-diff.ts +279 -0
- package/server-managed/repo-path.ts +21 -0
- package/server-managed/report-migration.ts +35 -0
- package/server-managed/report-query.ts +10 -0
- package/server-managed/report-response.ts +16 -0
- package/server-managed/report-sources.ts +178 -0
- package/server-managed/report-store.ts +85 -0
- package/server-managed/repository-actions.ts +218 -0
- package/server-managed/repository-agents.ts +355 -0
- package/server-managed/repository-aliases.ts +91 -0
- package/server-managed/repository-browser.ts +294 -0
- package/server-managed/repository-changes.ts +284 -0
- package/server-managed/repository-code.ts +215 -0
- package/server-managed/repository-dependency-reviews.ts +165 -0
- package/server-managed/repository-discovery.ts +82 -0
- package/server-managed/repository-history.ts +432 -0
- package/server-managed/repository-issues.ts +195 -0
- package/server-managed/repository-policy.ts +27 -0
- package/server-managed/repository-pulls.ts +156 -0
- package/server-managed/repository-pulse.ts +460 -0
- package/server-managed/repository-risk-assess.ts +105 -0
- package/server-managed/repository-risk-audit.ts +79 -0
- package/server-managed/repository-risk-read.ts +170 -0
- package/server-managed/repository-risk.ts +491 -0
- package/server-managed/repository-squashes.ts +168 -0
- package/server-managed/request-timing.ts +63 -0
- package/server-managed/reviewed-dependencies.ts +150 -0
- package/server-managed/revisions.ts +35 -0
- package/server-managed/scan-downloads.ts +87 -0
- package/server-managed/session.ts +132 -0
- package/server-managed/slugs.ts +38 -0
- package/server-managed/solidity-entry-points.ts +71 -0
- package/server-managed/sql-postgres.ts +34 -0
- package/server-managed/sql.ts +88 -0
- package/server-managed/static.ts +28 -0
- package/server-managed/storage-command.ts +16 -0
- package/server-managed/storage-db.ts +183 -0
- package/server-managed/storage-encryption.ts +290 -0
- package/server-managed/storage-maintenance.ts +259 -0
- package/server-managed/storage-payload.ts +121 -0
- package/server-managed/storage-stores.ts +91 -0
- package/server-managed/storage.ts +82 -0
- package/server-managed/team-app.ts +134 -0
- package/server-managed/team-catalog.ts +7 -0
- package/server-managed/team-feed.ts +180 -0
- package/server-managed/team-npm-scopes.ts +106 -0
- package/server-managed/team-reports.ts +307 -0
- package/server-managed/triage-response.ts +16 -0
- package/server-managed/type-only-worker.js +19 -0
- package/server-managed/type-only.ts +105 -0
- package/server-managed/uploads.ts +106 -0
- package/server-managed/upstream-cache.ts +243 -0
- package/server-managed/view-as.ts +58 -0
- package/server-managed/workspace-shares.ts +169 -0
- package/server.ts +70 -0
- package/server/index.ts +0 -480
- package/server/lifecycle.ts +0 -208
- /package/{LICENSE → LICENSE-MIT} +0 -0
- /package/{server → server-e2e}/auth.ts +0 -0
- /package/{server → server-e2e}/config.example.json +0 -0
- /package/{server → server-e2e}/objstore/blob-fs.ts +0 -0
- /package/{server → server-e2e}/objstore/fs.ts +0 -0
- /package/{server → server-e2e}/objstore/sign.ts +0 -0
- /package/{server → server-e2e}/peer.ts +0 -0
- /package/{server → server-e2e}/sse-session.ts +0 -0
- /package/{server → server-e2e}/util.ts +0 -0
- /package/{server → server-e2e}/validation.ts +0 -0
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
// Team content is filtered as one workspace, with separate report envelopes.
|
|
2
|
+
// Bounded snapshots retain filtered content and visibility for repeated reads.
|
|
3
|
+
import { mergeLinkGroups } from './link-reports.ts'
|
|
4
|
+
import { Buffer } from 'node:buffer'
|
|
5
|
+
import { createHash } from 'node:crypto'
|
|
6
|
+
import { backfillFindingIds, reportEntries, stampSecurityGroups } from '@preventive/report'
|
|
7
|
+
import { managedFindingSourcePaths, readManagedReport } from '../common/managed/report-content.ts'
|
|
8
|
+
import { filterReportData, projectFinding } from '../common/managed/report-filter.ts'
|
|
9
|
+
import { revalidationLinks, securityLinks } from '../common/security-links.js'
|
|
10
|
+
import { treeDependencyDirectory } from '../client/dependency-paths.js'
|
|
11
|
+
import { liteReportData } from '../common/managed/finding-detail.js'
|
|
12
|
+
import type { BlobStore } from './blob-store.ts'
|
|
13
|
+
import type { ManagedDb, TeamReportAccessSnapshot } from './db.ts'
|
|
14
|
+
import { triageWireEntry } from './triage-response.ts'
|
|
15
|
+
import { visibleManagedIssues } from './managed-issues.ts'
|
|
16
|
+
import { MAX_REPORT_QUERY_BYTES, MAX_REPORT_QUERY_COUNT } from './report-query.ts'
|
|
17
|
+
|
|
18
|
+
type Finding = Record<string, unknown>
|
|
19
|
+
type TeamReport = { id: string; filename: string; data: unknown; repo: { github: string | null; directory: string } }
|
|
20
|
+
type ReportVisibility = { ids: Set<string>; sourcePaths: Set<string> }
|
|
21
|
+
type Visibility = Map<string, ReportVisibility>
|
|
22
|
+
const VISIBILITY_CACHE_MAX_ITEMS = 250_000
|
|
23
|
+
const VISIBILITY_CACHE_MAX_SNAPSHOTS = 32
|
|
24
|
+
const caches = new WeakMap<ManagedDb, Map<string, Visibility>>()
|
|
25
|
+
const REPORT_CACHE_BYTES = 16 * 1024 * 1024
|
|
26
|
+
// A workspace's encoded reports, the same without their findings' detail for
|
|
27
|
+
// the board (`lite`), and its reports' file trees apart.
|
|
28
|
+
type EncodedWorkspace = { reports: Buffer; lite: Buffer; trees: Buffer }
|
|
29
|
+
export type TeamResponsePart = keyof EncodedWorkspace
|
|
30
|
+
// The part a team's `reports` or `trees` route asks for.
|
|
31
|
+
export const teamResponsePart = (route: string, url: URL): TeamResponsePart =>
|
|
32
|
+
route === 'trees' ? 'trees' : url.searchParams.get('lite') === '1' ? 'lite' : 'reports'
|
|
33
|
+
const reportCaches = new WeakMap<ManagedDb, Map<string, EncodedWorkspace>>()
|
|
34
|
+
const encodedBytes = ({ reports, lite, trees }: EncodedWorkspace) => reports.length + lite.length + trees.length
|
|
35
|
+
// A workspace's report reads at once. Its reports' sizes are bounded together
|
|
36
|
+
// before any is read, and each is held parsed till the build ends, so more at
|
|
37
|
+
// once costs only the raw bytes in flight: a cold build of a team's reports
|
|
38
|
+
// is so many storage round trips one after another.
|
|
39
|
+
const WORKSPACE_READS_AT_ONCE = 16
|
|
40
|
+
function retainReports(db: ManagedDb, key: string, encoded: EncodedWorkspace) {
|
|
41
|
+
if (encodedBytes(encoded) > REPORT_CACHE_BYTES) return
|
|
42
|
+
let cache = reportCaches.get(db)
|
|
43
|
+
if (!cache) { cache = new Map(); reportCaches.set(db, cache) }
|
|
44
|
+
cache.delete(key)
|
|
45
|
+
let total = encodedBytes(encoded) + [...cache.values()].reduce((sum, value) => sum + encodedBytes(value), 0)
|
|
46
|
+
for (const [oldest, value] of cache) {
|
|
47
|
+
if (total <= REPORT_CACHE_BYTES && cache.size < VISIBILITY_CACHE_MAX_SNAPSHOTS) break
|
|
48
|
+
cache.delete(oldest); total -= encodedBytes(value)
|
|
49
|
+
}
|
|
50
|
+
cache.set(key, encoded)
|
|
51
|
+
}
|
|
52
|
+
function visibilityWeight(visible: Visibility): number {
|
|
53
|
+
return [...visible.values()].reduce((sum, v) => sum + v.ids.size + v.sourcePaths.size, 0)
|
|
54
|
+
}
|
|
55
|
+
function retainVisibility(db: ManagedDb, key: string, visible: Visibility): void {
|
|
56
|
+
let cache = caches.get(db)
|
|
57
|
+
// Replacement contributes its weight once and does not evict a peer merely
|
|
58
|
+
// because the cache already holds the maximum number of snapshots.
|
|
59
|
+
cache?.delete(key)
|
|
60
|
+
const incoming = visibilityWeight(visible)
|
|
61
|
+
if (incoming > VISIBILITY_CACHE_MAX_ITEMS) return
|
|
62
|
+
if (!cache) { cache = new Map(); caches.set(db, cache) }
|
|
63
|
+
let total = incoming + [...cache.values()].reduce((sum, entry) => sum + visibilityWeight(entry), 0)
|
|
64
|
+
for (const [oldest, entry] of cache) {
|
|
65
|
+
if (cache.size < VISIBILITY_CACHE_MAX_SNAPSHOTS && total <= VISIBILITY_CACHE_MAX_ITEMS) break
|
|
66
|
+
cache.delete(oldest)
|
|
67
|
+
total -= visibilityWeight(entry)
|
|
68
|
+
}
|
|
69
|
+
cache.set(key, visible)
|
|
70
|
+
}
|
|
71
|
+
export class TeamReportsError extends Error {
|
|
72
|
+
status: number
|
|
73
|
+
constructor(status: number, error: string) { super(error); this.status = status }
|
|
74
|
+
}
|
|
75
|
+
export function teamSnapshotKey(snapshot: TeamReportAccessSnapshot): string {
|
|
76
|
+
return JSON.stringify([snapshot.user.id, snapshot.user.role, snapshot.teamId, snapshot.hiddenReportId ?? null, snapshot.reports, snapshot.repositories, snapshot.linkRevision])
|
|
77
|
+
}
|
|
78
|
+
// What a built workspace is kept by: everything its build reads, and nothing
|
|
79
|
+
// of who asked. A user's role and grants decide which reports (and which
|
|
80
|
+
// hidden preview) their snapshot holds and each one's permissions, all of it
|
|
81
|
+
// in the key, so members whose snapshots agree share one build.
|
|
82
|
+
function workspaceKey(snapshot: TeamReportAccessSnapshot): string {
|
|
83
|
+
return JSON.stringify([snapshot.teamId, snapshot.hiddenReportId ?? null, snapshot.reports, snapshot.repositories, snapshot.linkRevision])
|
|
84
|
+
}
|
|
85
|
+
export async function teamSnapshot(db: ManagedDb, sessionId: string, teamId: string, reportId: string | null = null): Promise<TeamReportAccessSnapshot> {
|
|
86
|
+
const snapshot = await db.getTeamReportAccessSnapshot(sessionId, Date.now(), teamId, reportId)
|
|
87
|
+
if (!snapshot) throw new TeamReportsError(401, 'unauthenticated')
|
|
88
|
+
if (!snapshot.teamId) throw new TeamReportsError(404, 'no-team')
|
|
89
|
+
if (reportId !== null && !snapshot.reports.some(report => report.id === reportId)) throw new TeamReportsError(404, 'no-report')
|
|
90
|
+
return snapshot
|
|
91
|
+
}
|
|
92
|
+
export async function recheckTeam(db: ManagedDb, sessionId: string, snapshot: TeamReportAccessSnapshot): Promise<void> {
|
|
93
|
+
if (teamSnapshotKey(await teamSnapshot(db, sessionId, snapshot.teamId!, snapshot.reportId)) !== teamSnapshotKey(snapshot)) throw new TeamReportsError(404, 'workspace-changed')
|
|
94
|
+
}
|
|
95
|
+
function groupsOf(data: unknown): Finding[][] {
|
|
96
|
+
return (reportEntries(data) ?? []).map(entry => (Array.isArray(entry) ? entry : [entry]).filter(
|
|
97
|
+
(finding): finding is Finding => finding !== null && typeof finding === 'object' && !Array.isArray(finding),
|
|
98
|
+
))
|
|
99
|
+
}
|
|
100
|
+
function linksOf(data: unknown): string[][] | null {
|
|
101
|
+
const record = data as { source?: string; links?: string[][] }
|
|
102
|
+
if (record?.source !== 'links' || !Array.isArray(record.links)) return null
|
|
103
|
+
if (!record.links.every(ids => Array.isArray(ids) && ids.every(id => typeof id === 'string'))) throw new TeamReportsError(422, 'unreadable-links')
|
|
104
|
+
return record.links
|
|
105
|
+
}
|
|
106
|
+
// A report's file tree goes apart, asked for by its graph or Files alone
|
|
107
|
+
// (loadTeamTreesResponse). The report tells in its place how many files it
|
|
108
|
+
// holds (`treeFiles`) and the dependency directory it names
|
|
109
|
+
// (`treeDependencies`), which classifies findings as the tree itself would
|
|
110
|
+
// (client/dependency-paths.js). Each file's is sent without the hash of its
|
|
111
|
+
// tree (`treeHash`), which no team view reads.
|
|
112
|
+
function withoutTree(report: TeamReport): string | null {
|
|
113
|
+
const tree = (report.data as { tree?: unknown } | null)?.tree
|
|
114
|
+
if (!tree || typeof tree !== 'object' || Array.isArray(tree)) return null
|
|
115
|
+
const { tree: _tree, ...data } = report.data as Finding
|
|
116
|
+
const named = treeDependencyDirectory(tree)
|
|
117
|
+
report.data = { ...data, treeFiles: Object.keys(tree).length, ...(named ? { treeDependencies: named } : {}) }
|
|
118
|
+
const files = Object.entries(tree).map(([file, entry]) => {
|
|
119
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return [file, entry]
|
|
120
|
+
const { treeHash: _treeHash, ...kept } = entry as Finding
|
|
121
|
+
return [file, kept]
|
|
122
|
+
})
|
|
123
|
+
return `${JSON.stringify(report.id)}:${JSON.stringify(Object.fromEntries(files))}`
|
|
124
|
+
}
|
|
125
|
+
async function buildTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<EncodedWorkspace & { visible: Visibility }> {
|
|
126
|
+
if (snapshot.reports.length > MAX_REPORT_QUERY_COUNT || snapshot.reports.reduce((n, r) => n + r.byteSize, 0) > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
|
|
127
|
+
const reports: TeamReport[] = []
|
|
128
|
+
let inputBytes = 0, next = 0
|
|
129
|
+
// Bound both remote reads and raw buffers, retaining stable report order for
|
|
130
|
+
// the cross-report classification that follows. Finish workers on failure.
|
|
131
|
+
const results = await Promise.allSettled(Array.from({ length: Math.min(WORKSPACE_READS_AT_ONCE, snapshot.reports.length) }, async () => {
|
|
132
|
+
while (next < snapshot.reports.length) {
|
|
133
|
+
const index = next++
|
|
134
|
+
const access = snapshot.reports[index]!
|
|
135
|
+
|
|
136
|
+
const bytes = await store.get(access.id)
|
|
137
|
+
if (!bytes) throw new TeamReportsError(503, 'unavailable')
|
|
138
|
+
inputBytes += bytes.length
|
|
139
|
+
if (inputBytes > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
|
|
140
|
+
const parsed = readManagedReport(bytes.toString('utf8'), access.filename)
|
|
141
|
+
if (!parsed.data) throw new TeamReportsError(422, 'unreadable-report')
|
|
142
|
+
// Backfill before filtering so links, annotations and the client agree on
|
|
143
|
+
// IDs even when hiding a component would change the original row shape.
|
|
144
|
+
await backfillFindingIds(groupsOf(parsed.data).flat())
|
|
145
|
+
reports[index] = { id: access.id, filename: access.filename, data: parsed.data, repo: access.repo }
|
|
146
|
+
}
|
|
147
|
+
}))
|
|
148
|
+
for (const result of results) if (result.status === 'rejected') throw result.reason
|
|
149
|
+
const globalLinks = snapshot.linkRevision ? await db.getEnabledLinkGroups(snapshot.linkRevision) : []
|
|
150
|
+
const groups = reports.flatMap(report => groupsOf(report.data).map(group => group.map(f => projectFinding(f, report.data as Finding)!)))
|
|
151
|
+
// Unknown IDs can connect known findings. App findings link to the inputs
|
|
152
|
+
// they revalidated in other reports, as the App view counts them.
|
|
153
|
+
stampSecurityGroups(groups, { linkedIds: securityLinks([
|
|
154
|
+
...globalLinks, ...reports.flatMap(report => linksOf(report.data) ?? []), ...revalidationLinks(groups.flat()),
|
|
155
|
+
]).linkedIds })
|
|
156
|
+
const securityIds = new Set(groups.flat().filter(f => f['isSecurity'] === true).map(f => String(f['id'])))
|
|
157
|
+
const allIds = new Set<string>(), visible: Visibility = new Map()
|
|
158
|
+
for (let i = 0; i < reports.length; i++) {
|
|
159
|
+
const access = snapshot.reports[i]!, report = reports[i]!
|
|
160
|
+
report.data = filterReportData(report.data, access.permissions, access.repo, securityIds)
|
|
161
|
+
const ids = new Set<string>()
|
|
162
|
+
for (const finding of groupsOf(report.data).flat()) {
|
|
163
|
+
// Preserve linked security classification for the UI lens, including a
|
|
164
|
+
// linked component that a dependency permission will hide from the UI.
|
|
165
|
+
finding['isSecurity'] = securityIds.has(String(finding['id']))
|
|
166
|
+
// The hash of its file's tree, which nothing a team view shows reads.
|
|
167
|
+
delete finding['treeHash']
|
|
168
|
+
if (typeof finding['id'] === 'string') { ids.add(finding['id']); allIds.add(finding['id']) }
|
|
169
|
+
}
|
|
170
|
+
visible.set(report.id, { ids, sourcePaths: managedFindingSourcePaths(groupsOf(report.data).flat()) })
|
|
171
|
+
}
|
|
172
|
+
for (const report of reports) {
|
|
173
|
+
const links = linksOf(report.data)
|
|
174
|
+
// Hidden previews are independently authorized; preserve a hidden links
|
|
175
|
+
// document even though its referenced findings are outside this response.
|
|
176
|
+
if (links && !snapshot.hiddenReportId) report.data = { source: 'links', findings: [], links: links.map(ids => [...new Set(ids.filter(id => allIds.has(id)))]).filter(ids => ids.length >= 2) }
|
|
177
|
+
}
|
|
178
|
+
const trees = Buffer.from(`{"trees":{${reports.map(withoutTree).filter(tree => tree !== null).join(',')}}}`)
|
|
179
|
+
const links = mergeLinkGroups(globalLinks, allIds)
|
|
180
|
+
const encoded = {
|
|
181
|
+
// Bounded together, as when the trees were sent with their reports.
|
|
182
|
+
reports: encodeReports(reports, links, trees.length),
|
|
183
|
+
// The board's, its findings without their detail (common/managed/finding-detail.js).
|
|
184
|
+
lite: encodeReports(reports.map(report => ({ ...report, data: liteReportData(report.data) })), links, 0),
|
|
185
|
+
trees,
|
|
186
|
+
}
|
|
187
|
+
retainVisibility(db, workspaceKey(snapshot), visible)
|
|
188
|
+
retainReports(db, workspaceKey(snapshot), encoded)
|
|
189
|
+
return { ...encoded, visible }
|
|
190
|
+
}
|
|
191
|
+
// `{ reports, links }` as a response sends them, `spent` bytes already
|
|
192
|
+
// counted against the bound.
|
|
193
|
+
function encodeReports(reports: readonly TeamReport[], links: readonly string[][], spent: number): Buffer {
|
|
194
|
+
let outputBytes = spent + 14
|
|
195
|
+
const parts = [Buffer.from('{"reports":[')]
|
|
196
|
+
for (const [index, report] of reports.entries()) {
|
|
197
|
+
const bytes = Buffer.from(JSON.stringify(report))
|
|
198
|
+
outputBytes += bytes.length + 1
|
|
199
|
+
if (outputBytes > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
|
|
200
|
+
if (index) parts.push(Buffer.from(','))
|
|
201
|
+
parts.push(bytes)
|
|
202
|
+
}
|
|
203
|
+
const suffix = Buffer.from(links.length > 0 ? `],"links":${JSON.stringify(links)}}` : ']}')
|
|
204
|
+
if (outputBytes + suffix.length > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
|
|
205
|
+
parts.push(suffix)
|
|
206
|
+
return Buffer.concat(parts)
|
|
207
|
+
}
|
|
208
|
+
const pendingWorkspaces = new WeakMap<ManagedDb, Map<string, ReturnType<typeof buildTeamWorkspace>>>()
|
|
209
|
+
async function loadTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot) {
|
|
210
|
+
let pending = pendingWorkspaces.get(db)
|
|
211
|
+
if (!pending) { pending = new Map(); pendingWorkspaces.set(db, pending) }
|
|
212
|
+
const key = workspaceKey(snapshot)
|
|
213
|
+
const existing = pending.get(key)
|
|
214
|
+
if (existing) return existing
|
|
215
|
+
const job = buildTeamWorkspace(db, store, snapshot)
|
|
216
|
+
pending.set(key, job)
|
|
217
|
+
try { return await job } finally { pending.delete(key) }
|
|
218
|
+
}
|
|
219
|
+
// Treat the encoded body as immutable and recheck access before sending it.
|
|
220
|
+
// Warm HTTP reads need neither a parsed copy nor another serialization pass.
|
|
221
|
+
// Its `lite` reports and its `trees`, by report ID (withoutTree), are of the
|
|
222
|
+
// same snapshot and take the same recheck.
|
|
223
|
+
export async function loadTeamReportsResponse(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot,
|
|
224
|
+
part: TeamResponsePart = 'reports'): Promise<Buffer> {
|
|
225
|
+
const cached = reportCaches.get(db)?.get(workspaceKey(snapshot))
|
|
226
|
+
return cached?.[part] ?? (await loadTeamWorkspace(db, store, snapshot))[part]
|
|
227
|
+
}
|
|
228
|
+
export async function loadTeamReports(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<TeamReport[]> {
|
|
229
|
+
// Object consumers get their own copy, including when sharing a cold load.
|
|
230
|
+
const body = await loadTeamReportsResponse(db, store, snapshot)
|
|
231
|
+
return (JSON.parse(body.toString('utf8')) as { reports: TeamReport[] }).reports
|
|
232
|
+
}
|
|
233
|
+
async function teamVisibility(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<Visibility> {
|
|
234
|
+
let visible = caches.get(db)?.get(workspaceKey(snapshot))
|
|
235
|
+
if (!visible) {
|
|
236
|
+
// A large snapshot may deliberately remain uncached, or be evicted by
|
|
237
|
+
// another request. Authorization uses this load's result in either case.
|
|
238
|
+
visible = (await loadTeamWorkspace(db, store, snapshot)).visible
|
|
239
|
+
}
|
|
240
|
+
return visible
|
|
241
|
+
}
|
|
242
|
+
export async function teamWorkspaceFindingIds(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<Set<string>> {
|
|
243
|
+
const ids = new Set<string>()
|
|
244
|
+
for (const report of (await teamVisibility(db, store, snapshot)).values()) for (const id of report.ids) ids.add(id)
|
|
245
|
+
return ids
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// What a focused team's annotations are at, for the snapshot reading them, as
|
|
249
|
+
// its feed tells it (team-feed.ts serveUserTeamFeed) and its annotations
|
|
250
|
+
// answer it (loadTeamAnnotations): the snapshot's key and the revision of
|
|
251
|
+
// every visible finding's annotations (getAnnotationRevision), its workspace's
|
|
252
|
+
// findings being `ids` (teamWorkspaceFindingIds).
|
|
253
|
+
export async function teamAnnotationRevision(db: ManagedDb, snapshot: TeamReportAccessSnapshot, ids: readonly string[]): Promise<string> {
|
|
254
|
+
return JSON.stringify([teamSnapshotKey(snapshot), await db.getAnnotationRevision(ids)])
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// A revision as the client is told it: a digest, telling only whether what it
|
|
258
|
+
// covers changed, none of the finding IDs, batches or issues it is read from.
|
|
259
|
+
export const annotationRevisionDigest = (revision: string): string => createHash('sha256').update(revision).digest('base64url')
|
|
260
|
+
export async function teamReportVisibility(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot, reportId: string): Promise<ReportVisibility> {
|
|
261
|
+
if (!snapshot.reports.some(r => r.id === reportId)) throw new TeamReportsError(404, 'no-report')
|
|
262
|
+
const visible = await teamVisibility(db, store, snapshot)
|
|
263
|
+
return visible.get(reportId)!
|
|
264
|
+
}
|
|
265
|
+
// A report's visibility for the session's current snapshot. A workspace built
|
|
266
|
+
// (or waited for) here outlives that snapshot's read, so the team's access is
|
|
267
|
+
// read again after it; one kept is answered with no read in between, which
|
|
268
|
+
// leaves nothing for another snapshot read to guard.
|
|
269
|
+
async function sessionReportVisibility(db: ManagedDb, store: BlobStore, sessionId: string, teamId: string, reportId: string): Promise<ReportVisibility> {
|
|
270
|
+
const snapshot = await teamSnapshot(db, sessionId, teamId, reportId)
|
|
271
|
+
const kept = caches.get(db)?.get(workspaceKey(snapshot))?.get(reportId)
|
|
272
|
+
if (kept) return kept
|
|
273
|
+
const visible = await teamReportVisibility(db, store, snapshot, reportId)
|
|
274
|
+
await recheckTeam(db, sessionId, snapshot)
|
|
275
|
+
return visible
|
|
276
|
+
}
|
|
277
|
+
export async function teamFindingIds(db: ManagedDb, store: BlobStore, sessionId: string, teamId: string, reportId: string): Promise<Set<string>> {
|
|
278
|
+
return (await sessionReportVisibility(db, store, sessionId, teamId, reportId)).ids
|
|
279
|
+
}
|
|
280
|
+
export async function teamSourcePaths(db: ManagedDb, store: BlobStore, sessionId: string, teamId: string, reportId: string): Promise<Set<string>> {
|
|
281
|
+
return (await sessionReportVisibility(db, store, sessionId, teamId, reportId)).sourcePaths
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// Serialize shared annotation bodies once. Reports carry only the annotated
|
|
285
|
+
// finding IDs visible in that report; callers revalidate access before sending.
|
|
286
|
+
// With them, the digest of the revision the whole workspace's are at, as the
|
|
287
|
+
// team's feed tells it (`revision`), read before them: a write between leaves
|
|
288
|
+
// it older than what is sent, so the feed's next differs and the client reads
|
|
289
|
+
// again, never the reverse.
|
|
290
|
+
export async function loadTeamAnnotations(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot, reportId: string | null = null) {
|
|
291
|
+
// Restrict annotation reads only after classifying the complete workspace:
|
|
292
|
+
// links and findings in other reports can affect this report's visibility.
|
|
293
|
+
if (reportId !== null && !snapshot.reports.some(r => r.id === reportId)) throw new TeamReportsError(404, 'no-report')
|
|
294
|
+
const workspace = await teamVisibility(db, store, snapshot)
|
|
295
|
+
const visible = reportId === null ? workspace : new Map([[reportId, workspace.get(reportId)!]])
|
|
296
|
+
const all = new Set<string>(), ids = new Set<string>()
|
|
297
|
+
for (const report of visible.values()) for (const id of report.ids) ids.add(id)
|
|
298
|
+
for (const report of workspace.values()) for (const id of report.ids) all.add(id)
|
|
299
|
+
const revision = await teamAnnotationRevision(db, snapshot, [...all])
|
|
300
|
+
const { triage, comments, issues: savedIssues } = await db.getAnnotations([...ids])
|
|
301
|
+
const issues = Object.fromEntries(visibleManagedIssues(savedIssues, snapshot)
|
|
302
|
+
.map(issue => [issue.findingId, { url: issue.issueUrl!, autoFix: issue.autoFixUrl }]))
|
|
303
|
+
const entries = Object.fromEntries(triage.map(row => [row.findingId, triageWireEntry(row)]))
|
|
304
|
+
const annotated = new Set([...triage.map(row => row.findingId), ...comments.map(comment => comment.findingId), ...Object.keys(issues)])
|
|
305
|
+
const reports = Object.fromEntries([...visible].map(([id, report]) => [id, [...report.ids].filter(finding => annotated.has(finding))]))
|
|
306
|
+
return { reports, entries, comments, issues, revision: annotationRevisionDigest(revision) }
|
|
307
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { TriageRow } from './db.ts'
|
|
2
|
+
import { type TriageEntryPatch, isTriageBucket } from '../common/managed/triage.ts'
|
|
3
|
+
|
|
4
|
+
// A stored triage row (current state or a trail event) → its wire entry: only
|
|
5
|
+
// set fields present, `false` kept for flagged (the explicit un-flag tombstone
|
|
6
|
+
// must round-trip), and null for the row of a cleared entry (every field
|
|
7
|
+
// null) — the reader adopts the clear.
|
|
8
|
+
export function triageWireEntry(row: Pick<TriageRow, 'color' | 'triage' | 'comment' | 'fix' | 'flagged'>, legacyHistory = false): TriageEntryPatch | null {
|
|
9
|
+
const e: TriageEntryPatch = {}
|
|
10
|
+
if (row.color != null) e.color = row.color
|
|
11
|
+
if (isTriageBucket(row.triage)) e.triage = row.triage
|
|
12
|
+
if (legacyHistory && row.comment != null) e.comment = row.comment
|
|
13
|
+
if (row.fix != null) e.fix = row.fix
|
|
14
|
+
if (row.flagged != null) e.flagged = row.flagged
|
|
15
|
+
return Object.keys(e).length > 0 ? e : null
|
|
16
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Which of a comparison's pairs of files changed in types alone (type-only.ts),
|
|
2
|
+
// as @preventive/compare's compareTrees stamps a change `typeOnly`, off the
|
|
3
|
+
// server's event loop: each pair a tree of a file a side, a side not there
|
|
4
|
+
// none, its paths as they are, which say what a file is. Two paths pair as one
|
|
5
|
+
// change wherever compareTrees takes them for a rename at all; where it
|
|
6
|
+
// doesn't, the pair is types only where each of its two changes is: the file
|
|
7
|
+
// gone erasing whole, and the one come erasing whole or leaving a single bare
|
|
8
|
+
// `export {}`, which nothing loaded before. `[true, false, ...]` by pair.
|
|
9
|
+
import { parentPort, workerData } from 'node:worker_threads'
|
|
10
|
+
import { compareTrees } from '@preventive/compare'
|
|
11
|
+
|
|
12
|
+
const tree = file => (file ? [{ name: file.path, data: file.text }] : [])
|
|
13
|
+
const answer = workerData.pairs.map(([from, to]) => {
|
|
14
|
+
const changes = compareTrees(tree(from), tree(to), { paths: 'keys', lines: false, renames: Number.MIN_VALUE })
|
|
15
|
+
return changes.length > 0 && changes.every(change => change.typeOnly === true)
|
|
16
|
+
})
|
|
17
|
+
// Node worker messages have no browser target origin.
|
|
18
|
+
// oxlint-disable-next-line unicorn/require-post-message-target-origin
|
|
19
|
+
parentPort.postMessage(answer)
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// Compare's changes to TypeScript declaration files in types alone (GET
|
|
2
|
+
// /api/bundles/:id/type-only?other=, GET /api/npm/type-only): every file
|
|
3
|
+
// that changed between a comparison's two sides, added and removed ones too,
|
|
4
|
+
// paired as its viewer pairs them (rename-diff.ts comparedFiles), whose
|
|
5
|
+
// change @preventive/compare's compareTrees stamps
|
|
6
|
+
// `typeOnly` -- each side there a file named as a declaration file and
|
|
7
|
+
// proven to hold types alone but for a runtime, the module loads and
|
|
8
|
+
// `export {}` its stripping leaves, that the change leaves as it was, an
|
|
9
|
+
// added one's empty or a single bare `export {}` (proveTypeOnly) -- and
|
|
10
|
+
// nothing else. Proven in a worker, off the event loop, a comparison at a
|
|
11
|
+
// time per process, for at most a minute; told a line each, `{ path, hash,
|
|
12
|
+
// otherPath, otherHash }`, a side not there null, which the viewer checks
|
|
13
|
+
// name the files it shows. Access is the caller's to check.
|
|
14
|
+
import { Worker } from 'node:worker_threads'
|
|
15
|
+
import { type ComparedSide, PrettyError, type PrettyRequest } from './pretty-print.ts'
|
|
16
|
+
import { comparedFiles } from './rename-diff.ts'
|
|
17
|
+
|
|
18
|
+
// TypeScript files, by extension as the pretty-print cache reads them: none
|
|
19
|
+
// other is asked about.
|
|
20
|
+
const TYPESCRIPT = new Set(['ts', 'mts', 'cts'])
|
|
21
|
+
// Comparisons proven at once per process, and those waiting their turn past
|
|
22
|
+
// them, any more refused; the longest one may take, its wait included.
|
|
23
|
+
const MAX_ACTIVE_PROOFS = 1
|
|
24
|
+
const MAX_WAITING_PROOFS = 16
|
|
25
|
+
const PROOF_TIMEOUT_MS = 60_000
|
|
26
|
+
|
|
27
|
+
type File = PrettyRequest & { text: string }
|
|
28
|
+
export interface TypeOnlyLine { path: string | null; hash: string | null; otherPath: string | null; otherHash: string | null }
|
|
29
|
+
|
|
30
|
+
let active = 0
|
|
31
|
+
const waiting: Array<() => void> = []
|
|
32
|
+
|
|
33
|
+
async function turn(deadline: number): Promise<void> {
|
|
34
|
+
if (active < MAX_ACTIVE_PROOFS) { active++; return }
|
|
35
|
+
if (waiting.length >= MAX_WAITING_PROOFS) throw new PrettyError(429, 'proof-busy')
|
|
36
|
+
await new Promise<void>((resolve, reject) => {
|
|
37
|
+
const next = () => { clearTimeout(timer); resolve() }
|
|
38
|
+
const timer = setTimeout(() => {
|
|
39
|
+
waiting.splice(waiting.indexOf(next), 1)
|
|
40
|
+
reject(new PrettyError(504, 'proof-timeout'))
|
|
41
|
+
}, deadline - Date.now())
|
|
42
|
+
waiting.push(next)
|
|
43
|
+
})
|
|
44
|
+
}
|
|
45
|
+
function done() {
|
|
46
|
+
const next = waiting.shift()
|
|
47
|
+
if (next) next()
|
|
48
|
+
else active--
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Which of `pairs` compareTrees stamps types only, by index, in a worker
|
|
52
|
+
// settled once it is gone; none where it fails, as no proof should.
|
|
53
|
+
function proven(pairs: Array<[File | null, File | null]>, deadline: number): Promise<boolean[]> {
|
|
54
|
+
let worker: Worker
|
|
55
|
+
try {
|
|
56
|
+
worker = new Worker(new URL('./type-only-worker.js', import.meta.url), {
|
|
57
|
+
workerData: { pairs: pairs.map(sides => sides.map(file => file && { path: file.path, text: file.text })) },
|
|
58
|
+
// Node's type stripping, which the proof holds oxc's tree to, warns it is experimental.
|
|
59
|
+
execArgv: ['--disable-warning=ExperimentalWarning'], resourceLimits: { maxOldGenerationSizeMb: 1024 },
|
|
60
|
+
})
|
|
61
|
+
} catch (err) {
|
|
62
|
+
console.warn('managed: a type-only proof could not start:', err)
|
|
63
|
+
throw new PrettyError(503, 'unavailable')
|
|
64
|
+
}
|
|
65
|
+
return new Promise((resolve, reject) => {
|
|
66
|
+
let answer: boolean[] | null = null, failure = new PrettyError(422, 'unproven')
|
|
67
|
+
const timer = setTimeout(() => { failure = new PrettyError(504, 'proof-timeout'); void worker.terminate() }, deadline - Date.now())
|
|
68
|
+
worker.once('message', (told: unknown) => {
|
|
69
|
+
if (Array.isArray(told) && told.length === pairs.length && told.every(one => typeof one === 'boolean')) answer = told
|
|
70
|
+
void worker.terminate()
|
|
71
|
+
})
|
|
72
|
+
worker.once('error', err => { console.warn('managed: a type-only proof failed:', err) })
|
|
73
|
+
worker.once('exit', () => {
|
|
74
|
+
clearTimeout(timer)
|
|
75
|
+
if (answer) resolve(answer)
|
|
76
|
+
else reject(failure)
|
|
77
|
+
})
|
|
78
|
+
})
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// The files that changed between a comparison's two sides in types alone,
|
|
82
|
+
// as the viewer pairs them, a line each once all are proven, in path order.
|
|
83
|
+
export function typeOnlyFiles([base, other]: readonly [ComparedSide, ComparedSide]): AsyncGenerator<TypeOnlyLine> {
|
|
84
|
+
const typescript = ({ files }: ComparedSide) => new Map(files.filter(file => TYPESCRIPT.has(file.extension)).map(file => [file.path, file]))
|
|
85
|
+
const [olds, news] = [typescript(base), typescript(other)]
|
|
86
|
+
const { changed, onlyBase, onlyOther } = comparedFiles([base, other])
|
|
87
|
+
// A side there not a TypeScript file this reads leaves its pair out; one not there is null.
|
|
88
|
+
const pairs = [...changed.map(({ path, basePath = path }) => [olds.get(basePath), news.get(path)]),
|
|
89
|
+
...onlyBase.map(({ path, basePath = path }) => [olds.get(basePath), null]), ...onlyOther.map(({ path }) => [null, news.get(path)])]
|
|
90
|
+
.filter((sides): sides is [File | null, File | null] => sides.every(file => file !== undefined) && sides.some(file => file !== null) && sides[0]?.hash !== sides[1]?.hash)
|
|
91
|
+
.toSorted(([a, b], [c, d]) => {
|
|
92
|
+
const [x, y] = [(b ?? a)!.path, (d ?? c)!.path]
|
|
93
|
+
return x < y ? -1 : x > y ? 1 : 0
|
|
94
|
+
})
|
|
95
|
+
return (async function* () {
|
|
96
|
+
if (pairs.length === 0) return
|
|
97
|
+
const deadline = Date.now() + PROOF_TIMEOUT_MS
|
|
98
|
+
await turn(deadline)
|
|
99
|
+
let stamped
|
|
100
|
+
try { stamped = await proven(pairs, deadline) } finally { done() }
|
|
101
|
+
for (const [i, [from, to]] of pairs.entries()) {
|
|
102
|
+
if (stamped[i]) yield { path: from?.path ?? null, hash: from?.hash ?? null, otherPath: to?.path ?? null, otherHash: to?.hash ?? null }
|
|
103
|
+
}
|
|
104
|
+
})()
|
|
105
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Staging is shared storage, never process memory. Each part is bound to the
|
|
2
|
+
// authenticated session, destination and random upload ID. No bearer URL is
|
|
3
|
+
// exposed; all reads/writes still pass the managed role/origin/CSRF gates.
|
|
4
|
+
import { Buffer } from 'node:buffer'
|
|
5
|
+
import { createHash, generateKeyPairSync } from 'node:crypto'
|
|
6
|
+
import type { IncomingMessage } from 'node:http'
|
|
7
|
+
import type { BlobStore } from './blob-store.ts'
|
|
8
|
+
import { maxSealedBytes, openUpload } from '../common/managed/upload-seal.ts'
|
|
9
|
+
|
|
10
|
+
export const UPLOAD_CHUNK_BYTES = 3 * 1024 * 1024
|
|
11
|
+
export const MAX_UPLOAD_BYTES = 1024 * 1024 * 1024
|
|
12
|
+
// Staged parts may carry a sealed body, slightly larger than its content.
|
|
13
|
+
const MAX_STAGED_BYTES = maxSealedBytes(MAX_UPLOAD_BYTES)
|
|
14
|
+
const MAX_UPLOAD_PARTS = Math.ceil(MAX_STAGED_BYTES / UPLOAD_CHUNK_BYTES)
|
|
15
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u
|
|
16
|
+
export type UploadKind = 'reports' | 'bundles'
|
|
17
|
+
|
|
18
|
+
function partId(session: string, kind: UploadKind, id: string, index: number): string {
|
|
19
|
+
const hash = createHash('sha256').update(JSON.stringify([session, kind, id, index])).digest('hex').slice(0, 32)
|
|
20
|
+
return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20)}`
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function validUploadPart(id: string, index: number, maxBytes: number): boolean {
|
|
24
|
+
return UUID.test(id) && Number.isSafeInteger(index) && index >= 0 && index < Math.ceil(Math.min(maxSealedBytes(maxBytes), MAX_STAGED_BYTES) / UPLOAD_CHUNK_BYTES)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function validUpload(id: string, count: number): boolean {
|
|
28
|
+
return UUID.test(id) && Number.isSafeInteger(count) && count > 0
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Limit cleanup to indices that any configured server could have accepted,
|
|
32
|
+
// even if the request advertises an enormous count or the limit was lowered.
|
|
33
|
+
export async function deleteUpload(store: BlobStore, session: string, kind: UploadKind, id: string, count: number): Promise<void> {
|
|
34
|
+
if (!validUpload(id, count)) throw new Error('bad-upload')
|
|
35
|
+
const end = Math.min(count, MAX_UPLOAD_PARTS), errors = []
|
|
36
|
+
for (let start = 0; start < end; start += 8) {
|
|
37
|
+
const results = await Promise.allSettled(Array.from({ length: Math.min(8, end - start) },
|
|
38
|
+
(_, index) => store.delete(partId(session, kind, id, start + index))))
|
|
39
|
+
for (const result of results) if (result.status === 'rejected') errors.push(result.reason)
|
|
40
|
+
}
|
|
41
|
+
if (errors.length > 0) throw new AggregateError(errors, 'Upload cleanup failed')
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export async function putUploadPart(store: BlobStore, session: string, kind: UploadKind, id: string, index: number, bytes: Buffer): Promise<void> {
|
|
45
|
+
await store.put(partId(session, kind, id, index), bytes)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Parts read at once while an upload is put together.
|
|
49
|
+
const PARTS_READ_AT_ONCE = 6
|
|
50
|
+
|
|
51
|
+
// The staged upload `req` names, its parts read a few at a time into the one
|
|
52
|
+
// buffer it fills, each as long as its place in it. Its parts are deleted
|
|
53
|
+
// once read, or once it fails: awaited, unless `behind` takes that cleanup to
|
|
54
|
+
// go on past the response (http.ts), which staging maintenance backs up.
|
|
55
|
+
export async function readUpload(store: BlobStore, req: IncomingMessage, session: string, kind: UploadKind, maxBytes: number,
|
|
56
|
+
behind?: (cleanup: Promise<void>) => void): Promise<Buffer> {
|
|
57
|
+
const id = req.headers['x-upload-id']
|
|
58
|
+
const count = Number(req.headers['x-upload-parts']), size = Number(req.headers['x-upload-size'])
|
|
59
|
+
if (typeof id !== 'string' || !validUpload(id, count)) throw new Error('bad-upload')
|
|
60
|
+
try {
|
|
61
|
+
if (!Number.isSafeInteger(size) || size <= 0 || count !== Math.ceil(size / UPLOAD_CHUNK_BYTES)) throw new Error('bad-upload')
|
|
62
|
+
if (size > maxBytes || size > MAX_STAGED_BYTES) throw new Error('too-large')
|
|
63
|
+
// Every byte of it is written below before it is returned: each part
|
|
64
|
+
// must fill its place exactly, or the upload fails.
|
|
65
|
+
const body = Buffer.allocUnsafe(size)
|
|
66
|
+
let failed = false, next = 0
|
|
67
|
+
const readParts = async () => {
|
|
68
|
+
while (!failed && next < count) {
|
|
69
|
+
const index = next++
|
|
70
|
+
const bytes = await store.get(partId(session, kind, id, index))
|
|
71
|
+
const expected = Math.min(UPLOAD_CHUNK_BYTES, size - index * UPLOAD_CHUNK_BYTES)
|
|
72
|
+
if (!bytes || bytes.length !== expected) { failed = true; throw new Error('bad-upload') }
|
|
73
|
+
bytes.copy(body, index * UPLOAD_CHUNK_BYTES)
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
// Each read settled before its parts are deleted.
|
|
77
|
+
const results = await Promise.allSettled(Array.from({ length: Math.min(PARTS_READ_AT_ONCE, count) }, readParts))
|
|
78
|
+
const rejected = results.find(result => result.status === 'rejected')
|
|
79
|
+
if (rejected) throw rejected.reason
|
|
80
|
+
return body
|
|
81
|
+
} finally {
|
|
82
|
+
// These bytes can no longer be finalized. Failed/abandoned chunk uploads
|
|
83
|
+
// are also covered by automatic staging maintenance.
|
|
84
|
+
const cleanup = deleteUpload(store, session, kind, id, count).catch(err => console.warn('managed: upload cleanup failed:', err))
|
|
85
|
+
if (behind) behind(cleanup)
|
|
86
|
+
else await cleanup
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// A session's upload key pair, stored as its private JWK.
|
|
91
|
+
export function newUploadKey(): string {
|
|
92
|
+
const { privateKey } = generateKeyPairSync('x25519')
|
|
93
|
+
const { kty, crv, x, d } = privateKey.export({ format: 'jwk' })
|
|
94
|
+
return JSON.stringify({ kty, crv, x, d })
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// The raw public key (base64url) browsers seal this session's uploads to.
|
|
98
|
+
export function uploadPublicKey(key: string): string {
|
|
99
|
+
return (JSON.parse(key) as { x: string }).x
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export async function openSessionUpload(key: string | null | undefined, sealed: Buffer, maxBytes: number): Promise<Buffer> {
|
|
103
|
+
if (!key) throw new Error('bad-upload')
|
|
104
|
+
const bytes = await openUpload(new Uint8Array(sealed.buffer as ArrayBuffer, sealed.byteOffset, sealed.byteLength), JSON.parse(key) as JsonWebKey, maxBytes)
|
|
105
|
+
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
106
|
+
}
|