@preventive/triage 1.0.0-alpha.5 → 1.0.0-alpha.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/README.md +16 -0
- package/api/reap.ts +17 -0
- package/cli.js +6 -0
- package/client/dependency-paths.d.ts +3 -0
- package/client/dependency-paths.js +36 -0
- package/client/finding-link.js +305 -0
- package/client/linked-findings.d.ts +1 -0
- package/client/linked-findings.js +115 -0
- package/common/bundle-commit.d.ts +1 -0
- package/common/bundle-commit.js +4 -0
- package/common/bundle-compare-diff.d.ts +3 -0
- package/common/bundle-compare-diff.js +357 -0
- package/common/bundle-compare-inputs.d.ts +2 -0
- package/common/bundle-compare-inputs.js +65 -0
- package/common/bundle-compare-versions.js +172 -0
- package/common/bundle-comparison.d.ts +8 -0
- package/common/bundle-comparison.js +62 -0
- package/common/bundle-metadata.d.ts +15 -0
- package/common/bundle-metadata.js +226 -0
- package/common/bundle-pkg-of.d.ts +1 -0
- package/common/bundle-pkg-of.js +90 -0
- package/common/bundle-reasons.d.ts +2 -0
- package/common/bundle-reasons.js +21 -0
- package/common/bundle-source-links.d.ts +6 -0
- package/common/bundle-source-links.js +216 -0
- package/common/bundle-sourcemap.d.ts +10 -0
- package/common/bundle-sourcemap.js +100 -0
- package/common/bundle-sources.d.ts +7 -0
- package/common/bundle-sources.js +333 -0
- package/common/bundle-stats.js +41 -0
- package/common/bundle-tabs.js +1 -0
- package/common/code-language.js +65 -0
- package/common/default-scan-models.ts +31 -0
- package/common/dependency-reviews.d.ts +18 -0
- package/common/dependency-reviews.js +64 -0
- package/common/finding-duplicates.js +58 -0
- package/common/finding-filters.js +274 -0
- package/common/finding-id.js +47 -0
- package/common/github-issue-labels.d.ts +1 -0
- package/common/github-issue-labels.js +11 -0
- package/common/github-pr.ts +56 -0
- package/common/gzip.js +59 -0
- package/common/line-diff.d.ts +8 -0
- package/common/line-diff.js +57 -0
- package/common/linked-workspace-groups.js +63 -0
- package/common/managed/agent-tasks.ts +292 -0
- package/common/managed/comments.ts +34 -0
- package/common/managed/finding-detail.d.ts +3 -0
- package/common/managed/finding-detail.js +66 -0
- package/common/managed/github-agents.ts +114 -0
- package/common/managed/github-logins.ts +49 -0
- package/common/managed/npm-packages.d.ts +7 -0
- package/common/managed/npm-packages.js +51 -0
- package/common/managed/permissions.ts +31 -0
- package/common/managed/report-content.ts +72 -0
- package/common/managed/report-filter.ts +109 -0
- package/common/managed/repository-alias.ts +65 -0
- package/common/managed/roles.ts +35 -0
- package/common/managed/routes.d.ts +2 -0
- package/common/managed/routes.js +359 -0
- package/common/managed/scan-models.ts +6 -0
- package/common/managed/team-app.d.ts +6 -0
- package/common/managed/team-app.js +47 -0
- package/common/managed/triage.ts +83 -0
- package/common/managed/upload-seal.ts +126 -0
- package/common/pretty-print.d.ts +5 -0
- package/common/pretty-print.js +24 -0
- package/common/revalidation-conflicts.js +48 -0
- package/common/revalidation-input-groups.js +56 -0
- package/common/save-error-reason.ts +20 -7
- package/common/scan-server.ts +13 -0
- package/common/security-links.d.ts +5 -0
- package/common/security-links.js +33 -0
- package/common/server-info.ts +46 -0
- package/common/utf8.d.ts +3 -0
- package/common/utf8.js +45 -0
- package/common/workspace-app.js +39 -0
- package/common/workspace-groups.js +176 -0
- package/out/activity.js +4 -0
- package/out/brotli-fallback.js +3 -3
- package/out/chunk-2K6UN3PC.js +3 -0
- package/out/chunk-2XUEZHIP.js +28 -0
- package/out/chunk-2ZDZWLFY.js +1 -0
- package/out/chunk-4HI2MSAV.js +17 -0
- package/out/chunk-6LNQTKAV.js +52 -0
- package/out/chunk-7BKKYVWR.js +7 -0
- package/out/chunk-7C7DWL7I.js +1 -0
- package/out/chunk-7JBSSRD7.js +3 -0
- package/out/chunk-7VAWINQF.js +6 -0
- package/out/chunk-C4Z6ZBXV.js +10 -0
- package/out/chunk-FMXWXDR4.js +36 -0
- package/out/chunk-GDPCLSOS.js +2 -0
- package/out/chunk-GO7FT4OD.js +1 -0
- package/out/chunk-HT2SSA3V.js +1 -0
- package/out/chunk-IPAH4LKF.js +232 -0
- package/out/chunk-IYPY47MC.js +1 -0
- package/out/chunk-IZ7POWI3.js +18 -0
- package/out/chunk-JCIIBLKR.js +1 -0
- package/out/chunk-KOGNQDSJ.js +1 -0
- package/out/chunk-NRPC74AM.js +1 -0
- package/out/chunk-OOJM4CTU.js +1 -0
- package/out/chunk-PBZPYIG7.js +1 -0
- package/out/chunk-PNHBZGDD.js +1 -0
- package/out/chunk-PWFUOLZK.js +17 -0
- package/out/chunk-PXK7EU4V.js +1 -0
- package/out/chunk-QB4CBOX2.js +1 -0
- package/out/chunk-R3NZFXVR.js +1 -0
- package/out/chunk-R5O6FAW4.js +2 -0
- package/out/chunk-SXV2T5C4.js +80 -0
- package/out/chunk-TQJDYHVE.js +2 -0
- package/out/chunk-TUII34K2.js +1 -0
- package/out/chunk-VDRXHIHN.js +3 -0
- package/out/chunk-VFWMQ3LG.js +2 -0
- package/out/chunk-VH4XMZIW.js +2 -0
- package/out/chunk-WCPJDAYA.js +10 -0
- package/out/chunk-WFYLMF6Z.js +10 -0
- package/out/chunk-WRAJVD35.js +1 -0
- package/out/chunk-WYLQWYLA.js +1 -0
- package/out/client-managed-import.js +4 -0
- package/out/client-managed.js +33 -0
- package/out/client-sync.js +11 -17
- package/out/compare.js +35 -0
- package/out/dialogs.js +55 -0
- package/out/github-issue-authorized.html +3 -0
- package/out/github-issue-authorized.js +1 -0
- package/out/graph.js +17 -4
- package/out/index.html +90 -8
- package/out/prism.js +14 -10
- package/out/scan.js +2 -0
- package/out/stasis.svg +20 -0
- package/out/terminal.js +366 -39
- package/out/view.css +1 -1
- package/out/view.js +19 -144
- package/package.json +364 -59
- package/risk/actions/findings.js +161 -0
- package/risk/actions/resolve.js +219 -0
- package/risk/actions/scripts/commands.js +182 -0
- package/risk/actions/scripts/grade.js +113 -0
- package/risk/actions/scripts/packages.js +123 -0
- package/risk/actions/scripts/shell.js +169 -0
- package/risk/actions/uses.js +76 -0
- package/risk/actions/workflow.js +110 -0
- package/risk/files/characters.js +196 -0
- package/risk/files/hosts.js +205 -0
- package/risk/files/readability.js +177 -0
- package/risk/files/runs-code.js +615 -0
- package/risk/files/script.js +118 -0
- package/risk/files/stamps.js +47 -0
- package/risk/index.d.ts +254 -0
- package/risk/index.js +11 -0
- package/risk/npm/assess.js +285 -0
- package/risk/npm/combined.js +68 -0
- package/risk/npm/compare.js +257 -0
- package/risk/npm/dependencies.js +172 -0
- package/risk/npm/diff.js +79 -0
- package/risk/npm/downloads.js +84 -0
- package/risk/npm/files.js +445 -0
- package/risk/npm/held.js +63 -0
- package/risk/npm/install-scripts.js +34 -0
- package/risk/npm/malware.js +36 -0
- package/risk/npm/manifest.js +65 -0
- package/risk/npm/new-dependency.js +60 -0
- package/risk/npm/old-version.js +108 -0
- package/risk/npm/registry.js +388 -0
- package/risk/npm/release.js +85 -0
- package/risk/npm/score.js +59 -0
- package/risk/npm/specs.js +124 -0
- package/risk/npm/tarball.js +96 -0
- package/server-common/database-config.ts +16 -0
- package/server-common/initialize.ts +18 -0
- package/server-common/npm-advisories.ts +101 -0
- package/{server → server-common}/origin.ts +5 -5
- package/server-common/reap.ts +48 -0
- package/server-common/scan-config.ts +19 -0
- package/server-common/standalone.ts +29 -0
- package/server-common/storage-crypto.ts +182 -0
- package/server-common/storage-log.ts +38 -0
- package/server-common/vercel-blob.ts +103 -0
- package/server-e2e/app.ts +487 -0
- package/{server → server-e2e}/bus-receiver.ts +2 -1
- package/{server → server-e2e}/cli.js +9 -4
- package/{server → server-e2e}/config.ts +50 -42
- package/{server → server-e2e}/db-neon.ts +2 -2
- package/{server → server-e2e}/db-revision-sql.ts +1 -1
- package/{server → server-e2e}/db-stmt.ts +2 -2
- package/{server → server-e2e}/db.ts +7 -7
- package/{server → server-e2e}/http.ts +44 -13
- package/{server → server-e2e}/hub.ts +40 -9
- package/server-e2e/index.ts +17 -0
- package/server-e2e/lifecycle.ts +95 -0
- package/{server → server-e2e}/neon-driver.ts +2 -2
- package/{server → server-e2e}/npm-proxy.ts +12 -145
- package/{server → server-e2e}/objstore/blob-vercel.ts +10 -114
- package/{server → server-e2e}/objstore/blob.ts +1 -1
- package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
- package/{server → server-e2e}/objstore/handlers.ts +9 -8
- package/{server → server-e2e}/objstore/init.ts +19 -27
- package/{server → server-e2e}/objstore/reaper.ts +1 -1
- package/server-e2e/objstore/rest-deny.ts +28 -0
- package/server-e2e/objstore/rest-mint.ts +224 -0
- package/{server → server-e2e}/objstore/rest.ts +3 -3
- package/{server → server-e2e}/objstore/store-neon.ts +6 -10
- package/{server → server-e2e}/objstore/store.ts +7 -7
- package/{server → server-e2e}/objstore/tokens.ts +2 -2
- package/{server → server-e2e}/pubsub.ts +12 -9
- package/{server → server-e2e}/sign.ts +4 -4
- package/{server → server-e2e}/sse-server.ts +60 -33
- package/{server → server-e2e}/static.ts +31 -21
- package/{server → server-e2e}/sync-handlers.ts +64 -30
- package/{server → server-e2e}/ws-server.ts +20 -9
- package/server-managed/activity.ts +258 -0
- package/server-managed/agent-merges.ts +109 -0
- package/server-managed/app-agents.ts +23 -0
- package/server-managed/avatar-store.ts +13 -0
- package/server-managed/blob-store.ts +17 -0
- package/server-managed/brotli.ts +10 -0
- package/server-managed/bundle-advisories.ts +120 -0
- package/server-managed/bundle-advisory-inventory.ts +77 -0
- package/server-managed/bundle-build-diagnostics.d.ts +10 -0
- package/server-managed/bundle-build-diagnostics.js +23 -0
- package/server-managed/bundle-build-leases.ts +41 -0
- package/server-managed/bundle-build-worker.js +115 -0
- package/server-managed/bundle-build.ts +200 -0
- package/server-managed/bundle-cache.ts +211 -0
- package/server-managed/bundle-catalog.ts +29 -0
- package/server-managed/bundle-commits.ts +140 -0
- package/server-managed/bundle-store.ts +24 -0
- package/server-managed/bundle-summary-cache.ts +130 -0
- package/server-managed/bundle.ts +119 -0
- package/server-managed/cache-storage.ts +20 -0
- package/server-managed/cli.js +21 -0
- package/server-managed/combined.ts +47 -0
- package/server-managed/comments.ts +151 -0
- package/server-managed/commit-coauthors.ts +106 -0
- package/server-managed/commit-lockfiles.ts +76 -0
- package/server-managed/comparison.ts +52 -0
- package/server-managed/config.ts +164 -0
- package/server-managed/content-access.ts +15 -0
- package/server-managed/crypto.ts +25 -0
- package/server-managed/db-methods.ts +1923 -0
- package/server-managed/db-neon.ts +468 -0
- package/server-managed/db-schema.ts +238 -0
- package/server-managed/db-table-names.ts +22 -0
- package/server-managed/db.ts +143 -0
- package/server-managed/dependabot-access.ts +60 -0
- package/server-managed/dependabot-alerts.ts +185 -0
- package/server-managed/dependency-review-comments.ts +98 -0
- package/server-managed/dependency-reviews.ts +364 -0
- package/server-managed/github-accounts.ts +60 -0
- package/server-managed/github-actions.ts +71 -0
- package/server-managed/github-app.ts +376 -0
- package/server-managed/github-errors.ts +70 -0
- package/server-managed/github-graphql.ts +39 -0
- package/server-managed/github-issue-links.ts +140 -0
- package/server-managed/github-issue-oauth.ts +47 -0
- package/server-managed/github-issues.ts +152 -0
- package/server-managed/github-link-oauth.ts +69 -0
- package/server-managed/github-links.ts +204 -0
- package/server-managed/github-metadata.ts +213 -0
- package/server-managed/github-oauth.ts +238 -0
- package/server-managed/github-pulls.ts +227 -0
- package/server-managed/github-requests.ts +136 -0
- package/server-managed/github-users.ts +30 -0
- package/server-managed/http-response.ts +76 -0
- package/server-managed/http.ts +5503 -0
- package/server-managed/import-triage.ts +51 -0
- package/server-managed/index.ts +192 -0
- package/server-managed/kept-briefly.ts +37 -0
- package/server-managed/kept.ts +28 -0
- package/server-managed/link-reports.ts +162 -0
- package/server-managed/listed-activity.ts +39 -0
- package/server-managed/locked-packages.ts +152 -0
- package/server-managed/lockfile-scope.ts +49 -0
- package/server-managed/lru.ts +30 -0
- package/server-managed/managed-issues.ts +78 -0
- package/server-managed/management-catalog.ts +45 -0
- package/server-managed/management.ts +99 -0
- package/server-managed/npm-delta-worker.js +23 -0
- package/server-managed/npm-delta.ts +194 -0
- package/server-managed/npm-insights.ts +449 -0
- package/server-managed/npm-kept.ts +225 -0
- package/server-managed/npm-loads.ts +264 -0
- package/server-managed/npm-manifest.ts +115 -0
- package/server-managed/npm-packages.ts +454 -0
- package/server-managed/npm-patch.ts +325 -0
- package/server-managed/npm-provenance.ts +123 -0
- package/server-managed/npm-risk.ts +285 -0
- package/server-managed/object-storage-disk.ts +148 -0
- package/server-managed/object-storage-vercel.ts +67 -0
- package/server-managed/object-storage.ts +75 -0
- package/server-managed/package-entry-points.ts +51 -0
- package/server-managed/pretty-print.ts +434 -0
- package/server-managed/public-workspace.ts +238 -0
- package/server-managed/pull-comment-reviews.ts +161 -0
- package/server-managed/pull-dependencies.ts +351 -0
- package/server-managed/pull-dependency-reviews.ts +334 -0
- package/server-managed/pull-reviews.ts +205 -0
- package/server-managed/queue.ts +39 -0
- package/server-managed/release-commits.ts +84 -0
- package/server-managed/rename-diff-worker.js +26 -0
- package/server-managed/rename-diff.ts +279 -0
- package/server-managed/repo-path.ts +21 -0
- package/server-managed/report-migration.ts +35 -0
- package/server-managed/report-query.ts +10 -0
- package/server-managed/report-response.ts +16 -0
- package/server-managed/report-sources.ts +178 -0
- package/server-managed/report-store.ts +85 -0
- package/server-managed/repository-actions.ts +218 -0
- package/server-managed/repository-agents.ts +448 -0
- package/server-managed/repository-aliases.ts +91 -0
- package/server-managed/repository-browser.ts +294 -0
- package/server-managed/repository-changes.ts +284 -0
- package/server-managed/repository-code.ts +215 -0
- package/server-managed/repository-dependency-reviews.ts +166 -0
- package/server-managed/repository-discovery.ts +82 -0
- package/server-managed/repository-history.ts +432 -0
- package/server-managed/repository-issues.ts +195 -0
- package/server-managed/repository-policy.ts +27 -0
- package/server-managed/repository-pulls.ts +212 -0
- package/server-managed/repository-pulse.ts +460 -0
- package/server-managed/repository-risk-assess.ts +105 -0
- package/server-managed/repository-risk-audit.ts +79 -0
- package/server-managed/repository-risk-read.ts +170 -0
- package/server-managed/repository-risk.ts +491 -0
- package/server-managed/repository-squashes.ts +168 -0
- package/server-managed/request-timing.ts +63 -0
- package/server-managed/reviewed-dependencies.ts +149 -0
- package/server-managed/revisions.ts +35 -0
- package/server-managed/scan-downloads.ts +87 -0
- package/server-managed/session.ts +132 -0
- package/server-managed/slugs.ts +38 -0
- package/server-managed/solidity-entry-points.ts +71 -0
- package/server-managed/sql-postgres.ts +34 -0
- package/server-managed/sql.ts +88 -0
- package/server-managed/static.ts +28 -0
- package/server-managed/storage-command.ts +16 -0
- package/server-managed/storage-db.ts +183 -0
- package/server-managed/storage-encryption.ts +290 -0
- package/server-managed/storage-maintenance.ts +259 -0
- package/server-managed/storage-payload.ts +121 -0
- package/server-managed/storage-stores.ts +91 -0
- package/server-managed/storage.ts +82 -0
- package/server-managed/team-app.ts +134 -0
- package/server-managed/team-catalog.ts +7 -0
- package/server-managed/team-feed.ts +180 -0
- package/server-managed/team-npm-scopes.ts +106 -0
- package/server-managed/team-reports.ts +319 -0
- package/server-managed/triage-response.ts +16 -0
- package/server-managed/type-only-worker.js +19 -0
- package/server-managed/type-only.ts +105 -0
- package/server-managed/uploads.ts +106 -0
- package/server-managed/upstream-cache.ts +243 -0
- package/server-managed/view-as.ts +58 -0
- package/server-managed/workspace-shares.ts +169 -0
- package/server.ts +70 -0
- package/server/index.ts +0 -480
- package/server/lifecycle.ts +0 -208
- /package/{LICENSE → LICENSE-MIT} +0 -0
- /package/{server → server-e2e}/auth.ts +0 -0
- /package/{server → server-e2e}/config.example.json +0 -0
- /package/{server → server-e2e}/objstore/blob-fs.ts +0 -0
- /package/{server → server-e2e}/objstore/fs.ts +0 -0
- /package/{server → server-e2e}/objstore/sign.ts +0 -0
- /package/{server → server-e2e}/peer.ts +0 -0
- /package/{server → server-e2e}/sse-session.ts +0 -0
- /package/{server → server-e2e}/util.ts +0 -0
- /package/{server → server-e2e}/validation.ts +0 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { setImmediate } from 'node:timers/promises'
|
|
2
|
+
import { bundleCommitHash } from '../common/bundle-commit.js'
|
|
3
|
+
import type { BundleSummary } from './bundle-cache.ts'
|
|
4
|
+
import type { ManagedDb } from './db.ts'
|
|
5
|
+
import { type GithubCommit, githubCommitKey } from './github-metadata.ts'
|
|
6
|
+
|
|
7
|
+
const MAX_MESSAGE_LENGTH = 65_536
|
|
8
|
+
const MAX_NAME_LENGTH = 256
|
|
9
|
+
// GitHub logins, and the `[bot]` accounts apps commit as.
|
|
10
|
+
const GITHUB_LOGIN = /^[A-Za-z\d][A-Za-z\d-]{0,38}(?:\[bot\])?$/u
|
|
11
|
+
const BACKFILL_LIMIT = 4
|
|
12
|
+
const RETRY_MS = 5 * 60_000
|
|
13
|
+
|
|
14
|
+
export type CommitDetails = Omit<GithubCommit, 'key' | 'fetchedAt'>
|
|
15
|
+
// What viewers see of a cached commit: its subject in place of the message.
|
|
16
|
+
export type CatalogCommitDetails = Omit<CommitDetails, 'message'> & { subject: string }
|
|
17
|
+
// What a catalog sends with a bundle whose summary records a commit, when
|
|
18
|
+
// the cache holds its details, tags pointing to it, or both. `github` is the
|
|
19
|
+
// repository they were cached for: the one the bundle is stored at.
|
|
20
|
+
export interface BundleCommitInfo { sha: string; github: string; tags: string[]; details: CatalogCommitDetails | null }
|
|
21
|
+
interface CatalogBundle { integrity: string; repoId: number | null; repoFullName: string | null }
|
|
22
|
+
type Summaries = ReadonlyMap<string, { summary: BundleSummary | null }>
|
|
23
|
+
export interface CatalogCommit { repoId: number; sha: string; key: string }
|
|
24
|
+
export interface CommitReader { commitDetails(sha: string): Promise<CommitDetails | null> }
|
|
25
|
+
|
|
26
|
+
function truncate(value: string, max: number): string {
|
|
27
|
+
if (value.length <= max) return value
|
|
28
|
+
// Never leave half of a surrogate pair at the end.
|
|
29
|
+
return value.slice(0, /[\uD800-\uDBFF]/u.test(value[max - 1]!) ? max - 1 : max)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function name(value: unknown): string | null {
|
|
33
|
+
return typeof value === 'string' && value.trim() ? truncate(value.trim(), MAX_NAME_LENGTH) : null
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function date(value: unknown): number | null {
|
|
37
|
+
const time = typeof value === 'string' ? Date.parse(value) : NaN
|
|
38
|
+
return Number.isSafeInteger(time) ? time : null
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// `body` is a GitHub list-commits page that starts at `sha`, so its first
|
|
42
|
+
// commit is that one, without the files GET /commits/:sha would add.
|
|
43
|
+
export function parseGithubCommit(sha: string, body: unknown): CommitDetails | null {
|
|
44
|
+
const item = Array.isArray(body) ? body[0] as {
|
|
45
|
+
sha?: unknown; author?: { login?: unknown } | null
|
|
46
|
+
commit?: { message?: unknown; author?: { name?: unknown; date?: unknown } | null; committer?: { date?: unknown } | null }
|
|
47
|
+
} | null : null
|
|
48
|
+
if (item?.sha !== sha || typeof item.commit?.message !== 'string') return null
|
|
49
|
+
const login = item.author?.login
|
|
50
|
+
return {
|
|
51
|
+
message: truncate(item.commit.message, MAX_MESSAGE_LENGTH),
|
|
52
|
+
authorName: name(item.commit.author?.name),
|
|
53
|
+
authorLogin: typeof login === 'string' && GITHUB_LOGIN.test(login) ? login : null,
|
|
54
|
+
authoredAt: date(item.commit.author?.date),
|
|
55
|
+
committedAt: date(item.commit.committer?.date),
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// A commit's subject, as `git log --format=%s` gives it: the message's first
|
|
60
|
+
// paragraph on one line, here without the Claude-Session lines Claude Code
|
|
61
|
+
// adds. Viewers see no more of the message, so catalogs send no more.
|
|
62
|
+
export function commitSubject(message: string): string {
|
|
63
|
+
const paragraph = message.replaceAll('\r\n', '\n').trim().split(/\n[ \t]*\n/u, 1)[0] ?? ''
|
|
64
|
+
return paragraph.split('\n').map(line => line.trim()).filter(line => line && !/^Claude-Session:/iu.test(line)).join(' ')
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// The commit a bundle's own summary records, in the repository it is stored at.
|
|
68
|
+
function catalogCommit(bundle: CatalogBundle, summaries: Summaries): CatalogCommit | null {
|
|
69
|
+
const sha = bundleCommitHash(summaries.get(bundle.integrity)?.summary?.commit)
|
|
70
|
+
return sha && bundle.repoId != null ? { repoId: bundle.repoId, sha, key: githubCommitKey(bundle.repoId, sha) } : null
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Reads only the cache: viewing a bundle never requests its commit's tags,
|
|
74
|
+
// and a commit missing from the cache is left for backfillBundleCommits.
|
|
75
|
+
// Call it before the catalog's final access check: `commitInfo` answers for
|
|
76
|
+
// the bundles of that last snapshot without another await, and has nothing
|
|
77
|
+
// for a commit or repository this read did not cover.
|
|
78
|
+
export async function bundleCommits(db: ManagedDb, bundles: readonly CatalogBundle[], summaries: Summaries) {
|
|
79
|
+
const wanted = new Map<string, CatalogCommit>()
|
|
80
|
+
for (const bundle of bundles) {
|
|
81
|
+
const commit = catalogCommit(bundle, summaries)
|
|
82
|
+
if (commit) wanted.set(commit.key, commit)
|
|
83
|
+
}
|
|
84
|
+
const keys = [...wanted.keys()]
|
|
85
|
+
const details = new Map<string, CatalogCommitDetails>()
|
|
86
|
+
const tags = new Map<string, string[]>()
|
|
87
|
+
if (keys.length > 0) {
|
|
88
|
+
for (const { key, fetchedAt: _fetchedAt, message, ...commit } of await db.listGithubCommits(keys)) {
|
|
89
|
+
details.set(key, { subject: commitSubject(message), ...commit })
|
|
90
|
+
}
|
|
91
|
+
for (const tag of await db.listGithubCommitTags(keys)) tags.set(tag.key, [...tags.get(tag.key) ?? [], tag.name])
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
commitInfo(bundle: CatalogBundle): BundleCommitInfo | null {
|
|
95
|
+
const commit = catalogCommit(bundle, summaries)
|
|
96
|
+
if (!commit || !bundle.repoFullName || (!details.has(commit.key) && !tags.has(commit.key))) return null
|
|
97
|
+
return { sha: commit.sha, github: bundle.repoFullName, tags: tags.get(commit.key) ?? [], details: details.get(commit.key) ?? null }
|
|
98
|
+
},
|
|
99
|
+
missing: [...wanted.values()].filter(commit => !details.has(commit.key)),
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Store the details of a commit, read with the reader's verified repository
|
|
104
|
+
// access, unless the cache already has them. Best effort: true once cached.
|
|
105
|
+
export async function cacheCommitDetails(db: ManagedDb, reader: CommitReader, repoId: number, sha: string): Promise<boolean> {
|
|
106
|
+
const key = githubCommitKey(repoId, sha)
|
|
107
|
+
try {
|
|
108
|
+
if ((await db.listGithubCommits([key])).length > 0) return true
|
|
109
|
+
const details = await reader.commitDetails(sha)
|
|
110
|
+
if (details) await db.setGithubCommits([{ key, ...details, fetchedAt: Date.now() }])
|
|
111
|
+
return details !== null
|
|
112
|
+
} catch { return false }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const active = new Set<string>()
|
|
116
|
+
const retries = new Map<string, number>()
|
|
117
|
+
|
|
118
|
+
// Called after the catalog response is sent: the next one has the details.
|
|
119
|
+
// Reads each commit with the viewer's own repository access, so a viewer who
|
|
120
|
+
// cannot read it on GitHub leaves it for another (retried after a delay for
|
|
121
|
+
// them). Commits never change, so each is stored once and kept.
|
|
122
|
+
export async function backfillBundleCommits(db: ManagedDb, userId: string, missing: readonly CatalogCommit[],
|
|
123
|
+
readerFor: (repoId: number) => Promise<CommitReader | null>): Promise<void> {
|
|
124
|
+
if (missing.length === 0) return
|
|
125
|
+
await setImmediate()
|
|
126
|
+
const now = Date.now()
|
|
127
|
+
for (const [key, retryAt] of retries) if (retryAt <= now) retries.delete(key)
|
|
128
|
+
const batch = missing.filter(commit => !active.has(commit.key) && !retries.has(`${userId}\n${commit.key}`)).slice(0, BACKFILL_LIMIT)
|
|
129
|
+
for (const commit of batch) active.add(commit.key)
|
|
130
|
+
try {
|
|
131
|
+
for (const repoId of new Set(batch.map(commit => commit.repoId))) {
|
|
132
|
+
const reader = await readerFor(repoId).catch(() => null)
|
|
133
|
+
for (const commit of batch.filter(entry => entry.repoId === repoId)) {
|
|
134
|
+
if (!reader || !await cacheCommitDetails(db, reader, repoId, commit.sha)) retries.set(`${userId}\n${commit.key}`, Date.now() + RETRY_MS)
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
} finally {
|
|
138
|
+
for (const commit of batch) active.delete(commit.key)
|
|
139
|
+
}
|
|
140
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Sourcemaps are stored once as Brotli. The DB retains the uploaded filename,
|
|
2
|
+
// byte size and integrity so compression does not change report links/dedup.
|
|
3
|
+
import type { Buffer } from 'node:buffer'
|
|
4
|
+
import type { BlobStore, OpenedBlob } from './blob-store.ts'
|
|
5
|
+
import { encodeBrotli } from './brotli.ts'
|
|
6
|
+
|
|
7
|
+
export interface BundleStore {
|
|
8
|
+
put(id: string, bytes: Buffer, kind: string | null): Promise<string | null>
|
|
9
|
+
get(id: string, kind: string | null): Promise<Buffer | null>
|
|
10
|
+
open(id: string, kind: string | null): Promise<OpenedBlob | null>
|
|
11
|
+
delete(id: string): Promise<void>
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function createBundleStore(archives: BlobStore, sourcemaps: BlobStore): BundleStore {
|
|
15
|
+
const storage = (kind: string | null) => kind === 'sourcemap' ? sourcemaps : archives
|
|
16
|
+
return {
|
|
17
|
+
async put(id, bytes, kind) {
|
|
18
|
+
return storage(kind).put(id, kind === 'sourcemap' ? await encodeBrotli(bytes) : bytes)
|
|
19
|
+
},
|
|
20
|
+
get: (id, kind) => storage(kind).get(id),
|
|
21
|
+
open: (id, kind) => storage(kind).open(id),
|
|
22
|
+
async delete(id) { await Promise.all([archives.delete(id), sourcemaps.delete(id)]) },
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
2
|
+
import { setImmediate } from 'node:timers/promises'
|
|
3
|
+
import { CacheMissError } from './cache-storage.ts'
|
|
4
|
+
import { decodeUtf8 } from '../common/utf8.js'
|
|
5
|
+
import { bundleCommitHash } from '../common/bundle-commit.js'
|
|
6
|
+
import type { BundleCacheRecord, BundleCacheStorage, BundleSummary } from './bundle-cache.ts'
|
|
7
|
+
|
|
8
|
+
// Version 3 adds the optional source commit; version 4 counts lines of code
|
|
9
|
+
// without blank lines; version 5 adds a Stasis bundle's format version and
|
|
10
|
+
// versioned package count. Older bundles use bounded backfill; until it
|
|
11
|
+
// reaches one, catalogs serve its version 4 summary, which lacks only those.
|
|
12
|
+
export const SUMMARY_FILENAME = 'v5-summary.json'
|
|
13
|
+
const PREVIOUS_SUMMARY_FILENAME = 'v4-summary.json'
|
|
14
|
+
const RETRY_MS = 5 * 60_000
|
|
15
|
+
const BACKFILL_LIMIT = 4
|
|
16
|
+
type CachedSummary = BundleSummary | { retryAt: number } | { previous: BundleSummary }
|
|
17
|
+
|
|
18
|
+
async function readCurrentSummary(storage: BundleCacheStorage, id: string): Promise<CachedSummary> {
|
|
19
|
+
try { return await readSummary(storage, id, SUMMARY_FILENAME) }
|
|
20
|
+
catch (error) {
|
|
21
|
+
if (!(error instanceof CacheMissError)) throw error
|
|
22
|
+
const previous = await readSummary(storage, id, PREVIOUS_SUMMARY_FILENAME)
|
|
23
|
+
return 'retryAt' in previous ? previous : { previous }
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async function readSummary(storage: BundleCacheStorage, id: string, file: string): Promise<BundleSummary | { retryAt: number }> {
|
|
28
|
+
const cached = await storage.open(id, file)
|
|
29
|
+
try {
|
|
30
|
+
const chunks: Buffer[] = []
|
|
31
|
+
let size = 0
|
|
32
|
+
for await (const chunk of cached.stream) {
|
|
33
|
+
size += chunk.length
|
|
34
|
+
if (size > 1024) throw new Error('Invalid bundle summary')
|
|
35
|
+
chunks.push(Buffer.from(chunk))
|
|
36
|
+
}
|
|
37
|
+
const summary = JSON.parse(decodeUtf8(Buffer.concat(chunks))) as BundleSummary | { retryAt: number }
|
|
38
|
+
if (summary && 'retryAt' in summary && Number.isSafeInteger(summary.retryAt) && summary.retryAt > 0) return summary
|
|
39
|
+
const counts = summary as BundleSummary
|
|
40
|
+
if (!counts || ![counts.files, counts.codeFiles, counts.lines].every(value => Number.isSafeInteger(value) && value >= 0)
|
|
41
|
+
|| counts.codeFiles > counts.files || (counts.commit !== undefined && !bundleCommitHash(counts.commit))
|
|
42
|
+
|| [counts.stasisVersion, counts.versionedPackages].some(value => value !== undefined && !(Number.isSafeInteger(value) && value >= 0))) throw new Error('Invalid bundle summary')
|
|
43
|
+
return counts
|
|
44
|
+
} finally { cached.stream.destroy() }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function createBundleSummaryCache(storage: BundleCacheStorage, build: (record: BundleCacheRecord) => Promise<BundleSummary>, exists: (id: string) => Promise<unknown>) {
|
|
48
|
+
const entries = new Map<string, { id: string; value: CachedSummary }>()
|
|
49
|
+
const reads = new Map<string, { id: string; job: Promise<CachedSummary | null> }>()
|
|
50
|
+
const active = new Map<string, Promise<void>>()
|
|
51
|
+
let pending: Promise<void> | null = null
|
|
52
|
+
function remember(record: BundleCacheRecord, value: CachedSummary) {
|
|
53
|
+
// A completed build supersedes any older storage read still in flight.
|
|
54
|
+
reads.delete(record.integrity)
|
|
55
|
+
// Summaries are tiny and immutable by integrity. Refuse overflow admission
|
|
56
|
+
// instead of evicting the next hit during a scan larger than the cache.
|
|
57
|
+
if (entries.size >= 4096 && !entries.has(record.integrity)) return
|
|
58
|
+
entries.set(record.integrity, { id: record.id, value })
|
|
59
|
+
}
|
|
60
|
+
function read(record: BundleCacheRecord): Promise<CachedSummary | null> {
|
|
61
|
+
const known = entries.get(record.integrity)
|
|
62
|
+
if (known) return Promise.resolve(known.value)
|
|
63
|
+
const existing = reads.get(record.integrity)
|
|
64
|
+
if (existing) return existing.job
|
|
65
|
+
// Concurrent catalogs share the decoded summary, never the response stream.
|
|
66
|
+
// Only in-flight work is shared: misses and failures remain retryable.
|
|
67
|
+
const job: Promise<CachedSummary | null> = readCurrentSummary(storage, record.id).then(value => {
|
|
68
|
+
if (reads.get(record.integrity)?.job === job) remember(record, value)
|
|
69
|
+
return entries.get(record.integrity)?.value ?? value
|
|
70
|
+
}).catch(error => {
|
|
71
|
+
if (error instanceof CacheMissError) return null
|
|
72
|
+
throw error
|
|
73
|
+
}).finally(() => { if (reads.get(record.integrity)?.job === job) reads.delete(record.integrity) })
|
|
74
|
+
reads.set(record.integrity, { id: record.id, job })
|
|
75
|
+
return job
|
|
76
|
+
}
|
|
77
|
+
async function publish(record: BundleCacheRecord, value: CachedSummary) {
|
|
78
|
+
if (!await exists(record.id)) return
|
|
79
|
+
await storage.put(record.id, SUMMARY_FILENAME, Buffer.from(JSON.stringify(value)))
|
|
80
|
+
if (!await exists(record.id)) { await storage.delete(record.id); return }
|
|
81
|
+
remember(record, value)
|
|
82
|
+
}
|
|
83
|
+
async function buildOne(record: BundleCacheRecord) {
|
|
84
|
+
try { await publish(record, await build(record)) }
|
|
85
|
+
catch { await publish(record, { retryAt: Date.now() + RETRY_MS }).catch(() => {}) }
|
|
86
|
+
}
|
|
87
|
+
async function summaryStatus(record: BundleCacheRecord) {
|
|
88
|
+
const value = ['stasis', 'sourcemap'].includes(record.kind ?? '') ? await read(record) : null
|
|
89
|
+
return { summary: !value || 'retryAt' in value ? null : 'previous' in value ? value.previous : value,
|
|
90
|
+
summaryRetryAt: value && 'retryAt' in value ? value.retryAt : null }
|
|
91
|
+
}
|
|
92
|
+
async function backfill(records: readonly BundleCacheRecord[]) {
|
|
93
|
+
// Start after the catalog has been sent. Keep this work awaited by the
|
|
94
|
+
// request owner (including serverless), with no detached tasks or backlog.
|
|
95
|
+
await setImmediate()
|
|
96
|
+
let attempts = 0
|
|
97
|
+
for (const record of records) {
|
|
98
|
+
if (!['stasis', 'sourcemap'].includes(record.kind ?? '')) continue
|
|
99
|
+
const cached = await read(record).catch(() => null)
|
|
100
|
+
if (cached && !('previous' in cached) && (!('retryAt' in cached) || cached.retryAt > Date.now())) continue
|
|
101
|
+
if (!await exists(record.id)) continue
|
|
102
|
+
if (attempts++ >= BACKFILL_LIMIT) break
|
|
103
|
+
const job = buildOne(record)
|
|
104
|
+
active.set(record.id, job)
|
|
105
|
+
try { await job } finally { active.delete(record.id) }
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
remember,
|
|
110
|
+
summaryStatus,
|
|
111
|
+
async summary(record: BundleCacheRecord): Promise<BundleSummary | null> {
|
|
112
|
+
return (await summaryStatus(record)).summary
|
|
113
|
+
},
|
|
114
|
+
backfill(records: readonly BundleCacheRecord[]) {
|
|
115
|
+
if (pending) return Promise.resolve() // Another catalog already owns the bounded batch.
|
|
116
|
+
const job = backfill(records).finally(() => { if (pending === job) pending = null })
|
|
117
|
+
pending = job
|
|
118
|
+
return job
|
|
119
|
+
},
|
|
120
|
+
async forget(id: string) {
|
|
121
|
+
// Only this bundle can republish its cache during deletion. A stalled
|
|
122
|
+
// backfill elsewhere must not delay removing these bytes or responding.
|
|
123
|
+
await active.get(id)
|
|
124
|
+
for (const [hash, entry] of entries) if (entry.id === id) entries.delete(hash)
|
|
125
|
+
// Do not wait for remote reads during deletion. Their callers still own
|
|
126
|
+
// and drain them, but a late response must not repopulate this cache.
|
|
127
|
+
for (const [hash, entry] of reads) if (entry.id === id) reads.delete(hash)
|
|
128
|
+
},
|
|
129
|
+
}
|
|
130
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
// Bundle helpers for the managed server: the content-addressed identity used to
|
|
2
|
+
// dedupe stored bundles and to auto-link reports to them.
|
|
3
|
+
//
|
|
4
|
+
// A report's top-level `bundleHashes` lists the `sha512-<base64>` integrities
|
|
5
|
+
// the analyzer ran against (see ui/view/ingest.js); a stored bundle's identity
|
|
6
|
+
// is that same integrity computed from its bytes. Matching the two is how a
|
|
7
|
+
// report auto-links to its bundle.
|
|
8
|
+
import { createHash } from 'node:crypto'
|
|
9
|
+
import type { Buffer } from 'node:buffer'
|
|
10
|
+
import { StringDecoder } from 'node:string_decoder'
|
|
11
|
+
import { brotliDecompress, createBrotliDecompress } from 'node:zlib'
|
|
12
|
+
import { promisify } from 'node:util'
|
|
13
|
+
import { parseBundleContents } from '../common/bundle-metadata.js'
|
|
14
|
+
import { bundleFilesAsMap } from '../common/bundle-sources.js'
|
|
15
|
+
import { commonFileDirectory } from '../common/managed/repository-alias.ts'
|
|
16
|
+
import { decodeUtf8 } from '../common/utf8.js'
|
|
17
|
+
|
|
18
|
+
// `sha512-<base64>` identity for a bundle's bytes. MUST stay byte-identical to
|
|
19
|
+
// the client's common/integrity.js (SHA-512 → standard base64 WITH padding) so
|
|
20
|
+
// a report's `bundleHashes` entries match a stored bundle's computed integrity.
|
|
21
|
+
// (The client uses `Uint8Array.toBase64()`, whose default is standard base64 +
|
|
22
|
+
// padding — the same bytes node:crypto's base64 digest emits.)
|
|
23
|
+
export function bundleIntegrity(bytes: Buffer): string {
|
|
24
|
+
return `sha512-${createHash('sha512').update(bytes).digest('base64')}`
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Classify an uploaded bundle by filename — mirrors ui/view/ingest.js
|
|
28
|
+
// `bundleKind`: sourcemap (.map) or stasis (stasis.code.br / .stasis.code.br).
|
|
29
|
+
// null for anything else (stored anyway; the kind is informational).
|
|
30
|
+
export function bundleKind(filename: string): 'sourcemap' | 'stasis' | null {
|
|
31
|
+
// Browsers insert a duplicate-download counter before the final extension.
|
|
32
|
+
const lower = filename.toLowerCase().replace(/ \(\d+\)(?=\.[^.]*$)/u, '')
|
|
33
|
+
if (lower.endsWith('.map')) return 'sourcemap'
|
|
34
|
+
if (lower === 'stasis.code.br' || lower.endsWith('.stasis.code.br')) return 'stasis'
|
|
35
|
+
return null
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface BundleRepo { github?: unknown; directory?: unknown }
|
|
39
|
+
const HEADER_KEYS = new Set(['version', 'config', 'repo', 'package'])
|
|
40
|
+
const MAX_HEADER_BYTES = 64 * 1024
|
|
41
|
+
|
|
42
|
+
// Stasis writes origin metadata before entries, formats and source bodies.
|
|
43
|
+
// undefined means another chunk is needed; null means the header has no repo.
|
|
44
|
+
function headerRepo(text: string): BundleRepo | null | undefined {
|
|
45
|
+
let depth = 0, escaped = false, expectKey = false, quoted = false
|
|
46
|
+
let key = '', start = 0, valueStart = 0
|
|
47
|
+
for (let i = 0; i < text.length; i++) {
|
|
48
|
+
const c = text[i]
|
|
49
|
+
if (quoted) {
|
|
50
|
+
if (escaped) { escaped = false; continue }
|
|
51
|
+
if (c === '\\') { escaped = true; continue }
|
|
52
|
+
if (c !== '"') continue
|
|
53
|
+
quoted = false
|
|
54
|
+
if (depth === 1 && expectKey) {
|
|
55
|
+
key = JSON.parse(text.slice(start, i + 1)) as string
|
|
56
|
+
if (!HEADER_KEYS.has(key)) return null
|
|
57
|
+
expectKey = false
|
|
58
|
+
}
|
|
59
|
+
continue
|
|
60
|
+
}
|
|
61
|
+
if (c === '"') { quoted = true; start = i; continue }
|
|
62
|
+
if (c === '{' || c === '[') {
|
|
63
|
+
if (depth === 0 && c !== '{') return null
|
|
64
|
+
depth++
|
|
65
|
+
if (depth === 1) expectKey = true
|
|
66
|
+
} else if (depth === 1 && (c === ',' || c === '}')) {
|
|
67
|
+
if (key === 'repo') {
|
|
68
|
+
const repo: unknown = JSON.parse(text.slice(valueStart, i))
|
|
69
|
+
return repo != null && typeof repo === 'object' && !Array.isArray(repo) ? repo as BundleRepo : null
|
|
70
|
+
}
|
|
71
|
+
if (c === '}') return null
|
|
72
|
+
expectKey = true
|
|
73
|
+
} else if (c === '}' || c === ']') depth--
|
|
74
|
+
else if (depth === 1 && c === ':') valueStart = i + 1
|
|
75
|
+
else if (depth === 0 && c?.trim()) return null
|
|
76
|
+
}
|
|
77
|
+
return undefined
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Decode only a bounded prefix, stopping as soon as repo is read or the
|
|
81
|
+
// header ends. Unknown/malformed archives remain uploadable as opaque bytes.
|
|
82
|
+
export async function bundleRepo(bytes: Buffer): Promise<BundleRepo | null> {
|
|
83
|
+
const stream = createBrotliDecompress({ chunkSize: 4096 })
|
|
84
|
+
const decoder = new StringDecoder('utf8')
|
|
85
|
+
let size = 0, text = ''
|
|
86
|
+
stream.end(bytes)
|
|
87
|
+
try {
|
|
88
|
+
for await (const chunk of stream) {
|
|
89
|
+
size += chunk.length
|
|
90
|
+
if (size > MAX_HEADER_BYTES) return null
|
|
91
|
+
text += decoder.write(chunk)
|
|
92
|
+
const repo = headerRepo(text)
|
|
93
|
+
if (repo !== undefined) return repo
|
|
94
|
+
}
|
|
95
|
+
return null
|
|
96
|
+
} catch { return null }
|
|
97
|
+
finally { stream.destroy() }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const decompress = promisify(brotliDecompress)
|
|
101
|
+
// Same decoded size bound as the bundle metadata cache; only reached when an
|
|
102
|
+
// alias can use a shared file suffix. Invalid/opaque bundles keep header defaults.
|
|
103
|
+
export async function bundleFilePrefix(bytes: Buffer): Promise<string> {
|
|
104
|
+
try {
|
|
105
|
+
const decoded = await decompress(bytes, { maxOutputLength: 512 * 1024 * 1024 })
|
|
106
|
+
const details = parseBundleContents(decodeUtf8(decoded), { integrity: '', kind: 'stasis', size: bytes.length })
|
|
107
|
+
return commonFileDirectory(bundleFilesAsMap(details).keys())
|
|
108
|
+
} catch { return '' }
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Extract the bundle integrities a report declares (its top-level
|
|
112
|
+
// `bundleHashes`), from the JSON it parsed to (readManagedReport's, as is).
|
|
113
|
+
// Non-JSON reports (markdown / CSV, given as null) or a missing field → [].
|
|
114
|
+
export function reportBundleHashes(data: unknown): string[] {
|
|
115
|
+
if (data == null || typeof data !== 'object') return []
|
|
116
|
+
const hashes = (data as { bundleHashes?: unknown }).bundleHashes
|
|
117
|
+
if (!Array.isArray(hashes)) return []
|
|
118
|
+
return hashes.filter((x): x is string => typeof x === 'string' && x !== '')
|
|
119
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Buffer } from 'node:buffer'
|
|
2
|
+
import type { OpenedBlob } from './blob-store.ts'
|
|
3
|
+
|
|
4
|
+
export class CacheMissError extends Error {
|
|
5
|
+
constructor() { super('Managed cache unavailable') }
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
// Keys are internal relative paths; delete removes a whole directory of
|
|
9
|
+
// variants, but for one directory within it where `keep` names one.
|
|
10
|
+
export interface CacheStorage {
|
|
11
|
+
exists(key: string): Promise<boolean>
|
|
12
|
+
put(key: string, bytes: Buffer): Promise<void>
|
|
13
|
+
open(key: string): Promise<OpenedBlob>
|
|
14
|
+
delete(prefix: string, keep?: string): Promise<void>
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function validateCacheKey(key: string): string {
|
|
18
|
+
if (!key.split('/').every(part => /^[a-z0-9][a-z0-9.-]*$/iu.test(part))) throw new Error('Invalid cache key')
|
|
19
|
+
return key
|
|
20
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Executable entry for the managed-server bin. See server-e2e/cli.js for the
|
|
3
|
+
// full rationale: Node refuses to strip TypeScript types from files under
|
|
4
|
+
// node_modules, so once this package is installed as a dependency
|
|
5
|
+
// server-managed/index.ts can't be executed directly. Register the in-thread
|
|
6
|
+
// type-stripping hook FIRST, then load the server via a dynamic import (a
|
|
7
|
+
// static import would be fetched before the hook is active) and start it.
|
|
8
|
+
//
|
|
9
|
+
// Imported here rather than run as the process entry, so index.ts's
|
|
10
|
+
// `import.meta.main` auto-start gate stays off and we call `start()` ourselves.
|
|
11
|
+
import '../strip-types-loader.js'
|
|
12
|
+
const command = process.argv[2]
|
|
13
|
+
if (command === '--storage-encryption-status') {
|
|
14
|
+
const { storageCommand } = await import('./storage-command.ts')
|
|
15
|
+
await storageCommand(process.argv[3])
|
|
16
|
+
} else if (command) {
|
|
17
|
+
throw new Error(`Unknown command: ${command}`)
|
|
18
|
+
} else {
|
|
19
|
+
const { start } = await import('./index.ts')
|
|
20
|
+
await start()
|
|
21
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// Compose the existing HTTP apps on e2e's listener, including its WS/SSE
|
|
2
|
+
// transports and static UI. Managed handles its API, discovery and page HTML;
|
|
3
|
+
// assets and other requests fall through to the unchanged e2e HTTP handler.
|
|
4
|
+
import { resolve } from 'node:path'
|
|
5
|
+
import type { Server } from 'node:http'
|
|
6
|
+
import { withReap } from '../server-common/reap.ts'
|
|
7
|
+
import { loadConfig } from '../server-e2e/config.ts'
|
|
8
|
+
import { createE2eApp } from '../server-e2e/app.ts'
|
|
9
|
+
import { loadManagedConfig } from './config.ts'
|
|
10
|
+
import { LOGIN_PATH } from './github-oauth.ts'
|
|
11
|
+
import { createManagedApp, logManagedStartup } from './index.ts'
|
|
12
|
+
|
|
13
|
+
export async function init(mode: 'managed+e2e' | 'e2e+managed', { ui = true }: { ui?: boolean } = {}): Promise<Server> {
|
|
14
|
+
const config = loadManagedConfig({ combined: true })
|
|
15
|
+
const e2eConfig = loadConfig()
|
|
16
|
+
if (!e2eConfig.neonUrl && !config.neonUrl && config.dbPath !== ':memory:' && resolve(config.dbPath) === resolve(e2eConfig.dbPath)) {
|
|
17
|
+
throw new Error('MANAGED_DB_PATH must differ from e2e DB_PATH in combined mode.')
|
|
18
|
+
}
|
|
19
|
+
const e2e = await createE2eApp(e2eConfig, { ui })
|
|
20
|
+
// Replace the assembled server's single
|
|
21
|
+
// request listener; adding a second would write two responses per request.
|
|
22
|
+
const handlers = e2e.httpServer.listeners('request')
|
|
23
|
+
const next = handlers[0]
|
|
24
|
+
if (handlers.length !== 1 || !next) throw new Error('Expected one e2e HTTP request handler')
|
|
25
|
+
const managed = await createManagedApp(config, {
|
|
26
|
+
next: e2e.handleRequest,
|
|
27
|
+
isShuttingDown: e2e.isShuttingDown,
|
|
28
|
+
ui,
|
|
29
|
+
serverInfo: {
|
|
30
|
+
mode, managed: { loginPath: LOGIN_PATH, cookieName: config.sessionCookieName },
|
|
31
|
+
...(e2eConfig.deepviewScanServer ? { deepviewScanServer: e2eConfig.deepviewScanServer } : {}),
|
|
32
|
+
},
|
|
33
|
+
}).catch(async err => {
|
|
34
|
+
await e2e.httpServer[Symbol.asyncDispose]()
|
|
35
|
+
throw err
|
|
36
|
+
})
|
|
37
|
+
e2e.onShutdown(managed.close)
|
|
38
|
+
e2e.httpServer.removeListener('request', next)
|
|
39
|
+
e2e.httpServer.on('request', withReap(managed.handleRequest,
|
|
40
|
+
{ e2e: e2e.reap, managed: managed.reap }, { isShuttingDown: e2e.isShuttingDown }))
|
|
41
|
+
e2e.httpServer.once('listening', () => {
|
|
42
|
+
const address = e2e.httpServer.address()
|
|
43
|
+
const port = typeof address === 'object' && address ? address.port : config.port
|
|
44
|
+
logManagedStartup(config, port, mode)
|
|
45
|
+
})
|
|
46
|
+
return e2e.httpServer
|
|
47
|
+
}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto'
|
|
2
|
+
import type { DatabaseSync } from 'node:sqlite'
|
|
3
|
+
import { type ManagedComment, canDeleteComment } from '../common/managed/comments.ts'
|
|
4
|
+
import type { ManagedSql } from './sql.ts'
|
|
5
|
+
|
|
6
|
+
export interface CommentInput {
|
|
7
|
+
findingId: string
|
|
8
|
+
body: string
|
|
9
|
+
// Runtime imports may omit authorship, including future admin UI imports.
|
|
10
|
+
// Ordinary comment posts always supply the authenticated user as author.
|
|
11
|
+
authorId: string | null
|
|
12
|
+
authorLogin: string | null
|
|
13
|
+
// The user performing an import need not be the author of its comments.
|
|
14
|
+
// Omitted for ordinary posts, where the author is also the acting user.
|
|
15
|
+
actor?: { id: string; login: string }
|
|
16
|
+
// Omitted dates default to the action time; null preserves unknown dates.
|
|
17
|
+
createdAt?: number | null
|
|
18
|
+
updatedAt?: number | null
|
|
19
|
+
reportId?: string | null
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface CommentStore {
|
|
23
|
+
listComments(findingIds: readonly string[]): Promise<ManagedComment[]>
|
|
24
|
+
listCommentedFindingIds(findingIds: readonly string[]): Promise<string[]>
|
|
25
|
+
getComment(id: string): Promise<ManagedComment | null>
|
|
26
|
+
createComment(input: CommentInput, now: number): Promise<ManagedComment>
|
|
27
|
+
editComment(id: string, authorId: string, authorLogin: string, body: string, version: number, reportId: string, now: number): Promise<ManagedComment | 'conflict' | 'forbidden' | null>
|
|
28
|
+
deleteComment(id: string, actorId: string, actorLogin: string, version: number, reportId: string, now: number): Promise<'deleted' | 'conflict' | 'forbidden' | null>
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const COMMENT_COLUMNS = `id TEXT PRIMARY KEY, finding_id TEXT NOT NULL, body TEXT NOT NULL,
|
|
32
|
+
author_id TEXT REFERENCES managed_user(id) ON DELETE SET NULL,
|
|
33
|
+
author_login TEXT, created_at INTEGER, updated_at INTEGER,
|
|
34
|
+
version INTEGER NOT NULL DEFAULT 1`
|
|
35
|
+
|
|
36
|
+
export const COMMENT_SCHEMA = `
|
|
37
|
+
CREATE TABLE IF NOT EXISTS managed_finding_comment (${COMMENT_COLUMNS}) STRICT;
|
|
38
|
+
CREATE INDEX IF NOT EXISTS finding_comment_finding_idx ON managed_finding_comment(finding_id, created_at, id);
|
|
39
|
+
CREATE TABLE IF NOT EXISTS managed_finding_comment_event (
|
|
40
|
+
seq INTEGER PRIMARY KEY, comment_id TEXT NOT NULL, finding_id TEXT NOT NULL,
|
|
41
|
+
actor_id TEXT REFERENCES managed_user(id) ON DELETE SET NULL,
|
|
42
|
+
actor_login TEXT, action TEXT NOT NULL, at INTEGER NOT NULL,
|
|
43
|
+
report_id TEXT, report TEXT, repo TEXT
|
|
44
|
+
) STRICT;
|
|
45
|
+
CREATE INDEX IF NOT EXISTS finding_comment_event_finding_idx ON managed_finding_comment_event(finding_id, seq);
|
|
46
|
+
CREATE INDEX IF NOT EXISTS finding_comment_event_actor_at_idx ON managed_finding_comment_event(actor_id, at);
|
|
47
|
+
CREATE INDEX IF NOT EXISTS finding_comment_event_at_idx ON managed_finding_comment_event(at, seq);
|
|
48
|
+
`
|
|
49
|
+
|
|
50
|
+
export function initCommentMethods(db: DatabaseSync): void {
|
|
51
|
+
db.exec(COMMENT_SCHEMA)
|
|
52
|
+
migrateCommentDates(db)
|
|
53
|
+
// updated_by belongs to the whole triage row, not necessarily its comment.
|
|
54
|
+
// Preserve legacy text without guessing an author. Clearing the old column
|
|
55
|
+
// in the same transaction makes the migration safe to restart. Use a fresh
|
|
56
|
+
// ID so a later legacy-server write cannot collide with an earlier import.
|
|
57
|
+
db.exec('BEGIN')
|
|
58
|
+
try {
|
|
59
|
+
db.exec(`INSERT INTO managed_finding_comment (id, finding_id, body, created_at, updated_at)
|
|
60
|
+
SELECT 'legacy:' || lower(hex(randomblob(16))), finding_id, comment, updated_at, updated_at
|
|
61
|
+
FROM managed_finding_triage WHERE comment IS NOT NULL AND comment != '';
|
|
62
|
+
UPDATE managed_finding_triage SET comment = NULL WHERE comment IS NOT NULL;`)
|
|
63
|
+
db.exec('COMMIT')
|
|
64
|
+
} catch (err) {
|
|
65
|
+
db.exec('ROLLBACK')
|
|
66
|
+
throw err
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function commentMethods(db: ManagedSql): CommentStore {
|
|
71
|
+
const fields = `c.id, c.finding_id AS findingId, c.body, c.author_id AS authorId,
|
|
72
|
+
COALESCE(u.login, c.author_login) AS authorLogin,
|
|
73
|
+
c.created_at AS createdAt, c.updated_at AS updatedAt, c.version`
|
|
74
|
+
const select = db.prepare(`SELECT ${fields} FROM managed_finding_comment c
|
|
75
|
+
LEFT JOIN managed_user u ON u.id = c.author_id WHERE c.id = ?`)
|
|
76
|
+
const list = db.prepare(`SELECT ${fields} FROM managed_finding_comment c
|
|
77
|
+
LEFT JOIN managed_user u ON u.id = c.author_id
|
|
78
|
+
WHERE c.finding_id IN (SELECT value FROM json_each(?)) ORDER BY c.created_at NULLS FIRST, c.id`)
|
|
79
|
+
const insert = db.prepare(`INSERT INTO managed_finding_comment
|
|
80
|
+
(id, finding_id, body, author_id, author_login, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
|
81
|
+
const update = db.prepare(`UPDATE managed_finding_comment SET body = ?, updated_at = ?, version = version + 1
|
|
82
|
+
WHERE id = ? AND author_id = ? AND version = ?`)
|
|
83
|
+
const remove = db.prepare('DELETE FROM managed_finding_comment WHERE id = ? AND (author_id = ? OR (author_id IS NULL AND ? IS NULL)) AND version = ?')
|
|
84
|
+
const userRole = db.prepare('SELECT role FROM managed_user WHERE id = ?')
|
|
85
|
+
// Deleted comments still have audit events which explicit annotation purges
|
|
86
|
+
// must find, including when no current triage/comment record remains.
|
|
87
|
+
const annotated = db.prepare(`SELECT finding_id AS id FROM managed_finding_comment WHERE finding_id IN (SELECT value FROM json_each(?))
|
|
88
|
+
UNION SELECT finding_id AS id FROM managed_finding_comment_event WHERE finding_id IN (SELECT value FROM json_each(?))`)
|
|
89
|
+
const event = db.prepare(`INSERT INTO managed_finding_comment_event
|
|
90
|
+
(comment_id, finding_id, actor_id, actor_login, action, at, report_id, report, repo)
|
|
91
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, (SELECT filename FROM managed_report WHERE id = ?),
|
|
92
|
+
(SELECT p.full_name FROM managed_report r JOIN managed_selected_repo p ON p.repo_id = r.repo_id WHERE r.id = ?))`)
|
|
93
|
+
const read = async (id: string) => (await select.get(id)) as ManagedComment | undefined
|
|
94
|
+
return {
|
|
95
|
+
async listComments(ids) { return ids.length > 0 ? (await list.all(JSON.stringify(ids))) as unknown as ManagedComment[] : [] },
|
|
96
|
+
async listCommentedFindingIds(ids) {
|
|
97
|
+
const json = JSON.stringify(ids)
|
|
98
|
+
return ((await annotated.all(json, json)) as { id: string }[]).map(row => row.id)
|
|
99
|
+
},
|
|
100
|
+
async getComment(id) { return (await read(id)) ?? null },
|
|
101
|
+
async createComment(input, now) {
|
|
102
|
+
const id = randomUUID()
|
|
103
|
+
const createdAt = input.createdAt === undefined ? now : input.createdAt
|
|
104
|
+
const updatedAt = input.updatedAt === undefined ? createdAt : input.updatedAt
|
|
105
|
+
await insert.run(id, input.findingId, input.body, input.authorId, input.authorLogin, createdAt, updatedAt)
|
|
106
|
+
await event.run(id, input.findingId, input.actor?.id ?? input.authorId, input.actor?.login ?? input.authorLogin, 'added a comment', now,
|
|
107
|
+
input.reportId ?? null, input.reportId ?? null, input.reportId ?? null)
|
|
108
|
+
return (await read(id))!
|
|
109
|
+
},
|
|
110
|
+
async editComment(id, authorId, authorLogin, body, version, reportId, now) {
|
|
111
|
+
// Identity/version checks and the event share the operation transaction.
|
|
112
|
+
const current = await read(id)
|
|
113
|
+
if (!current || current.authorId !== authorId || current.version !== version || current.body === body) {
|
|
114
|
+
if (!current) return null
|
|
115
|
+
if (current.authorId !== authorId) return 'forbidden'
|
|
116
|
+
if (current.version !== version) return 'conflict'
|
|
117
|
+
return current
|
|
118
|
+
}
|
|
119
|
+
await update.run(body, now, id, authorId, version)
|
|
120
|
+
await event.run(id, current.findingId, authorId, authorLogin, 'edited a comment', now, reportId, reportId, reportId)
|
|
121
|
+
return (await read(id))!
|
|
122
|
+
},
|
|
123
|
+
async deleteComment(id, actorId, actorLogin, version, reportId, now) {
|
|
124
|
+
const current = await read(id)
|
|
125
|
+
const role = (await userRole.get(actorId) as { role: string } | undefined)?.role ?? 'none'
|
|
126
|
+
const allowed = current != null && canDeleteComment(current, { id: actorId, role })
|
|
127
|
+
if (!current || !allowed || current.version !== version) {
|
|
128
|
+
if (!current) return null
|
|
129
|
+
return allowed ? 'conflict' : 'forbidden'
|
|
130
|
+
}
|
|
131
|
+
await remove.run(id, current.authorId, current.authorId, version)
|
|
132
|
+
await event.run(id, current.findingId, actorId, actorLogin, 'deleted a comment', now, reportId, reportId, reportId)
|
|
133
|
+
return 'deleted'
|
|
134
|
+
},
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function migrateCommentDates(db: DatabaseSync): void {
|
|
139
|
+
const columns = db.prepare('PRAGMA table_info(managed_finding_comment)').all() as { name: string; notnull: number }[]
|
|
140
|
+
if (!columns.some(column => ['created_at', 'updated_at'].includes(column.name) && column.notnull)) return
|
|
141
|
+
db.exec('BEGIN')
|
|
142
|
+
try {
|
|
143
|
+
db.exec(`CREATE TABLE managed_finding_comment_nullable_dates (${COMMENT_COLUMNS}) STRICT;
|
|
144
|
+
INSERT INTO managed_finding_comment_nullable_dates (id, finding_id, body, author_id, author_login, created_at, updated_at, version)
|
|
145
|
+
SELECT id, finding_id, body, author_id, author_login, created_at, updated_at, version FROM managed_finding_comment;
|
|
146
|
+
DROP TABLE managed_finding_comment;
|
|
147
|
+
ALTER TABLE managed_finding_comment_nullable_dates RENAME TO managed_finding_comment;
|
|
148
|
+
CREATE INDEX finding_comment_finding_idx ON managed_finding_comment(finding_id, created_at, id);`)
|
|
149
|
+
db.exec('COMMIT')
|
|
150
|
+
} catch (err) { db.exec('ROLLBACK'); throw err }
|
|
151
|
+
}
|