@preventive/triage 1.0.0-alpha.4 → 1.0.0-alpha.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/README.md +16 -0
- package/api/reap.ts +17 -0
- package/cli.js +6 -0
- package/client/dependency-paths.d.ts +2 -0
- package/client/dependency-paths.js +24 -0
- package/client/finding-link.js +305 -0
- package/client/linked-findings.d.ts +1 -0
- package/client/linked-findings.js +112 -0
- package/common/bundle-commit.d.ts +1 -0
- package/common/bundle-commit.js +4 -0
- package/common/bundle-compare-diff.d.ts +3 -0
- package/common/bundle-compare-diff.js +522 -0
- package/common/bundle-compare-inputs.d.ts +2 -0
- package/common/bundle-compare-inputs.js +65 -0
- package/common/bundle-metadata.d.ts +14 -0
- package/common/bundle-metadata.js +226 -0
- package/common/bundle-pkg-of.d.ts +1 -0
- package/common/bundle-pkg-of.js +90 -0
- package/common/bundle-reasons.d.ts +2 -0
- package/common/bundle-reasons.js +21 -0
- package/common/bundle-source-links.d.ts +6 -0
- package/common/bundle-source-links.js +216 -0
- package/common/bundle-sourcemap.d.ts +10 -0
- package/common/bundle-sourcemap.js +100 -0
- package/common/bundle-sources.d.ts +7 -0
- package/common/bundle-sources.js +310 -0
- package/common/bundle-stats.js +41 -0
- package/common/bundle-tabs.js +1 -0
- package/common/code-language.js +65 -0
- package/common/default-scan-models.ts +31 -0
- package/common/dependency-reviews.d.ts +23 -0
- package/common/dependency-reviews.js +81 -0
- package/common/finding-duplicates.js +58 -0
- package/common/finding-filters.js +274 -0
- package/common/finding-id.js +47 -0
- package/common/github-issue-labels.d.ts +1 -0
- package/common/github-issue-labels.js +11 -0
- package/common/github-pr.ts +56 -0
- package/common/gzip.js +59 -0
- package/common/line-diff.d.ts +8 -0
- package/common/line-diff.js +56 -0
- package/common/linked-workspace-groups.js +63 -0
- package/common/managed/comments.ts +34 -0
- package/common/managed/npm-packages.d.ts +5 -0
- package/common/managed/npm-packages.js +34 -0
- package/common/managed/permissions.ts +31 -0
- package/common/managed/report-content.ts +72 -0
- package/common/managed/report-filter.ts +108 -0
- package/common/managed/repository-alias.ts +65 -0
- package/common/managed/roles.ts +28 -0
- package/common/managed/routes.d.ts +2 -0
- package/common/managed/routes.js +254 -0
- package/common/managed/scan-models.ts +6 -0
- package/common/managed/team-app.d.ts +2 -0
- package/common/managed/team-app.js +31 -0
- package/common/managed/triage.ts +83 -0
- package/common/managed/upload-seal.ts +126 -0
- package/common/pretty-print.d.ts +5 -0
- package/common/pretty-print.js +24 -0
- package/common/revalidation-conflicts.js +48 -0
- package/common/revalidation-input-groups.js +56 -0
- package/common/save-error-reason.ts +20 -7
- package/common/scan-server.ts +13 -0
- package/common/server-info.ts +35 -0
- package/common/utf8.d.ts +3 -0
- package/common/utf8.js +45 -0
- package/common/workspace-app.js +23 -0
- package/common/workspace-groups.js +176 -0
- package/out/brotli-fallback.js +3 -3
- package/out/chunk-2KBF5TZ7.js +2 -0
- package/out/chunk-2XUEZHIP.js +28 -0
- package/out/chunk-35VE6CDH.js +2 -0
- package/out/chunk-3BZ7JERB.js +1 -0
- package/out/chunk-4F6JSF57.js +3 -0
- package/out/chunk-4XCDFGDP.js +10 -0
- package/out/chunk-AQ75QITE.js +1 -0
- package/out/chunk-B2RQY6OP.js +2 -0
- package/out/chunk-C4Z6ZBXV.js +10 -0
- package/out/chunk-CMQA4DM6.js +1 -0
- package/out/chunk-CVVJAXVY.js +2 -0
- package/out/chunk-DIJJNTRH.js +10 -0
- package/out/chunk-GJ4IUCNS.js +1 -0
- package/out/chunk-GNXOKIGP.js +80 -0
- package/out/chunk-GZ4MS3TE.js +13 -0
- package/out/chunk-IISEF2BZ.js +1 -0
- package/out/chunk-IU3CCINR.js +3 -0
- package/out/chunk-K3X5EJBQ.js +2 -0
- package/out/chunk-KEX2ATEW.js +2 -0
- package/out/chunk-KYCNJVB7.js +1 -0
- package/out/chunk-MYDDTA7W.js +225 -0
- package/out/chunk-NRPC74AM.js +1 -0
- package/out/chunk-OOJM4CTU.js +1 -0
- package/out/chunk-PNHBZGDD.js +1 -0
- package/out/chunk-R5O6FAW4.js +2 -0
- package/out/chunk-RFZQQXSG.js +17 -0
- package/out/chunk-RSHMVYNS.js +1 -0
- package/out/chunk-TGANXVTA.js +1 -0
- package/out/chunk-TUII34K2.js +1 -0
- package/out/chunk-TZ5KB2Y5.js +10 -0
- package/out/chunk-UN5ZSYLE.js +1 -0
- package/out/chunk-UNTRGE26.js +2 -0
- package/out/chunk-UXNUAIIC.js +33 -0
- package/out/chunk-UYAXVRVL.js +1 -0
- package/out/chunk-V3ME6FI5.js +2 -0
- package/out/chunk-V5O7M2YT.js +1 -0
- package/out/chunk-VIHGO77Y.js +1 -0
- package/out/chunk-VQJCXGIY.js +7 -0
- package/out/chunk-W4J3MGLB.js +1 -0
- package/out/chunk-WGG7KQUH.js +1 -0
- package/out/chunk-WZDTZE64.js +1 -0
- package/out/chunk-XYDGU6BM.js +51 -0
- package/out/chunk-XYFXXB2G.js +10 -0
- package/out/chunk-YVZH5SHP.js +1 -0
- package/out/chunk-ZBBOBQ43.js +1 -0
- package/out/chunk-ZMQQCGF6.js +2 -0
- package/out/client-managed-import.js +4 -0
- package/out/client-managed.js +37 -0
- package/out/client-sync.js +10 -13
- package/out/github-issue-authorized.html +3 -0
- package/out/github-issue-authorized.js +1 -0
- package/out/graph.js +17 -4
- package/out/index.html +90 -8
- package/out/prism.js +6 -2
- package/out/stasis.svg +20 -0
- package/out/terminal.js +365 -39
- package/out/view.css +1 -1
- package/out/view.js +83 -117
- package/package.json +297 -59
- package/risk/files/characters.js +196 -0
- package/risk/files/hosts.js +205 -0
- package/risk/files/readability.js +171 -0
- package/risk/files/runs-code.js +358 -0
- package/risk/files/script.js +114 -0
- package/risk/files/stamps.js +46 -0
- package/risk/index.d.ts +142 -0
- package/risk/index.js +7 -0
- package/risk/npm/assess.js +219 -0
- package/risk/npm/combined.js +65 -0
- package/risk/npm/compare.js +228 -0
- package/risk/npm/dependencies.js +153 -0
- package/risk/npm/diff.js +43 -0
- package/risk/npm/downloads.js +84 -0
- package/risk/npm/files.js +34 -0
- package/risk/npm/held.js +60 -0
- package/risk/npm/install-scripts.js +34 -0
- package/risk/npm/malware.js +36 -0
- package/risk/npm/manifest.js +65 -0
- package/risk/npm/new-dependency.js +30 -0
- package/risk/npm/old-version.js +108 -0
- package/risk/npm/registry.js +282 -0
- package/risk/npm/release.js +85 -0
- package/risk/npm/specs.js +124 -0
- package/risk/npm/tarball.js +113 -0
- package/server-common/database-config.ts +16 -0
- package/server-common/initialize.ts +18 -0
- package/server-common/npm-advisories.ts +101 -0
- package/{server → server-common}/origin.ts +5 -5
- package/server-common/reap.ts +48 -0
- package/server-common/scan-config.ts +19 -0
- package/server-common/standalone.ts +29 -0
- package/server-common/storage-crypto.ts +182 -0
- package/server-common/storage-log.ts +38 -0
- package/server-common/vercel-blob.ts +103 -0
- package/server-e2e/app.ts +486 -0
- package/{server → server-e2e}/auth.ts +5 -1
- package/{server → server-e2e}/bus-receiver.ts +2 -1
- package/{server → server-e2e}/cli.js +9 -4
- package/{server → server-e2e}/config.ts +56 -35
- package/{server → server-e2e}/db-neon.ts +2 -2
- package/{server → server-e2e}/db-revision-sql.ts +1 -1
- package/{server → server-e2e}/db-stmt.ts +2 -2
- package/{server → server-e2e}/db.ts +7 -7
- package/{server → server-e2e}/http.ts +116 -18
- package/{server → server-e2e}/hub.ts +40 -9
- package/server-e2e/index.ts +17 -0
- package/server-e2e/lifecycle.ts +95 -0
- package/{server → server-e2e}/neon-driver.ts +2 -2
- package/{server → server-e2e}/npm-proxy.ts +12 -145
- package/{server → server-e2e}/objstore/blob-vercel.ts +10 -114
- package/{server → server-e2e}/objstore/blob.ts +1 -1
- package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
- package/{server → server-e2e}/objstore/handlers.ts +9 -8
- package/{server → server-e2e}/objstore/init.ts +35 -18
- package/{server → server-e2e}/objstore/reaper.ts +1 -1
- package/server-e2e/objstore/rest-deny.ts +28 -0
- package/server-e2e/objstore/rest-mint.ts +224 -0
- package/{server → server-e2e}/objstore/rest.ts +27 -26
- package/{server → server-e2e}/objstore/sign.ts +105 -0
- package/{server → server-e2e}/objstore/store-neon.ts +6 -10
- package/{server → server-e2e}/objstore/store.ts +7 -7
- package/{server → server-e2e}/objstore/tokens.ts +2 -2
- package/{server → server-e2e}/pubsub.ts +12 -9
- package/{server → server-e2e}/sign.ts +4 -4
- package/{server → server-e2e}/sse-server.ts +102 -68
- package/{server → server-e2e}/sse-session.ts +28 -8
- package/{server → server-e2e}/static.ts +31 -21
- package/server-e2e/sync-handlers.ts +408 -0
- package/{server → server-e2e}/ws-server.ts +20 -9
- package/server-managed/activity.ts +245 -0
- package/server-managed/avatar-store.ts +12 -0
- package/server-managed/blob-store.ts +17 -0
- package/server-managed/brotli.ts +10 -0
- package/server-managed/bundle-advisories.ts +120 -0
- package/server-managed/bundle-advisory-inventory.ts +77 -0
- package/server-managed/bundle-build-diagnostics.d.ts +10 -0
- package/server-managed/bundle-build-diagnostics.js +23 -0
- package/server-managed/bundle-build-leases.ts +41 -0
- package/server-managed/bundle-build-worker.js +111 -0
- package/server-managed/bundle-build.ts +198 -0
- package/server-managed/bundle-cache.ts +199 -0
- package/server-managed/bundle-catalog.ts +29 -0
- package/server-managed/bundle-commits.ts +140 -0
- package/server-managed/bundle-store.ts +24 -0
- package/server-managed/bundle-summary-cache.ts +130 -0
- package/server-managed/bundle.ts +120 -0
- package/server-managed/cache-storage.ts +19 -0
- package/server-managed/cli.js +21 -0
- package/server-managed/combined.ts +46 -0
- package/server-managed/comments.ts +151 -0
- package/server-managed/config.ts +158 -0
- package/server-managed/content-access.ts +15 -0
- package/server-managed/crypto.ts +25 -0
- package/server-managed/db-methods.ts +1725 -0
- package/server-managed/db-neon.ts +367 -0
- package/server-managed/db-schema.ts +227 -0
- package/server-managed/db-table-names.ts +22 -0
- package/server-managed/db.ts +132 -0
- package/server-managed/dependency-reviews.ts +248 -0
- package/server-managed/github-app.ts +391 -0
- package/server-managed/github-errors.ts +68 -0
- package/server-managed/github-issue-links.ts +127 -0
- package/server-managed/github-issue-oauth.ts +47 -0
- package/server-managed/github-issues.ts +152 -0
- package/server-managed/github-metadata.ts +213 -0
- package/server-managed/github-oauth.ts +235 -0
- package/server-managed/github-pulls.ts +227 -0
- package/server-managed/http-response.ts +50 -0
- package/server-managed/http.ts +3472 -0
- package/server-managed/import-triage.ts +51 -0
- package/server-managed/index.ts +187 -0
- package/server-managed/kept.ts +28 -0
- package/server-managed/link-reports.ts +158 -0
- package/server-managed/locked-packages.ts +85 -0
- package/server-managed/lru.ts +30 -0
- package/server-managed/managed-issues.ts +75 -0
- package/server-managed/management-catalog.ts +45 -0
- package/server-managed/management.ts +99 -0
- package/server-managed/npm-insights.ts +440 -0
- package/server-managed/npm-kept.ts +65 -0
- package/server-managed/npm-loads.ts +167 -0
- package/server-managed/npm-manifest.ts +115 -0
- package/server-managed/npm-packages.ts +378 -0
- package/server-managed/npm-provenance.ts +123 -0
- package/server-managed/npm-risk.ts +177 -0
- package/server-managed/object-storage-disk.ts +148 -0
- package/server-managed/object-storage-vercel.ts +67 -0
- package/server-managed/object-storage.ts +74 -0
- package/server-managed/package-entry-points.ts +51 -0
- package/server-managed/pretty-print.ts +352 -0
- package/server-managed/public-workspace.ts +220 -0
- package/server-managed/rename-diff-worker.js +26 -0
- package/server-managed/rename-diff.ts +270 -0
- package/server-managed/repo-path.ts +21 -0
- package/server-managed/report-migration.ts +35 -0
- package/server-managed/report-query.ts +10 -0
- package/server-managed/report-response.ts +16 -0
- package/server-managed/report-sources.ts +178 -0
- package/server-managed/report-store.ts +47 -0
- package/server-managed/repository-aliases.ts +91 -0
- package/server-managed/repository-browser.ts +153 -0
- package/server-managed/repository-discovery.ts +82 -0
- package/server-managed/repository-policy.ts +27 -0
- package/server-managed/repository-risk.ts +418 -0
- package/server-managed/revisions.ts +35 -0
- package/server-managed/session.ts +92 -0
- package/server-managed/slugs.ts +38 -0
- package/server-managed/solidity-entry-points.ts +69 -0
- package/server-managed/sql-postgres.ts +31 -0
- package/server-managed/sql.ts +76 -0
- package/server-managed/static.ts +28 -0
- package/server-managed/storage-command.ts +16 -0
- package/server-managed/storage-db.ts +175 -0
- package/server-managed/storage-encryption.ts +279 -0
- package/server-managed/storage-maintenance.ts +256 -0
- package/server-managed/storage-payload.ts +103 -0
- package/server-managed/storage-stores.ts +85 -0
- package/server-managed/storage.ts +78 -0
- package/server-managed/team-app.ts +94 -0
- package/server-managed/team-catalog.ts +7 -0
- package/server-managed/team-feed.ts +173 -0
- package/server-managed/team-npm-scopes.ts +77 -0
- package/server-managed/team-reports.ts +228 -0
- package/server-managed/triage-response.ts +16 -0
- package/server-managed/type-only-worker.js +19 -0
- package/server-managed/type-only.ts +103 -0
- package/server-managed/uploads.ts +86 -0
- package/server-managed/upstream-cache.ts +215 -0
- package/server-managed/view-as.ts +58 -0
- package/server-managed/workspace-shares.ts +166 -0
- package/server.ts +50 -0
- package/server/index.ts +0 -477
- package/server/lifecycle.ts +0 -204
- package/server/sync-handlers.ts +0 -320
- /package/{LICENSE → LICENSE-MIT} +0 -0
- /package/{server → server-e2e}/config.example.json +0 -0
- /package/{server → server-e2e}/objstore/blob-fs.ts +0 -0
- /package/{server → server-e2e}/objstore/fs.ts +0 -0
- /package/{server → server-e2e}/peer.ts +0 -0
- /package/{server → server-e2e}/util.ts +0 -0
- /package/{server → server-e2e}/validation.ts +0 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import type { ManagedDb } from './db-methods.ts'
|
|
3
|
+
import type { ManagedComment } from '../common/managed/comments.ts'
|
|
4
|
+
import type { TriageEntryPatch } from '../common/managed/triage.ts'
|
|
5
|
+
import { triageWireEntry } from './triage-response.ts'
|
|
6
|
+
|
|
7
|
+
export interface ImportTriageSnapshot {
|
|
8
|
+
entry: TriageEntryPatch | null
|
|
9
|
+
comments: ManagedComment[]
|
|
10
|
+
version: string
|
|
11
|
+
}
|
|
12
|
+
export interface ImportTriageStore {
|
|
13
|
+
getImportTriage(ids: readonly string[]): Promise<Record<string, ImportTriageSnapshot>>
|
|
14
|
+
importTriage(entries: [string, TriageEntryPatch | null][], expected: Record<string, string>, actor: { id: string; login: string }, reportId: string, now: number): Promise<boolean>
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// Raw store methods share one transaction for comparison, triage, and comments.
|
|
18
|
+
export function importTriageMethods(store: Pick<ManagedDb, 'getReport' | 'listTriage' | 'listComments' | 'setTriageEntries' | 'createComment'>): ImportTriageStore {
|
|
19
|
+
async function snapshot(ids: readonly string[]) {
|
|
20
|
+
const rows = new Map((await store.listTriage(ids)).map(row => [row.findingId, row]))
|
|
21
|
+
const comments = await store.listComments(ids)
|
|
22
|
+
return Object.fromEntries(ids.map(id => {
|
|
23
|
+
const row = rows.get(id)
|
|
24
|
+
const entry = row ? triageWireEntry(row) : null
|
|
25
|
+
const own = comments.filter(comment => comment.findingId === id)
|
|
26
|
+
const version = createHash('sha256').update(JSON.stringify([entry, row?.updatedAt, own])).digest('hex')
|
|
27
|
+
return [id, { entry, comments: own, version }]
|
|
28
|
+
}))
|
|
29
|
+
}
|
|
30
|
+
return {
|
|
31
|
+
getImportTriage: snapshot,
|
|
32
|
+
async importTriage(entries, expected, actor, reportId, now) {
|
|
33
|
+
// A report deleted after the HTTP membership check must not leave newly
|
|
34
|
+
// imported annotations behind. Check existence in the write transaction.
|
|
35
|
+
if (!reportId || !(await store.getReport(reportId))) return false
|
|
36
|
+
const current = await snapshot(entries.map(([id]) => id))
|
|
37
|
+
if (entries.some(([id]) => current[id]!.version !== expected[id])) return false
|
|
38
|
+
const changes: [string, TriageEntryPatch | null][] = []
|
|
39
|
+
for (const [id, value] of entries) {
|
|
40
|
+
const { comment, ...entry } = value ?? {}
|
|
41
|
+
changes.push([id, Object.keys(entry).length > 0 ? entry : null])
|
|
42
|
+
if (comment && !current[id]!.comments.some(existing => existing.body === comment)) {
|
|
43
|
+
await store.createComment({ findingId: id, body: comment, authorId: null, authorLogin: null,
|
|
44
|
+
actor, createdAt: null, updatedAt: null, reportId }, now)
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
await store.setTriageEntries(changes, actor.id, actor.login, now, reportId)
|
|
48
|
+
return true
|
|
49
|
+
},
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// Managed HTTP app and standalone boot. The combined launcher mounts this
|
|
2
|
+
// same app on e2e's listener; storage, routing and cleanup stay here.
|
|
3
|
+
import { randomUUID } from 'node:crypto'
|
|
4
|
+
import { type Server, createServer } from 'node:http'
|
|
5
|
+
import { fileURLToPath } from 'node:url'
|
|
6
|
+
import { createOriginGate } from '../server-common/origin.ts'
|
|
7
|
+
import { managedStorageLines } from '../server-common/storage-log.ts'
|
|
8
|
+
import { runReapers, withReap } from '../server-common/reap.ts'
|
|
9
|
+
import { startServer } from '../server-common/standalone.ts'
|
|
10
|
+
import { initializeApp } from '../server-common/initialize.ts'
|
|
11
|
+
import { type ManagedConfig, loadManagedConfig } from './config.ts'
|
|
12
|
+
import { type ManagedHttpDeps, createManagedRequestHandler } from './http.ts'
|
|
13
|
+
import { setNpmTarballCache } from './npm-loads.ts'
|
|
14
|
+
import { setNpmDocumentCache } from './npm-packages.ts'
|
|
15
|
+
import { loadManagedStatic } from './static.ts'
|
|
16
|
+
import { openManagedStorage } from './storage.ts'
|
|
17
|
+
import { setCacheDir } from '@preventive/upstream/npm.js'
|
|
18
|
+
|
|
19
|
+
// Housekeeping runs on ordinary traffic too, including serverless instances
|
|
20
|
+
// without a timer or cron. Failed sweeps retry on later traffic after a minute.
|
|
21
|
+
const REAP_INTERVAL_MS = 3_600_000
|
|
22
|
+
const REAP_RETRY_MS = 60_000
|
|
23
|
+
|
|
24
|
+
type ManagedAppOptions = Partial<Pick<ManagedHttpDeps, 'next' | 'serverInfo' | 'isShuttingDown'>>
|
|
25
|
+
|
|
26
|
+
export async function createManagedApp(config: ManagedConfig, options: ManagedAppOptions = {}) {
|
|
27
|
+
// Audits keep upstream's records on disk where the config has a cache
|
|
28
|
+
// directory; bundle builds set it in their own worker. The npm viewer
|
|
29
|
+
// keeps the tarballs it downloads there too, beside theirs, and public
|
|
30
|
+
// packages' whole documents (npm-packages.ts).
|
|
31
|
+
setCacheDir(config.upstreamCacheDir ?? false)
|
|
32
|
+
setNpmTarballCache(config.upstreamCacheDir ?? null)
|
|
33
|
+
setNpmDocumentCache(config.upstreamCacheDir ?? null)
|
|
34
|
+
// Load the optional peer only for serverless managed deployments. Embedding
|
|
35
|
+
// hosts can discard listener promises, so retain maintenance inside the app.
|
|
36
|
+
const waitUntil = config.serverless ? (await import('@vercel/functions')).waitUntil : undefined
|
|
37
|
+
return await initializeApp(rollback => assembleManagedApp(config, options, rollback, waitUntil))
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOptions, rollback: AsyncDisposableStack, waitUntil?: (promise: Promise<unknown>) => void) {
|
|
41
|
+
const storage = await openManagedStorage(config)
|
|
42
|
+
const { db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, prettyCache, npmCacheStorage } = storage
|
|
43
|
+
rollback.defer(() => db.close())
|
|
44
|
+
const originGate = createOriginGate(config.host, config.trustProxyEnv)
|
|
45
|
+
|
|
46
|
+
let shuttingDown = false
|
|
47
|
+
const isShuttingDown = () => shuttingDown || options.isShuttingDown?.() === true
|
|
48
|
+
const inFlight = new Set<Promise<unknown>>()
|
|
49
|
+
function track(p: Promise<unknown>): void {
|
|
50
|
+
inFlight.add(p)
|
|
51
|
+
p.finally(() => inFlight.delete(p)).catch(() => {})
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const serveStatic = loadManagedStatic(fileURLToPath(new URL('../out/', import.meta.url)), {
|
|
55
|
+
indexOnly: options.next != null, scanServer: options.serverInfo?.deepviewScanServer ?? null,
|
|
56
|
+
})
|
|
57
|
+
const routeRequest = createManagedRequestHandler({
|
|
58
|
+
...options, config, db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, prettyCache, npmCacheStorage, originGate, serveStatic,
|
|
59
|
+
...(storage.uploadStore ? { uploadStore: storage.uploadStore } : {}),
|
|
60
|
+
isShuttingDown, track,
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
let cleanup: Promise<void> | undefined, nextCleanupAt = 0
|
|
64
|
+
const maintenance = new AbortController()
|
|
65
|
+
function reap(automatic = false): Promise<void> {
|
|
66
|
+
if (cleanup) return cleanup
|
|
67
|
+
if (isShuttingDown()) return Promise.resolve()
|
|
68
|
+
const startedAt = Date.now()
|
|
69
|
+
nextCleanupAt = startedAt + REAP_INTERVAL_MS
|
|
70
|
+
let sessions = 0, uploads = 0
|
|
71
|
+
cleanup = (async () => {
|
|
72
|
+
const owner = automatic && config.serverless ? randomUUID() : null
|
|
73
|
+
if (owner && !await db.claimMaintenanceLease(owner, startedAt, startedAt + Math.max(300_000, (config.storageEncryptionMigrateMaxMs ?? 0) + 60_000), config.storageEncryptionMigrate)) return false
|
|
74
|
+
let succeeded = false
|
|
75
|
+
try {
|
|
76
|
+
await runReapers({
|
|
77
|
+
sessions: async () => { sessions = await db.deleteExpiredSessions(Date.now()) },
|
|
78
|
+
...(storage.reapUploads ? { uploads: async () => { uploads = await storage.reapUploads!(maintenance.signal) } } : {}),
|
|
79
|
+
...('reapPrettyCopies' in storage ? { prettyCopies: () => storage.reapPrettyCopies(maintenance.signal) } : {}),
|
|
80
|
+
...('reapStorage' in storage ? { storage: () => storage.reapStorage(maintenance.signal) } : {}),
|
|
81
|
+
})
|
|
82
|
+
succeeded = true
|
|
83
|
+
return true
|
|
84
|
+
} finally {
|
|
85
|
+
if (owner) await db.finishMaintenanceLease(owner, Date.now() + (succeeded ? REAP_INTERVAL_MS : REAP_RETRY_MS))
|
|
86
|
+
}
|
|
87
|
+
})().then(ran => {
|
|
88
|
+
nextCleanupAt = Date.now() + REAP_INTERVAL_MS
|
|
89
|
+
if (!ran) return
|
|
90
|
+
return console.info(`managed-reaper: removed ${sessions} expired session(s), ${uploads} stale upload part(s) in ${Date.now() - startedAt}ms`)
|
|
91
|
+
}, err => {
|
|
92
|
+
nextCleanupAt = Date.now() + REAP_RETRY_MS
|
|
93
|
+
throw err
|
|
94
|
+
}).finally(() => { cleanup = undefined })
|
|
95
|
+
track(cleanup)
|
|
96
|
+
return cleanup
|
|
97
|
+
}
|
|
98
|
+
function automaticReap(): Promise<void> {
|
|
99
|
+
if (isShuttingDown() || cleanup || Date.now() < nextCleanupAt) return Promise.resolve()
|
|
100
|
+
return reap(true).catch(err => console.warn('managed-reaper: cleanup failed:', err))
|
|
101
|
+
}
|
|
102
|
+
async function handleRequest(req: Parameters<typeof routeRequest>[0], res: Parameters<typeof routeRequest>[1]): Promise<void> {
|
|
103
|
+
// Register before routing can end the response, including when init()'s
|
|
104
|
+
// listener is mounted in Fastify rather than using api/managed.ts.
|
|
105
|
+
const pendingMaintenance = automaticReap(); waitUntil?.(pendingMaintenance)
|
|
106
|
+
try { await routeRequest(req, res) }
|
|
107
|
+
finally { await pendingMaintenance }
|
|
108
|
+
}
|
|
109
|
+
const gcTimer = config.serverless ? null : setInterval(() => {
|
|
110
|
+
void automaticReap()
|
|
111
|
+
}, REAP_INTERVAL_MS)
|
|
112
|
+
rollback.defer(stop)
|
|
113
|
+
|
|
114
|
+
function stop(): void {
|
|
115
|
+
shuttingDown = true
|
|
116
|
+
maintenance.abort()
|
|
117
|
+
if (gcTimer) clearInterval(gcTimer)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async function close(): Promise<void> {
|
|
121
|
+
stop()
|
|
122
|
+
if (inFlight.size > 0) await Promise.allSettled([...inFlight])
|
|
123
|
+
await db.close()
|
|
124
|
+
}
|
|
125
|
+
return { handleRequest, reap, isShuttingDown, stop, close }
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function logManagedStartup(config: ManagedConfig, port: number, mode = 'managed'): void {
|
|
129
|
+
console.log([
|
|
130
|
+
`DeepView managed server (${mode}):`,
|
|
131
|
+
` HTTP: http://${config.host}:${port}/`,
|
|
132
|
+
...managedStorageLines(config),
|
|
133
|
+
].join('\n'))
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export async function init(config = loadManagedConfig()): Promise<Server> {
|
|
137
|
+
const app = await createManagedApp(config)
|
|
138
|
+
const server = createServer(withReap(app.handleRequest, { managed: app.reap }, { isShuttingDown: app.isShuttingDown }))
|
|
139
|
+
let disposal: Promise<void> | undefined
|
|
140
|
+
function dispose(): Promise<void> {
|
|
141
|
+
if (!disposal) {
|
|
142
|
+
app.stop()
|
|
143
|
+
disposal = (async () => {
|
|
144
|
+
try {
|
|
145
|
+
if (server.listening) {
|
|
146
|
+
await new Promise<void>((resolve, reject) => {
|
|
147
|
+
server.close(err => { if (err) reject(err); else resolve() })
|
|
148
|
+
})
|
|
149
|
+
}
|
|
150
|
+
} finally {
|
|
151
|
+
try { await app.close() }
|
|
152
|
+
finally {
|
|
153
|
+
server.off('error', onError)
|
|
154
|
+
server.off('close', onClose)
|
|
155
|
+
server.off('listening', onListening)
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
})()
|
|
159
|
+
}
|
|
160
|
+
return disposal
|
|
161
|
+
}
|
|
162
|
+
// Embedding hosts can await full cleanup even when they only copied the
|
|
163
|
+
// request listener and never bound this server. Repeated disposal is safe.
|
|
164
|
+
server[Symbol.asyncDispose] = dispose
|
|
165
|
+
|
|
166
|
+
function onError(): void { onClose() }
|
|
167
|
+
function onClose(): void {
|
|
168
|
+
if (!disposal) void dispose().catch(err => { console.error('Managed server cleanup failed:', err) })
|
|
169
|
+
}
|
|
170
|
+
function onListening(): void {
|
|
171
|
+
const address = server.address()
|
|
172
|
+
const port = typeof address === 'object' && address ? address.port : config.port
|
|
173
|
+
logManagedStartup(config, port)
|
|
174
|
+
}
|
|
175
|
+
server.on('error', onError)
|
|
176
|
+
server.once('close', onClose)
|
|
177
|
+
server.on('listening', onListening)
|
|
178
|
+
return server
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
export async function start(): Promise<void> {
|
|
182
|
+
const config = loadManagedConfig()
|
|
183
|
+
const server = await init(config)
|
|
184
|
+
startServer(server, config)
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
if (import.meta.main) await start()
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// Answers kept for an hour, by what they answer (npm-insights.ts,
|
|
2
|
+
// npm-provenance.ts), at most `max` of them, the oldest asked going first;
|
|
3
|
+
// one still being asked for is shared, and one that failed, or that `whole`
|
|
4
|
+
// says is partial, is dropped once it is answered. `peek` gives one already
|
|
5
|
+
// answered, asking for nothing; undefined where none is.
|
|
6
|
+
const KEPT_MS = 60 * 60_000
|
|
7
|
+
|
|
8
|
+
export function kept<T>(max: number, whole: (value: T) => boolean = () => true) {
|
|
9
|
+
const answers = new Map<string, { at: number; value: Promise<T>; answered?: { value: T } }>()
|
|
10
|
+
const get = (key: string, ask: () => Promise<T>): Promise<T> => {
|
|
11
|
+
const now = Date.now()
|
|
12
|
+
const hit = answers.get(key)
|
|
13
|
+
if (hit && now - hit.at < KEPT_MS) return hit.value
|
|
14
|
+
const value = ask()
|
|
15
|
+
const entry: { at: number; value: Promise<T>; answered?: { value: T } } = { at: now, value }
|
|
16
|
+
answers.delete(key)
|
|
17
|
+
answers.set(key, entry)
|
|
18
|
+
while (answers.size > max) answers.delete(answers.keys().next().value!)
|
|
19
|
+
const drop = () => { if (answers.get(key) === entry) answers.delete(key) }
|
|
20
|
+
value.then(answer => { if (whole(answer)) entry.answered = { value: answer }; else drop(); return answer }, drop)
|
|
21
|
+
return value
|
|
22
|
+
}
|
|
23
|
+
const peek = (key: string): T | undefined => {
|
|
24
|
+
const hit = answers.get(key)
|
|
25
|
+
return hit?.answered && Date.now() - hit.at < KEPT_MS ? hit.answered.value : undefined
|
|
26
|
+
}
|
|
27
|
+
return Object.assign(get, { peek })
|
|
28
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
// Global deduplication reports live entirely in SQL. The payload is authenticated
|
|
2
|
+
// ciphertext bound to its row, using the deployment's existing storage key.
|
|
3
|
+
import { Buffer } from 'node:buffer'
|
|
4
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
5
|
+
import { type StorageKey, unwrapStorageValue, wrapStorageValue } from '../server-common/storage-crypto.ts'
|
|
6
|
+
import { parseLinkedFindings } from '../client/linked-findings.js'
|
|
7
|
+
import type { ManagedSql } from './sql.ts'
|
|
8
|
+
import { ManagedMutationError } from './management.ts'
|
|
9
|
+
|
|
10
|
+
export const LINK_REPORT_SCHEMA = `
|
|
11
|
+
CREATE TABLE IF NOT EXISTS managed_link_report (
|
|
12
|
+
id TEXT PRIMARY KEY, filename TEXT NOT NULL, sha256 TEXT NOT NULL UNIQUE,
|
|
13
|
+
encrypted_groups TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 1,
|
|
14
|
+
group_count INTEGER NOT NULL, finding_count INTEGER NOT NULL,
|
|
15
|
+
uploaded_by TEXT REFERENCES managed_user(id) ON DELETE SET NULL,
|
|
16
|
+
uploaded_by_login TEXT, uploaded_at INTEGER NOT NULL
|
|
17
|
+
) STRICT;
|
|
18
|
+
`
|
|
19
|
+
export interface LinkReport {
|
|
20
|
+
id: string; filename: string; enabled: boolean; groupCount: number; findingCount: number
|
|
21
|
+
uploadedByLogin: string | null; uploadedAt: number
|
|
22
|
+
}
|
|
23
|
+
export interface LinkReportStore {
|
|
24
|
+
listLegacyLinkReports(): Promise<{ id: string; filename: string }[]>
|
|
25
|
+
migrateLinkReport(id: string, content: string): Promise<boolean>
|
|
26
|
+
listLinkReports(): Promise<LinkReport[]>
|
|
27
|
+
getLinkReport(sessionId: string, id: string): Promise<LinkReport & { groups: string[][] }>
|
|
28
|
+
getLinkRevision(): Promise<string>
|
|
29
|
+
getEnabledLinkGroups(revision: string): Promise<string[][]>
|
|
30
|
+
importLinkReport(sessionId: string, filename: string, content: string): Promise<{ report: LinkReport; reused: boolean }>
|
|
31
|
+
setLinkReportEnabled(sessionId: string, id: string, enabled: boolean): Promise<void>
|
|
32
|
+
}
|
|
33
|
+
const columns = `id, filename, enabled, group_count AS groupCount, finding_count AS findingCount,
|
|
34
|
+
uploaded_by_login AS uploadedByLogin, uploaded_at AS uploadedAt`
|
|
35
|
+
const identity = (id: string) => `managed_link_report.encrypted_groups:${id}`
|
|
36
|
+
function decryptGroups(key: StorageKey | null, id: string, payload: string): string[][] {
|
|
37
|
+
if (!key) throw new ManagedMutationError(503, 'storage-encryption-required')
|
|
38
|
+
const plain = unwrapStorageValue(key, identity(id), payload)
|
|
39
|
+
try { return JSON.parse(plain.toString('utf8')) as string[][] } finally { plain.fill(0) }
|
|
40
|
+
}
|
|
41
|
+
export async function linkRevision(db: ManagedSql): Promise<string> {
|
|
42
|
+
const rows = await db.prepare('SELECT id, sha256 FROM managed_link_report WHERE enabled = 1 ORDER BY id').all()
|
|
43
|
+
return rows.length > 0 ? createHash('sha256').update(JSON.stringify(rows)).digest('base64url') : ''
|
|
44
|
+
}
|
|
45
|
+
function parseLinkReport(content: string) {
|
|
46
|
+
const parsed = parseLinkedFindings(content)
|
|
47
|
+
if (parsed) return parsed
|
|
48
|
+
// Older managed exports may carry the normalized links envelope.
|
|
49
|
+
try {
|
|
50
|
+
const data = JSON.parse(content)
|
|
51
|
+
if (data?.source !== 'links' || !Array.isArray(data.links) || !Array.isArray(data.findings) || data.findings.length > 0) return null
|
|
52
|
+
return data.links.length === 0 ? { groups: [], skipped: 0 } : parseLinkedFindings(JSON.stringify(data.links))
|
|
53
|
+
} catch { return null }
|
|
54
|
+
}
|
|
55
|
+
export function linkReportMethods(db: ManagedSql, key: StorageKey | null): LinkReportStore {
|
|
56
|
+
const map = (row: Omit<LinkReport, 'enabled'> & { enabled: number }): LinkReport => ({ ...row, enabled: row.enabled === 1 })
|
|
57
|
+
async function authorize(sessionId: string) {
|
|
58
|
+
const user = await db.prepare(`SELECT u.id, u.login, u.role FROM managed_session s JOIN managed_user u ON u.id = s.user_id
|
|
59
|
+
WHERE s.id = ? AND s.expires_at > ?`).get(sessionId, Date.now()) as { id: string; login: string; role: string } | undefined
|
|
60
|
+
if (!user) throw new ManagedMutationError(401, 'unauthenticated')
|
|
61
|
+
if (user.role !== 'admin') throw new ManagedMutationError(403, 'forbidden')
|
|
62
|
+
return user
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
async listLegacyLinkReports() {
|
|
66
|
+
return await db.prepare("SELECT id, filename FROM managed_report WHERE analyzer = 'links' ORDER BY id").all() as { id: string; filename: string }[]
|
|
67
|
+
},
|
|
68
|
+
async migrateLinkReport(id, content) {
|
|
69
|
+
const row = await db.prepare(`SELECT filename, sha256, visible, uploaded_by AS uploadedBy,
|
|
70
|
+
uploaded_by_login AS uploadedByLogin, uploaded_at AS uploadedAt FROM managed_report WHERE id = ? AND analyzer = 'links'`).get(id) as {
|
|
71
|
+
filename: string; sha256: string; visible: number; uploadedBy: string | null; uploadedByLogin: string | null; uploadedAt: number
|
|
72
|
+
} | undefined
|
|
73
|
+
if (!row) return false
|
|
74
|
+
if (!key) throw new Error('Managed link reports require MANAGED_STORAGE_ENCRYPTION_KEY')
|
|
75
|
+
if (createHash('sha256').update(content).digest('base64url') !== row.sha256) throw new Error('Legacy link report content changed')
|
|
76
|
+
const parsed = parseLinkReport(content)
|
|
77
|
+
if (!parsed) throw new Error('Invalid legacy link report')
|
|
78
|
+
const plain = Buffer.from(JSON.stringify(parsed.groups))
|
|
79
|
+
try {
|
|
80
|
+
const sha256 = createHash('sha256').update(plain).digest('base64url')
|
|
81
|
+
await db.prepare(`INSERT INTO managed_link_report
|
|
82
|
+
(id, filename, sha256, encrypted_groups, enabled, group_count, finding_count, uploaded_by, uploaded_by_login, uploaded_at)
|
|
83
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(sha256) DO NOTHING`).run(
|
|
84
|
+
id, row.filename, sha256, wrapStorageValue(key, identity(id), plain), row.visible,
|
|
85
|
+
parsed.groups.length, new Set(parsed.groups.flat()).size, row.uploadedBy, row.uploadedByLogin, row.uploadedAt)
|
|
86
|
+
await db.prepare('DELETE FROM managed_report WHERE id = ?').run(id)
|
|
87
|
+
} finally { plain.fill(0) }
|
|
88
|
+
return true
|
|
89
|
+
},
|
|
90
|
+
async listLinkReports() {
|
|
91
|
+
return (await db.prepare(`SELECT ${columns} FROM managed_link_report ORDER BY uploaded_at DESC, id`).all() as Parameters<typeof map>[0][]).map(map)
|
|
92
|
+
},
|
|
93
|
+
async getLinkReport(sessionId, id) {
|
|
94
|
+
await authorize(sessionId)
|
|
95
|
+
const row = await db.prepare(`SELECT ${columns}, encrypted_groups AS payload FROM managed_link_report WHERE id = ?`).get(id) as (Parameters<typeof map>[0] & { payload: string }) | undefined
|
|
96
|
+
if (!row) throw new ManagedMutationError(404, 'no-link-report')
|
|
97
|
+
const { payload, ...report } = row
|
|
98
|
+
return { ...map(report), groups: decryptGroups(key, id, payload) }
|
|
99
|
+
},
|
|
100
|
+
getLinkRevision: () => linkRevision(db),
|
|
101
|
+
async getEnabledLinkGroups(revision) {
|
|
102
|
+
if (await linkRevision(db) !== revision) throw new ManagedMutationError(409, 'deduplication-changed')
|
|
103
|
+
const rows = await db.prepare('SELECT id, encrypted_groups AS payload FROM managed_link_report WHERE enabled = 1 ORDER BY id').all() as { id: string; payload: string }[]
|
|
104
|
+
if (rows.length > 0 && !key) throw new Error('Managed link reports require MANAGED_STORAGE_ENCRYPTION_KEY')
|
|
105
|
+
return rows.flatMap(row => decryptGroups(key, row.id, row.payload))
|
|
106
|
+
},
|
|
107
|
+
async importLinkReport(sessionId, filename, content) {
|
|
108
|
+
const user = await authorize(sessionId)
|
|
109
|
+
if (!key) throw new ManagedMutationError(503, 'storage-encryption-required')
|
|
110
|
+
const parsed = parseLinkReport(content)
|
|
111
|
+
if (!parsed || parsed.skipped) throw new ManagedMutationError(400, 'invalid-link-report')
|
|
112
|
+
const plain = Buffer.from(JSON.stringify(parsed.groups))
|
|
113
|
+
const sha256 = createHash('sha256').update(plain).digest('base64url')
|
|
114
|
+
try {
|
|
115
|
+
const existing = await db.prepare(`SELECT ${columns} FROM managed_link_report WHERE sha256 = ?`).get(sha256) as Parameters<typeof map>[0] | undefined
|
|
116
|
+
if (existing) return { report: map(existing), reused: true }
|
|
117
|
+
const id = randomUUID(), uploadedAt = Date.now()
|
|
118
|
+
const findingCount = new Set(parsed.groups.flat()).size, groupCount = parsed.groups.length
|
|
119
|
+
await db.prepare(`INSERT INTO managed_link_report
|
|
120
|
+
(id, filename, sha256, encrypted_groups, group_count, finding_count, uploaded_by, uploaded_by_login, uploaded_at)
|
|
121
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, filename, sha256, wrapStorageValue(key, identity(id), plain), groupCount, findingCount, user.id, user.login, uploadedAt)
|
|
122
|
+
return { report: { id, filename, enabled: true, groupCount, findingCount, uploadedByLogin: user.login, uploadedAt }, reused: false }
|
|
123
|
+
} finally { plain.fill(0) }
|
|
124
|
+
},
|
|
125
|
+
async setLinkReportEnabled(sessionId, id, enabled) {
|
|
126
|
+
await authorize(sessionId)
|
|
127
|
+
if (!(await db.prepare('UPDATE managed_link_report SET enabled = ? WHERE id = ?').run(+enabled, id)).changes) throw new ManagedMutationError(404, 'no-link-report')
|
|
128
|
+
},
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Merge complete components BEFORE projecting them onto the response. An ID
|
|
133
|
+
// absent from every returned report can still bridge two visible findings.
|
|
134
|
+
export function mergeLinkGroups(groups: readonly string[][], present?: ReadonlySet<string>): string[][] {
|
|
135
|
+
const parents = new Map<string, string>()
|
|
136
|
+
const root = (id: string): string => {
|
|
137
|
+
if (!parents.has(id)) parents.set(id, id)
|
|
138
|
+
let value = id
|
|
139
|
+
while (parents.get(value) !== value) {
|
|
140
|
+
parents.set(value, parents.get(parents.get(value)!)!)
|
|
141
|
+
value = parents.get(value)!
|
|
142
|
+
}
|
|
143
|
+
return value
|
|
144
|
+
}
|
|
145
|
+
for (const ids of groups) {
|
|
146
|
+
if (ids.length === 0) continue
|
|
147
|
+
const first = root(ids[0]!)
|
|
148
|
+
for (const id of ids.slice(1)) { const other = root(id); if (other !== first) parents.set(other, first) }
|
|
149
|
+
}
|
|
150
|
+
const merged = new Map<string, string[]>()
|
|
151
|
+
for (const id of parents.keys()) {
|
|
152
|
+
if (present && !present.has(id)) continue
|
|
153
|
+
const key = root(id)
|
|
154
|
+
const ids = merged.get(key) ?? []
|
|
155
|
+
ids.push(id); merged.set(key, ids)
|
|
156
|
+
}
|
|
157
|
+
return [...merged.values()].filter(ids => ids.length >= 2)
|
|
158
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// The packages a lockfile of npm's ecosystem locks, as @preventive/lockfile
|
|
2
|
+
// reads package-lock.json and npm-shrinkwrap.json of `lockfileVersion: 3`,
|
|
3
|
+
// pnpm-lock.yaml of pnpm 9 to 12 and Yarn 1's yarn.lock, each with where it
|
|
4
|
+
// is installed from (repository-risk.ts).
|
|
5
|
+
import { LockfileError, parseNpmLockfile } from '@preventive/lockfile/npm.js'
|
|
6
|
+
import { parsePnpmLockfile } from '@preventive/lockfile/pnpm.js'
|
|
7
|
+
import { parseYarn1Lockfile } from '@preventive/lockfile/yarn1.js'
|
|
8
|
+
import type { LockfileFormat } from '@preventive/upstream/github.js'
|
|
9
|
+
|
|
10
|
+
export interface LockedPackage {
|
|
11
|
+
name: string
|
|
12
|
+
// None for a git dependency whose package.json has none.
|
|
13
|
+
version: string | null
|
|
14
|
+
// Where it is installed from: npm's registry, another host's tarball with
|
|
15
|
+
// an integrity, such as a company's mirror of npm's, a git repository, a
|
|
16
|
+
// tarball with none, or a file of the project.
|
|
17
|
+
source: 'registry' | 'mirror' | 'git' | 'tarball' | 'file'
|
|
18
|
+
// Where one from elsewhere than npm's registry comes from, without any
|
|
19
|
+
// credentials its URL has.
|
|
20
|
+
from?: string
|
|
21
|
+
// A mirror's tarball's, as the lockfile has it: npm's package of its name
|
|
22
|
+
// and version only where npm's integrity is among it.
|
|
23
|
+
integrity?: string
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// A package as each reader resolves one, its name the one it is published
|
|
27
|
+
// under, and a registry tarball's URL held to its name and version.
|
|
28
|
+
type Resolution =
|
|
29
|
+
| { type: 'tarball'; tarball: string | undefined; integrity: string | undefined; gitHosted?: boolean }
|
|
30
|
+
| { type: 'git'; repo: string }
|
|
31
|
+
| { type: 'directory' }
|
|
32
|
+
interface ReadPackage { name: string; version: string | undefined; resolution: Resolution | undefined }
|
|
33
|
+
|
|
34
|
+
// The packages of a lockfile of `format`, as its reader resolves them.
|
|
35
|
+
// Versions go unchecked against ranges, as only what is locked is asked for.
|
|
36
|
+
function packagesOf(format: LockfileFormat, text: string): ReadPackage[] {
|
|
37
|
+
if (format === 'npm') return Object.values(parseNpmLockfile(text, { checkVersions: false }).packages)
|
|
38
|
+
if (format === 'pnpm') {
|
|
39
|
+
const { lockfile, env } = parsePnpmLockfile(text)
|
|
40
|
+
return [...Object.values(env?.packages ?? {}), ...Object.values(lockfile?.packages ?? {})]
|
|
41
|
+
}
|
|
42
|
+
if (format !== 'yarn') throw new TypeError(`lockedPackages: ${format} lockfiles are not read`)
|
|
43
|
+
// Yarn 2 and later write another format, which only the text tells apart.
|
|
44
|
+
if (/^__metadata:/mu.test(text)) throw new LockfileError('a yarn.lock of Yarn 2 or later, which is not read')
|
|
45
|
+
// Yarn 1 names an alias, `my-q@npm:q@^1.5.1`, by its own name.
|
|
46
|
+
return Object.values(parseYarn1Lockfile(text, { checkVersions: false }).packages)
|
|
47
|
+
.map(pkg => ({ ...pkg, name: /@npm:((?:@[^/@]+\/)?[^/@]+)/u.exec(pkg.patterns[0] ?? '')?.[1] ?? pkg.name }))
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// npm's registry, by either of its names, and yarn's mirror of it, as
|
|
51
|
+
// @preventive/lockfile holds their tarballs to a package's name and version.
|
|
52
|
+
const REGISTRY_HOSTS = new Set(['registry.npmjs.org', 'registry.npmjs.com', 'registry.yarnpkg.com'])
|
|
53
|
+
const isRegistry = (url: URL) => url.protocol === 'https:' && !url.port && !url.username && !url.password
|
|
54
|
+
&& REGISTRY_HOSTS.has(url.hostname.replace(/\.+$/u, ''))
|
|
55
|
+
// Where a URL is, as it may be shown: its credentials, query and fragment
|
|
56
|
+
// left out.
|
|
57
|
+
const shown = (url: URL | null) => url ? { from: `${url.protocol}//${url.host}${url.pathname}` } : {}
|
|
58
|
+
|
|
59
|
+
// Where a package read comes from (LockedPackage); null for one a package
|
|
60
|
+
// bundles, with no resolution, or one of the project's own directories.
|
|
61
|
+
function locked({ name, version, resolution }: ReadPackage): LockedPackage | null {
|
|
62
|
+
const at = { name, version: version ?? null }
|
|
63
|
+
if (resolution === undefined || resolution.type === 'directory') return null
|
|
64
|
+
if (resolution.type === 'git') return { ...at, source: 'git', ...shown(URL.parse(resolution.repo)) }
|
|
65
|
+
const { tarball, integrity } = resolution
|
|
66
|
+
if (resolution.gitHosted) return { ...at, source: 'git', ...shown(URL.parse(tarball ?? '')) }
|
|
67
|
+
if (tarball === undefined) return { ...at, source: 'registry' }
|
|
68
|
+
if (tarball.startsWith('file:')) return { ...at, source: 'file' }
|
|
69
|
+
const url = URL.parse(tarball)
|
|
70
|
+
if (url && isRegistry(url)) return { ...at, source: 'registry' }
|
|
71
|
+
return integrity ? { ...at, ...shown(url), source: 'mirror', integrity } : { ...at, ...shown(url), source: 'tarball' }
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// The packages a lockfile of `format` (npm, pnpm or yarn, as
|
|
75
|
+
// findRepoLockfiles names them) locks, each once, in the lockfile's order.
|
|
76
|
+
// Throws the reader's LockfileError or YamlError for one it refuses.
|
|
77
|
+
export function lockedPackages(format: LockfileFormat, text: string): LockedPackage[] {
|
|
78
|
+
const found = new Map<string, LockedPackage>()
|
|
79
|
+
for (const pkg of packagesOf(format, text)) {
|
|
80
|
+
const one = locked(pkg)
|
|
81
|
+
const key = one && `${one.source}\0${one.name}@${one.version ?? ''}\0${one.from ?? ''}\0${one.integrity ?? ''}`
|
|
82
|
+
if (one && key && !found.has(key)) found.set(key, one)
|
|
83
|
+
}
|
|
84
|
+
return [...found.values()]
|
|
85
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// Values by key, the least recently used going first once their weights
|
|
2
|
+
// (`weight`, 1 each unless given) are past `limit`.
|
|
3
|
+
export interface Lru<V> { get(key: string): V | undefined; set(key: string, value: V): void }
|
|
4
|
+
export function lru<V>(limit: number, weight: (value: V) => number = () => 1): Lru<V> {
|
|
5
|
+
const kept = new Map<string, V>()
|
|
6
|
+
let total = 0
|
|
7
|
+
const forget = (key: string) => {
|
|
8
|
+
const value = kept.get(key)
|
|
9
|
+
if (value === undefined) return
|
|
10
|
+
kept.delete(key)
|
|
11
|
+
total -= weight(value)
|
|
12
|
+
}
|
|
13
|
+
return {
|
|
14
|
+
get(key) {
|
|
15
|
+
const value = kept.get(key)
|
|
16
|
+
if (value !== undefined) { kept.delete(key); kept.set(key, value) }
|
|
17
|
+
return value
|
|
18
|
+
},
|
|
19
|
+
set(key, value) {
|
|
20
|
+
forget(key)
|
|
21
|
+
if (weight(value) > limit) return
|
|
22
|
+
kept.set(key, value)
|
|
23
|
+
total += weight(value)
|
|
24
|
+
for (const first of kept.keys()) {
|
|
25
|
+
if (total <= limit) break
|
|
26
|
+
forget(first)
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type { ManagedSql } from './sql.ts'
|
|
2
|
+
import type { TeamReportAccessSnapshot } from './db.ts'
|
|
3
|
+
import { parseGithubPrUrl } from '../common/github-pr.ts'
|
|
4
|
+
|
|
5
|
+
// One immutable reference per finding, shared across reports and teams. Keep
|
|
6
|
+
// reservations across crashes/uncertain GitHub responses to prevent duplicates.
|
|
7
|
+
// No triage/import/admin patch accepts these fields.
|
|
8
|
+
export const MANAGED_ISSUE_SCHEMA = `
|
|
9
|
+
CREATE TABLE IF NOT EXISTS managed_finding_issue (
|
|
10
|
+
finding_id TEXT PRIMARY KEY,
|
|
11
|
+
repo_id INTEGER NOT NULL,
|
|
12
|
+
repository TEXT NOT NULL,
|
|
13
|
+
request_id TEXT NOT NULL,
|
|
14
|
+
issue_url TEXT,
|
|
15
|
+
auto_fix_url TEXT,
|
|
16
|
+
auto_fix_checked_at INTEGER,
|
|
17
|
+
created_by TEXT REFERENCES managed_user(id) ON DELETE SET NULL,
|
|
18
|
+
created_at INTEGER NOT NULL
|
|
19
|
+
) STRICT;
|
|
20
|
+
`
|
|
21
|
+
export interface ManagedIssue {
|
|
22
|
+
findingId: string
|
|
23
|
+
repoId: number
|
|
24
|
+
repository: string
|
|
25
|
+
requestId: string
|
|
26
|
+
issueUrl: string | null
|
|
27
|
+
autoFixUrl: string | null
|
|
28
|
+
autoFixCheckedAt: number | null
|
|
29
|
+
createdBy: string | null
|
|
30
|
+
createdAt: number
|
|
31
|
+
}
|
|
32
|
+
export interface ManagedIssueStore {
|
|
33
|
+
getManagedIssue(findingId: string): Promise<ManagedIssue | null>
|
|
34
|
+
listManagedIssues(findingIds: readonly string[]): Promise<ManagedIssue[]>
|
|
35
|
+
claimManagedIssue(issue: Omit<ManagedIssue, 'issueUrl' | 'autoFixUrl' | 'autoFixCheckedAt'>): Promise<boolean>
|
|
36
|
+
finishManagedIssue(findingId: string, requestId: string, url: string): Promise<boolean>
|
|
37
|
+
releaseManagedIssue(findingId: string, requestId: string): Promise<void>
|
|
38
|
+
setManagedIssueAutoFix(findingId: string, issueUrl: string, previous: string | null, next: string | null, checkedAt: number): Promise<boolean>
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// A repeated finding ID never grants access to another team's repository.
|
|
42
|
+
export function visibleManagedIssues(issues: ManagedIssue[], snapshot: TeamReportAccessSnapshot) {
|
|
43
|
+
if (snapshot.user.id.startsWith('share:')) return []
|
|
44
|
+
const repos = new Set(snapshot.repositories.map(repo => repo.repoId))
|
|
45
|
+
const names = new Set(snapshot.repositories.map(repo => repo.github.toLowerCase()))
|
|
46
|
+
return issues.filter(issue => issue.issueUrl && repos.has(issue.repoId)).map(issue => {
|
|
47
|
+
const ref = parseGithubPrUrl(issue.autoFixUrl)
|
|
48
|
+
return { ...issue, autoFixUrl: ref && names.has(ref.repo.toLowerCase()) ? issue.autoFixUrl : null }
|
|
49
|
+
})
|
|
50
|
+
}
|
|
51
|
+
export function managedIssueMethods(db: ManagedSql): ManagedIssueStore {
|
|
52
|
+
const fields = `finding_id AS findingId, repo_id AS repoId, repository, request_id AS requestId,
|
|
53
|
+
issue_url AS issueUrl, auto_fix_url AS autoFixUrl, auto_fix_checked_at AS autoFixCheckedAt, created_by AS createdBy, created_at AS createdAt`
|
|
54
|
+
const get = db.prepare(`SELECT ${fields} FROM managed_finding_issue WHERE finding_id = ?`)
|
|
55
|
+
const list = db.prepare(`SELECT ${fields} FROM managed_finding_issue WHERE finding_id IN (SELECT value FROM json_each(?)) ORDER BY finding_id`)
|
|
56
|
+
const autoFix = db.prepare(`UPDATE managed_finding_issue SET auto_fix_url = ?, auto_fix_checked_at = ?
|
|
57
|
+
WHERE finding_id = ? AND issue_url = ? AND COALESCE(auto_fix_url, '') = COALESCE(?, '')
|
|
58
|
+
AND (auto_fix_checked_at IS NULL OR auto_fix_checked_at <= ?)`)
|
|
59
|
+
const claim = db.prepare(`INSERT INTO managed_finding_issue (finding_id, repo_id, repository, request_id, created_by, created_at)
|
|
60
|
+
VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (finding_id) DO NOTHING`)
|
|
61
|
+
const finish = db.prepare(`UPDATE managed_finding_issue SET issue_url = ? WHERE finding_id = ? AND request_id = ? AND issue_url IS NULL`)
|
|
62
|
+
const release = db.prepare(`DELETE FROM managed_finding_issue WHERE finding_id = ? AND request_id = ? AND issue_url IS NULL`)
|
|
63
|
+
return {
|
|
64
|
+
async getManagedIssue(findingId) { return (await get.get(findingId) as ManagedIssue | undefined) ?? null },
|
|
65
|
+
async listManagedIssues(ids) { return await list.all(JSON.stringify(ids)) as ManagedIssue[] },
|
|
66
|
+
async setManagedIssueAutoFix(findingId, issueUrl, previous, next, checkedAt) {
|
|
67
|
+
return Number((await autoFix.run(next, checkedAt, findingId, issueUrl, previous, checkedAt)).changes) > 0
|
|
68
|
+
},
|
|
69
|
+
async claimManagedIssue(issue) {
|
|
70
|
+
return Number((await claim.run(issue.findingId, issue.repoId, issue.repository, issue.requestId, issue.createdBy, issue.createdAt)).changes) > 0
|
|
71
|
+
},
|
|
72
|
+
async finishManagedIssue(findingId, requestId, url) { return Number((await finish.run(url, findingId, requestId)).changes) > 0 },
|
|
73
|
+
async releaseManagedIssue(findingId, requestId) { await release.run(findingId, requestId) },
|
|
74
|
+
}
|
|
75
|
+
}
|