@preventive/triage 1.0.0-alpha.26 → 1.0.0-alpha.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/LICENSE-APACHE +202 -0
  2. package/client/linked-findings.js +1 -1
  3. package/common/bundle-metadata.js +12 -10
  4. package/common/code-language.js +17 -2
  5. package/common/default-scan-models.ts +3 -3
  6. package/common/finding-id.js +1 -1
  7. package/common/managed/report-content.ts +1 -1
  8. package/common/managed/report-filter.ts +1 -1
  9. package/out/client-managed-import.js +24 -24
  10. package/out/client-managed.js +93 -68
  11. package/out/graph.js +11 -11
  12. package/out/index.html +1 -1
  13. package/out/prism.js +2 -2
  14. package/out/stasis.svg +0 -25
  15. package/out/terminal.js +163 -163
  16. package/out/view.css +1 -1
  17. package/out/view.js +223 -190
  18. package/package.json +29 -29
  19. package/server-common/storage-crypto.ts +182 -0
  20. package/server-common/storage-log.ts +5 -1
  21. package/server-common/vercel-blob.ts +6 -3
  22. package/server-e2e/static.ts +3 -4
  23. package/server-managed/avatar-store.ts +1 -40
  24. package/server-managed/blob-store.ts +3 -52
  25. package/server-managed/bundle-advisories.ts +53 -0
  26. package/server-managed/bundle-advisory-inventory.ts +75 -0
  27. package/server-managed/bundle-build-leases.ts +41 -0
  28. package/server-managed/bundle-build-worker.js +44 -0
  29. package/server-managed/bundle-build.ts +141 -0
  30. package/server-managed/bundle-cache.ts +22 -41
  31. package/server-managed/bundle-store.ts +3 -7
  32. package/server-managed/bundle-summary-cache.ts +27 -9
  33. package/server-managed/bundle.ts +66 -1
  34. package/server-managed/cache-storage.ts +4 -25
  35. package/server-managed/cli.js +10 -2
  36. package/server-managed/config.ts +12 -1
  37. package/server-managed/content-access.ts +1 -1
  38. package/server-managed/db-methods.ts +163 -40
  39. package/server-managed/db-neon.ts +124 -30
  40. package/server-managed/db-schema.ts +2 -0
  41. package/server-managed/db.ts +11 -1
  42. package/server-managed/github-app.ts +10 -15
  43. package/server-managed/github-errors.ts +21 -0
  44. package/server-managed/github-issues.ts +1 -1
  45. package/server-managed/github-metadata.ts +34 -1
  46. package/server-managed/github-pulls.ts +142 -30
  47. package/server-managed/http.ts +278 -167
  48. package/server-managed/index.ts +26 -9
  49. package/server-managed/management-catalog.ts +45 -0
  50. package/server-managed/object-storage-disk.ts +144 -0
  51. package/server-managed/object-storage-vercel.ts +60 -0
  52. package/server-managed/object-storage.ts +71 -0
  53. package/server-managed/public-workspace.ts +20 -14
  54. package/server-managed/report-migration.ts +1 -1
  55. package/server-managed/report-sources.ts +14 -4
  56. package/server-managed/repository-browser.ts +8 -1
  57. package/server-managed/repository-discovery.ts +1 -1
  58. package/server-managed/revisions.ts +35 -0
  59. package/server-managed/solidity-entry-points.ts +69 -0
  60. package/server-managed/sql-postgres.ts +1 -0
  61. package/server-managed/sql.ts +14 -8
  62. package/server-managed/storage-command.ts +16 -0
  63. package/server-managed/storage-db.ts +175 -0
  64. package/server-managed/storage-encryption.ts +211 -0
  65. package/server-managed/storage-maintenance.ts +197 -0
  66. package/server-managed/storage-payload.ts +72 -0
  67. package/server-managed/storage-stores.ts +81 -0
  68. package/server-managed/storage.ts +48 -24
  69. package/server-managed/team-feed.ts +50 -5
  70. package/server-managed/team-reports.ts +85 -22
  71. package/server-managed/uploads.ts +2 -2
  72. package/server-managed/workspace-shares.ts +15 -3
  73. package/report/index.js +0 -260
  74. package/report/src/finding-id.js +0 -80
  75. package/report/src/finding.js +0 -314
  76. package/report/src/labels.js +0 -35
  77. package/report/src/md-structure.js +0 -471
  78. package/report/src/md-text.js +0 -167
  79. package/report/src/meta.js +0 -76
  80. package/report/src/parse-codex.js +0 -149
  81. package/report/src/parse-deepsec.js +0 -199
  82. package/report/src/parse-deepview-fields.js +0 -375
  83. package/report/src/parse-deepview-md.js +0 -186
  84. package/report/src/parse-md-id.js +0 -137
  85. package/report/src/parse-md.js +0 -323
  86. package/report/src/parse-piolium-id.js +0 -79
  87. package/report/src/parse-piolium-rows.js +0 -126
  88. package/report/src/parse-piolium-tokens.js +0 -185
  89. package/report/src/parse-piolium.js +0 -399
  90. package/report/src/security.js +0 -63
  91. package/report/src/severity.js +0 -18
  92. package/report/src/utf8.js +0 -21
  93. package/report/src/write-md-finding.js +0 -273
  94. package/report/src/write-md.js +0 -291
  95. package/server-managed/blob-vercel.ts +0 -127
  96. /package/{LICENSE → LICENSE-MIT} +0 -0
@@ -1,11 +1,12 @@
1
1
  // Team content is filtered as one workspace, with separate report envelopes.
2
- // Only the visibility index is retained in memory for annotations/source reads.
2
+ // Bounded snapshots retain filtered content and visibility for repeated reads.
3
3
  import { Buffer } from 'node:buffer'
4
- import { backfillFindingIds, reportEntries, stampSecurityGroups } from '../report/index.js'
4
+ import { backfillFindingIds, reportEntries, stampSecurityGroups } from '@preventive/report'
5
5
  import { managedFindingSourcePaths, readManagedReport } from '../common/managed/report-content.ts'
6
6
  import { filterReportData, projectFinding } from '../common/managed/report-filter.ts'
7
7
  import type { BlobStore } from './blob-store.ts'
8
8
  import type { ManagedDb, TeamReportAccessSnapshot } from './db.ts'
9
+ import { triageWireEntry } from './triage-response.ts'
9
10
  import { MAX_REPORT_QUERY_BYTES, MAX_REPORT_QUERY_COUNT } from './report-query.ts'
10
11
 
11
12
  type Finding = Record<string, unknown>
@@ -15,6 +16,20 @@ type Visibility = Map<string, ReportVisibility>
15
16
  const VISIBILITY_CACHE_MAX_ITEMS = 250_000
16
17
  const VISIBILITY_CACHE_MAX_SNAPSHOTS = 32
17
18
  const caches = new WeakMap<ManagedDb, Map<string, Visibility>>()
19
+ const REPORT_CACHE_BYTES = 16 * 1024 * 1024
20
+ const reportCaches = new WeakMap<ManagedDb, Map<string, Buffer>>()
21
+ function retainReports(db: ManagedDb, key: string, bytes: Buffer) {
22
+ if (bytes.length > REPORT_CACHE_BYTES) return
23
+ let cache = reportCaches.get(db)
24
+ if (!cache) { cache = new Map(); reportCaches.set(db, cache) }
25
+ cache.delete(key)
26
+ let total = bytes.length + [...cache.values()].reduce((sum, value) => sum + value.length, 0)
27
+ for (const [oldest, value] of cache) {
28
+ if (total <= REPORT_CACHE_BYTES && cache.size < VISIBILITY_CACHE_MAX_SNAPSHOTS) break
29
+ cache.delete(oldest); total -= value.length
30
+ }
31
+ cache.set(key, bytes)
32
+ }
18
33
  function visibilityWeight(visible: Visibility): number {
19
34
  return [...visible.values()].reduce((sum, v) => sum + v.ids.size + v.sourcePaths.size, 0)
20
35
  }
@@ -61,24 +76,30 @@ function linksOf(data: unknown): string[][] | null {
61
76
  if (!record.links.every(ids => Array.isArray(ids) && ids.every(id => typeof id === 'string'))) throw new TeamReportsError(422, 'unreadable-links')
62
77
  return record.links
63
78
  }
64
- async function loadTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<{ reports: TeamReport[]; visible: Visibility }> {
79
+ async function buildTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<{ body: Buffer; visible: Visibility }> {
65
80
  if (snapshot.reports.length > MAX_REPORT_QUERY_COUNT || snapshot.reports.reduce((n, r) => n + r.byteSize, 0) > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
66
81
  const reports: TeamReport[] = []
67
- let inputBytes = 0
68
- // Retain parsed reports for cross-report classification; release raw bytes
69
- // after each parse. No unbounded burst of remote reads or duplicate buffers.
70
- for (const access of snapshot.reports) {
71
- const bytes = await store.get(access.id)
72
- if (!bytes) throw new TeamReportsError(503, 'unavailable')
73
- inputBytes += bytes.length
74
- if (inputBytes > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
75
- const parsed = readManagedReport(bytes.toString('utf8'), access.filename)
76
- if (!parsed.data) throw new TeamReportsError(422, 'unreadable-report')
77
- // Backfill before filtering so links, annotations and the client agree on
78
- // IDs even when hiding a component would change the original row shape.
79
- await backfillFindingIds(groupsOf(parsed.data).flat())
80
- reports.push({ id: access.id, filename: access.filename, data: parsed.data, repo: access.repo })
81
- }
82
+ let inputBytes = 0, next = 0
83
+ // Bound both remote reads and raw buffers, retaining stable report order for
84
+ // the cross-report classification that follows. Finish workers on failure.
85
+ const results = await Promise.allSettled(Array.from({ length: Math.min(4, snapshot.reports.length) }, async () => {
86
+ while (next < snapshot.reports.length) {
87
+ const index = next++
88
+ const access = snapshot.reports[index]!
89
+
90
+ const bytes = await store.get(access.id)
91
+ if (!bytes) throw new TeamReportsError(503, 'unavailable')
92
+ inputBytes += bytes.length
93
+ if (inputBytes > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
94
+ const parsed = readManagedReport(bytes.toString('utf8'), access.filename)
95
+ if (!parsed.data) throw new TeamReportsError(422, 'unreadable-report')
96
+ // Backfill before filtering so links, annotations and the client agree on
97
+ // IDs even when hiding a component would change the original row shape.
98
+ await backfillFindingIds(groupsOf(parsed.data).flat())
99
+ reports[index] = { id: access.id, filename: access.filename, data: parsed.data, repo: access.repo }
100
+ }
101
+ }))
102
+ for (const result of results) if (result.status === 'rejected') throw result.reason
82
103
  const edges = new Map<string, Set<string>>()
83
104
  const connect = (a: string, b: string) => {
84
105
  if (!edges.has(a)) edges.set(a, new Set())
@@ -112,15 +133,41 @@ async function loadTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: Team
112
133
  if (links) report.data = { source: 'links', findings: [], links: links.map(ids => [...new Set(ids.filter(id => allIds.has(id)))]).filter(ids => ids.length >= 2) }
113
134
  }
114
135
  let outputBytes = 14
115
- for (const report of reports) {
116
- outputBytes += Buffer.byteLength(JSON.stringify(report)) + 1
136
+ const parts = [Buffer.from('{"reports":[')]
137
+ for (const [index, report] of reports.entries()) {
138
+ const bytes = Buffer.from(JSON.stringify(report))
139
+ outputBytes += bytes.length + 1
117
140
  if (outputBytes > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
141
+ if (index) parts.push(Buffer.from(','))
142
+ parts.push(bytes)
118
143
  }
144
+ parts.push(Buffer.from(']}'))
145
+ const body = Buffer.concat(parts)
119
146
  retainVisibility(db, teamSnapshotKey(snapshot), visible)
120
- return { reports, visible }
147
+ retainReports(db, teamSnapshotKey(snapshot), body)
148
+ return { body, visible }
149
+ }
150
+ const pendingWorkspaces = new WeakMap<ManagedDb, Map<string, ReturnType<typeof buildTeamWorkspace>>>()
151
+ async function loadTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot) {
152
+ let pending = pendingWorkspaces.get(db)
153
+ if (!pending) { pending = new Map(); pendingWorkspaces.set(db, pending) }
154
+ const key = teamSnapshotKey(snapshot)
155
+ const existing = pending.get(key)
156
+ if (existing) return existing
157
+ const job = buildTeamWorkspace(db, store, snapshot)
158
+ pending.set(key, job)
159
+ try { return await job } finally { pending.delete(key) }
160
+ }
161
+ // Treat the encoded body as immutable and recheck access before sending it.
162
+ // Warm HTTP reads need neither a parsed copy nor another serialization pass.
163
+ export async function loadTeamReportsResponse(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<Buffer> {
164
+ const cached = reportCaches.get(db)?.get(teamSnapshotKey(snapshot))
165
+ return cached ?? (await loadTeamWorkspace(db, store, snapshot)).body
121
166
  }
122
167
  export async function loadTeamReports(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<TeamReport[]> {
123
- return (await loadTeamWorkspace(db, store, snapshot)).reports
168
+ // Object consumers get their own copy, including when sharing a cold load.
169
+ const body = await loadTeamReportsResponse(db, store, snapshot)
170
+ return (JSON.parse(body.toString('utf8')) as { reports: TeamReport[] }).reports
124
171
  }
125
172
  async function teamVisibility(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<Visibility> {
126
173
  const key = teamSnapshotKey(snapshot)
@@ -154,3 +201,19 @@ export async function teamSourcePaths(db: ManagedDb, store: BlobStore, sessionId
154
201
  await recheckTeam(db, sessionId, snapshot)
155
202
  return visible.sourcePaths
156
203
  }
204
+
205
+ // Serialize shared annotation bodies once. Reports carry only the annotated
206
+ // finding IDs visible in that report; callers revalidate access before sending.
207
+ export async function loadTeamAnnotations(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot, reportId: string | null = null) {
208
+ // Restrict annotation reads only after classifying the complete workspace:
209
+ // links and findings in other reports can affect this report's visibility.
210
+ const visible = reportId === null ? await teamVisibility(db, store, snapshot)
211
+ : new Map([[reportId, await teamReportVisibility(db, store, snapshot, reportId)]])
212
+ const ids = new Set<string>()
213
+ for (const report of visible.values()) for (const id of report.ids) ids.add(id)
214
+ const { triage, comments } = await db.getAnnotations([...ids])
215
+ const entries = Object.fromEntries(triage.map(row => [row.findingId, triageWireEntry(row)]))
216
+ const annotated = new Set([...triage.map(row => row.findingId), ...comments.map(comment => comment.findingId)])
217
+ const reports = Object.fromEntries([...visible].map(([id, report]) => [id, [...report.ids].filter(finding => annotated.has(finding))]))
218
+ return { reports, entries, comments }
219
+ }
@@ -38,8 +38,8 @@ export async function deleteUpload(store: BlobStore, session: string, kind: Uplo
38
38
  if (errors.length > 0) throw new AggregateError(errors, 'Upload cleanup failed')
39
39
  }
40
40
 
41
- export function putUploadPart(store: BlobStore, session: string, kind: UploadKind, id: string, index: number, bytes: Buffer): Promise<void> {
42
- return store.put(partId(session, kind, id, index), bytes)
41
+ export async function putUploadPart(store: BlobStore, session: string, kind: UploadKind, id: string, index: number, bytes: Buffer): Promise<void> {
42
+ await store.put(partId(session, kind, id, index), bytes)
43
43
  }
44
44
 
45
45
  export async function readUpload(store: BlobStore, req: IncomingMessage, session: string, kind: UploadKind, maxBytes: number): Promise<Buffer> {
@@ -31,6 +31,7 @@ export interface ManagedWorkspaceShare extends WorkspaceShareInfo {
31
31
  teamName: string
32
32
  teamSlug: string
33
33
  }
34
+ export type WorkspaceShareFeedState = { grant: string; catalog: number; annotations: number }
34
35
  export interface WorkspaceShareStore {
35
36
  createWorkspaceShare(sessionId: string, now: number, teamId: string, tokenHash: string, permissions?: TeamUserPermissions): Promise<boolean>
36
37
  listWorkspaceShares(sessionId: string, now: number, teamId: string): Promise<WorkspaceShareInfo[] | null>
@@ -38,6 +39,7 @@ export interface WorkspaceShareStore {
38
39
  updateWorkspaceShare(sessionId: string, now: number, teamId: string, id: string, permissions: TeamUserPermissions): Promise<boolean>
39
40
  revokeWorkspaceShares(sessionId: string, now: number, teamId: string, id?: string): Promise<boolean>
40
41
  getWorkspaceShare(tokenHash: string): Promise<WorkspaceShareSnapshot | null>
42
+ getWorkspaceShareFeedState(tokenHash: string): Promise<WorkspaceShareFeedState | null>
41
43
  }
42
44
 
43
45
  // A share is a capability, never a user session. Its issuer must still manage
@@ -63,11 +65,17 @@ function shareQueries(db: ManagedSql) {
63
65
  JOIN managed_user u ON u.id = s.created_by JOIN managed_team t ON t.id = s.team_id
64
66
  WHERE ? = 'admin' OR EXISTS (SELECT 1 FROM managed_team_user tu WHERE tu.user_id = ? AND tu.team_id = s.team_id)
65
67
  ORDER BY t.name, t.id, s.created_at DESC, s.token_hash`)
66
- const share = db.prepare(`SELECT t.id, t.slug, t.name, s.dependencies, s.security FROM managed_workspace_share s
68
+ const shareGrant = `FROM managed_workspace_share s
67
69
  JOIN managed_user u ON u.id = s.created_by
68
70
  JOIN managed_team t ON t.id = s.team_id
69
71
  WHERE s.token_hash = ? AND (u.role = 'admin' OR (u.role = 'manage'
70
- AND EXISTS (SELECT 1 FROM managed_team_user tu WHERE tu.user_id = u.id AND tu.team_id = s.team_id)))`)
72
+ AND EXISTS (SELECT 1 FROM managed_team_user tu WHERE tu.user_id = u.id AND tu.team_id = s.team_id)))`
73
+ const share = db.prepare(`SELECT t.id, t.slug, t.name, s.dependencies, s.security ${shareGrant}`)
74
+ // Validate the capability and its issuer every poll; global counters are
75
+ // internal hints that let unchanged feeds skip rebuilding the full catalog.
76
+ const feed = db.prepare(`SELECT t.id, s.dependencies, s.security,
77
+ (SELECT catalog FROM managed_change_revision WHERE id = 1) AS catalog,
78
+ (SELECT annotations FROM managed_change_revision WHERE id = 1) AS annotations ${shareGrant}`)
71
79
  const repositories = db.prepare(`SELECT tr.repo_id AS repoId, sr.full_name AS github, tr.path
72
80
  FROM managed_team_repo tr JOIN managed_selected_repo sr ON sr.repo_id = tr.repo_id
73
81
  WHERE tr.team_id = ? ORDER BY tr.repo_id, tr.path`)
@@ -85,7 +93,7 @@ function shareQueries(db: ManagedSql) {
85
93
  JOIN managed_selected_repo sr ON sr.repo_id = b.repo_id
86
94
  WHERE tr.team_id = ? AND (tr.path = '' OR b.repo_directory = tr.path
87
95
  OR substr(b.repo_directory, 1, length(tr.path) + 1) = tr.path || '/') ORDER BY b.id`)
88
- return { manager, insert, remove, update, list, session, all, share, repositories, reports, bundles }
96
+ return { manager, insert, remove, update, list, session, all, share, feed, repositories, reports, bundles }
89
97
  }
90
98
 
91
99
  function shareInfo<T extends { dependencies: number; security: number }>(row: T) {
@@ -123,6 +131,10 @@ export function workspaceShareMethods(db: ManagedSql): WorkspaceShareStore {
123
131
  const result = await q.remove.run(teamId, id ?? null, id ?? null)
124
132
  return id === undefined || result.changes > 0
125
133
  },
134
+ async getWorkspaceShareFeedState(tokenHash) {
135
+ const row = await q.feed.get(tokenHash) as { id: string; dependencies: number; security: number; catalog: number; annotations: number } | undefined
136
+ return row ? { grant: JSON.stringify([row.id, row.dependencies, row.security]), catalog: row.catalog, annotations: row.annotations } : null
137
+ },
126
138
  async getWorkspaceShare(tokenHash) {
127
139
  const grant = await q.share.get(tokenHash) as { id: string; slug: string; name: string; dependencies: number; security: number } | undefined
128
140
  if (!grant) return null
package/report/index.js DELETED
@@ -1,260 +0,0 @@
1
- // The report library — one door to every report format this project
2
- // reads, and to the one it writes.
3
- //
4
- // A "report" is whatever an analyzer wrote: the JSON dump this
5
- // project's own analyzer emits, or one of the shapes other tools
6
- // produce — DeepSec and Piolium markdown, Claude Security markdown,
7
- // Codex CSV — or the markdown document this library itself writes
8
- // (write-md.js), read back by parse-deepview-md.js. The parsers beside
9
- // this file each recognise exactly one of those and know nothing about
10
- // each other; this module is the dispatch over them, so "which formats
11
- // do we read, and in what order" is answered in one place instead of
12
- // once per call site.
13
- //
14
- // import { loadFindings, writeMarkdown } from '../report/index.js'
15
- // const report = await loadFindings(text)
16
- // // → { format, data, findings: [ … with ids ] } | null
17
- // const md = writeMarkdown({ title, groups: report.findings.map((f) => [f]) })
18
- //
19
- // Three entry points for reading, in rising order of how much they do:
20
- //
21
- // detectFormat — name the format, parse nothing further
22
- // readReport — the parsed report, or the reason it isn't one
23
- // loadFindings — parsed, flattened, and every finding carrying an id
24
- //
25
- // `analyzeReport` is `readReport` for a file list — entry count and
26
- // producer — `reportEntries` is a report's entry list whichever of the
27
- // two names it goes under (`findings` or `groups`), for a caller that
28
- // has to keep the grouping rather than flatten it, and
29
- // `backfillFindingIds` is the id step on its own, for a caller that
30
- // has to interleave something with it.
31
- //
32
- // And one for writing: `writeMarkdown` takes findings — the parsers'
33
- // own objects, grouped as the viewer groups them — with whatever the
34
- // caller knows about the selection and the reader's annotations, and
35
- // writes the markdown document the Download button saves (write-md.js
36
- // for the document, write-md-finding.js for one finding, labels.js for
37
- // the words). Both directions read a finding through finding.js, so
38
- // what a parser produced and what the writer prints agree on what a
39
- // finding IS — and the document is itself a report the readers above
40
- // take (parse-deepview-md.js), so what was written can be loaded
41
- // again, with the ids and fields it left with.
42
- //
43
- // Codex is the one format the content doesn't name: its export is a
44
- // CSV, and a CSV is a container — one row per finding across several
45
- // scans — rather than a report. `detectFormat` recognises it by the
46
- // FILENAME (`.csv`) when given one; the readers are single-report and
47
- // don't take it. A codex export goes through `parseCodexCsvToScans`,
48
- // which splits it into one JSON-shaped report per scan, and each of
49
- // those reads through the readers like any other JSON report.
50
- //
51
- // THIS FILE IS THE WHOLE SURFACE. The modules live in `src/` and the
52
- // package exports one path — `@preventive/report`, this file — so
53
- // everything a caller may hold is named here, in one list, and
54
- // everything else is free to move, split or be renamed without
55
- // breaking anyone. A consumer that wants `fenceRanges` or
56
- // `findingTitle` imports it from here beside `loadFindings`; there is
57
- // no second, deeper way in, inside this repo or out of it.
58
- //
59
- // That is a deliberate trade against the old shape, where every module
60
- // was its own entry point. What it costs is the ability to reach past
61
- // this list; what it buys is that the list IS the contract. Nothing is
62
- // pulled in that a caller doesn't use: every module here is
63
- // side-effect-free (`sideEffects: false` in package.json — the whole
64
- // file is declarations), so a bundler drops what a caller never names,
65
- // and `ui/view/format.js` still rides its lazily-loaded chunk.
66
- //
67
- // This directory is its own package (see package.json beside this
68
- // file) and imports nothing outside itself: no DOM, no app state, no
69
- // storage, nothing from the rest of the repo. Text in, data out — and
70
- // data in, text out. That is what makes it reusable outside the
71
- // viewer — the analyzer stamps its ids with the same `findingId` the
72
- // viewer derives them with, so both sides agree on what a finding IS
73
- // — and `node --test` in this directory runs its suite with nothing
74
- // else installed — and nothing outside this directory reaches into it,
75
- // tests included: every test of this library lives in `report/tests/`.
76
- // Those come through the door like any other caller, except where they
77
- // exercise an internal this file doesn't export; those name `../src/`,
78
- // which is what they are testing.
79
-
80
- import { parseDeepsecFindings } from './src/parse-deepsec.js'
81
- import { parseDeepviewMarkdown } from './src/parse-deepview-md.js'
82
- import { parseMarkdownFindings } from './src/parse-md.js'
83
- import { parsePioliumFindings } from './src/parse-piolium.js'
84
- import { deriveFindingId } from './src/finding-id.js'
85
- import { normalizeFindingSeverity } from './src/severity.js'
86
-
87
- // The rest of the surface, so a consumer needs one import: the codex
88
- // splitter, the id helpers the analyzer shares with the viewer, and the
89
- // run-meta projection a caller applies to the findings it loads.
90
- export { parseCodexCsvToScans } from './src/parse-codex.js'
91
- export { computeFileHash, deriveFindingId, findingId } from './src/finding-id.js'
92
- export { META_FIELDS, inheritReportMeta, repoDirectory, reportRepoGithub } from './src/meta.js'
93
- export { stampSecurityGroups } from './src/security.js'
94
- // The writing side: the document writer, and the label tables it
95
- // spells the app's enumerations with, for the viewer's surfaces that
96
- // describe the same things in prose.
97
- export { writeMarkdown } from './src/write-md.js'
98
- export { COLOR_LABELS, SEVERITY_LABELS, SOURCE_LABELS, TRIAGE_LABELS, severityLabel } from './src/labels.js'
99
-
100
- // Reading a finding: what a finding IS, asked of one. The card, the
101
- // row, the filters and the writer all ask the same questions of the
102
- // same object — which tier does this display under, what is its name,
103
- // where does it sit, what did the pass say about it — and they ask
104
- // them here, so a parser's output and every surface that renders it
105
- // can't drift on the answers.
106
- export {
107
- REVALIDATE_KINDS, SEVERITIES, SEVERITY_ORDER, correctedVariants, descriptionSections,
108
- displayedSeverity, effectiveSeverity, evidenceNote, findingDisplayName, findingTitle,
109
- firstLine, hasSeverityCorrection, isAppFinding, isSecurityFinding, locationLabel, prettyModel, revalidateKindOf,
110
- runMetaLine, splitDescription, stripExportMarker, titledDescription,
111
- } from './src/finding.js'
112
-
113
- // The structural-markdown helpers, for a caller rendering the prose a
114
- // parser handed back: where the fences are (so a `## ` inside a
115
- // snippet stays in the snippet), and the escapes markdown puts on a
116
- // name. The viewer's own markdown rendering (ui/view/format.js,
117
- // export-view-chunks.js) reads the document's shape with these rather
118
- // than keeping a second, subtly different set.
119
- export { fenceRanges, inFence, unescapeMd } from './src/md-structure.js'
120
- export { isHttpUrl } from './src/md-text.js'
121
-
122
- // The markdown chain, in dispatch order: tightest guard first. This
123
- // library's own document opens on a marker line no other format has;
124
- // DeepSec keys off `## SEVERITY (n)` and Piolium off its `# Security
125
- // Audit Report` / `## Technical Findings Detail` headings, while
126
- // parse-md accepts any `# Title` document — so it has to stay last or
127
- // it would swallow the others. Each returns the standard `{ type,
128
- // findings, … }` shape, or null when the text isn't its format.
129
- //
130
- // `format` is this library's name for the document's producer. For the
131
- // three foreign markdown formats it matches the `source` marker the
132
- // parser stamps on what it returns, which is what the viewer reads for
133
- // its header label. 'deepview-md' is the exception that proves the
134
- // rule: the document is this library's, but its findings came from
135
- // whichever producer the document names, and THAT is the `source` it
136
- // carries back (none for the analyzer's own runs). 'json' has no
137
- // marker (the analyzer's own dump carries `type` instead).
138
- const MARKDOWN_FORMATS = [
139
- ['deepview-md', parseDeepviewMarkdown],
140
- ['deepsec', parseDeepsecFindings],
141
- ['piolium', parsePioliumFindings],
142
- ['claude-security', parseMarkdownFindings],
143
- ]
144
-
145
- // A report's entries: `findings`, or `groups` for a pre-deduplicated
146
- // dump. Each entry is one finding or a Finding[] group. Null when the
147
- // document carries neither as an array — which is how a JSON file that
148
- // isn't a report at all (or a report with a malformed list) is told
149
- // apart from an empty one.
150
- //
151
- // Exported because a report is two shapes and only one of them is
152
- // called `findings`: a caller reading `data.findings` alone sees an
153
- // empty report wherever the entries are groups — which is every
154
- // deduplicated dump, and every export of a view that merged a finding
155
- // reported twice (parse-deepview-md.js writes `groups` for exactly
156
- // those). `loadFindings` is the answer for a caller that wants the
157
- // member findings; this is the one for a caller that has to keep the
158
- // grouping, as the viewer's ingest does.
159
- export function reportEntries(data) {
160
- if (Array.isArray(data?.findings)) return data.findings
161
- if (Array.isArray(data?.groups)) return data.groups
162
- return null
163
- }
164
-
165
- // Which producer wrote `content` — 'json' / 'deepview-md' / 'deepsec' /
166
- // 'piolium' / 'claude-security' / 'codex', or null when nothing
167
- // recognises it.
168
- //
169
- // `filename` is optional and decides only codex: a `.csv` is a codex
170
- // export, and the content is not consulted for it (nothing in a CSV's
171
- // text says whose it is, and no other format arrives as one). Every
172
- // other format is named from the content alone, so a `.md` holding a
173
- // JSON dump is 'json'. Case-insensitive on the extension; strip any
174
- // download-duplicate suffix (`report (1).csv`) before calling if the
175
- // name can carry one after the extension.
176
- export function detectFormat(content, filename) {
177
- if (typeof filename === 'string' && /\.csv$/iu.test(filename)) return 'codex'
178
- return readReport(content).format
179
- }
180
-
181
- // Parse `content` in whichever format it turns out to be. JSON first —
182
- // the analyzer's native dump is the common case and the only format
183
- // with a cheap, total test — then the markdown chain when `JSON.parse`
184
- // throws. A JSON document counts as a report only when it carries a
185
- // `findings` (or `groups`) array: anything else that parses is some
186
- // other JSON file, not an empty report.
187
- //
188
- // Returns `{ data, format, reason }`: the parsed report and its
189
- // format, or `data: null` with `reason` saying why in one sentence —
190
- // which a caller reporting "this file isn't a report" can show as is.
191
- // The usual cause is a truncated or malformed JSON dump rather than an
192
- // unknown format, so the JSON error rides along in that sentence.
193
- export function readReport(content) {
194
- let jsonError
195
- try {
196
- const data = JSON.parse(content)
197
- if (reportEntries(data)) {
198
- for (const finding of flattenFindings(reportEntries(data))) normalizeFindingSeverity(finding)
199
- return { data, format: 'json', reason: null }
200
- }
201
- return { data: null, format: null, reason: 'JSON, but not a report: no findings array' }
202
- } catch (err) {
203
- jsonError = err
204
- }
205
- for (const [format, parse] of MARKDOWN_FORMATS) {
206
- const data = parse(content)
207
- if (data) return { data, format, reason: null }
208
- }
209
- return {
210
- data: null,
211
- format: null,
212
- reason: `Not JSON, and not a recognized markdown format. (JSON error: ${jsonError.message})`,
213
- }
214
- }
215
-
216
- // How many entries `content` holds and who produced it, without
217
- // flattening anything or deriving a single id — what a file list wants
218
- // for a badge next to a name. `count` is ENTRIES, not findings: an
219
- // entry is either one finding or a pre-deduplicated group of them, and
220
- // the entry count is what a user sees as rows.
221
- export function analyzeReport(content) {
222
- const { data } = readReport(content)
223
- if (!data) return { count: 0, recognized: false }
224
- return { count: reportEntries(data).length, source: data.source, recognized: true }
225
- }
226
-
227
- // Entries → member findings. A group contributes its members; falsy
228
- // and non-object entries (a malformed list's stray strings and nulls)
229
- // are dropped rather than handed on as findings.
230
- function flattenFindings(entries) {
231
- return entries.flat().filter((f) => f && typeof f === 'object')
232
- }
233
-
234
- // Fill in `f.id` for any finding that lacks one, deriving it from the
235
- // same fingerprint the analyzer stamps. Mutates in place. Findings
236
- // whose id can't be derived (a host without crypto.subtle) are left
237
- // untouched. Batched via Promise.all — sequential awaits would
238
- // serialise hundreds of crypto.subtle.digest calls for no reason.
239
- export async function backfillFindingIds(findings) {
240
- // Managed responses may arrive as already-parsed objects.
241
- for (const finding of findings) normalizeFindingSeverity(finding)
242
- const idLess = findings.filter((f) => !f.id)
243
- if (idLess.length === 0) return
244
- const derived = await Promise.all(idLess.map(deriveFindingId))
245
- idLess.forEach((f, i) => { if (derived[i]) f.id = derived[i] })
246
- }
247
-
248
- // Recognise, flatten, and give every finding an id — the whole read
249
- // path in one call. Returns `{ format, data, findings }`, or null when
250
- // nothing recognises the text. The findings are the parser's own
251
- // objects (not copies), so a caller that means to keep them can, and
252
- // one projecting run meta onto them has `inheritReportMeta` and `data`
253
- // to hand.
254
- export async function loadFindings(content) {
255
- const { data, format } = readReport(content)
256
- if (!data) return null
257
- const findings = flattenFindings(reportEntries(data))
258
- await backfillFindingIds(findings)
259
- return { format, data, findings }
260
- }
@@ -1,80 +0,0 @@
1
- // Finding ids, stamped by the analyzer onto its JSON output and filled
2
- // in by the viewer for findings that arrive without one. Web Crypto is
3
- // the common surface — `crypto.subtle` exists in modern Node and in
4
- // secure browser contexts — so one implementation runs in both.
5
- //
6
- // Two reports from the same source give a finding the same id; an edit
7
- // to its description or its source invalidates it.
8
-
9
- import { encodeUtf8 } from './utf8.js'
10
-
11
- function toHex(bytes) {
12
- return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('')
13
- }
14
-
15
- // The file-content hash the JSON output format uses. sha512 because the
16
- // id below hashes a string that already includes it, so a collision here
17
- // would propagate into an id collision. Padded base64 with the SRI-style
18
- // tag. `btoa` rather than `Uint8Array#toBase64`, still flagged in Node.
19
- export async function computeFileHash(source) {
20
- const bytes = typeof source === 'string' ? encodeUtf8(source) : source
21
- const digest = new Uint8Array(await crypto.subtle.digest('SHA-512', bytes))
22
- return `sha512-${btoa(String.fromCodePoint(...digest))}`
23
- }
24
-
25
- // A fingerprint hashed into a v4-shaped UUID: derived, not random, but
26
- // the shape lets a downstream tool treat it as an opaque id.
27
- async function fingerprintToId(fingerprint) {
28
- const bytes = encodeUtf8(JSON.stringify(fingerprint))
29
- const digest = await crypto.subtle.digest('SHA-256', bytes)
30
- const u = new Uint8Array(digest, 0, 16)
31
- // version 4: 0100xxxx
32
- u[6] = (u[6] & 0x0f) | 0x40
33
- // variant 1: 10xxxxxx
34
- u[8] = (u[8] & 0x3f) | 0x80
35
- const hex = toHex(u)
36
- return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`
37
- }
38
-
39
- // Stable per-finding id from the (severity, description, fileHash) triple
40
- // the analyzer emits. fileHash being undefined is fine — JSON.stringify
41
- // drops undefined keys, so a finding with no hash keys off the pair and
42
- // re-runs over the same source yield the same ids.
43
- export function findingId(severity, description, fileHash) {
44
- return fingerprintToId({ severity, description, fileHash })
45
- }
46
-
47
- // An id derived from a finding, on the first discriminator it carries.
48
- // null when `crypto.subtle` is unavailable (some `file://` setups), so
49
- // the caller can fall back to a session-local id — the UI still works,
50
- // without persistent triage on those findings.
51
- //
52
- // In order:
53
- // - _idBasis — a FROZEN fingerprint a parser stamped, used verbatim;
54
- // it exists so a change to the rendered description
55
- // can't re-key stored triage (parse-md-id.js).
56
- // - fileHash — as `findingId` above.
57
- // - location — a markdown import's url, also stable.
58
- // - file/line — last resort for a JSON finding with neither: not what
59
- // the spec prescribes, but better than collapsing two
60
- // unrelated findings onto one id.
61
- export async function deriveFindingId(f) {
62
- if (typeof crypto?.subtle?.digest !== 'function') return null
63
- const fingerprint = fingerprintOf(f)
64
- try {
65
- return await fingerprintToId(fingerprint)
66
- } catch {
67
- return null
68
- }
69
- }
70
-
71
- // The choice above as the object that gets hashed. Key order is part of
72
- // the id — JSON.stringify keeps insertion order — so every shape lists
73
- // severity and description first.
74
- export function fingerprintOf(f) {
75
- if (f._idBasis) return f._idBasis
76
- const { severity, description } = f
77
- if (f.fileHash) return { severity, description, fileHash: f.fileHash }
78
- if (f.location) return { severity, description, location: f.location }
79
- return { severity, description, file: f.file, line: f.line }
80
- }