@preventive/triage 1.0.0-alpha.26 → 1.0.0-alpha.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/client/linked-findings.js +1 -1
- package/common/bundle-metadata.js +12 -10
- package/common/code-language.js +17 -2
- package/common/default-scan-models.ts +3 -3
- package/common/finding-id.js +1 -1
- package/common/managed/report-content.ts +1 -1
- package/common/managed/report-filter.ts +1 -1
- package/out/client-managed-import.js +24 -24
- package/out/client-managed.js +93 -68
- package/out/graph.js +11 -11
- package/out/index.html +1 -1
- package/out/prism.js +2 -2
- package/out/stasis.svg +0 -25
- package/out/terminal.js +163 -163
- package/out/view.css +1 -1
- package/out/view.js +223 -190
- package/package.json +29 -29
- package/server-common/storage-crypto.ts +182 -0
- package/server-common/storage-log.ts +5 -1
- package/server-common/vercel-blob.ts +6 -3
- package/server-e2e/static.ts +3 -4
- package/server-managed/avatar-store.ts +1 -40
- package/server-managed/blob-store.ts +3 -52
- package/server-managed/bundle-advisories.ts +53 -0
- package/server-managed/bundle-advisory-inventory.ts +75 -0
- package/server-managed/bundle-build-leases.ts +41 -0
- package/server-managed/bundle-build-worker.js +44 -0
- package/server-managed/bundle-build.ts +141 -0
- package/server-managed/bundle-cache.ts +22 -41
- package/server-managed/bundle-store.ts +3 -7
- package/server-managed/bundle-summary-cache.ts +27 -9
- package/server-managed/bundle.ts +66 -1
- package/server-managed/cache-storage.ts +4 -25
- package/server-managed/cli.js +10 -2
- package/server-managed/config.ts +12 -1
- package/server-managed/content-access.ts +1 -1
- package/server-managed/db-methods.ts +163 -40
- package/server-managed/db-neon.ts +124 -30
- package/server-managed/db-schema.ts +2 -0
- package/server-managed/db.ts +11 -1
- package/server-managed/github-app.ts +10 -15
- package/server-managed/github-errors.ts +21 -0
- package/server-managed/github-issues.ts +1 -1
- package/server-managed/github-metadata.ts +34 -1
- package/server-managed/github-pulls.ts +142 -30
- package/server-managed/http.ts +278 -167
- package/server-managed/index.ts +26 -9
- package/server-managed/management-catalog.ts +45 -0
- package/server-managed/object-storage-disk.ts +144 -0
- package/server-managed/object-storage-vercel.ts +60 -0
- package/server-managed/object-storage.ts +71 -0
- package/server-managed/public-workspace.ts +20 -14
- package/server-managed/report-migration.ts +1 -1
- package/server-managed/report-sources.ts +14 -4
- package/server-managed/repository-browser.ts +8 -1
- package/server-managed/repository-discovery.ts +1 -1
- package/server-managed/revisions.ts +35 -0
- package/server-managed/solidity-entry-points.ts +69 -0
- package/server-managed/sql-postgres.ts +1 -0
- package/server-managed/sql.ts +14 -8
- package/server-managed/storage-command.ts +16 -0
- package/server-managed/storage-db.ts +175 -0
- package/server-managed/storage-encryption.ts +211 -0
- package/server-managed/storage-maintenance.ts +197 -0
- package/server-managed/storage-payload.ts +72 -0
- package/server-managed/storage-stores.ts +81 -0
- package/server-managed/storage.ts +48 -24
- package/server-managed/team-feed.ts +50 -5
- package/server-managed/team-reports.ts +85 -22
- package/server-managed/uploads.ts +2 -2
- package/server-managed/workspace-shares.ts +15 -3
- package/report/index.js +0 -260
- package/report/src/finding-id.js +0 -80
- package/report/src/finding.js +0 -314
- package/report/src/labels.js +0 -35
- package/report/src/md-structure.js +0 -471
- package/report/src/md-text.js +0 -167
- package/report/src/meta.js +0 -76
- package/report/src/parse-codex.js +0 -149
- package/report/src/parse-deepsec.js +0 -199
- package/report/src/parse-deepview-fields.js +0 -375
- package/report/src/parse-deepview-md.js +0 -186
- package/report/src/parse-md-id.js +0 -137
- package/report/src/parse-md.js +0 -323
- package/report/src/parse-piolium-id.js +0 -79
- package/report/src/parse-piolium-rows.js +0 -126
- package/report/src/parse-piolium-tokens.js +0 -185
- package/report/src/parse-piolium.js +0 -399
- package/report/src/security.js +0 -63
- package/report/src/severity.js +0 -18
- package/report/src/utf8.js +0 -21
- package/report/src/write-md-finding.js +0 -273
- package/report/src/write-md.js +0 -291
- package/server-managed/blob-vercel.ts +0 -127
- /package/{LICENSE → LICENSE-MIT} +0 -0
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
// Team content is filtered as one workspace, with separate report envelopes.
|
|
2
|
-
//
|
|
2
|
+
// Bounded snapshots retain filtered content and visibility for repeated reads.
|
|
3
3
|
import { Buffer } from 'node:buffer'
|
|
4
|
-
import { backfillFindingIds, reportEntries, stampSecurityGroups } from '
|
|
4
|
+
import { backfillFindingIds, reportEntries, stampSecurityGroups } from '@preventive/report'
|
|
5
5
|
import { managedFindingSourcePaths, readManagedReport } from '../common/managed/report-content.ts'
|
|
6
6
|
import { filterReportData, projectFinding } from '../common/managed/report-filter.ts'
|
|
7
7
|
import type { BlobStore } from './blob-store.ts'
|
|
8
8
|
import type { ManagedDb, TeamReportAccessSnapshot } from './db.ts'
|
|
9
|
+
import { triageWireEntry } from './triage-response.ts'
|
|
9
10
|
import { MAX_REPORT_QUERY_BYTES, MAX_REPORT_QUERY_COUNT } from './report-query.ts'
|
|
10
11
|
|
|
11
12
|
type Finding = Record<string, unknown>
|
|
@@ -15,6 +16,20 @@ type Visibility = Map<string, ReportVisibility>
|
|
|
15
16
|
const VISIBILITY_CACHE_MAX_ITEMS = 250_000
|
|
16
17
|
const VISIBILITY_CACHE_MAX_SNAPSHOTS = 32
|
|
17
18
|
const caches = new WeakMap<ManagedDb, Map<string, Visibility>>()
|
|
19
|
+
const REPORT_CACHE_BYTES = 16 * 1024 * 1024
|
|
20
|
+
const reportCaches = new WeakMap<ManagedDb, Map<string, Buffer>>()
|
|
21
|
+
function retainReports(db: ManagedDb, key: string, bytes: Buffer) {
|
|
22
|
+
if (bytes.length > REPORT_CACHE_BYTES) return
|
|
23
|
+
let cache = reportCaches.get(db)
|
|
24
|
+
if (!cache) { cache = new Map(); reportCaches.set(db, cache) }
|
|
25
|
+
cache.delete(key)
|
|
26
|
+
let total = bytes.length + [...cache.values()].reduce((sum, value) => sum + value.length, 0)
|
|
27
|
+
for (const [oldest, value] of cache) {
|
|
28
|
+
if (total <= REPORT_CACHE_BYTES && cache.size < VISIBILITY_CACHE_MAX_SNAPSHOTS) break
|
|
29
|
+
cache.delete(oldest); total -= value.length
|
|
30
|
+
}
|
|
31
|
+
cache.set(key, bytes)
|
|
32
|
+
}
|
|
18
33
|
function visibilityWeight(visible: Visibility): number {
|
|
19
34
|
return [...visible.values()].reduce((sum, v) => sum + v.ids.size + v.sourcePaths.size, 0)
|
|
20
35
|
}
|
|
@@ -61,24 +76,30 @@ function linksOf(data: unknown): string[][] | null {
|
|
|
61
76
|
if (!record.links.every(ids => Array.isArray(ids) && ids.every(id => typeof id === 'string'))) throw new TeamReportsError(422, 'unreadable-links')
|
|
62
77
|
return record.links
|
|
63
78
|
}
|
|
64
|
-
async function
|
|
79
|
+
async function buildTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<{ body: Buffer; visible: Visibility }> {
|
|
65
80
|
if (snapshot.reports.length > MAX_REPORT_QUERY_COUNT || snapshot.reports.reduce((n, r) => n + r.byteSize, 0) > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
|
|
66
81
|
const reports: TeamReport[] = []
|
|
67
|
-
let inputBytes = 0
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
+
let inputBytes = 0, next = 0
|
|
83
|
+
// Bound both remote reads and raw buffers, retaining stable report order for
|
|
84
|
+
// the cross-report classification that follows. Finish workers on failure.
|
|
85
|
+
const results = await Promise.allSettled(Array.from({ length: Math.min(4, snapshot.reports.length) }, async () => {
|
|
86
|
+
while (next < snapshot.reports.length) {
|
|
87
|
+
const index = next++
|
|
88
|
+
const access = snapshot.reports[index]!
|
|
89
|
+
|
|
90
|
+
const bytes = await store.get(access.id)
|
|
91
|
+
if (!bytes) throw new TeamReportsError(503, 'unavailable')
|
|
92
|
+
inputBytes += bytes.length
|
|
93
|
+
if (inputBytes > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
|
|
94
|
+
const parsed = readManagedReport(bytes.toString('utf8'), access.filename)
|
|
95
|
+
if (!parsed.data) throw new TeamReportsError(422, 'unreadable-report')
|
|
96
|
+
// Backfill before filtering so links, annotations and the client agree on
|
|
97
|
+
// IDs even when hiding a component would change the original row shape.
|
|
98
|
+
await backfillFindingIds(groupsOf(parsed.data).flat())
|
|
99
|
+
reports[index] = { id: access.id, filename: access.filename, data: parsed.data, repo: access.repo }
|
|
100
|
+
}
|
|
101
|
+
}))
|
|
102
|
+
for (const result of results) if (result.status === 'rejected') throw result.reason
|
|
82
103
|
const edges = new Map<string, Set<string>>()
|
|
83
104
|
const connect = (a: string, b: string) => {
|
|
84
105
|
if (!edges.has(a)) edges.set(a, new Set())
|
|
@@ -112,15 +133,41 @@ async function loadTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: Team
|
|
|
112
133
|
if (links) report.data = { source: 'links', findings: [], links: links.map(ids => [...new Set(ids.filter(id => allIds.has(id)))]).filter(ids => ids.length >= 2) }
|
|
113
134
|
}
|
|
114
135
|
let outputBytes = 14
|
|
115
|
-
|
|
116
|
-
|
|
136
|
+
const parts = [Buffer.from('{"reports":[')]
|
|
137
|
+
for (const [index, report] of reports.entries()) {
|
|
138
|
+
const bytes = Buffer.from(JSON.stringify(report))
|
|
139
|
+
outputBytes += bytes.length + 1
|
|
117
140
|
if (outputBytes > MAX_REPORT_QUERY_BYTES) throw new TeamReportsError(413, 'batch-too-large')
|
|
141
|
+
if (index) parts.push(Buffer.from(','))
|
|
142
|
+
parts.push(bytes)
|
|
118
143
|
}
|
|
144
|
+
parts.push(Buffer.from(']}'))
|
|
145
|
+
const body = Buffer.concat(parts)
|
|
119
146
|
retainVisibility(db, teamSnapshotKey(snapshot), visible)
|
|
120
|
-
|
|
147
|
+
retainReports(db, teamSnapshotKey(snapshot), body)
|
|
148
|
+
return { body, visible }
|
|
149
|
+
}
|
|
150
|
+
const pendingWorkspaces = new WeakMap<ManagedDb, Map<string, ReturnType<typeof buildTeamWorkspace>>>()
|
|
151
|
+
async function loadTeamWorkspace(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot) {
|
|
152
|
+
let pending = pendingWorkspaces.get(db)
|
|
153
|
+
if (!pending) { pending = new Map(); pendingWorkspaces.set(db, pending) }
|
|
154
|
+
const key = teamSnapshotKey(snapshot)
|
|
155
|
+
const existing = pending.get(key)
|
|
156
|
+
if (existing) return existing
|
|
157
|
+
const job = buildTeamWorkspace(db, store, snapshot)
|
|
158
|
+
pending.set(key, job)
|
|
159
|
+
try { return await job } finally { pending.delete(key) }
|
|
160
|
+
}
|
|
161
|
+
// Treat the encoded body as immutable and recheck access before sending it.
|
|
162
|
+
// Warm HTTP reads need neither a parsed copy nor another serialization pass.
|
|
163
|
+
export async function loadTeamReportsResponse(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<Buffer> {
|
|
164
|
+
const cached = reportCaches.get(db)?.get(teamSnapshotKey(snapshot))
|
|
165
|
+
return cached ?? (await loadTeamWorkspace(db, store, snapshot)).body
|
|
121
166
|
}
|
|
122
167
|
export async function loadTeamReports(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<TeamReport[]> {
|
|
123
|
-
|
|
168
|
+
// Object consumers get their own copy, including when sharing a cold load.
|
|
169
|
+
const body = await loadTeamReportsResponse(db, store, snapshot)
|
|
170
|
+
return (JSON.parse(body.toString('utf8')) as { reports: TeamReport[] }).reports
|
|
124
171
|
}
|
|
125
172
|
async function teamVisibility(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot): Promise<Visibility> {
|
|
126
173
|
const key = teamSnapshotKey(snapshot)
|
|
@@ -154,3 +201,19 @@ export async function teamSourcePaths(db: ManagedDb, store: BlobStore, sessionId
|
|
|
154
201
|
await recheckTeam(db, sessionId, snapshot)
|
|
155
202
|
return visible.sourcePaths
|
|
156
203
|
}
|
|
204
|
+
|
|
205
|
+
// Serialize shared annotation bodies once. Reports carry only the annotated
|
|
206
|
+
// finding IDs visible in that report; callers revalidate access before sending.
|
|
207
|
+
export async function loadTeamAnnotations(db: ManagedDb, store: BlobStore, snapshot: TeamReportAccessSnapshot, reportId: string | null = null) {
|
|
208
|
+
// Restrict annotation reads only after classifying the complete workspace:
|
|
209
|
+
// links and findings in other reports can affect this report's visibility.
|
|
210
|
+
const visible = reportId === null ? await teamVisibility(db, store, snapshot)
|
|
211
|
+
: new Map([[reportId, await teamReportVisibility(db, store, snapshot, reportId)]])
|
|
212
|
+
const ids = new Set<string>()
|
|
213
|
+
for (const report of visible.values()) for (const id of report.ids) ids.add(id)
|
|
214
|
+
const { triage, comments } = await db.getAnnotations([...ids])
|
|
215
|
+
const entries = Object.fromEntries(triage.map(row => [row.findingId, triageWireEntry(row)]))
|
|
216
|
+
const annotated = new Set([...triage.map(row => row.findingId), ...comments.map(comment => comment.findingId)])
|
|
217
|
+
const reports = Object.fromEntries([...visible].map(([id, report]) => [id, [...report.ids].filter(finding => annotated.has(finding))]))
|
|
218
|
+
return { reports, entries, comments }
|
|
219
|
+
}
|
|
@@ -38,8 +38,8 @@ export async function deleteUpload(store: BlobStore, session: string, kind: Uplo
|
|
|
38
38
|
if (errors.length > 0) throw new AggregateError(errors, 'Upload cleanup failed')
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
export function putUploadPart(store: BlobStore, session: string, kind: UploadKind, id: string, index: number, bytes: Buffer): Promise<void> {
|
|
42
|
-
|
|
41
|
+
export async function putUploadPart(store: BlobStore, session: string, kind: UploadKind, id: string, index: number, bytes: Buffer): Promise<void> {
|
|
42
|
+
await store.put(partId(session, kind, id, index), bytes)
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
export async function readUpload(store: BlobStore, req: IncomingMessage, session: string, kind: UploadKind, maxBytes: number): Promise<Buffer> {
|
|
@@ -31,6 +31,7 @@ export interface ManagedWorkspaceShare extends WorkspaceShareInfo {
|
|
|
31
31
|
teamName: string
|
|
32
32
|
teamSlug: string
|
|
33
33
|
}
|
|
34
|
+
export type WorkspaceShareFeedState = { grant: string; catalog: number; annotations: number }
|
|
34
35
|
export interface WorkspaceShareStore {
|
|
35
36
|
createWorkspaceShare(sessionId: string, now: number, teamId: string, tokenHash: string, permissions?: TeamUserPermissions): Promise<boolean>
|
|
36
37
|
listWorkspaceShares(sessionId: string, now: number, teamId: string): Promise<WorkspaceShareInfo[] | null>
|
|
@@ -38,6 +39,7 @@ export interface WorkspaceShareStore {
|
|
|
38
39
|
updateWorkspaceShare(sessionId: string, now: number, teamId: string, id: string, permissions: TeamUserPermissions): Promise<boolean>
|
|
39
40
|
revokeWorkspaceShares(sessionId: string, now: number, teamId: string, id?: string): Promise<boolean>
|
|
40
41
|
getWorkspaceShare(tokenHash: string): Promise<WorkspaceShareSnapshot | null>
|
|
42
|
+
getWorkspaceShareFeedState(tokenHash: string): Promise<WorkspaceShareFeedState | null>
|
|
41
43
|
}
|
|
42
44
|
|
|
43
45
|
// A share is a capability, never a user session. Its issuer must still manage
|
|
@@ -63,11 +65,17 @@ function shareQueries(db: ManagedSql) {
|
|
|
63
65
|
JOIN managed_user u ON u.id = s.created_by JOIN managed_team t ON t.id = s.team_id
|
|
64
66
|
WHERE ? = 'admin' OR EXISTS (SELECT 1 FROM managed_team_user tu WHERE tu.user_id = ? AND tu.team_id = s.team_id)
|
|
65
67
|
ORDER BY t.name, t.id, s.created_at DESC, s.token_hash`)
|
|
66
|
-
const
|
|
68
|
+
const shareGrant = `FROM managed_workspace_share s
|
|
67
69
|
JOIN managed_user u ON u.id = s.created_by
|
|
68
70
|
JOIN managed_team t ON t.id = s.team_id
|
|
69
71
|
WHERE s.token_hash = ? AND (u.role = 'admin' OR (u.role = 'manage'
|
|
70
|
-
AND EXISTS (SELECT 1 FROM managed_team_user tu WHERE tu.user_id = u.id AND tu.team_id = s.team_id)))`
|
|
72
|
+
AND EXISTS (SELECT 1 FROM managed_team_user tu WHERE tu.user_id = u.id AND tu.team_id = s.team_id)))`
|
|
73
|
+
const share = db.prepare(`SELECT t.id, t.slug, t.name, s.dependencies, s.security ${shareGrant}`)
|
|
74
|
+
// Validate the capability and its issuer every poll; global counters are
|
|
75
|
+
// internal hints that let unchanged feeds skip rebuilding the full catalog.
|
|
76
|
+
const feed = db.prepare(`SELECT t.id, s.dependencies, s.security,
|
|
77
|
+
(SELECT catalog FROM managed_change_revision WHERE id = 1) AS catalog,
|
|
78
|
+
(SELECT annotations FROM managed_change_revision WHERE id = 1) AS annotations ${shareGrant}`)
|
|
71
79
|
const repositories = db.prepare(`SELECT tr.repo_id AS repoId, sr.full_name AS github, tr.path
|
|
72
80
|
FROM managed_team_repo tr JOIN managed_selected_repo sr ON sr.repo_id = tr.repo_id
|
|
73
81
|
WHERE tr.team_id = ? ORDER BY tr.repo_id, tr.path`)
|
|
@@ -85,7 +93,7 @@ function shareQueries(db: ManagedSql) {
|
|
|
85
93
|
JOIN managed_selected_repo sr ON sr.repo_id = b.repo_id
|
|
86
94
|
WHERE tr.team_id = ? AND (tr.path = '' OR b.repo_directory = tr.path
|
|
87
95
|
OR substr(b.repo_directory, 1, length(tr.path) + 1) = tr.path || '/') ORDER BY b.id`)
|
|
88
|
-
return { manager, insert, remove, update, list, session, all, share, repositories, reports, bundles }
|
|
96
|
+
return { manager, insert, remove, update, list, session, all, share, feed, repositories, reports, bundles }
|
|
89
97
|
}
|
|
90
98
|
|
|
91
99
|
function shareInfo<T extends { dependencies: number; security: number }>(row: T) {
|
|
@@ -123,6 +131,10 @@ export function workspaceShareMethods(db: ManagedSql): WorkspaceShareStore {
|
|
|
123
131
|
const result = await q.remove.run(teamId, id ?? null, id ?? null)
|
|
124
132
|
return id === undefined || result.changes > 0
|
|
125
133
|
},
|
|
134
|
+
async getWorkspaceShareFeedState(tokenHash) {
|
|
135
|
+
const row = await q.feed.get(tokenHash) as { id: string; dependencies: number; security: number; catalog: number; annotations: number } | undefined
|
|
136
|
+
return row ? { grant: JSON.stringify([row.id, row.dependencies, row.security]), catalog: row.catalog, annotations: row.annotations } : null
|
|
137
|
+
},
|
|
126
138
|
async getWorkspaceShare(tokenHash) {
|
|
127
139
|
const grant = await q.share.get(tokenHash) as { id: string; slug: string; name: string; dependencies: number; security: number } | undefined
|
|
128
140
|
if (!grant) return null
|
package/report/index.js
DELETED
|
@@ -1,260 +0,0 @@
|
|
|
1
|
-
// The report library — one door to every report format this project
|
|
2
|
-
// reads, and to the one it writes.
|
|
3
|
-
//
|
|
4
|
-
// A "report" is whatever an analyzer wrote: the JSON dump this
|
|
5
|
-
// project's own analyzer emits, or one of the shapes other tools
|
|
6
|
-
// produce — DeepSec and Piolium markdown, Claude Security markdown,
|
|
7
|
-
// Codex CSV — or the markdown document this library itself writes
|
|
8
|
-
// (write-md.js), read back by parse-deepview-md.js. The parsers beside
|
|
9
|
-
// this file each recognise exactly one of those and know nothing about
|
|
10
|
-
// each other; this module is the dispatch over them, so "which formats
|
|
11
|
-
// do we read, and in what order" is answered in one place instead of
|
|
12
|
-
// once per call site.
|
|
13
|
-
//
|
|
14
|
-
// import { loadFindings, writeMarkdown } from '../report/index.js'
|
|
15
|
-
// const report = await loadFindings(text)
|
|
16
|
-
// // → { format, data, findings: [ … with ids ] } | null
|
|
17
|
-
// const md = writeMarkdown({ title, groups: report.findings.map((f) => [f]) })
|
|
18
|
-
//
|
|
19
|
-
// Three entry points for reading, in rising order of how much they do:
|
|
20
|
-
//
|
|
21
|
-
// detectFormat — name the format, parse nothing further
|
|
22
|
-
// readReport — the parsed report, or the reason it isn't one
|
|
23
|
-
// loadFindings — parsed, flattened, and every finding carrying an id
|
|
24
|
-
//
|
|
25
|
-
// `analyzeReport` is `readReport` for a file list — entry count and
|
|
26
|
-
// producer — `reportEntries` is a report's entry list whichever of the
|
|
27
|
-
// two names it goes under (`findings` or `groups`), for a caller that
|
|
28
|
-
// has to keep the grouping rather than flatten it, and
|
|
29
|
-
// `backfillFindingIds` is the id step on its own, for a caller that
|
|
30
|
-
// has to interleave something with it.
|
|
31
|
-
//
|
|
32
|
-
// And one for writing: `writeMarkdown` takes findings — the parsers'
|
|
33
|
-
// own objects, grouped as the viewer groups them — with whatever the
|
|
34
|
-
// caller knows about the selection and the reader's annotations, and
|
|
35
|
-
// writes the markdown document the Download button saves (write-md.js
|
|
36
|
-
// for the document, write-md-finding.js for one finding, labels.js for
|
|
37
|
-
// the words). Both directions read a finding through finding.js, so
|
|
38
|
-
// what a parser produced and what the writer prints agree on what a
|
|
39
|
-
// finding IS — and the document is itself a report the readers above
|
|
40
|
-
// take (parse-deepview-md.js), so what was written can be loaded
|
|
41
|
-
// again, with the ids and fields it left with.
|
|
42
|
-
//
|
|
43
|
-
// Codex is the one format the content doesn't name: its export is a
|
|
44
|
-
// CSV, and a CSV is a container — one row per finding across several
|
|
45
|
-
// scans — rather than a report. `detectFormat` recognises it by the
|
|
46
|
-
// FILENAME (`.csv`) when given one; the readers are single-report and
|
|
47
|
-
// don't take it. A codex export goes through `parseCodexCsvToScans`,
|
|
48
|
-
// which splits it into one JSON-shaped report per scan, and each of
|
|
49
|
-
// those reads through the readers like any other JSON report.
|
|
50
|
-
//
|
|
51
|
-
// THIS FILE IS THE WHOLE SURFACE. The modules live in `src/` and the
|
|
52
|
-
// package exports one path — `@preventive/report`, this file — so
|
|
53
|
-
// everything a caller may hold is named here, in one list, and
|
|
54
|
-
// everything else is free to move, split or be renamed without
|
|
55
|
-
// breaking anyone. A consumer that wants `fenceRanges` or
|
|
56
|
-
// `findingTitle` imports it from here beside `loadFindings`; there is
|
|
57
|
-
// no second, deeper way in, inside this repo or out of it.
|
|
58
|
-
//
|
|
59
|
-
// That is a deliberate trade against the old shape, where every module
|
|
60
|
-
// was its own entry point. What it costs is the ability to reach past
|
|
61
|
-
// this list; what it buys is that the list IS the contract. Nothing is
|
|
62
|
-
// pulled in that a caller doesn't use: every module here is
|
|
63
|
-
// side-effect-free (`sideEffects: false` in package.json — the whole
|
|
64
|
-
// file is declarations), so a bundler drops what a caller never names,
|
|
65
|
-
// and `ui/view/format.js` still rides its lazily-loaded chunk.
|
|
66
|
-
//
|
|
67
|
-
// This directory is its own package (see package.json beside this
|
|
68
|
-
// file) and imports nothing outside itself: no DOM, no app state, no
|
|
69
|
-
// storage, nothing from the rest of the repo. Text in, data out — and
|
|
70
|
-
// data in, text out. That is what makes it reusable outside the
|
|
71
|
-
// viewer — the analyzer stamps its ids with the same `findingId` the
|
|
72
|
-
// viewer derives them with, so both sides agree on what a finding IS
|
|
73
|
-
// — and `node --test` in this directory runs its suite with nothing
|
|
74
|
-
// else installed — and nothing outside this directory reaches into it,
|
|
75
|
-
// tests included: every test of this library lives in `report/tests/`.
|
|
76
|
-
// Those come through the door like any other caller, except where they
|
|
77
|
-
// exercise an internal this file doesn't export; those name `../src/`,
|
|
78
|
-
// which is what they are testing.
|
|
79
|
-
|
|
80
|
-
import { parseDeepsecFindings } from './src/parse-deepsec.js'
|
|
81
|
-
import { parseDeepviewMarkdown } from './src/parse-deepview-md.js'
|
|
82
|
-
import { parseMarkdownFindings } from './src/parse-md.js'
|
|
83
|
-
import { parsePioliumFindings } from './src/parse-piolium.js'
|
|
84
|
-
import { deriveFindingId } from './src/finding-id.js'
|
|
85
|
-
import { normalizeFindingSeverity } from './src/severity.js'
|
|
86
|
-
|
|
87
|
-
// The rest of the surface, so a consumer needs one import: the codex
|
|
88
|
-
// splitter, the id helpers the analyzer shares with the viewer, and the
|
|
89
|
-
// run-meta projection a caller applies to the findings it loads.
|
|
90
|
-
export { parseCodexCsvToScans } from './src/parse-codex.js'
|
|
91
|
-
export { computeFileHash, deriveFindingId, findingId } from './src/finding-id.js'
|
|
92
|
-
export { META_FIELDS, inheritReportMeta, repoDirectory, reportRepoGithub } from './src/meta.js'
|
|
93
|
-
export { stampSecurityGroups } from './src/security.js'
|
|
94
|
-
// The writing side: the document writer, and the label tables it
|
|
95
|
-
// spells the app's enumerations with, for the viewer's surfaces that
|
|
96
|
-
// describe the same things in prose.
|
|
97
|
-
export { writeMarkdown } from './src/write-md.js'
|
|
98
|
-
export { COLOR_LABELS, SEVERITY_LABELS, SOURCE_LABELS, TRIAGE_LABELS, severityLabel } from './src/labels.js'
|
|
99
|
-
|
|
100
|
-
// Reading a finding: what a finding IS, asked of one. The card, the
|
|
101
|
-
// row, the filters and the writer all ask the same questions of the
|
|
102
|
-
// same object — which tier does this display under, what is its name,
|
|
103
|
-
// where does it sit, what did the pass say about it — and they ask
|
|
104
|
-
// them here, so a parser's output and every surface that renders it
|
|
105
|
-
// can't drift on the answers.
|
|
106
|
-
export {
|
|
107
|
-
REVALIDATE_KINDS, SEVERITIES, SEVERITY_ORDER, correctedVariants, descriptionSections,
|
|
108
|
-
displayedSeverity, effectiveSeverity, evidenceNote, findingDisplayName, findingTitle,
|
|
109
|
-
firstLine, hasSeverityCorrection, isAppFinding, isSecurityFinding, locationLabel, prettyModel, revalidateKindOf,
|
|
110
|
-
runMetaLine, splitDescription, stripExportMarker, titledDescription,
|
|
111
|
-
} from './src/finding.js'
|
|
112
|
-
|
|
113
|
-
// The structural-markdown helpers, for a caller rendering the prose a
|
|
114
|
-
// parser handed back: where the fences are (so a `## ` inside a
|
|
115
|
-
// snippet stays in the snippet), and the escapes markdown puts on a
|
|
116
|
-
// name. The viewer's own markdown rendering (ui/view/format.js,
|
|
117
|
-
// export-view-chunks.js) reads the document's shape with these rather
|
|
118
|
-
// than keeping a second, subtly different set.
|
|
119
|
-
export { fenceRanges, inFence, unescapeMd } from './src/md-structure.js'
|
|
120
|
-
export { isHttpUrl } from './src/md-text.js'
|
|
121
|
-
|
|
122
|
-
// The markdown chain, in dispatch order: tightest guard first. This
|
|
123
|
-
// library's own document opens on a marker line no other format has;
|
|
124
|
-
// DeepSec keys off `## SEVERITY (n)` and Piolium off its `# Security
|
|
125
|
-
// Audit Report` / `## Technical Findings Detail` headings, while
|
|
126
|
-
// parse-md accepts any `# Title` document — so it has to stay last or
|
|
127
|
-
// it would swallow the others. Each returns the standard `{ type,
|
|
128
|
-
// findings, … }` shape, or null when the text isn't its format.
|
|
129
|
-
//
|
|
130
|
-
// `format` is this library's name for the document's producer. For the
|
|
131
|
-
// three foreign markdown formats it matches the `source` marker the
|
|
132
|
-
// parser stamps on what it returns, which is what the viewer reads for
|
|
133
|
-
// its header label. 'deepview-md' is the exception that proves the
|
|
134
|
-
// rule: the document is this library's, but its findings came from
|
|
135
|
-
// whichever producer the document names, and THAT is the `source` it
|
|
136
|
-
// carries back (none for the analyzer's own runs). 'json' has no
|
|
137
|
-
// marker (the analyzer's own dump carries `type` instead).
|
|
138
|
-
const MARKDOWN_FORMATS = [
|
|
139
|
-
['deepview-md', parseDeepviewMarkdown],
|
|
140
|
-
['deepsec', parseDeepsecFindings],
|
|
141
|
-
['piolium', parsePioliumFindings],
|
|
142
|
-
['claude-security', parseMarkdownFindings],
|
|
143
|
-
]
|
|
144
|
-
|
|
145
|
-
// A report's entries: `findings`, or `groups` for a pre-deduplicated
|
|
146
|
-
// dump. Each entry is one finding or a Finding[] group. Null when the
|
|
147
|
-
// document carries neither as an array — which is how a JSON file that
|
|
148
|
-
// isn't a report at all (or a report with a malformed list) is told
|
|
149
|
-
// apart from an empty one.
|
|
150
|
-
//
|
|
151
|
-
// Exported because a report is two shapes and only one of them is
|
|
152
|
-
// called `findings`: a caller reading `data.findings` alone sees an
|
|
153
|
-
// empty report wherever the entries are groups — which is every
|
|
154
|
-
// deduplicated dump, and every export of a view that merged a finding
|
|
155
|
-
// reported twice (parse-deepview-md.js writes `groups` for exactly
|
|
156
|
-
// those). `loadFindings` is the answer for a caller that wants the
|
|
157
|
-
// member findings; this is the one for a caller that has to keep the
|
|
158
|
-
// grouping, as the viewer's ingest does.
|
|
159
|
-
export function reportEntries(data) {
|
|
160
|
-
if (Array.isArray(data?.findings)) return data.findings
|
|
161
|
-
if (Array.isArray(data?.groups)) return data.groups
|
|
162
|
-
return null
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
// Which producer wrote `content` — 'json' / 'deepview-md' / 'deepsec' /
|
|
166
|
-
// 'piolium' / 'claude-security' / 'codex', or null when nothing
|
|
167
|
-
// recognises it.
|
|
168
|
-
//
|
|
169
|
-
// `filename` is optional and decides only codex: a `.csv` is a codex
|
|
170
|
-
// export, and the content is not consulted for it (nothing in a CSV's
|
|
171
|
-
// text says whose it is, and no other format arrives as one). Every
|
|
172
|
-
// other format is named from the content alone, so a `.md` holding a
|
|
173
|
-
// JSON dump is 'json'. Case-insensitive on the extension; strip any
|
|
174
|
-
// download-duplicate suffix (`report (1).csv`) before calling if the
|
|
175
|
-
// name can carry one after the extension.
|
|
176
|
-
export function detectFormat(content, filename) {
|
|
177
|
-
if (typeof filename === 'string' && /\.csv$/iu.test(filename)) return 'codex'
|
|
178
|
-
return readReport(content).format
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
// Parse `content` in whichever format it turns out to be. JSON first —
|
|
182
|
-
// the analyzer's native dump is the common case and the only format
|
|
183
|
-
// with a cheap, total test — then the markdown chain when `JSON.parse`
|
|
184
|
-
// throws. A JSON document counts as a report only when it carries a
|
|
185
|
-
// `findings` (or `groups`) array: anything else that parses is some
|
|
186
|
-
// other JSON file, not an empty report.
|
|
187
|
-
//
|
|
188
|
-
// Returns `{ data, format, reason }`: the parsed report and its
|
|
189
|
-
// format, or `data: null` with `reason` saying why in one sentence —
|
|
190
|
-
// which a caller reporting "this file isn't a report" can show as is.
|
|
191
|
-
// The usual cause is a truncated or malformed JSON dump rather than an
|
|
192
|
-
// unknown format, so the JSON error rides along in that sentence.
|
|
193
|
-
export function readReport(content) {
|
|
194
|
-
let jsonError
|
|
195
|
-
try {
|
|
196
|
-
const data = JSON.parse(content)
|
|
197
|
-
if (reportEntries(data)) {
|
|
198
|
-
for (const finding of flattenFindings(reportEntries(data))) normalizeFindingSeverity(finding)
|
|
199
|
-
return { data, format: 'json', reason: null }
|
|
200
|
-
}
|
|
201
|
-
return { data: null, format: null, reason: 'JSON, but not a report: no findings array' }
|
|
202
|
-
} catch (err) {
|
|
203
|
-
jsonError = err
|
|
204
|
-
}
|
|
205
|
-
for (const [format, parse] of MARKDOWN_FORMATS) {
|
|
206
|
-
const data = parse(content)
|
|
207
|
-
if (data) return { data, format, reason: null }
|
|
208
|
-
}
|
|
209
|
-
return {
|
|
210
|
-
data: null,
|
|
211
|
-
format: null,
|
|
212
|
-
reason: `Not JSON, and not a recognized markdown format. (JSON error: ${jsonError.message})`,
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
// How many entries `content` holds and who produced it, without
|
|
217
|
-
// flattening anything or deriving a single id — what a file list wants
|
|
218
|
-
// for a badge next to a name. `count` is ENTRIES, not findings: an
|
|
219
|
-
// entry is either one finding or a pre-deduplicated group of them, and
|
|
220
|
-
// the entry count is what a user sees as rows.
|
|
221
|
-
export function analyzeReport(content) {
|
|
222
|
-
const { data } = readReport(content)
|
|
223
|
-
if (!data) return { count: 0, recognized: false }
|
|
224
|
-
return { count: reportEntries(data).length, source: data.source, recognized: true }
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
// Entries → member findings. A group contributes its members; falsy
|
|
228
|
-
// and non-object entries (a malformed list's stray strings and nulls)
|
|
229
|
-
// are dropped rather than handed on as findings.
|
|
230
|
-
function flattenFindings(entries) {
|
|
231
|
-
return entries.flat().filter((f) => f && typeof f === 'object')
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
// Fill in `f.id` for any finding that lacks one, deriving it from the
|
|
235
|
-
// same fingerprint the analyzer stamps. Mutates in place. Findings
|
|
236
|
-
// whose id can't be derived (a host without crypto.subtle) are left
|
|
237
|
-
// untouched. Batched via Promise.all — sequential awaits would
|
|
238
|
-
// serialise hundreds of crypto.subtle.digest calls for no reason.
|
|
239
|
-
export async function backfillFindingIds(findings) {
|
|
240
|
-
// Managed responses may arrive as already-parsed objects.
|
|
241
|
-
for (const finding of findings) normalizeFindingSeverity(finding)
|
|
242
|
-
const idLess = findings.filter((f) => !f.id)
|
|
243
|
-
if (idLess.length === 0) return
|
|
244
|
-
const derived = await Promise.all(idLess.map(deriveFindingId))
|
|
245
|
-
idLess.forEach((f, i) => { if (derived[i]) f.id = derived[i] })
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
// Recognise, flatten, and give every finding an id — the whole read
|
|
249
|
-
// path in one call. Returns `{ format, data, findings }`, or null when
|
|
250
|
-
// nothing recognises the text. The findings are the parser's own
|
|
251
|
-
// objects (not copies), so a caller that means to keep them can, and
|
|
252
|
-
// one projecting run meta onto them has `inheritReportMeta` and `data`
|
|
253
|
-
// to hand.
|
|
254
|
-
export async function loadFindings(content) {
|
|
255
|
-
const { data, format } = readReport(content)
|
|
256
|
-
if (!data) return null
|
|
257
|
-
const findings = flattenFindings(reportEntries(data))
|
|
258
|
-
await backfillFindingIds(findings)
|
|
259
|
-
return { format, data, findings }
|
|
260
|
-
}
|
package/report/src/finding-id.js
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
// Finding ids, stamped by the analyzer onto its JSON output and filled
|
|
2
|
-
// in by the viewer for findings that arrive without one. Web Crypto is
|
|
3
|
-
// the common surface — `crypto.subtle` exists in modern Node and in
|
|
4
|
-
// secure browser contexts — so one implementation runs in both.
|
|
5
|
-
//
|
|
6
|
-
// Two reports from the same source give a finding the same id; an edit
|
|
7
|
-
// to its description or its source invalidates it.
|
|
8
|
-
|
|
9
|
-
import { encodeUtf8 } from './utf8.js'
|
|
10
|
-
|
|
11
|
-
function toHex(bytes) {
|
|
12
|
-
return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('')
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
// The file-content hash the JSON output format uses. sha512 because the
|
|
16
|
-
// id below hashes a string that already includes it, so a collision here
|
|
17
|
-
// would propagate into an id collision. Padded base64 with the SRI-style
|
|
18
|
-
// tag. `btoa` rather than `Uint8Array#toBase64`, still flagged in Node.
|
|
19
|
-
export async function computeFileHash(source) {
|
|
20
|
-
const bytes = typeof source === 'string' ? encodeUtf8(source) : source
|
|
21
|
-
const digest = new Uint8Array(await crypto.subtle.digest('SHA-512', bytes))
|
|
22
|
-
return `sha512-${btoa(String.fromCodePoint(...digest))}`
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
// A fingerprint hashed into a v4-shaped UUID: derived, not random, but
|
|
26
|
-
// the shape lets a downstream tool treat it as an opaque id.
|
|
27
|
-
async function fingerprintToId(fingerprint) {
|
|
28
|
-
const bytes = encodeUtf8(JSON.stringify(fingerprint))
|
|
29
|
-
const digest = await crypto.subtle.digest('SHA-256', bytes)
|
|
30
|
-
const u = new Uint8Array(digest, 0, 16)
|
|
31
|
-
// version 4: 0100xxxx
|
|
32
|
-
u[6] = (u[6] & 0x0f) | 0x40
|
|
33
|
-
// variant 1: 10xxxxxx
|
|
34
|
-
u[8] = (u[8] & 0x3f) | 0x80
|
|
35
|
-
const hex = toHex(u)
|
|
36
|
-
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
// Stable per-finding id from the (severity, description, fileHash) triple
|
|
40
|
-
// the analyzer emits. fileHash being undefined is fine — JSON.stringify
|
|
41
|
-
// drops undefined keys, so a finding with no hash keys off the pair and
|
|
42
|
-
// re-runs over the same source yield the same ids.
|
|
43
|
-
export function findingId(severity, description, fileHash) {
|
|
44
|
-
return fingerprintToId({ severity, description, fileHash })
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
// An id derived from a finding, on the first discriminator it carries.
|
|
48
|
-
// null when `crypto.subtle` is unavailable (some `file://` setups), so
|
|
49
|
-
// the caller can fall back to a session-local id — the UI still works,
|
|
50
|
-
// without persistent triage on those findings.
|
|
51
|
-
//
|
|
52
|
-
// In order:
|
|
53
|
-
// - _idBasis — a FROZEN fingerprint a parser stamped, used verbatim;
|
|
54
|
-
// it exists so a change to the rendered description
|
|
55
|
-
// can't re-key stored triage (parse-md-id.js).
|
|
56
|
-
// - fileHash — as `findingId` above.
|
|
57
|
-
// - location — a markdown import's url, also stable.
|
|
58
|
-
// - file/line — last resort for a JSON finding with neither: not what
|
|
59
|
-
// the spec prescribes, but better than collapsing two
|
|
60
|
-
// unrelated findings onto one id.
|
|
61
|
-
export async function deriveFindingId(f) {
|
|
62
|
-
if (typeof crypto?.subtle?.digest !== 'function') return null
|
|
63
|
-
const fingerprint = fingerprintOf(f)
|
|
64
|
-
try {
|
|
65
|
-
return await fingerprintToId(fingerprint)
|
|
66
|
-
} catch {
|
|
67
|
-
return null
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// The choice above as the object that gets hashed. Key order is part of
|
|
72
|
-
// the id — JSON.stringify keeps insertion order — so every shape lists
|
|
73
|
-
// severity and description first.
|
|
74
|
-
export function fingerprintOf(f) {
|
|
75
|
-
if (f._idBasis) return f._idBasis
|
|
76
|
-
const { severity, description } = f
|
|
77
|
-
if (f.fileHash) return { severity, description, fileHash: f.fileHash }
|
|
78
|
-
if (f.location) return { severity, description, location: f.location }
|
|
79
|
-
return { severity, description, file: f.file, line: f.line }
|
|
80
|
-
}
|