@preventive/triage 1.0.0-alpha.2 → 1.0.0-alpha.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/reap.ts +17 -0
- package/cli.js +6 -0
- package/client/finding-link.js +305 -0
- package/client/linked-findings.d.ts +1 -0
- package/client/linked-findings.js +111 -0
- package/common/bundle-metadata.d.ts +10 -0
- package/common/bundle-metadata.js +177 -0
- package/common/bundle-reasons.d.ts +2 -0
- package/common/bundle-reasons.js +21 -0
- package/common/bundle-sources.d.ts +3 -0
- package/common/bundle-sources.js +284 -0
- package/common/bundle-stats.js +41 -0
- package/common/bundle-tabs.js +1 -0
- package/common/code-language.js +36 -0
- package/common/default-scan-models.ts +30 -0
- package/common/finding-id.js +47 -0
- package/common/github-pr.ts +56 -0
- package/common/managed/comments.ts +34 -0
- package/common/managed/permissions.ts +35 -0
- package/common/managed/report-content.ts +42 -0
- package/common/managed/report-filter.ts +108 -0
- package/common/managed/roles.ts +28 -0
- package/common/managed/routes.d.ts +2 -0
- package/common/managed/routes.js +121 -0
- package/common/managed/scan-models.ts +6 -0
- package/common/managed/triage.ts +83 -0
- package/common/save-error-reason.ts +20 -7
- package/common/scan-server.ts +13 -0
- package/common/server-info.ts +33 -0
- package/common/utf8.d.ts +3 -0
- package/common/utf8.js +45 -0
- package/out/brotli-fallback.js +3 -3
- package/out/client-managed-import.js +81 -0
- package/out/client-managed.js +110 -0
- package/out/client-sync.js +16 -13
- package/out/graph.js +30 -4
- package/out/index.html +55 -8
- package/out/prism.js +2 -2
- package/out/stasis.svg +45 -0
- package/out/terminal.js +273 -39
- package/out/view.css +1 -1
- package/out/view.js +198 -62
- package/package.json +179 -55
- package/report/index.js +254 -0
- package/report/src/finding-id.js +80 -0
- package/report/src/finding.js +312 -0
- package/report/src/labels.js +33 -0
- package/report/src/md-structure.js +471 -0
- package/report/src/md-text.js +167 -0
- package/report/src/meta.js +76 -0
- package/report/src/parse-codex.js +147 -0
- package/report/src/parse-deepsec.js +197 -0
- package/report/src/parse-deepview-fields.js +375 -0
- package/report/src/parse-deepview-md.js +185 -0
- package/report/src/parse-md-id.js +137 -0
- package/report/src/parse-md.js +322 -0
- package/report/src/parse-piolium-id.js +79 -0
- package/report/src/parse-piolium-rows.js +131 -0
- package/report/src/parse-piolium-tokens.js +175 -0
- package/report/src/parse-piolium.js +400 -0
- package/report/src/security.js +63 -0
- package/report/src/utf8.js +21 -0
- package/report/src/write-md-finding.js +273 -0
- package/report/src/write-md.js +291 -0
- package/server-common/database-config.ts +16 -0
- package/server-common/initialize.ts +18 -0
- package/server-common/npm-advisories.ts +101 -0
- package/{server → server-common}/origin.ts +5 -5
- package/server-common/reap.ts +48 -0
- package/server-common/scan-config.ts +19 -0
- package/server-common/standalone.ts +29 -0
- package/server-common/storage-log.ts +34 -0
- package/server-common/vercel-blob.ts +110 -0
- package/server-e2e/app.ts +485 -0
- package/{server → server-e2e}/auth.ts +5 -1
- package/{server → server-e2e}/bus-receiver.ts +9 -8
- package/{server → server-e2e}/cli.js +9 -4
- package/{server → server-e2e}/config.ts +54 -39
- package/{server → server-e2e}/db-neon.ts +2 -2
- package/{server → server-e2e}/db-revision-sql.ts +7 -10
- package/{server → server-e2e}/db-stmt.ts +2 -2
- package/{server → server-e2e}/db.ts +96 -135
- package/{server → server-e2e}/http.ts +110 -12
- package/{server → server-e2e}/hub.ts +44 -14
- package/server-e2e/index.ts +17 -0
- package/server-e2e/lifecycle.ts +95 -0
- package/{server → server-e2e}/neon-driver.ts +2 -2
- package/{server → server-e2e}/npm-proxy.ts +11 -144
- package/{server → server-e2e}/objstore/blob-fs.ts +6 -8
- package/{server → server-e2e}/objstore/blob-vercel.ts +49 -141
- package/{server → server-e2e}/objstore/blob.ts +24 -9
- package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
- package/{server → server-e2e}/objstore/handlers.ts +19 -20
- package/{server → server-e2e}/objstore/init.ts +53 -27
- package/{server → server-e2e}/objstore/reaper.ts +31 -11
- package/server-e2e/objstore/rest-deny.ts +28 -0
- package/server-e2e/objstore/rest-mint.ts +224 -0
- package/{server → server-e2e}/objstore/rest.ts +110 -93
- package/{server → server-e2e}/objstore/sign.ts +105 -0
- package/{server → server-e2e}/objstore/store-neon.ts +10 -14
- package/{server → server-e2e}/objstore/store.ts +98 -118
- package/{server → server-e2e}/objstore/tokens.ts +9 -12
- package/{server → server-e2e}/peer.ts +7 -9
- package/{server → server-e2e}/pubsub.ts +29 -36
- package/{server → server-e2e}/sign.ts +12 -14
- package/{server → server-e2e}/sse-server.ts +105 -73
- package/{server → server-e2e}/sse-session.ts +30 -16
- package/{server → server-e2e}/static.ts +36 -29
- package/server-e2e/sync-handlers.ts +408 -0
- package/{server → server-e2e}/util.ts +9 -0
- package/{server → server-e2e}/ws-server.ts +29 -23
- package/server-managed/activity.ts +231 -0
- package/server-managed/avatar-store.ts +51 -0
- package/server-managed/blob-store.ts +66 -0
- package/server-managed/blob-vercel.ts +125 -0
- package/server-managed/brotli.ts +10 -0
- package/server-managed/bundle-cache.ts +185 -0
- package/server-managed/bundle-catalog.ts +29 -0
- package/server-managed/bundle-store.ts +28 -0
- package/server-managed/bundle-summary-cache.ts +97 -0
- package/server-managed/bundle.ts +39 -0
- package/server-managed/cache-storage.ts +40 -0
- package/server-managed/cli.js +13 -0
- package/server-managed/combined.ts +46 -0
- package/server-managed/comments.ts +151 -0
- package/server-managed/config.ts +144 -0
- package/server-managed/content-access.ts +15 -0
- package/server-managed/crypto.ts +25 -0
- package/server-managed/db-methods.ts +1330 -0
- package/server-managed/db-neon.ts +171 -0
- package/server-managed/db-schema.ts +203 -0
- package/server-managed/db-table-names.ts +22 -0
- package/server-managed/db.ts +109 -0
- package/server-managed/github-app.ts +332 -0
- package/server-managed/github-metadata.ts +65 -0
- package/server-managed/github-oauth.ts +215 -0
- package/server-managed/github-pulls.ts +115 -0
- package/server-managed/http-response.ts +18 -0
- package/server-managed/http.ts +2043 -0
- package/server-managed/import-triage.ts +48 -0
- package/server-managed/index.ts +135 -0
- package/server-managed/public-workspace.ts +150 -0
- package/server-managed/repo-path.ts +21 -0
- package/server-managed/report-migration.ts +35 -0
- package/server-managed/report-query.ts +4 -0
- package/server-managed/report-response.ts +16 -0
- package/server-managed/report-sources.ts +154 -0
- package/server-managed/repository-discovery.ts +82 -0
- package/server-managed/repository-policy.ts +25 -0
- package/server-managed/session.ts +78 -0
- package/server-managed/slugs.ts +38 -0
- package/server-managed/sql-postgres.ts +30 -0
- package/server-managed/sql.ts +61 -0
- package/server-managed/static.ts +28 -0
- package/server-managed/storage.ts +34 -0
- package/server-managed/team-catalog.ts +7 -0
- package/server-managed/team-feed.ts +128 -0
- package/server-managed/team-reports.ts +156 -0
- package/server-managed/triage-response.ts +16 -0
- package/server-managed/uploads.ts +47 -0
- package/server-managed/workspace-shares.ts +150 -0
- package/server.ts +50 -0
- package/server/index.ts +0 -481
- package/server/lifecycle.ts +0 -204
- package/server/sync-handlers.ts +0 -327
- /package/{server → server-e2e}/config.example.json +0 -0
- /package/{server → server-e2e}/objstore/fs.ts +0 -0
- /package/{server → server-e2e}/validation.ts +0 -0
|
@@ -2,15 +2,16 @@
|
|
|
2
2
|
// vars, the optional config.json) into one immutable `Config`, failing
|
|
3
3
|
// loud on malformed values so a typo surfaces at startup rather than
|
|
4
4
|
// deep in `node:net` / at the first token verification. Pure parsing —
|
|
5
|
-
// no backends opened
|
|
6
|
-
//
|
|
7
|
-
// result and does the wiring (backend selection, password HMAC, …).
|
|
5
|
+
// no backends opened or process termination. Invalid configuration throws so
|
|
6
|
+
// embedding hosts can recover; app.ts handles backend selection and wiring.
|
|
8
7
|
|
|
9
8
|
import { Buffer } from 'node:buffer'
|
|
10
9
|
import { readFileSync } from 'node:fs'
|
|
11
|
-
import {
|
|
10
|
+
import { env } from 'node:process'
|
|
12
11
|
import { fileURLToPath } from 'node:url'
|
|
13
12
|
import { dirname, join } from 'node:path'
|
|
13
|
+
import { configuredScanServer } from '../server-common/scan-config.ts'
|
|
14
|
+
import { databaseUrls } from '../server-common/database-config.ts'
|
|
14
15
|
|
|
15
16
|
export type Config = {
|
|
16
17
|
port: number
|
|
@@ -18,6 +19,7 @@ export type Config = {
|
|
|
18
19
|
dbPath: string
|
|
19
20
|
objstoreDir: string
|
|
20
21
|
reapIntervalMs: number
|
|
22
|
+
reapDisabled: boolean
|
|
21
23
|
maxInflightPerSocket: number
|
|
22
24
|
debug: boolean
|
|
23
25
|
neonUrl: string | null
|
|
@@ -25,9 +27,10 @@ export type Config = {
|
|
|
25
27
|
tokenSecret: Uint8Array<ArrayBuffer> | null
|
|
26
28
|
password: string | null
|
|
27
29
|
trustProxyEnv: string | undefined
|
|
30
|
+
deepviewScanServer: string | null
|
|
28
31
|
}
|
|
29
32
|
|
|
30
|
-
// Parse + range-validate an integer env var,
|
|
33
|
+
// Parse + range-validate an integer env var, throwing a clear
|
|
31
34
|
// up-front message on a malformed value — a NaN from `Number("abc")`
|
|
32
35
|
// otherwise surfaces as a confusing crash deep inside `node:net`
|
|
33
36
|
// (`WebSocketServer({ port: NaN })`) or a 0-ms `setInterval` loop. An
|
|
@@ -38,20 +41,28 @@ function intEnv(name: string, def: number, min: number, max: number, hint = ''):
|
|
|
38
41
|
const raw = env[name]
|
|
39
42
|
const n = raw == null ? def : Number(raw)
|
|
40
43
|
if (!Number.isSafeInteger(n) || n < min || n > max) {
|
|
41
|
-
|
|
42
|
-
process.exit(1)
|
|
44
|
+
throw new Error(`Invalid ${name}: ${raw} — must be an integer in [${min}, ${max}].${hint ? ` ${hint}` : ''}`)
|
|
43
45
|
}
|
|
44
46
|
return n
|
|
45
47
|
}
|
|
46
48
|
|
|
47
|
-
const HELP = `Usage: node server/index.ts
|
|
49
|
+
export const HELP = `Usage: node server-e2e/index.ts
|
|
48
50
|
Environment:
|
|
49
51
|
PORT listen port (default 8765)
|
|
50
52
|
HOST bind host (default 127.0.0.1)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
DEEPVIEW_SCAN_SERVER optional HTTP(S) scan-service URL; advertised
|
|
54
|
+
in /api/config and allowed by the UI CSP.
|
|
55
|
+
Unset by default (no external scan service).
|
|
56
|
+
DB_PATH sqlite file (default: server-e2e/data/data.db);
|
|
57
|
+
ignored when an e2e database URL is set
|
|
58
|
+
DATABASE_URL shared Neon Postgres URL for enabled modes.
|
|
59
|
+
Cannot be combined with E2E_DATABASE_URL
|
|
60
|
+
or MANAGED_DATABASE_URL.
|
|
61
|
+
E2E_DATABASE_URL e2e-only Neon URL, used without DATABASE_URL.
|
|
62
|
+
Combined mode also requires
|
|
63
|
+
MANAGED_DATABASE_URL; mixing Neon and
|
|
64
|
+
SQLite backends is rejected.
|
|
65
|
+
Either e2e URL selects Neon instead of
|
|
55
66
|
SQLite. Requires the optional peer dep
|
|
56
67
|
@neondatabase/serverless. The Neon
|
|
57
68
|
pairing additionally requires
|
|
@@ -60,12 +71,12 @@ Environment:
|
|
|
60
71
|
local-FS bytes cannot back a multi-
|
|
61
72
|
replica DB plane.
|
|
62
73
|
BLOB_READ_WRITE_TOKEN Vercel Blob R/W token (private store).
|
|
63
|
-
Required when
|
|
74
|
+
Required when an e2e database URL is set;
|
|
64
75
|
ignored otherwise. Requires the optional
|
|
65
76
|
peer dep @vercel/blob.
|
|
66
77
|
OBJSTORE_TOKEN_SECRET Base64 (32 bytes) HMAC secret for REST
|
|
67
|
-
bearer tokens. REQUIRED
|
|
68
|
-
|
|
78
|
+
bearer tokens. REQUIRED in Neon mode
|
|
79
|
+
(multi-replica deployments: a
|
|
69
80
|
token minted on one replica's WS plane
|
|
70
81
|
must validate on another replica's REST
|
|
71
82
|
plane). Optional under SQLite (a fresh
|
|
@@ -75,9 +86,15 @@ Environment:
|
|
|
75
86
|
OBJSTORE_DIR object store root (default: ./objstore
|
|
76
87
|
next to DB_PATH). Used by the local-FS
|
|
77
88
|
byte plane only; ignored when
|
|
78
|
-
|
|
79
|
-
|
|
89
|
+
an e2e database URL selects Neon
|
|
90
|
+
(bytes live in Vercel Blob).
|
|
80
91
|
OBJSTORE_REAP_INTERVAL_MS orphan reaper period (default 600000)
|
|
92
|
+
OBJSTORE_REAP_DISABLED set '1' / 'true' to disable automatic orphan
|
|
93
|
+
sweeps (boot and periodic). Explicit /api/reap
|
|
94
|
+
remains available. Schedule cleanup externally
|
|
95
|
+
to avoid accumulating orphaned bytes.
|
|
96
|
+
CRON_SECRET bearer secret for GET /api/reap (fails closed
|
|
97
|
+
when unset). Runs cleanup for enabled modes.
|
|
81
98
|
TRUST_PROXY set '1' / 'true' to honour X-Forwarded-
|
|
82
99
|
Host / X-Forwarded-Proto when computing
|
|
83
100
|
the same-origin gate's expected origin.
|
|
@@ -95,7 +112,7 @@ Environment:
|
|
|
95
112
|
that need to deterministically
|
|
96
113
|
exercise the cap.
|
|
97
114
|
CONFIG_PATH operator config JSON path (default:
|
|
98
|
-
server/config.json). Currently the
|
|
115
|
+
server-e2e/config.json). Currently the
|
|
99
116
|
only field is { "password": "..." }
|
|
100
117
|
which gates first-action creation of
|
|
101
118
|
a new workspace on the
|
|
@@ -113,11 +130,11 @@ function readServerConfigFile(path: string): ServerConfigFile {
|
|
|
113
130
|
let raw: string
|
|
114
131
|
try { raw = readFileSync(path, 'utf8') } catch (err) {
|
|
115
132
|
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return {}
|
|
116
|
-
|
|
133
|
+
throw new Error(`Failed to read ${path}: ${(err as Error)?.message ?? err}`, { cause: err })
|
|
117
134
|
}
|
|
118
135
|
try { return JSON.parse(raw) as ServerConfigFile }
|
|
119
136
|
catch (err) {
|
|
120
|
-
|
|
137
|
+
throw new Error(`Failed to parse ${path} as JSON: ${(err as Error)?.message ?? err}`, { cause: err })
|
|
121
138
|
}
|
|
122
139
|
}
|
|
123
140
|
|
|
@@ -132,20 +149,17 @@ function decodeTokenSecret(raw: string): Uint8Array<ArrayBuffer> {
|
|
|
132
149
|
// typo-detector below would fail with a misleading message.
|
|
133
150
|
const trimmed = raw.trim()
|
|
134
151
|
if (trimmed.length === 0) {
|
|
135
|
-
|
|
136
|
-
process.exit(1)
|
|
152
|
+
throw new Error('OBJSTORE_TOKEN_SECRET is empty after trimming whitespace')
|
|
137
153
|
}
|
|
138
154
|
const decoded = Buffer.from(trimmed, 'base64')
|
|
139
155
|
const reencoded = decoded.toString('base64')
|
|
140
156
|
const norm = (s: string): string => s.replace(/=+$/u, '')
|
|
141
157
|
if (norm(reencoded) !== norm(trimmed)) {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
process.exit(1)
|
|
158
|
+
throw new Error('OBJSTORE_TOKEN_SECRET contains non-base64 characters (likely a typo, e.g. base64url chars in a base64 secret).\n' +
|
|
159
|
+
'Regenerate with: node -e \'console.log(require("crypto").randomBytes(32).toString("base64"))\'')
|
|
145
160
|
}
|
|
146
161
|
if (decoded.byteLength !== 32) {
|
|
147
|
-
|
|
148
|
-
process.exit(1)
|
|
162
|
+
throw new Error(`OBJSTORE_TOKEN_SECRET must decode to 32 bytes (got ${decoded.byteLength})`)
|
|
149
163
|
}
|
|
150
164
|
// Copy into a fresh ArrayBuffer so the type matches
|
|
151
165
|
// `Uint8Array<ArrayBuffer>` (Buffer may be SharedArrayBuffer-backed).
|
|
@@ -156,41 +170,42 @@ export function loadConfig(): Config {
|
|
|
156
170
|
// 0 = OS-assigned ephemeral port (the test harness boots with PORT=0).
|
|
157
171
|
const port = intEnv('PORT', 8765, 0, 65535)
|
|
158
172
|
const host = env['HOST'] ?? '127.0.0.1'
|
|
159
|
-
// `fileURLToPath` decodes percent-escapes / non-ASCII path segments
|
|
160
|
-
//
|
|
173
|
+
// `fileURLToPath` decodes percent-escapes / non-ASCII path segments;
|
|
174
|
+
// `new URL(...).pathname` would leave `%20` raw.
|
|
161
175
|
const dbPath = env['DB_PATH'] ?? fileURLToPath(new URL('./data/data.db', import.meta.url))
|
|
162
176
|
// `path.join` so a Windows DB_PATH doesn't get a mixed-separator child.
|
|
163
177
|
const objstoreDir = env['OBJSTORE_DIR'] ?? join(dirname(dbPath), 'objstore')
|
|
164
178
|
// No practical upper bound beyond the safe-integer range.
|
|
165
179
|
const reapIntervalMs = intEnv('OBJSTORE_REAP_INTERVAL_MS', 10 * 60 * 1000, 1, Number.MAX_SAFE_INTEGER)
|
|
180
|
+
// Off-switch for automatic orphan reaping (both the boot sweep AND the
|
|
181
|
+
// periodic timer). '1' / 'true' (case-insensitive) → disabled; anything
|
|
182
|
+
// else, including unset, leaves it ON. Same boolean shape as TRUST_PROXY.
|
|
183
|
+
const reapDisabledEnv = env['OBJSTORE_REAP_DISABLED']
|
|
184
|
+
const reapDisabled = reapDisabledEnv === '1' || reapDisabledEnv?.toLowerCase() === 'true'
|
|
166
185
|
const debug = env['DEBUG'] === '1'
|
|
167
186
|
|
|
168
187
|
const configPath = env['CONFIG_PATH'] ?? fileURLToPath(new URL('./config.json', import.meta.url))
|
|
169
188
|
const serverConfig = readServerConfigFile(configPath)
|
|
170
189
|
const rawPassword = serverConfig.password
|
|
171
190
|
if (rawPassword != null && typeof rawPassword !== 'string') {
|
|
172
|
-
|
|
191
|
+
throw new Error(`Invalid ${configPath}: "password" must be a string or null`)
|
|
173
192
|
}
|
|
174
193
|
const password = rawPassword ?? null
|
|
175
194
|
// Upper bound 65_536 — bounds memory under hostile load; a deployer
|
|
176
195
|
// passing MAX_SAFE_INTEGER would silently defeat the cap. Validated
|
|
177
|
-
// here
|
|
178
|
-
//
|
|
196
|
+
// here, after the config.json / password parse, to keep the
|
|
197
|
+
// error-precedence order.
|
|
179
198
|
const maxInflightPerSocket = intEnv('MAX_INFLIGHT_PER_SOCKET', 64, 1, 65_536)
|
|
180
199
|
|
|
181
|
-
|
|
182
|
-
console.log(HELP)
|
|
183
|
-
process.exit(0)
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
const neonUrl = env['DATABASE_URL'] ?? null
|
|
200
|
+
const neonUrl = databaseUrls().e2e
|
|
187
201
|
const blobToken = env['BLOB_READ_WRITE_TOKEN'] ?? null
|
|
188
202
|
const tokenSecretB64 = env['OBJSTORE_TOKEN_SECRET'] ?? null
|
|
189
203
|
const tokenSecret = tokenSecretB64 ? decodeTokenSecret(tokenSecretB64) : null
|
|
190
204
|
|
|
191
205
|
return {
|
|
192
|
-
port, host, dbPath, objstoreDir, reapIntervalMs, maxInflightPerSocket,
|
|
206
|
+
port, host, dbPath, objstoreDir, reapIntervalMs, reapDisabled, maxInflightPerSocket,
|
|
193
207
|
debug, neonUrl, blobToken, tokenSecret, password,
|
|
194
208
|
trustProxyEnv: env['TRUST_PROXY'],
|
|
209
|
+
deepviewScanServer: configuredScanServer(env['DEEPVIEW_SCAN_SERVER']),
|
|
195
210
|
}
|
|
196
211
|
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// dialect on the wire.
|
|
5
5
|
//
|
|
6
6
|
// `@neondatabase/serverless` is an OPTIONAL peer dep — selected by
|
|
7
|
-
// the
|
|
7
|
+
// the configured database URL branch in `server-e2e/index.ts`. The peer dep
|
|
8
8
|
// itself is loaded lazily via the dynamic `import()` inside
|
|
9
9
|
// `openNeonDb` below, so a SQLite-only deployment never installs it
|
|
10
10
|
// (`autoInstallPeers: false` in `pnpm-workspace.yaml`) and never
|
|
@@ -340,7 +340,7 @@ export async function openNeonDb(connectionString: string): Promise<Handle> {
|
|
|
340
340
|
// in-process Postgres (PGlite) via `mock.module`: that hook can only
|
|
341
341
|
// intercept a specifier it can RESOLVE, and the optional peer dep
|
|
342
342
|
// isn't installed in a SQLite-only checkout. The wrapper path always
|
|
343
|
-
// resolves — see `server/neon-driver.ts`. Cast through `unknown`
|
|
343
|
+
// resolves — see `server-e2e/neon-driver.ts`. Cast through `unknown`
|
|
344
344
|
// because the wrapper's `export *` re-exports a `@ts-ignore`'d
|
|
345
345
|
// (possibly-absent) module, so tsc can't see `neon`'s type here.
|
|
346
346
|
const mod = (await import('./neon-driver.ts')) as unknown as { neon: (url: string) => NeonSql }
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
// Shared SQL + row-mapping for the `workspace_revision` chain, used by
|
|
2
2
|
// BOTH backends — `./db.ts` (SQLite) and `./db-neon.ts` (Neon/Postgres).
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// duplication is collapsed here so a query edit can't silently drift
|
|
6
|
-
// between backends.
|
|
3
|
+
// Single source of truth (modulo `?`↔`$N` placeholders) so a query edit
|
|
4
|
+
// can't silently drift between backends.
|
|
7
5
|
//
|
|
8
6
|
// Single source of truth, in `$N` (Postgres) form:
|
|
9
7
|
// • the read queries (`headFor`, `seqOfId`, `lastKeyframeSeq`, the
|
|
@@ -60,11 +58,10 @@ export function numOrNull(v: unknown): number | null {
|
|
|
60
58
|
// `Record<string, unknown>` rows whose `keyframe` may be a number OR (on
|
|
61
59
|
// a future driver change) a string; `node:sqlite` hands back native
|
|
62
60
|
// numbers. The `num`/`numOrNull` coercion is safe over both — a native
|
|
63
|
-
// `0`/`1` integer passes through unchanged,
|
|
64
|
-
//
|
|
65
|
-
//
|
|
66
|
-
//
|
|
67
|
-
// 0 / 1 via the `=== 1` check the chain-broadcast contract relies on.
|
|
61
|
+
// `0`/`1` integer passes through unchanged, while Neon rows keep their
|
|
62
|
+
// defensive string→number coercion. `base` is the only nullable column
|
|
63
|
+
// (first revision); `keyframe` collapses to a strict 0 / 1 via the
|
|
64
|
+
// `=== 1` check the chain-broadcast contract relies on.
|
|
68
65
|
export function mapRevisionRow(r: Record<string, unknown>): RevisionRow {
|
|
69
66
|
return {
|
|
70
67
|
base: (r['base'] as string | null) ?? null,
|
|
@@ -109,7 +106,7 @@ export const CHAIN_FROM_SQL =
|
|
|
109
106
|
export const REVISION_EXISTS_SQL =
|
|
110
107
|
`SELECT 1 AS one FROM workspace_revision WHERE workspace_tag = $1 AND id = $2`
|
|
111
108
|
// Fetch a single revision row by content-addressed id. The cross-instance
|
|
112
|
-
// pubsub (server/pubsub.ts) NOTIFY payload carries only `(tag, revisionId)`
|
|
109
|
+
// pubsub (server-e2e/pubsub.ts) NOTIFY payload carries only `(tag, revisionId)`
|
|
113
110
|
// because the full `workspace-state` broadcast envelope is bounded by
|
|
114
111
|
// `MAX_CIPHERTEXT_LEN` (2 MiB) and Postgres NOTIFY caps payloads at ~8 KB.
|
|
115
112
|
// The receiver re-fetches the row from this shared table to construct the
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// Shared async-statement primitives. Both `server/db.ts` (workspace_revision
|
|
2
|
-
// chain) and `server/objstore/store.ts` (objstore tables) expose Handles
|
|
1
|
+
// Shared async-statement primitives. Both `server-e2e/db.ts` (workspace_revision
|
|
2
|
+
// chain) and `server-e2e/objstore/store.ts` (objstore tables) expose Handles
|
|
3
3
|
// whose statements look like `{ get(...) → Promise<…>, all(...) → Promise<[…]>,
|
|
4
4
|
// run(...) → Promise<void> }`. The underlying `node:sqlite` driver is
|
|
5
5
|
// synchronous; the wrappers below catch sync errors and route them through
|
|
@@ -10,49 +10,30 @@
|
|
|
10
10
|
// `base` points at the previous revision's `id` (or null for the
|
|
11
11
|
// first revision in a workspace).
|
|
12
12
|
//
|
|
13
|
-
// `keyframe` is `1` for a revision the client emits with
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
// `
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
// reaches this column. Client-driven: the server only stores what
|
|
22
|
-
// the client sent and treats keyframes as catch-up roots when a
|
|
23
|
-
// from=null subscriber arrives.
|
|
13
|
+
// `keyframe` is `1` for a revision the client emits with full state
|
|
14
|
+
// baked in (rather than a delta). The wire flag is covered by the
|
|
15
|
+
// signature, so the column value MUST match the signed canonical
|
|
16
|
+
// bytes: `canonicalSave` (server-e2e/sign.ts, via `handleSave`) encodes
|
|
17
|
+
// `keyframe ? '1' : ''` into the bytes `verifyEd25519` checks, so a
|
|
18
|
+
// mismatched wire flag fails verify and never reaches this column.
|
|
19
|
+
// Client-driven: the server stores what the client sent and treats
|
|
20
|
+
// keyframes as catch-up roots when a from=null subscriber arrives.
|
|
24
21
|
//
|
|
25
22
|
// `node:sqlite` is the built-in driver (Node ≥ 22 experimental,
|
|
26
|
-
// stable in 24+)
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
// off a sync `node:sqlite` call.
|
|
23
|
+
// stable in 24+), synchronous under the hood; the Handle wraps each
|
|
24
|
+
// prepared statement so call sites `await` uniformly — async-ready
|
|
25
|
+
// surface for a future async DB backend (every op resolves in the
|
|
26
|
+
// current microtask off a sync call).
|
|
31
27
|
//
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
//
|
|
41
|
-
// can interleave mid-statement, and the head-check + MAX(seq) read
|
|
42
|
-
// from ONE consistent snapshot. That single-snapshot property is
|
|
43
|
-
// what makes a per-tag lock redundant: the lock formerly existed
|
|
44
|
-
// only to stop a chain fork where a racer read `head` from one
|
|
45
|
-
// snapshot but `MAX(seq)` from a LATER one (after a sibling
|
|
46
|
-
// committed) and inserted (seq=N+2, base=X) alongside the winner's
|
|
47
|
-
// (seq=N+1, base=X) — same base, different seq, no PK conflict. With
|
|
48
|
-
// both reads inside one statement that interleaving is impossible:
|
|
49
|
-
// a racer's snapshot is either before the winner's commit (→ same
|
|
50
|
-
// seq=N+1 → the UNIQUE(workspace_tag, seq) PK rejects the second →
|
|
51
|
-
// recovery → stale-base) or after it (→ head ≠ base → no insert →
|
|
52
|
-
// stale-base). Exactly one commits; the loser gets stale-base.
|
|
53
|
-
// SQLite also serialises writers internally, and the PK backstops
|
|
54
|
-
// the unsupported multi-connection case. See `commitRevisionSqlite`
|
|
55
|
-
// for the full fork-safety argument.
|
|
28
|
+
// Operations being async, two handlers can interleave across an
|
|
29
|
+
// `await`. `commitRevision` (below) takes NO in-process lock — it
|
|
30
|
+
// folds the dup recheck, base-equality check, MAX(seq) and INSERT
|
|
31
|
+
// into ONE gated INSERT (`commitRevisionSqlite`). `node:sqlite` runs
|
|
32
|
+
// that statement to completion without yielding, so its head-check +
|
|
33
|
+
// MAX(seq) read ONE snapshot, which is what makes a per-tag lock
|
|
34
|
+
// redundant. SQLite also serialises writers internally, and the PK
|
|
35
|
+
// backstops the unsupported multi-connection case. See
|
|
36
|
+
// `commitRevisionSqlite` for the full fork-safety argument.
|
|
56
37
|
|
|
57
38
|
import { DatabaseSync } from 'node:sqlite'
|
|
58
39
|
import { mkdirSync } from 'node:fs'
|
|
@@ -64,24 +45,20 @@ import {
|
|
|
64
45
|
mapRevisionRow, toSqlitePlaceholders,
|
|
65
46
|
} from './db-revision-sql.ts'
|
|
66
47
|
|
|
67
|
-
// `CHECK (keyframe IN (0, 1))` is the value-domain guard
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
//
|
|
71
|
-
//
|
|
72
|
-
// 0
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
//
|
|
76
|
-
// column TYPE. The CHECK closes the value-domain half, giving SQLite
|
|
77
|
-
// the protection the Neon schema's identical `CHECK (keyframe IN
|
|
78
|
-
// (0, 1))` carries (see `db-neon.ts`).
|
|
48
|
+
// `CHECK (keyframe IN (0, 1))` is the value-domain guard. STRICT
|
|
49
|
+
// (the table marker) enforces the column TYPE (an INTEGER stays an
|
|
50
|
+
// INTEGER) but NOT its value range: `keyframe = 2` is a valid integer
|
|
51
|
+
// STRICT accepts, which `mapRevisionRow`'s `=== 1` check then coerces
|
|
52
|
+
// back to 0. That divergence from the signed canonical (only ever
|
|
53
|
+
// 0 / 1) poisons chain-replay verifies for any recomputing peer — the
|
|
54
|
+
// same operator-with-direct-DB-write vector the STRICT guard in
|
|
55
|
+
// `openDbInner` catches for TYPE. The CHECK closes the value-domain
|
|
56
|
+
// half, matching the Neon schema's identical CHECK (see `db-neon.ts`).
|
|
79
57
|
//
|
|
80
|
-
//
|
|
81
|
-
//
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
// one, and a future column edit can't drift the two apart.
|
|
58
|
+
// Parenthesised column + constraint body (plus STRICT marker), shared
|
|
59
|
+
// by the initial `CREATE TABLE` and the `migrateAddKeyframeCheck`
|
|
60
|
+
// rebuild below — so a rebuilt table is byte-identical in shape to a
|
|
61
|
+
// fresh one and a future column edit can't drift the two apart.
|
|
85
62
|
const WORKSPACE_REVISION_DEF = `(
|
|
86
63
|
workspace_tag TEXT NOT NULL,
|
|
87
64
|
seq INTEGER NOT NULL,
|
|
@@ -105,10 +82,10 @@ const SCHEMA = `
|
|
|
105
82
|
${WORKSPACE_REVISION_TAG_ID_INDEX};
|
|
106
83
|
`
|
|
107
84
|
|
|
108
|
-
// Row shape
|
|
109
|
-
//
|
|
110
|
-
// to a strict boolean before broadcasting
|
|
111
|
-
//
|
|
85
|
+
// Row shape from the chain queries. `keyframe` is stored as INTEGER
|
|
86
|
+
// (0 / 1); the raw row carries the integer — `chainForWire` in
|
|
87
|
+
// server-e2e/index.ts normalises to a strict boolean before broadcasting.
|
|
88
|
+
// `base` is nullable on the very first revision.
|
|
112
89
|
export type RevisionRow = {
|
|
113
90
|
base: string | null
|
|
114
91
|
id: string
|
|
@@ -119,9 +96,8 @@ export type RevisionRow = {
|
|
|
119
96
|
}
|
|
120
97
|
|
|
121
98
|
// Input to `commitRevision`. `keyframe` is a strict boolean here —
|
|
122
|
-
// the canonical-payload contract uses `=== true
|
|
123
|
-
//
|
|
124
|
-
// STRICT INTEGER column.
|
|
99
|
+
// the canonical-payload contract uses `=== true`; the storage path
|
|
100
|
+
// coerces to 0 / 1 before hitting the STRICT INTEGER column.
|
|
125
101
|
export type RevisionInsert = {
|
|
126
102
|
tag: string
|
|
127
103
|
id: string
|
|
@@ -142,37 +118,31 @@ export type CommitResult =
|
|
|
142
118
|
| { kind: 'duplicate' }
|
|
143
119
|
| { kind: 'stale-base'; head: string | null }
|
|
144
120
|
|
|
145
|
-
//
|
|
146
|
-
//
|
|
121
|
+
// Pre-prepared statements + the underlying connection, held for the
|
|
122
|
+
// process lifetime; `close()` runs from `shutdown()`.
|
|
147
123
|
//
|
|
148
|
-
// `db` is the raw `DatabaseSync
|
|
149
|
-
//
|
|
150
|
-
//
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
//
|
|
124
|
+
// `db` is the raw `DatabaseSync`, SQLite-only — the Neon backend
|
|
125
|
+
// (`./db-neon.ts`) constructs a Handle with `db` unset. Callers that
|
|
126
|
+
// reach into `db` directly (e.g. `openObjstore`, test-only fixture
|
|
127
|
+
// SQL) are SQLite-coupled by construction; passing them a Neon-backed
|
|
128
|
+
// Handle is the operator's mistake to catch at the database URL
|
|
129
|
+
// switch in `server-e2e/index.ts`.
|
|
154
130
|
//
|
|
155
|
-
// `tryCommit` is the backend-specific atomic-commit primitive
|
|
131
|
+
// `tryCommit` is the backend-specific atomic-commit primitive
|
|
156
132
|
// `commitRevision` dispatches through. SQLite runs one synchronous
|
|
157
|
-
// gated INSERT (
|
|
158
|
-
//
|
|
159
|
-
//
|
|
160
|
-
//
|
|
161
|
-
// READ-COMMITTED single-statement snapshot (the gated INSERT's
|
|
162
|
-
// head-check and MAX(seq) read one snapshot) plus the
|
|
163
|
-
// `UNIQUE(workspace_tag, seq)` PK to keep cross-replica racers from
|
|
164
|
-
// forking the chain — see `db-neon.ts`'s `tryCommitNeon`.
|
|
133
|
+
// gated INSERT (see `commitRevisionSqlite`); Neon wraps it in a
|
|
134
|
+
// pipelined transaction (see `db-neon.ts`'s `tryCommitNeon`). Both
|
|
135
|
+
// rely on a single-statement snapshot + the `UNIQUE(workspace_tag,
|
|
136
|
+
// seq)` PK for fork-safety; see those functions for the argument.
|
|
165
137
|
//
|
|
166
138
|
// `gatedInsert` is SQLite-only (like `db`): it backs
|
|
167
|
-
// `commitRevisionSqlite`'s single gated INSERT
|
|
168
|
-
//
|
|
169
|
-
//
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
//
|
|
173
|
-
//
|
|
174
|
-
// unique-violation / non-unique failure into the commit, the same
|
|
175
|
-
// recovery paths the Neon suite stages via `failNextCommit`.
|
|
139
|
+
// `commitRevisionSqlite`'s single gated INSERT. The Neon backend
|
|
140
|
+
// leaves it unset — its gated INSERT lives inside the pipelined
|
|
141
|
+
// `sql.transaction([...])`, not a standalone statement object. Kept
|
|
142
|
+
// on the Handle (not a module-private closure) so SQLite white-box
|
|
143
|
+
// tests can wrap `.get` to inject a unique-violation / non-unique
|
|
144
|
+
// failure into the commit, exercising the same recovery paths the
|
|
145
|
+
// Neon suite stages via `failNextCommit`.
|
|
176
146
|
export type Handle = {
|
|
177
147
|
db?: DatabaseSync
|
|
178
148
|
headFor: GetStmt<[string], { id: string }>
|
|
@@ -184,7 +154,7 @@ export type Handle = {
|
|
|
184
154
|
revisionExists: GetStmt<[string, string], unknown>
|
|
185
155
|
// Single-revision fetch by content-addressed id. The cross-instance
|
|
186
156
|
// pubsub receiver uses this to assemble a `workspace-state` from a
|
|
187
|
-
// NOTIFY hint (see `server/pubsub.ts`).
|
|
157
|
+
// NOTIFY hint (see `server-e2e/pubsub.ts`).
|
|
188
158
|
revisionById: GetStmt<[string, string], RevisionRow>
|
|
189
159
|
gatedInsert?: GetStmt<[string, string, string | null, number, string, string, string, number], { seq: number }>
|
|
190
160
|
tryCommit: (input: RevisionInsert) => Promise<CommitResult>
|
|
@@ -192,23 +162,22 @@ export type Handle = {
|
|
|
192
162
|
}
|
|
193
163
|
|
|
194
164
|
// Narrowing alias for the SQLite-backed Handle: `db` is guaranteed
|
|
195
|
-
//
|
|
165
|
+
// set. `openDb` returns this so call sites needing direct
|
|
196
166
|
// `DatabaseSync` access (e.g. `openObjstore(handle.db, …)` in
|
|
197
|
-
// `server/index.ts`'s SQLite branch)
|
|
198
|
-
//
|
|
199
|
-
// (`openNeonDb`) keeps the wider `db?: DatabaseSync` shape
|
|
200
|
-
//
|
|
201
|
-
//
|
|
167
|
+
// `server-e2e/index.ts`'s SQLite branch) reach `handle.db` without an
|
|
168
|
+
// optional-chain or non-null assertion. A Neon-backed Handle
|
|
169
|
+
// (`openNeonDb`) keeps the wider `db?: DatabaseSync` shape, so routing
|
|
170
|
+
// one into a SQLite-coupled call site is a compile-time error. Mirrors
|
|
171
|
+
// `server-e2e/objstore/store.ts`.
|
|
202
172
|
export type SqliteHandle = Handle & { db: DatabaseSync }
|
|
203
173
|
|
|
204
174
|
export function openDb(path: string): SqliteHandle {
|
|
205
175
|
mkdirSync(dirname(path), { recursive: true })
|
|
206
176
|
const db = new DatabaseSync(path)
|
|
207
|
-
//
|
|
208
|
-
//
|
|
209
|
-
//
|
|
210
|
-
//
|
|
211
|
-
// …) and re-run without a stale lock pinning the file.
|
|
177
|
+
// A throw between the DatabaseSync constructor and the return would
|
|
178
|
+
// leak the file / WAL / shm locks until process exit — close before
|
|
179
|
+
// re-raising so the operator can fix the cause (failed STRICT check,
|
|
180
|
+
// ALTER TABLE error, …) and re-run without a stale lock on the file.
|
|
212
181
|
try {
|
|
213
182
|
return openDbInner(db)
|
|
214
183
|
} catch (err) {
|
|
@@ -218,35 +187,30 @@ export function openDb(path: string): SqliteHandle {
|
|
|
218
187
|
}
|
|
219
188
|
|
|
220
189
|
function openDbInner(db: DatabaseSync): SqliteHandle {
|
|
221
|
-
// WAL gives concurrent readers + faster writes and survives
|
|
222
|
-
//
|
|
223
|
-
//
|
|
224
|
-
//
|
|
225
|
-
// tables later without revisiting init.
|
|
190
|
+
// WAL gives concurrent readers + faster writes and survives crashes
|
|
191
|
+
// between commits without corrupting the file. Foreign keys aren't
|
|
192
|
+
// needed here (single-table schema) but turning them on keeps the
|
|
193
|
+
// option to add referential tables later without revisiting init.
|
|
226
194
|
db.exec('PRAGMA journal_mode = WAL;')
|
|
227
195
|
// FULL (not NORMAL): the server emits `workspace-save-ack` BEFORE
|
|
228
|
-
// returning to the event loop after `commitRevision`.
|
|
229
|
-
//
|
|
230
|
-
//
|
|
231
|
-
//
|
|
232
|
-
//
|
|
233
|
-
//
|
|
234
|
-
// for the protocol's edit-driven write pattern (triage edits, not
|
|
235
|
-
// streaming throughput). Audit round-9 M1.
|
|
196
|
+
// returning to the event loop after `commitRevision`. NORMAL only
|
|
197
|
+
// fsyncs at WAL checkpoint, so a power loss between ack and the next
|
|
198
|
+
// checkpoint loses a row the originator + peers were told committed.
|
|
199
|
+
// FULL fsyncs per commit, matching the durability the ack implies.
|
|
200
|
+
// Trade-off is per-commit fsync latency, acceptable for the edit-
|
|
201
|
+
// driven write pattern (triage edits, not streaming). Audit round-9 M1.
|
|
236
202
|
db.exec('PRAGMA synchronous = FULL;')
|
|
237
203
|
db.exec('PRAGMA foreign_keys = ON;')
|
|
238
204
|
db.exec(SCHEMA)
|
|
239
205
|
// Fail-loud on a pre-existing non-STRICT table — `CREATE TABLE IF
|
|
240
|
-
// NOT EXISTS … STRICT` is a no-op when the table
|
|
241
|
-
//
|
|
242
|
-
//
|
|
243
|
-
//
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
//
|
|
247
|
-
//
|
|
248
|
-
// making every subsequent verify fail. Operator must migrate
|
|
249
|
-
// before this server boots.
|
|
206
|
+
// NOT EXISTS … STRICT` is a no-op when the table exists, so a
|
|
207
|
+
// deployment predating the STRICT marker keeps its non-STRICT shape.
|
|
208
|
+
// Without STRICT, an operator with direct DB write access could
|
|
209
|
+
// insert mis-typed rows (e.g. `keyframe = "1\nfoo"` text in the
|
|
210
|
+
// INTEGER column) and poison the chain: the signed canonical says
|
|
211
|
+
// `keyframe = 1`, but the stored text round-trips into the canonical
|
|
212
|
+
// as a different string, failing every subsequent verify. Operator
|
|
213
|
+
// must migrate before this server boots.
|
|
250
214
|
const meta = db.prepare(
|
|
251
215
|
`SELECT strict FROM pragma_table_list WHERE schema = 'main' AND name = 'workspace_revision'`,
|
|
252
216
|
).get() as { strict: number } | undefined
|
|
@@ -254,13 +218,11 @@ function openDbInner(db: DatabaseSync): SqliteHandle {
|
|
|
254
218
|
throw new Error('workspace_revision is non-STRICT — migrate via rename+create+copy before booting')
|
|
255
219
|
}
|
|
256
220
|
// Idempotent migration for DBs created before the keyframe column
|
|
257
|
-
// existed. Inspect the column list rather than
|
|
258
|
-
//
|
|
259
|
-
//
|
|
260
|
-
//
|
|
261
|
-
//
|
|
262
|
-
// the ALTER itself bubbles up as an open-time crash where the
|
|
263
|
-
// operator can act on it.
|
|
221
|
+
// existed. Inspect the column list rather than `try { ALTER } catch
|
|
222
|
+
// {}`: a blanket catch swallows ANY failure (lock contention, disk
|
|
223
|
+
// full, corrupt page) as "column already exists". ALTER only when
|
|
224
|
+
// the column is genuinely missing, so an ALTER failure bubbles up as
|
|
225
|
+
// an open-time crash the operator can act on.
|
|
264
226
|
const columns = db.prepare(`PRAGMA table_info(workspace_revision)`).all() as Array<{ name: string }>
|
|
265
227
|
if (!columns.some((c) => c.name === 'keyframe')) {
|
|
266
228
|
// ADD COLUMN carries the CHECK so a legacy DB migrating up lands
|
|
@@ -507,9 +469,8 @@ export function commitRevision(handle: Handle, input: RevisionInsert): Promise<C
|
|
|
507
469
|
// base gate fails → `stale-base`.
|
|
508
470
|
// • Two retransmits with the same id: the second's dup gate fails →
|
|
509
471
|
// `duplicate`.
|
|
510
|
-
//
|
|
511
|
-
//
|
|
512
|
-
// chainFrom-during-commits) which now pass with no lock present.
|
|
472
|
+
// Covered by the no-fork concurrency tests in `tests/server-db.test.js`
|
|
473
|
+
// (two/N concurrent same-base, mixed, chainFrom-during-commits).
|
|
513
474
|
//
|
|
514
475
|
// SQLite serialises writers internally even ACROSS connections, but a
|
|
515
476
|
// multi-connection deployment is unsupported regardless. The
|