@preventive/triage 1.0.0-alpha.2 → 1.0.0-alpha.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/api/reap.ts +17 -0
  2. package/cli.js +6 -0
  3. package/client/finding-link.js +305 -0
  4. package/client/linked-findings.d.ts +1 -0
  5. package/client/linked-findings.js +111 -0
  6. package/common/bundle-metadata.d.ts +10 -0
  7. package/common/bundle-metadata.js +177 -0
  8. package/common/bundle-reasons.d.ts +2 -0
  9. package/common/bundle-reasons.js +21 -0
  10. package/common/bundle-sources.d.ts +3 -0
  11. package/common/bundle-sources.js +284 -0
  12. package/common/bundle-stats.js +41 -0
  13. package/common/bundle-tabs.js +1 -0
  14. package/common/code-language.js +36 -0
  15. package/common/default-scan-models.ts +30 -0
  16. package/common/finding-id.js +47 -0
  17. package/common/github-pr.ts +56 -0
  18. package/common/managed/comments.ts +34 -0
  19. package/common/managed/permissions.ts +35 -0
  20. package/common/managed/report-content.ts +42 -0
  21. package/common/managed/report-filter.ts +108 -0
  22. package/common/managed/roles.ts +28 -0
  23. package/common/managed/routes.d.ts +2 -0
  24. package/common/managed/routes.js +121 -0
  25. package/common/managed/scan-models.ts +6 -0
  26. package/common/managed/triage.ts +83 -0
  27. package/common/save-error-reason.ts +20 -7
  28. package/common/scan-server.ts +13 -0
  29. package/common/server-info.ts +33 -0
  30. package/common/utf8.d.ts +3 -0
  31. package/common/utf8.js +45 -0
  32. package/out/brotli-fallback.js +3 -3
  33. package/out/client-managed-import.js +81 -0
  34. package/out/client-managed.js +110 -0
  35. package/out/client-sync.js +16 -13
  36. package/out/graph.js +30 -4
  37. package/out/index.html +55 -8
  38. package/out/prism.js +2 -2
  39. package/out/stasis.svg +45 -0
  40. package/out/terminal.js +273 -39
  41. package/out/view.css +1 -1
  42. package/out/view.js +198 -62
  43. package/package.json +179 -55
  44. package/report/index.js +254 -0
  45. package/report/src/finding-id.js +80 -0
  46. package/report/src/finding.js +312 -0
  47. package/report/src/labels.js +33 -0
  48. package/report/src/md-structure.js +471 -0
  49. package/report/src/md-text.js +167 -0
  50. package/report/src/meta.js +76 -0
  51. package/report/src/parse-codex.js +147 -0
  52. package/report/src/parse-deepsec.js +197 -0
  53. package/report/src/parse-deepview-fields.js +375 -0
  54. package/report/src/parse-deepview-md.js +185 -0
  55. package/report/src/parse-md-id.js +137 -0
  56. package/report/src/parse-md.js +322 -0
  57. package/report/src/parse-piolium-id.js +79 -0
  58. package/report/src/parse-piolium-rows.js +131 -0
  59. package/report/src/parse-piolium-tokens.js +175 -0
  60. package/report/src/parse-piolium.js +400 -0
  61. package/report/src/security.js +63 -0
  62. package/report/src/utf8.js +21 -0
  63. package/report/src/write-md-finding.js +273 -0
  64. package/report/src/write-md.js +291 -0
  65. package/server-common/database-config.ts +16 -0
  66. package/server-common/initialize.ts +18 -0
  67. package/server-common/npm-advisories.ts +101 -0
  68. package/{server → server-common}/origin.ts +5 -5
  69. package/server-common/reap.ts +48 -0
  70. package/server-common/scan-config.ts +19 -0
  71. package/server-common/standalone.ts +29 -0
  72. package/server-common/storage-log.ts +34 -0
  73. package/server-common/vercel-blob.ts +110 -0
  74. package/server-e2e/app.ts +485 -0
  75. package/{server → server-e2e}/auth.ts +5 -1
  76. package/{server → server-e2e}/bus-receiver.ts +9 -8
  77. package/{server → server-e2e}/cli.js +9 -4
  78. package/{server → server-e2e}/config.ts +54 -39
  79. package/{server → server-e2e}/db-neon.ts +2 -2
  80. package/{server → server-e2e}/db-revision-sql.ts +7 -10
  81. package/{server → server-e2e}/db-stmt.ts +2 -2
  82. package/{server → server-e2e}/db.ts +96 -135
  83. package/{server → server-e2e}/http.ts +110 -12
  84. package/{server → server-e2e}/hub.ts +44 -14
  85. package/server-e2e/index.ts +17 -0
  86. package/server-e2e/lifecycle.ts +95 -0
  87. package/{server → server-e2e}/neon-driver.ts +2 -2
  88. package/{server → server-e2e}/npm-proxy.ts +11 -144
  89. package/{server → server-e2e}/objstore/blob-fs.ts +6 -8
  90. package/{server → server-e2e}/objstore/blob-vercel.ts +49 -141
  91. package/{server → server-e2e}/objstore/blob.ts +24 -9
  92. package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
  93. package/{server → server-e2e}/objstore/handlers.ts +19 -20
  94. package/{server → server-e2e}/objstore/init.ts +53 -27
  95. package/{server → server-e2e}/objstore/reaper.ts +31 -11
  96. package/server-e2e/objstore/rest-deny.ts +28 -0
  97. package/server-e2e/objstore/rest-mint.ts +224 -0
  98. package/{server → server-e2e}/objstore/rest.ts +110 -93
  99. package/{server → server-e2e}/objstore/sign.ts +105 -0
  100. package/{server → server-e2e}/objstore/store-neon.ts +10 -14
  101. package/{server → server-e2e}/objstore/store.ts +98 -118
  102. package/{server → server-e2e}/objstore/tokens.ts +9 -12
  103. package/{server → server-e2e}/peer.ts +7 -9
  104. package/{server → server-e2e}/pubsub.ts +29 -36
  105. package/{server → server-e2e}/sign.ts +12 -14
  106. package/{server → server-e2e}/sse-server.ts +105 -73
  107. package/{server → server-e2e}/sse-session.ts +30 -16
  108. package/{server → server-e2e}/static.ts +36 -29
  109. package/server-e2e/sync-handlers.ts +408 -0
  110. package/{server → server-e2e}/util.ts +9 -0
  111. package/{server → server-e2e}/ws-server.ts +29 -23
  112. package/server-managed/activity.ts +231 -0
  113. package/server-managed/avatar-store.ts +51 -0
  114. package/server-managed/blob-store.ts +66 -0
  115. package/server-managed/blob-vercel.ts +125 -0
  116. package/server-managed/brotli.ts +10 -0
  117. package/server-managed/bundle-cache.ts +185 -0
  118. package/server-managed/bundle-catalog.ts +29 -0
  119. package/server-managed/bundle-store.ts +28 -0
  120. package/server-managed/bundle-summary-cache.ts +97 -0
  121. package/server-managed/bundle.ts +39 -0
  122. package/server-managed/cache-storage.ts +40 -0
  123. package/server-managed/cli.js +13 -0
  124. package/server-managed/combined.ts +46 -0
  125. package/server-managed/comments.ts +151 -0
  126. package/server-managed/config.ts +144 -0
  127. package/server-managed/content-access.ts +15 -0
  128. package/server-managed/crypto.ts +25 -0
  129. package/server-managed/db-methods.ts +1330 -0
  130. package/server-managed/db-neon.ts +171 -0
  131. package/server-managed/db-schema.ts +203 -0
  132. package/server-managed/db-table-names.ts +22 -0
  133. package/server-managed/db.ts +109 -0
  134. package/server-managed/github-app.ts +332 -0
  135. package/server-managed/github-metadata.ts +65 -0
  136. package/server-managed/github-oauth.ts +215 -0
  137. package/server-managed/github-pulls.ts +115 -0
  138. package/server-managed/http-response.ts +18 -0
  139. package/server-managed/http.ts +2043 -0
  140. package/server-managed/import-triage.ts +48 -0
  141. package/server-managed/index.ts +135 -0
  142. package/server-managed/public-workspace.ts +150 -0
  143. package/server-managed/repo-path.ts +21 -0
  144. package/server-managed/report-migration.ts +35 -0
  145. package/server-managed/report-query.ts +4 -0
  146. package/server-managed/report-response.ts +16 -0
  147. package/server-managed/report-sources.ts +154 -0
  148. package/server-managed/repository-discovery.ts +82 -0
  149. package/server-managed/repository-policy.ts +25 -0
  150. package/server-managed/session.ts +78 -0
  151. package/server-managed/slugs.ts +38 -0
  152. package/server-managed/sql-postgres.ts +30 -0
  153. package/server-managed/sql.ts +61 -0
  154. package/server-managed/static.ts +28 -0
  155. package/server-managed/storage.ts +34 -0
  156. package/server-managed/team-catalog.ts +7 -0
  157. package/server-managed/team-feed.ts +128 -0
  158. package/server-managed/team-reports.ts +156 -0
  159. package/server-managed/triage-response.ts +16 -0
  160. package/server-managed/uploads.ts +47 -0
  161. package/server-managed/workspace-shares.ts +150 -0
  162. package/server.ts +50 -0
  163. package/server/index.ts +0 -481
  164. package/server/lifecycle.ts +0 -204
  165. package/server/sync-handlers.ts +0 -327
  166. /package/{server → server-e2e}/config.example.json +0 -0
  167. /package/{server → server-e2e}/objstore/fs.ts +0 -0
  168. /package/{server → server-e2e}/validation.ts +0 -0
@@ -2,15 +2,16 @@
2
2
  // vars, the optional config.json) into one immutable `Config`, failing
3
3
  // loud on malformed values so a typo surfaces at startup rather than
4
4
  // deep in `node:net` / at the first token verification. Pure parsing —
5
- // no backends opened, no crypto keys derived, no side effects beyond
6
- // `--help` / fail-fast `process.exit`. index.ts destructures the
7
- // result and does the wiring (backend selection, password HMAC, …).
5
+ // no backends opened or process termination. Invalid configuration throws so
6
+ // embedding hosts can recover; app.ts handles backend selection and wiring.
8
7
 
9
8
  import { Buffer } from 'node:buffer'
10
9
  import { readFileSync } from 'node:fs'
11
- import { argv, env } from 'node:process'
10
+ import { env } from 'node:process'
12
11
  import { fileURLToPath } from 'node:url'
13
12
  import { dirname, join } from 'node:path'
13
+ import { configuredScanServer } from '../server-common/scan-config.ts'
14
+ import { databaseUrls } from '../server-common/database-config.ts'
14
15
 
15
16
  export type Config = {
16
17
  port: number
@@ -18,6 +19,7 @@ export type Config = {
18
19
  dbPath: string
19
20
  objstoreDir: string
20
21
  reapIntervalMs: number
22
+ reapDisabled: boolean
21
23
  maxInflightPerSocket: number
22
24
  debug: boolean
23
25
  neonUrl: string | null
@@ -25,9 +27,10 @@ export type Config = {
25
27
  tokenSecret: Uint8Array<ArrayBuffer> | null
26
28
  password: string | null
27
29
  trustProxyEnv: string | undefined
30
+ deepviewScanServer: string | null
28
31
  }
29
32
 
30
- // Parse + range-validate an integer env var, exiting with a clear
33
+ // Parse + range-validate an integer env var, throwing a clear
31
34
  // up-front message on a malformed value — a NaN from `Number("abc")`
32
35
  // otherwise surfaces as a confusing crash deep inside `node:net`
33
36
  // (`WebSocketServer({ port: NaN })`) or a 0-ms `setInterval` loop. An
@@ -38,20 +41,28 @@ function intEnv(name: string, def: number, min: number, max: number, hint = ''):
38
41
  const raw = env[name]
39
42
  const n = raw == null ? def : Number(raw)
40
43
  if (!Number.isSafeInteger(n) || n < min || n > max) {
41
- console.error(`Invalid ${name}: ${raw} — must be an integer in [${min}, ${max}].${hint ? ` ${hint}` : ''}`)
42
- process.exit(1)
44
+ throw new Error(`Invalid ${name}: ${raw} — must be an integer in [${min}, ${max}].${hint ? ` ${hint}` : ''}`)
43
45
  }
44
46
  return n
45
47
  }
46
48
 
47
- const HELP = `Usage: node server/index.ts
49
+ export const HELP = `Usage: node server-e2e/index.ts
48
50
  Environment:
49
51
  PORT listen port (default 8765)
50
52
  HOST bind host (default 127.0.0.1)
51
- DB_PATH sqlite file (default: server/data/data.db);
52
- ignored when DATABASE_URL is set
53
- DATABASE_URL Neon Postgres connection string; if set,
54
- selects the Neon backend instead of
53
+ DEEPVIEW_SCAN_SERVER optional HTTP(S) scan-service URL; advertised
54
+ in /api/config and allowed by the UI CSP.
55
+ Unset by default (no external scan service).
56
+ DB_PATH sqlite file (default: server-e2e/data/data.db);
57
+ ignored when an e2e database URL is set
58
+ DATABASE_URL shared Neon Postgres URL for enabled modes.
59
+ Cannot be combined with E2E_DATABASE_URL
60
+ or MANAGED_DATABASE_URL.
61
+ E2E_DATABASE_URL e2e-only Neon URL, used without DATABASE_URL.
62
+ Combined mode also requires
63
+ MANAGED_DATABASE_URL; mixing Neon and
64
+ SQLite backends is rejected.
65
+ Either e2e URL selects Neon instead of
55
66
  SQLite. Requires the optional peer dep
56
67
  @neondatabase/serverless. The Neon
57
68
  pairing additionally requires
@@ -60,12 +71,12 @@ Environment:
60
71
  local-FS bytes cannot back a multi-
61
72
  replica DB plane.
62
73
  BLOB_READ_WRITE_TOKEN Vercel Blob R/W token (private store).
63
- Required when DATABASE_URL is set;
74
+ Required when an e2e database URL is set;
64
75
  ignored otherwise. Requires the optional
65
76
  peer dep @vercel/blob.
66
77
  OBJSTORE_TOKEN_SECRET Base64 (32 bytes) HMAC secret for REST
67
- bearer tokens. REQUIRED when DATABASE_URL
68
- is set (multi-replica deployments: a
78
+ bearer tokens. REQUIRED in Neon mode
79
+ (multi-replica deployments: a
69
80
  token minted on one replica's WS plane
70
81
  must validate on another replica's REST
71
82
  plane). Optional under SQLite (a fresh
@@ -75,9 +86,15 @@ Environment:
75
86
  OBJSTORE_DIR object store root (default: ./objstore
76
87
  next to DB_PATH). Used by the local-FS
77
88
  byte plane only; ignored when
78
- DATABASE_URL + BLOB_READ_WRITE_TOKEN
79
- are set (bytes live in Vercel Blob).
89
+ an e2e database URL selects Neon
90
+ (bytes live in Vercel Blob).
80
91
  OBJSTORE_REAP_INTERVAL_MS orphan reaper period (default 600000)
92
+ OBJSTORE_REAP_DISABLED set '1' / 'true' to disable automatic orphan
93
+ sweeps (boot and periodic). Explicit /api/reap
94
+ remains available. Schedule cleanup externally
95
+ to avoid accumulating orphaned bytes.
96
+ CRON_SECRET bearer secret for GET /api/reap (fails closed
97
+ when unset). Runs cleanup for enabled modes.
81
98
  TRUST_PROXY set '1' / 'true' to honour X-Forwarded-
82
99
  Host / X-Forwarded-Proto when computing
83
100
  the same-origin gate's expected origin.
@@ -95,7 +112,7 @@ Environment:
95
112
  that need to deterministically
96
113
  exercise the cap.
97
114
  CONFIG_PATH operator config JSON path (default:
98
- server/config.json). Currently the
115
+ server-e2e/config.json). Currently the
99
116
  only field is { "password": "..." }
100
117
  which gates first-action creation of
101
118
  a new workspace on the
@@ -113,11 +130,11 @@ function readServerConfigFile(path: string): ServerConfigFile {
113
130
  let raw: string
114
131
  try { raw = readFileSync(path, 'utf8') } catch (err) {
115
132
  if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return {}
116
- console.error(`Failed to read ${path}:`, (err as Error)?.message ?? err); process.exit(1)
133
+ throw new Error(`Failed to read ${path}: ${(err as Error)?.message ?? err}`, { cause: err })
117
134
  }
118
135
  try { return JSON.parse(raw) as ServerConfigFile }
119
136
  catch (err) {
120
- console.error(`Failed to parse ${path} as JSON:`, (err as Error)?.message ?? err); process.exit(1)
137
+ throw new Error(`Failed to parse ${path} as JSON: ${(err as Error)?.message ?? err}`, { cause: err })
121
138
  }
122
139
  }
123
140
 
@@ -132,20 +149,17 @@ function decodeTokenSecret(raw: string): Uint8Array<ArrayBuffer> {
132
149
  // typo-detector below would fail with a misleading message.
133
150
  const trimmed = raw.trim()
134
151
  if (trimmed.length === 0) {
135
- console.error('OBJSTORE_TOKEN_SECRET is empty after trimming whitespace')
136
- process.exit(1)
152
+ throw new Error('OBJSTORE_TOKEN_SECRET is empty after trimming whitespace')
137
153
  }
138
154
  const decoded = Buffer.from(trimmed, 'base64')
139
155
  const reencoded = decoded.toString('base64')
140
156
  const norm = (s: string): string => s.replace(/=+$/u, '')
141
157
  if (norm(reencoded) !== norm(trimmed)) {
142
- console.error('OBJSTORE_TOKEN_SECRET contains non-base64 characters (likely a typo, e.g. base64url chars in a base64 secret).')
143
- console.error('Regenerate with: node -e \'console.log(require("crypto").randomBytes(32).toString("base64"))\'')
144
- process.exit(1)
158
+ throw new Error('OBJSTORE_TOKEN_SECRET contains non-base64 characters (likely a typo, e.g. base64url chars in a base64 secret).\n' +
159
+ 'Regenerate with: node -e \'console.log(require("crypto").randomBytes(32).toString("base64"))\'')
145
160
  }
146
161
  if (decoded.byteLength !== 32) {
147
- console.error(`OBJSTORE_TOKEN_SECRET must decode to 32 bytes (got ${decoded.byteLength})`)
148
- process.exit(1)
162
+ throw new Error(`OBJSTORE_TOKEN_SECRET must decode to 32 bytes (got ${decoded.byteLength})`)
149
163
  }
150
164
  // Copy into a fresh ArrayBuffer so the type matches
151
165
  // `Uint8Array<ArrayBuffer>` (Buffer may be SharedArrayBuffer-backed).
@@ -156,41 +170,42 @@ export function loadConfig(): Config {
156
170
  // 0 = OS-assigned ephemeral port (the test harness boots with PORT=0).
157
171
  const port = intEnv('PORT', 8765, 0, 65535)
158
172
  const host = env['HOST'] ?? '127.0.0.1'
159
- // `fileURLToPath` decodes percent-escapes / non-ASCII path segments
160
- // correctly (the older `new URL(...).pathname` left `%20` raw).
173
+ // `fileURLToPath` decodes percent-escapes / non-ASCII path segments;
174
+ // `new URL(...).pathname` would leave `%20` raw.
161
175
  const dbPath = env['DB_PATH'] ?? fileURLToPath(new URL('./data/data.db', import.meta.url))
162
176
  // `path.join` so a Windows DB_PATH doesn't get a mixed-separator child.
163
177
  const objstoreDir = env['OBJSTORE_DIR'] ?? join(dirname(dbPath), 'objstore')
164
178
  // No practical upper bound beyond the safe-integer range.
165
179
  const reapIntervalMs = intEnv('OBJSTORE_REAP_INTERVAL_MS', 10 * 60 * 1000, 1, Number.MAX_SAFE_INTEGER)
180
+ // Off-switch for automatic orphan reaping (both the boot sweep AND the
181
+ // periodic timer). '1' / 'true' (case-insensitive) → disabled; anything
182
+ // else, including unset, leaves it ON. Same boolean shape as TRUST_PROXY.
183
+ const reapDisabledEnv = env['OBJSTORE_REAP_DISABLED']
184
+ const reapDisabled = reapDisabledEnv === '1' || reapDisabledEnv?.toLowerCase() === 'true'
166
185
  const debug = env['DEBUG'] === '1'
167
186
 
168
187
  const configPath = env['CONFIG_PATH'] ?? fileURLToPath(new URL('./config.json', import.meta.url))
169
188
  const serverConfig = readServerConfigFile(configPath)
170
189
  const rawPassword = serverConfig.password
171
190
  if (rawPassword != null && typeof rawPassword !== 'string') {
172
- console.error(`Invalid ${configPath}: "password" must be a string or null`); process.exit(1)
191
+ throw new Error(`Invalid ${configPath}: "password" must be a string or null`)
173
192
  }
174
193
  const password = rawPassword ?? null
175
194
  // Upper bound 65_536 — bounds memory under hostile load; a deployer
176
195
  // passing MAX_SAFE_INTEGER would silently defeat the cap. Validated
177
- // here (after the config.json / password parse) to preserve the
178
- // pre-split error-precedence order.
196
+ // here, after the config.json / password parse, to keep the
197
+ // error-precedence order.
179
198
  const maxInflightPerSocket = intEnv('MAX_INFLIGHT_PER_SOCKET', 64, 1, 65_536)
180
199
 
181
- if (argv.includes('--help') || argv.includes('-h')) {
182
- console.log(HELP)
183
- process.exit(0)
184
- }
185
-
186
- const neonUrl = env['DATABASE_URL'] ?? null
200
+ const neonUrl = databaseUrls().e2e
187
201
  const blobToken = env['BLOB_READ_WRITE_TOKEN'] ?? null
188
202
  const tokenSecretB64 = env['OBJSTORE_TOKEN_SECRET'] ?? null
189
203
  const tokenSecret = tokenSecretB64 ? decodeTokenSecret(tokenSecretB64) : null
190
204
 
191
205
  return {
192
- port, host, dbPath, objstoreDir, reapIntervalMs, maxInflightPerSocket,
206
+ port, host, dbPath, objstoreDir, reapIntervalMs, reapDisabled, maxInflightPerSocket,
193
207
  debug, neonUrl, blobToken, tokenSecret, password,
194
208
  trustProxyEnv: env['TRUST_PROXY'],
209
+ deepviewScanServer: configuredScanServer(env['DEEPVIEW_SCAN_SERVER']),
195
210
  }
196
211
  }
@@ -4,7 +4,7 @@
4
4
  // dialect on the wire.
5
5
  //
6
6
  // `@neondatabase/serverless` is an OPTIONAL peer dep — selected by
7
- // the `DATABASE_URL` branch in `server/index.ts`. The peer dep
7
+ // the configured database URL branch in `server-e2e/index.ts`. The peer dep
8
8
  // itself is loaded lazily via the dynamic `import()` inside
9
9
  // `openNeonDb` below, so a SQLite-only deployment never installs it
10
10
  // (`autoInstallPeers: false` in `pnpm-workspace.yaml`) and never
@@ -340,7 +340,7 @@ export async function openNeonDb(connectionString: string): Promise<Handle> {
340
340
  // in-process Postgres (PGlite) via `mock.module`: that hook can only
341
341
  // intercept a specifier it can RESOLVE, and the optional peer dep
342
342
  // isn't installed in a SQLite-only checkout. The wrapper path always
343
- // resolves — see `server/neon-driver.ts`. Cast through `unknown`
343
+ // resolves — see `server-e2e/neon-driver.ts`. Cast through `unknown`
344
344
  // because the wrapper's `export *` re-exports a `@ts-ignore`'d
345
345
  // (possibly-absent) module, so tsc can't see `neon`'s type here.
346
346
  const mod = (await import('./neon-driver.ts')) as unknown as { neon: (url: string) => NeonSql }
@@ -1,9 +1,7 @@
1
1
  // Shared SQL + row-mapping for the `workspace_revision` chain, used by
2
2
  // BOTH backends — `./db.ts` (SQLite) and `./db-neon.ts` (Neon/Postgres).
3
- // The two backends previously carried byte-for-byte-equal query strings
4
- // (modulo `?`↔`$N` placeholders) and a copy of the same row mapper; that
5
- // duplication is collapsed here so a query edit can't silently drift
6
- // between backends.
3
+ // Single source of truth (modulo `?`↔`$N` placeholders) so a query edit
4
+ // can't silently drift between backends.
7
5
  //
8
6
  // Single source of truth, in `$N` (Postgres) form:
9
7
  // • the read queries (`headFor`, `seqOfId`, `lastKeyframeSeq`, the
@@ -60,11 +58,10 @@ export function numOrNull(v: unknown): number | null {
60
58
  // `Record<string, unknown>` rows whose `keyframe` may be a number OR (on
61
59
  // a future driver change) a string; `node:sqlite` hands back native
62
60
  // numbers. The `num`/`numOrNull` coercion is safe over both — a native
63
- // `0`/`1` integer passes through unchanged, so SQLite rows round-trip
64
- // identically to the bespoke pass-through they had before, while Neon
65
- // rows keep their defensive string→number coercion. `base` is the only
66
- // nullable column (first revision); `keyframe` collapses to a strict
67
- // 0 / 1 via the `=== 1` check the chain-broadcast contract relies on.
61
+ // `0`/`1` integer passes through unchanged, while Neon rows keep their
62
+ // defensive string→number coercion. `base` is the only nullable column
63
+ // (first revision); `keyframe` collapses to a strict 0 / 1 via the
64
+ // `=== 1` check the chain-broadcast contract relies on.
68
65
  export function mapRevisionRow(r: Record<string, unknown>): RevisionRow {
69
66
  return {
70
67
  base: (r['base'] as string | null) ?? null,
@@ -109,7 +106,7 @@ export const CHAIN_FROM_SQL =
109
106
  export const REVISION_EXISTS_SQL =
110
107
  `SELECT 1 AS one FROM workspace_revision WHERE workspace_tag = $1 AND id = $2`
111
108
  // Fetch a single revision row by content-addressed id. The cross-instance
112
- // pubsub (server/pubsub.ts) NOTIFY payload carries only `(tag, revisionId)`
109
+ // pubsub (server-e2e/pubsub.ts) NOTIFY payload carries only `(tag, revisionId)`
113
110
  // because the full `workspace-state` broadcast envelope is bounded by
114
111
  // `MAX_CIPHERTEXT_LEN` (2 MiB) and Postgres NOTIFY caps payloads at ~8 KB.
115
112
  // The receiver re-fetches the row from this shared table to construct the
@@ -1,5 +1,5 @@
1
- // Shared async-statement primitives. Both `server/db.ts` (workspace_revision
2
- // chain) and `server/objstore/store.ts` (objstore tables) expose Handles
1
+ // Shared async-statement primitives. Both `server-e2e/db.ts` (workspace_revision
2
+ // chain) and `server-e2e/objstore/store.ts` (objstore tables) expose Handles
3
3
  // whose statements look like `{ get(...) → Promise<…>, all(...) → Promise<[…]>,
4
4
  // run(...) → Promise<void> }`. The underlying `node:sqlite` driver is
5
5
  // synchronous; the wrappers below catch sync errors and route them through
@@ -10,49 +10,30 @@
10
10
  // `base` points at the previous revision's `id` (or null for the
11
11
  // first revision in a workspace).
12
12
  //
13
- // `keyframe` is `1` for a revision the client emits with the full
14
- // state baked in (rather than just a delta). The wire-level flag
15
- // is also covered by the signature, so the column value MUST match
16
- // what the signed canonical bytes claim — `canonicalSave` in
17
- // `server/sign.ts` (called from `handleSave` in `server/index.ts`)
18
- // encodes `keyframe ? '1' : ''` into the bytes that `verifyEd25519`
19
- // then checks against the wire-supplied signature, so a wire flag
20
- // that doesn't match what the signer hashed fails verify and never
21
- // reaches this column. Client-driven: the server only stores what
22
- // the client sent and treats keyframes as catch-up roots when a
23
- // from=null subscriber arrives.
13
+ // `keyframe` is `1` for a revision the client emits with full state
14
+ // baked in (rather than a delta). The wire flag is covered by the
15
+ // signature, so the column value MUST match the signed canonical
16
+ // bytes: `canonicalSave` (server-e2e/sign.ts, via `handleSave`) encodes
17
+ // `keyframe ? '1' : ''` into the bytes `verifyEd25519` checks, so a
18
+ // mismatched wire flag fails verify and never reaches this column.
19
+ // Client-driven: the server stores what the client sent and treats
20
+ // keyframes as catch-up roots when a from=null subscriber arrives.
24
21
  //
25
22
  // `node:sqlite` is the built-in driver (Node ≥ 22 experimental,
26
- // stable in 24+). The driver is synchronous under the hood; the
27
- // Handle wraps each prepared statement so call sites `await`
28
- // uniformly. This is async-ready surface for a future async DB
29
- // backend — every operation today resolves in the current microtask
30
- // off a sync `node:sqlite` call.
23
+ // stable in 24+), synchronous under the hood; the Handle wraps each
24
+ // prepared statement so call sites `await` uniformly — async-ready
25
+ // surface for a future async DB backend (every op resolves in the
26
+ // current microtask off a sync call).
31
27
  //
32
- // Because operations are now async, two handlers can interleave
33
- // across an `await`. `commitRevision` (below) does NOT take an
34
- // in-process lock — it folds the dup recheck, base-equality check,
35
- // MAX(seq) and INSERT into ONE gated INSERT statement
36
- // (`commitRevisionSqlite` below):
37
- // INSERT … SELECT COALESCE(MAX(seq),0)+1 … WHERE NOT EXISTS(dup)
38
- // AND head IS base RETURNING seq
39
- // `node:sqlite` is synchronous, so that single statement runs to
40
- // completion without yielding the event loop — no concurrent commit
41
- // can interleave mid-statement, and the head-check + MAX(seq) read
42
- // from ONE consistent snapshot. That single-snapshot property is
43
- // what makes a per-tag lock redundant: the lock formerly existed
44
- // only to stop a chain fork where a racer read `head` from one
45
- // snapshot but `MAX(seq)` from a LATER one (after a sibling
46
- // committed) and inserted (seq=N+2, base=X) alongside the winner's
47
- // (seq=N+1, base=X) — same base, different seq, no PK conflict. With
48
- // both reads inside one statement that interleaving is impossible:
49
- // a racer's snapshot is either before the winner's commit (→ same
50
- // seq=N+1 → the UNIQUE(workspace_tag, seq) PK rejects the second →
51
- // recovery → stale-base) or after it (→ head ≠ base → no insert →
52
- // stale-base). Exactly one commits; the loser gets stale-base.
53
- // SQLite also serialises writers internally, and the PK backstops
54
- // the unsupported multi-connection case. See `commitRevisionSqlite`
55
- // for the full fork-safety argument.
28
+ // Operations being async, two handlers can interleave across an
29
+ // `await`. `commitRevision` (below) takes NO in-process lock — it
30
+ // folds the dup recheck, base-equality check, MAX(seq) and INSERT
31
+ // into ONE gated INSERT (`commitRevisionSqlite`). `node:sqlite` runs
32
+ // that statement to completion without yielding, so its head-check +
33
+ // MAX(seq) read ONE snapshot, which is what makes a per-tag lock
34
+ // redundant. SQLite also serialises writers internally, and the PK
35
+ // backstops the unsupported multi-connection case. See
36
+ // `commitRevisionSqlite` for the full fork-safety argument.
56
37
 
57
38
  import { DatabaseSync } from 'node:sqlite'
58
39
  import { mkdirSync } from 'node:fs'
@@ -64,24 +45,20 @@ import {
64
45
  mapRevisionRow, toSqlitePlaceholders,
65
46
  } from './db-revision-sql.ts'
66
47
 
67
- // `CHECK (keyframe IN (0, 1))` is the value-domain guard on the
68
- // keyframe column. STRICT (the table marker) enforces the column's
69
- // TYPE — an INTEGER stays an INTEGER — but NOT its value range:
70
- // `keyframe = 2` is a perfectly valid integer that STRICT accepts,
71
- // which `mapRevisionRow`'s `=== 1` check then silently coerces back to
72
- // 0. That divergence between the stored row and the signed canonical
73
- // (which only ever encodes 0 / 1) poisons chain-replay verifies for
74
- // any peer who recomputes — the same operator-with-direct-DB-write
75
- // attack vector the STRICT guard in `openDbInner` catches for the
76
- // column TYPE. The CHECK closes the value-domain half, giving SQLite
77
- // the protection the Neon schema's identical `CHECK (keyframe IN
78
- // (0, 1))` carries (see `db-neon.ts`).
48
+ // `CHECK (keyframe IN (0, 1))` is the value-domain guard. STRICT
49
+ // (the table marker) enforces the column TYPE (an INTEGER stays an
50
+ // INTEGER) but NOT its value range: `keyframe = 2` is a valid integer
51
+ // STRICT accepts, which `mapRevisionRow`'s `=== 1` check then coerces
52
+ // back to 0. That divergence from the signed canonical (only ever
53
+ // 0 / 1) poisons chain-replay verifies for any recomputing peer — the
54
+ // same operator-with-direct-DB-write vector the STRICT guard in
55
+ // `openDbInner` catches for TYPE. The CHECK closes the value-domain
56
+ // half, matching the Neon schema's identical CHECK (see `db-neon.ts`).
79
57
  //
80
- // `WORKSPACE_REVISION_DEF` is the parenthesised column + constraint
81
- // body (plus the STRICT marker), shared by the initial `CREATE TABLE`
82
- // and the `migrateAddKeyframeCheck` rebuild below — so a table the
83
- // rebuild produces is byte-identical in shape to a freshly-created
84
- // one, and a future column edit can't drift the two apart.
58
+ // Parenthesised column + constraint body (plus STRICT marker), shared
59
+ // by the initial `CREATE TABLE` and the `migrateAddKeyframeCheck`
60
+ // rebuild below — so a rebuilt table is byte-identical in shape to a
61
+ // fresh one and a future column edit can't drift the two apart.
85
62
  const WORKSPACE_REVISION_DEF = `(
86
63
  workspace_tag TEXT NOT NULL,
87
64
  seq INTEGER NOT NULL,
@@ -105,10 +82,10 @@ const SCHEMA = `
105
82
  ${WORKSPACE_REVISION_TAG_ID_INDEX};
106
83
  `
107
84
 
108
- // Row shape returned by the chain queries. SQLite stores `keyframe`
109
- // as INTEGER (0 / 1); `chainForWire` in server/index.ts normalises
110
- // to a strict boolean before broadcasting, but the raw row carries
111
- // the integer. `base` is nullable on the very first revision.
85
+ // Row shape from the chain queries. `keyframe` is stored as INTEGER
86
+ // (0 / 1); the raw row carries the integer — `chainForWire` in
87
+ // server-e2e/index.ts normalises to a strict boolean before broadcasting.
88
+ // `base` is nullable on the very first revision.
112
89
  export type RevisionRow = {
113
90
  base: string | null
114
91
  id: string
@@ -119,9 +96,8 @@ export type RevisionRow = {
119
96
  }
120
97
 
121
98
  // Input to `commitRevision`. `keyframe` is a strict boolean here —
122
- // the canonical-payload contract uses `=== true`, and the storage
123
- // path coerces to 0 / 1 via `keyframe ? 1 : 0` before hitting the
124
- // STRICT INTEGER column.
99
+ // the canonical-payload contract uses `=== true`; the storage path
100
+ // coerces to 0 / 1 before hitting the STRICT INTEGER column.
125
101
  export type RevisionInsert = {
126
102
  tag: string
127
103
  id: string
@@ -142,37 +118,31 @@ export type CommitResult =
142
118
  | { kind: 'duplicate' }
143
119
  | { kind: 'stale-base'; head: string | null }
144
120
 
145
- // Bag of pre-prepared statements + the underlying connection.
146
- // Held for the process lifetime; `close()` runs from `shutdown()`.
121
+ // Pre-prepared statements + the underlying connection, held for the
122
+ // process lifetime; `close()` runs from `shutdown()`.
147
123
  //
148
- // `db` is the raw `DatabaseSync` and is SQLite-only. The Neon
149
- // backend (`./db-neon.ts`) constructs a Handle with `db` unset.
150
- // Callers that reach into `db` directly (e.g. `openObjstore`,
151
- // test-only fixture SQL) are SQLite-coupled by construction —
152
- // passing them a Neon-backed Handle is the operator's mistake to
153
- // catch at the `if (DATABASE_URL)` switch in `server/index.ts`.
124
+ // `db` is the raw `DatabaseSync`, SQLite-only — the Neon backend
125
+ // (`./db-neon.ts`) constructs a Handle with `db` unset. Callers that
126
+ // reach into `db` directly (e.g. `openObjstore`, test-only fixture
127
+ // SQL) are SQLite-coupled by construction; passing them a Neon-backed
128
+ // Handle is the operator's mistake to catch at the database URL
129
+ // switch in `server-e2e/index.ts`.
154
130
  //
155
- // `tryCommit` is the backend-specific atomic-commit primitive that
131
+ // `tryCommit` is the backend-specific atomic-commit primitive
156
132
  // `commitRevision` dispatches through. SQLite runs one synchronous
157
- // gated INSERT (no in-process lock — `node:sqlite` doesn't yield
158
- // mid-statement, so the head-check + MAX(seq) read one snapshot;
159
- // see `commitRevisionSqlite`). Neon wraps the dup-check + head-check
160
- // + gated INSERT in a pipelined transaction; it relies on Postgres'
161
- // READ-COMMITTED single-statement snapshot (the gated INSERT's
162
- // head-check and MAX(seq) read one snapshot) plus the
163
- // `UNIQUE(workspace_tag, seq)` PK to keep cross-replica racers from
164
- // forking the chain — see `db-neon.ts`'s `tryCommitNeon`.
133
+ // gated INSERT (see `commitRevisionSqlite`); Neon wraps it in a
134
+ // pipelined transaction (see `db-neon.ts`'s `tryCommitNeon`). Both
135
+ // rely on a single-statement snapshot + the `UNIQUE(workspace_tag,
136
+ // seq)` PK for fork-safety; see those functions for the argument.
165
137
  //
166
138
  // `gatedInsert` is SQLite-only (like `db`): it backs
167
- // `commitRevisionSqlite`'s single gated INSERT (the dup-gate +
168
- // head-equals-base-gate + server-assigned seq folded into one
169
- // statement, mirroring the Neon path's gated INSERT). The Neon
170
- // backend leaves it unset — its gated INSERT lives inside the
171
- // pipelined `sql.transaction([...])`, not a standalone statement
172
- // object. Kept on the Handle (rather than a module-private closure)
173
- // so the SQLite white-box tests can wrap `.get` to inject a
174
- // unique-violation / non-unique failure into the commit, the same
175
- // recovery paths the Neon suite stages via `failNextCommit`.
139
+ // `commitRevisionSqlite`'s single gated INSERT. The Neon backend
140
+ // leaves it unset — its gated INSERT lives inside the pipelined
141
+ // `sql.transaction([...])`, not a standalone statement object. Kept
142
+ // on the Handle (not a module-private closure) so SQLite white-box
143
+ // tests can wrap `.get` to inject a unique-violation / non-unique
144
+ // failure into the commit, exercising the same recovery paths the
145
+ // Neon suite stages via `failNextCommit`.
176
146
  export type Handle = {
177
147
  db?: DatabaseSync
178
148
  headFor: GetStmt<[string], { id: string }>
@@ -184,7 +154,7 @@ export type Handle = {
184
154
  revisionExists: GetStmt<[string, string], unknown>
185
155
  // Single-revision fetch by content-addressed id. The cross-instance
186
156
  // pubsub receiver uses this to assemble a `workspace-state` from a
187
- // NOTIFY hint (see `server/pubsub.ts`).
157
+ // NOTIFY hint (see `server-e2e/pubsub.ts`).
188
158
  revisionById: GetStmt<[string, string], RevisionRow>
189
159
  gatedInsert?: GetStmt<[string, string, string | null, number, string, string, string, number], { seq: number }>
190
160
  tryCommit: (input: RevisionInsert) => Promise<CommitResult>
@@ -192,23 +162,22 @@ export type Handle = {
192
162
  }
193
163
 
194
164
  // Narrowing alias for the SQLite-backed Handle: `db` is guaranteed
195
- // to be set. `openDb` returns this so call sites that need direct
165
+ // set. `openDb` returns this so call sites needing direct
196
166
  // `DatabaseSync` access (e.g. `openObjstore(handle.db, …)` in
197
- // `server/index.ts`'s SQLite branch) can reach `handle.db` without
198
- // an optional-chain or non-null assertion. A Neon-backed Handle
199
- // (`openNeonDb`) keeps the wider `db?: DatabaseSync` shape; routing
200
- // a Neon Handle into a SQLite-coupled call site is a compile-time
201
- // error. Mirrors the same pattern in `server/objstore/store.ts`.
167
+ // `server-e2e/index.ts`'s SQLite branch) reach `handle.db` without an
168
+ // optional-chain or non-null assertion. A Neon-backed Handle
169
+ // (`openNeonDb`) keeps the wider `db?: DatabaseSync` shape, so routing
170
+ // one into a SQLite-coupled call site is a compile-time error. Mirrors
171
+ // `server-e2e/objstore/store.ts`.
202
172
  export type SqliteHandle = Handle & { db: DatabaseSync }
203
173
 
204
174
  export function openDb(path: string): SqliteHandle {
205
175
  mkdirSync(dirname(path), { recursive: true })
206
176
  const db = new DatabaseSync(path)
207
- // Any throw between the DatabaseSync constructor and the return
208
- // would otherwise leak the underlying file / WAL / shm locks until
209
- // process exit — close before re-raising so the operator can fix
210
- // the underlying issue (failed STRICT check, ALTER TABLE error,
211
- // …) and re-run without a stale lock pinning the file.
177
+ // A throw between the DatabaseSync constructor and the return would
178
+ // leak the file / WAL / shm locks until process exit — close before
179
+ // re-raising so the operator can fix the cause (failed STRICT check,
180
+ // ALTER TABLE error, …) and re-run without a stale lock on the file.
212
181
  try {
213
182
  return openDbInner(db)
214
183
  } catch (err) {
@@ -218,35 +187,30 @@ export function openDb(path: string): SqliteHandle {
218
187
  }
219
188
 
220
189
  function openDbInner(db: DatabaseSync): SqliteHandle {
221
- // WAL gives concurrent readers + faster writes and survives
222
- // crashes between commits without corrupting the file. Foreign
223
- // keys aren't strictly needed here (single-table schema) but
224
- // turning them on preserves the option to add referential
225
- // tables later without revisiting init.
190
+ // WAL gives concurrent readers + faster writes and survives crashes
191
+ // between commits without corrupting the file. Foreign keys aren't
192
+ // needed here (single-table schema) but turning them on keeps the
193
+ // option to add referential tables later without revisiting init.
226
194
  db.exec('PRAGMA journal_mode = WAL;')
227
195
  // FULL (not NORMAL): the server emits `workspace-save-ack` BEFORE
228
- // returning to the event loop after `commitRevision`. With NORMAL,
229
- // SQLite only fsyncs at WAL checkpoint, so a power loss between
230
- // ack and the next checkpoint loses the row even though the
231
- // originator and broadcast peers were told the revision committed.
232
- // FULL fsyncs per commit; durability matches the contract the
233
- // ack implies. Trade-off is per-commit fsync latency, acceptable
234
- // for the protocol's edit-driven write pattern (triage edits, not
235
- // streaming throughput). Audit round-9 M1.
196
+ // returning to the event loop after `commitRevision`. NORMAL only
197
+ // fsyncs at WAL checkpoint, so a power loss between ack and the next
198
+ // checkpoint loses a row the originator + peers were told committed.
199
+ // FULL fsyncs per commit, matching the durability the ack implies.
200
+ // Trade-off is per-commit fsync latency, acceptable for the edit-
201
+ // driven write pattern (triage edits, not streaming). Audit round-9 M1.
236
202
  db.exec('PRAGMA synchronous = FULL;')
237
203
  db.exec('PRAGMA foreign_keys = ON;')
238
204
  db.exec(SCHEMA)
239
205
  // Fail-loud on a pre-existing non-STRICT table — `CREATE TABLE IF
240
- // NOT EXISTS … STRICT` is a no-op when the table already exists,
241
- // so a deployment that predates the STRICT marker would silently
242
- // keep its non-STRICT shape. Without STRICT, an operator with
243
- // direct DB write access could insert mis-typed rows (e.g. a
244
- // `keyframe = "1\nfoo"` text value in the INTEGER column) and
245
- // poison the chain — the signed canonical the client originally
246
- // hashed says `keyframe = 1`, but the stored `keyframe = "1\nfoo"`
247
- // round-trips back into the canonical as a different string,
248
- // making every subsequent verify fail. Operator must migrate
249
- // before this server boots.
206
+ // NOT EXISTS … STRICT` is a no-op when the table exists, so a
207
+ // deployment predating the STRICT marker keeps its non-STRICT shape.
208
+ // Without STRICT, an operator with direct DB write access could
209
+ // insert mis-typed rows (e.g. `keyframe = "1\nfoo"` text in the
210
+ // INTEGER column) and poison the chain: the signed canonical says
211
+ // `keyframe = 1`, but the stored text round-trips into the canonical
212
+ // as a different string, failing every subsequent verify. Operator
213
+ // must migrate before this server boots.
250
214
  const meta = db.prepare(
251
215
  `SELECT strict FROM pragma_table_list WHERE schema = 'main' AND name = 'workspace_revision'`,
252
216
  ).get() as { strict: number } | undefined
@@ -254,13 +218,11 @@ function openDbInner(db: DatabaseSync): SqliteHandle {
254
218
  throw new Error('workspace_revision is non-STRICT — migrate via rename+create+copy before booting')
255
219
  }
256
220
  // Idempotent migration for DBs created before the keyframe column
257
- // existed. Inspect the column list rather than catching every
258
- // ALTER error — the previous shape swallowed `try { ALTER } catch
259
- // {}` for ANY failure (lock contention, disk full, corrupt page),
260
- // masking real problems as "column already exists". Now we only
261
- // ALTER when the column is genuinely missing, and any failure of
262
- // the ALTER itself bubbles up as an open-time crash where the
263
- // operator can act on it.
221
+ // existed. Inspect the column list rather than `try { ALTER } catch
222
+ // {}`: a blanket catch swallows ANY failure (lock contention, disk
223
+ // full, corrupt page) as "column already exists". ALTER only when
224
+ // the column is genuinely missing, so an ALTER failure bubbles up as
225
+ // an open-time crash the operator can act on.
264
226
  const columns = db.prepare(`PRAGMA table_info(workspace_revision)`).all() as Array<{ name: string }>
265
227
  if (!columns.some((c) => c.name === 'keyframe')) {
266
228
  // ADD COLUMN carries the CHECK so a legacy DB migrating up lands
@@ -507,9 +469,8 @@ export function commitRevision(handle: Handle, input: RevisionInsert): Promise<C
507
469
  // base gate fails → `stale-base`.
508
470
  // • Two retransmits with the same id: the second's dup gate fails →
509
471
  // `duplicate`.
510
- // These are PROVEN green, unchanged, by the no-fork concurrency tests
511
- // in `tests/server-db.test.js` (two/N concurrent same-base, mixed,
512
- // chainFrom-during-commits) which now pass with no lock present.
472
+ // Covered by the no-fork concurrency tests in `tests/server-db.test.js`
473
+ // (two/N concurrent same-base, mixed, chainFrom-during-commits).
513
474
  //
514
475
  // SQLite serialises writers internally even ACROSS connections, but a
515
476
  // multi-connection deployment is unsupported regardless. The