@sixb/pg 0.1.13 → 0.1.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/auth-storage/auth-storage.d.ts +2 -1
  2. package/dist/auth-storage/auth-storage.d.ts.map +1 -1
  3. package/dist/auth-storage/rows.d.ts +1 -1
  4. package/dist/auth-storage/rows.d.ts.map +1 -1
  5. package/dist/index.d.ts +8 -1
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js +779 -357
  8. package/dist/index.js.map +16 -16
  9. package/dist/latest-run-query.d.ts +2 -0
  10. package/dist/latest-run-query.d.ts.map +1 -1
  11. package/dist/migrations.d.ts.map +1 -1
  12. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  13. package/dist/ontology-storage/shared.d.ts +12 -9
  14. package/dist/ontology-storage/shared.d.ts.map +1 -1
  15. package/dist/ontology-storage/source-cleanup.d.ts +7 -1
  16. package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
  17. package/dist/ontology-storage/source-roots.d.ts +8 -0
  18. package/dist/ontology-storage/source-roots.d.ts.map +1 -1
  19. package/dist/ontology-storage/sources.d.ts +8 -3
  20. package/dist/ontology-storage/sources.d.ts.map +1 -1
  21. package/dist/pg-client.d.ts +2 -0
  22. package/dist/pg-client.d.ts.map +1 -1
  23. package/dist/pg-projection-run-storage.d.ts +3 -1
  24. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  25. package/dist/storage-errors.d.ts +13 -3
  26. package/dist/storage-errors.d.ts.map +1 -1
  27. package/dist/transactions.d.ts +23 -1
  28. package/dist/transactions.d.ts.map +1 -1
  29. package/package.json +3 -3
  30. package/src/auth-storage/auth-storage.ts +43 -75
  31. package/src/auth-storage/rows.ts +1 -1
  32. package/src/index.ts +8 -1
  33. package/src/latest-run-query.ts +10 -2
  34. package/src/migrations/050-directory-group-memberships.sql +5 -0
  35. package/src/migrations/051-projection-run-supersession.sql +43 -0
  36. package/src/migrations/052-compact-source-storage.sql +118 -0
  37. package/src/migrations.ts +30 -23
  38. package/src/ontology-storage/materialization-state.ts +20 -46
  39. package/src/ontology-storage/shared.ts +18 -12
  40. package/src/ontology-storage/source-cleanup.ts +66 -22
  41. package/src/ontology-storage/source-roots.ts +164 -102
  42. package/src/ontology-storage/sources.ts +150 -120
  43. package/src/pg-client.ts +132 -2
  44. package/src/pg-projection-run-storage.ts +56 -2
  45. package/src/storage-errors.ts +101 -15
  46. package/src/transactions.ts +113 -20
@@ -9,15 +9,18 @@ import {
9
9
  isExactStagingManifest,
10
10
  reconcileSourceStageRows,
11
11
  type SourceStageRow,
12
+ sourceAssertionPayload,
12
13
  sourceConflict,
13
14
  sourceMaterializationIdentity,
14
15
  sourceStageRow,
15
16
  sourceStageRows,
16
- utf8SortKey,
17
17
  } from "@sixb/core/internal/ontology-storage-provider"
18
18
  import type {
19
+ AbandonRunSourceMaterializationInput,
19
20
  AbandonSourceMaterializationCandidateInput,
20
21
  AbandonSourceMaterializationInput,
22
+ AdoptedSourceMaterialization,
23
+ AdoptSourceMaterializationInput,
21
24
  AssertSourceMaterializationExecutionInput,
22
25
  BeginSourceMaterializationInput,
23
26
  CleanupTerminalSourceMaterializationsInput,
@@ -26,7 +29,7 @@ import type {
26
29
  MarkSourceMaterializationReadyInput,
27
30
  OntologySourceRecord,
28
31
  OntologySourceStorage,
29
- ReclaimSourceMaterializationInput,
32
+ PurgeAbandonedSourceMaterializationsInput,
30
33
  StageSourceRowsInput,
31
34
  StageSourceRowsResult,
32
35
  SummarizeTerminalSourceMaterializationsInput,
@@ -49,8 +52,8 @@ import {
49
52
  sourceRecord,
50
53
  toIsoString,
51
54
  } from "./shared"
52
- import { cleanupSourceVersions } from "./source-cleanup"
53
- import { assertSourceRootCoverage, stageSourceRoots } from "./source-roots"
55
+ import { cleanupSourceVersions, purgeAbandonedSourceVersions } from "./source-cleanup"
56
+ import { assertSourceRootCoverage, sourceAssertionColumns, stageSourceRoots } from "./source-roots"
54
57
 
55
58
  export class PgOntologySourceStorage implements OntologySourceStorage {
56
59
  constructor(private readonly runRootOperation: PgRootOperation) {}
@@ -88,7 +91,7 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
88
91
  const candidate = await this.getRunCandidate(sql, input)
89
92
  if (candidate) {
90
93
  throw sourceConflict(
91
- `Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; reclaim it before beginning another.`
94
+ `Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`
92
95
  )
93
96
  }
94
97
 
@@ -126,7 +129,7 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
126
129
  }
127
130
  if (await this.getRunCandidate(sql, input)) {
128
131
  throw sourceConflict(
129
- `Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; reclaim it before beginning another.`
132
+ `Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`
130
133
  )
131
134
  }
132
135
  throw sourceConflict(
@@ -154,10 +157,10 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
154
157
  }
155
158
 
156
159
  const rows = sourceStageRows(manifest.projection_kind, input.rows)
157
- const existing = await this.findStageRows(sql, input, rows)
160
+ const existing = await this.findStageRows(sql, manifest, rows)
158
161
  const { pending, unchanged } = reconcileSourceStageRows(rows, existing)
159
162
  await stageSourceRoots(sql, manifest, input)
160
- await this.insertStageRows(sql, input, pending)
163
+ await this.insertStageRows(sql, manifest, pending)
161
164
  return { inserted: pending.length, unchanged }
162
165
  })
163
166
  }
@@ -236,17 +239,50 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
236
239
  })
237
240
  }
238
241
 
242
+ async adopt(
243
+ input: AdoptSourceMaterializationInput
244
+ ): Promise<AdoptedSourceMaterialization | null> {
245
+ return this.runRootOperation(async (sql) => {
246
+ assertProjectAndSource(input)
247
+ assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken)
248
+ assertTimestamp(input.adoptedAt, "Source adoptedAt", true)
249
+ await this.assertExecution(sql, input)
250
+ const candidate = await this.lockRunCandidate(sql, input)
251
+ if (!candidate) return null
252
+ // Workers' clocks differ; adoption never moves the record back in time.
253
+ const [adopted] = await sql<PgOntologySourceRow[]>`
254
+ UPDATE ontology_sources
255
+ SET execution_token = ${input.execution.executionToken},
256
+ updated_at = GREATEST(updated_at, ${input.adoptedAt}::timestamptz)
257
+ WHERE project_id = ${candidate.project_id} AND source_id = ${candidate.source_id}
258
+ AND materialization_id = ${candidate.materialization_id}
259
+ AND status IN ('staging', 'ready')
260
+ RETURNING *
261
+ `
262
+ if (!adopted) {
263
+ throw sourceConflict(`Source materialization '${candidate.materialization_id}' changed.`)
264
+ }
265
+ return {
266
+ record: sourceRecord(adopted),
267
+ resumeStagingOrdinal:
268
+ adopted.status === "ready"
269
+ ? Number(adopted.root_count)
270
+ : await this.lastStagedOrdinal(sql, adopted),
271
+ }
272
+ })
273
+ }
274
+
239
275
  async abandon(input: AbandonSourceMaterializationCandidateInput): Promise<OntologySourceRecord>
240
- async abandon(input: ReclaimSourceMaterializationInput): Promise<OntologySourceRecord | null>
276
+ async abandon(input: AbandonRunSourceMaterializationInput): Promise<OntologySourceRecord | null>
241
277
  async abandon(input: AbandonSourceMaterializationInput): Promise<OntologySourceRecord | null> {
242
278
  return this.runRootOperation(async (sql) => {
243
279
  assertProjectAndSource(input)
244
280
  assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken)
245
281
  assertTimestamp(input.abandonedAt, "Source abandonedAt", true)
246
282
  await this.assertExecution(sql, input)
247
- return input.kind === "candidate"
248
- ? this.abandonCandidate(sql, input)
249
- : this.abandonForReclaim(sql, input)
283
+ if (input.kind === "candidate") return this.abandonCandidate(sql, input)
284
+ const candidate = await this.lockRunCandidate(sql, input)
285
+ return candidate ? this.transitionToAbandoned(sql, candidate, input.abandonedAt) : null
250
286
  })
251
287
  }
252
288
 
@@ -261,6 +297,16 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
261
297
  })
262
298
  }
263
299
 
300
+ async purgeAbandoned(
301
+ input: PurgeAbandonedSourceMaterializationsInput
302
+ ): Promise<CleanupTerminalSourceMaterializationsResult> {
303
+ return this.runRootOperation(async (sql) => {
304
+ assertNonblank(input.projectId, "Abandoned source purge project id")
305
+ assertPositiveInteger(input.limit, "Abandoned source purge limit")
306
+ return purgeAbandonedSourceVersions(sql, input)
307
+ })
308
+ }
309
+
264
310
  async summarizeTerminal(
265
311
  input: SummarizeTerminalSourceMaterializationsInput
266
312
  ): Promise<TerminalSourceMaterializationSummary> {
@@ -311,10 +357,15 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
311
357
  return this.transitionToAbandoned(sql, manifest, input.abandonedAt)
312
358
  }
313
359
 
314
- private async abandonForReclaim(
360
+ /** The run's one staging or ready candidate, locked, whichever execution staged it. */
361
+ private async lockRunCandidate(
315
362
  sql: SQLClient,
316
- input: ReclaimSourceMaterializationInput
317
- ): Promise<OntologySourceRecord | null> {
363
+ input: {
364
+ readonly projectId: string
365
+ readonly source: { readonly projectionId: string }
366
+ readonly execution: { readonly projectionRunId: string }
367
+ }
368
+ ): Promise<PgOntologySourceRow | null> {
318
369
  const candidates = await sql<PgOntologySourceRow[]>`
319
370
  SELECT * FROM ontology_sources
320
371
  WHERE project_id = ${input.projectId}
@@ -323,19 +374,20 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
323
374
  AND status IN ('staging', 'ready')
324
375
  FOR UPDATE
325
376
  `
326
- if (
327
- candidates.some((candidate) => candidate.execution_token === input.execution.executionToken)
328
- ) {
329
- throw sourceConflict(
330
- `Projection run '${input.execution.projectionRunId}' already has a source materialization owned by the current execution.`
331
- )
332
- }
333
377
  if (candidates.length > 1) {
334
378
  throw sourceConflict(
335
379
  `Projection run '${input.execution.projectionRunId}' has multiple nonterminal source materializations.`
336
380
  )
337
381
  }
338
- return candidates[0] ? this.transitionToAbandoned(sql, candidates[0], input.abandonedAt) : null
382
+ return candidates[0] ?? null
383
+ }
384
+
385
+ private async lastStagedOrdinal(sql: SQLClient, manifest: PgOntologySourceRow): Promise<number> {
386
+ const [last] = await sql<{ readonly ordinal: number | string | null }[]>`
387
+ SELECT MAX(staging_ordinal) AS ordinal FROM ontology_source_roots
388
+ WHERE version_id = ${manifest.version_id}
389
+ `
390
+ return last?.ordinal == null ? 0 : Number(last.ordinal)
339
391
  }
340
392
 
341
393
  private async transitionToAbandoned(
@@ -366,9 +418,7 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
366
418
  `Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`
367
419
  )
368
420
  }
369
- await sql`UPDATE ontology_source_roots SET retired_at = ${abandonedAt}
370
- WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
371
- AND materialization_id = ${manifest.materialization_id} AND NOT active`
421
+ // Its roots were never live and stay unpublished; purging removes them with the version.
372
422
  return sourceRecord(rows[0])
373
423
  }
374
424
 
@@ -378,6 +428,7 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
378
428
  rootCount: number,
379
429
  assertionCount: number
380
430
  ): Promise<void> {
431
+ const version = manifest.version_id
381
432
  const [counts] = await sql<
382
433
  {
383
434
  readonly assertions: number | string
@@ -387,15 +438,13 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
387
438
  readonly max_ordinal: number | string | null
388
439
  }[]
389
440
  >`
390
- SELECT (SELECT COUNT(*) FROM ontology_source_rows
391
- WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
392
- AND materialization_id = ${manifest.materialization_id}) AS assertions, COUNT(*) AS roots,
393
- COUNT(DISTINCT staging_ordinal) AS ordinals,
441
+ SELECT (SELECT COUNT(*) FROM ontology_source_roots AS roots
442
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
443
+ WHERE roots.version_id = ${version}) AS assertions,
444
+ COUNT(*) AS roots, COUNT(DISTINCT staging_ordinal) AS ordinals,
394
445
  MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
395
446
  FROM ontology_source_roots
396
- WHERE project_id = ${manifest.project_id}
397
- AND source_id = ${manifest.source_id}
398
- AND materialization_id = ${manifest.materialization_id}
447
+ WHERE version_id = ${version}
399
448
  `
400
449
  const assertions = Number(counts?.assertions ?? 0)
401
450
  const roots = Number(counts?.roots ?? 0)
@@ -414,39 +463,45 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
414
463
  }
415
464
  await assertSourceRootCoverage(sql, manifest)
416
465
 
466
+ // Roots store only their canonical key; compare its parts with each row's typed identity.
417
467
  const [invalid] =
418
468
  manifest.projection_kind === "link"
419
- ? await sql<{ readonly root_key: unknown }[]>`
420
- SELECT root_key
421
- FROM ontology_source_rows
422
- WHERE project_id = ${manifest.project_id}
423
- AND source_id = ${manifest.source_id}
424
- AND materialization_id = ${manifest.materialization_id}
425
- GROUP BY root_key
469
+ ? await sql<{ readonly root_key: string }[]>`
470
+ SELECT roots.root_key
471
+ FROM ontology_source_roots AS roots
472
+ CROSS JOIN LATERAL (SELECT roots.root_key::jsonb AS parts) AS root
473
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
474
+ WHERE roots.version_id = ${version}
475
+ GROUP BY roots.id, roots.root_key
426
476
  HAVING COUNT(*) <> 1
427
- OR BOOL_OR(root_kind <> 'link')
428
- OR BOOL_OR(entity_kind <> 'link')
429
- OR BOOL_OR(root_key <> entity_key)
477
+ OR BOOL_OR(root.parts->>0 <> 'link')
478
+ OR BOOL_OR(rows.entity_kind <> 'link')
479
+ OR BOOL_OR(rows.source_type_id <> root.parts->>1
480
+ OR rows.source_primary_id <> root.parts->>2 OR rows.link_id <> root.parts->>3
481
+ OR rows.target_type_id <> root.parts->>4 OR rows.target_primary_id <> root.parts->>5)
430
482
  LIMIT 1
431
483
  `
432
- : await sql<{ readonly root_key: unknown }[]>`
433
- SELECT root_key
434
- FROM ontology_source_rows
435
- WHERE project_id = ${manifest.project_id}
436
- AND source_id = ${manifest.source_id}
437
- AND materialization_id = ${manifest.materialization_id}
438
- GROUP BY root_key
439
- HAVING BOOL_OR(root_kind <> 'object')
484
+ : await sql<{ readonly root_key: string }[]>`
485
+ SELECT roots.root_key
486
+ FROM ontology_source_roots AS roots
487
+ CROSS JOIN LATERAL (SELECT roots.root_key::jsonb AS parts) AS root
488
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
489
+ WHERE roots.version_id = ${version}
490
+ GROUP BY roots.id, roots.root_key
491
+ HAVING BOOL_OR(root.parts->>0 <> 'object')
440
492
  OR COUNT(*) FILTER (
441
- WHERE entity_kind = 'object' AND entity_key = root_key
493
+ WHERE rows.entity_kind = 'object' AND rows.object_type_id = root.parts->>1
494
+ AND rows.primary_id = root.parts->>2
442
495
  ) <> 1
443
496
  OR COUNT(*) FILTER (
444
- WHERE (entity_kind = 'object' AND entity_key <> root_key)
445
- OR (entity_kind = 'link' AND (
446
- source_type_id <> root_object_type_id
447
- OR source_primary_id <> root_primary_id
448
- ))
497
+ WHERE (rows.entity_kind = 'object' AND (rows.object_type_id <> root.parts->>1
498
+ OR rows.primary_id <> root.parts->>2))
499
+ OR (rows.entity_kind = 'link' AND (rows.source_type_id <> root.parts->>1
500
+ OR rows.source_primary_id <> root.parts->>2))
449
501
  ) <> 0
502
+ OR COUNT(*) FILTER (WHERE rows.entity_kind = 'link') <> COUNT(DISTINCT (
503
+ rows.link_id, rows.target_type_id, rows.target_primary_id
504
+ )) FILTER (WHERE rows.entity_kind = 'link')
450
505
  LIMIT 1
451
506
  `
452
507
  if (invalid) {
@@ -458,24 +513,24 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
458
513
  }
459
514
  }
460
515
 
516
+ /** Already staged assertions sharing a root or an ordinal with `rows`. */
461
517
  private async findStageRows(
462
518
  sql: SQLClient,
463
- input: StageSourceRowsInput,
519
+ manifest: PgOntologySourceRow,
464
520
  rows: readonly SourceStageRow[]
465
521
  ): Promise<readonly SourceStageRow[]> {
466
522
  if (rows.length === 0) return []
467
- const rootSortKeys = [...new Set(rows.map((row) => utf8SortKey(row.rootKey)))]
523
+ const rootKeys = [...new Set(rows.map((row) => row.rootKey))]
468
524
  const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))]
469
- const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey(row.entityKey)))]
470
525
  const existing = await sql<PgOntologySourceAssertionRow[]>`
471
- SELECT * FROM ontology_source_rows
472
- WHERE project_id = ${input.projectId}
473
- AND source_id = ${input.source.projectionId}
474
- AND materialization_id = ${input.materializationId}
526
+ SELECT ${sourceAssertionColumns(sql)}
527
+ FROM ontology_source_roots AS roots
528
+ JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
529
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
530
+ WHERE roots.version_id = ${manifest.version_id}
475
531
  AND (
476
- root_sort_key = ANY(${sql.array(rootSortKeys)}::text[])
477
- OR staging_ordinal = ANY(${sql.array(ordinals)}::bigint[])
478
- OR entity_sort_key = ANY(${sql.array(entitySortKeys)}::text[])
532
+ roots.root_key = ANY(${sql.array(rootKeys)}::text[])
533
+ OR roots.staging_ordinal = ANY(${sql.array(ordinals)}::bigint[])
479
534
  )
480
535
  `
481
536
  return existing.map((row) => sourceStageRow(sourceAssertion(row)))
@@ -483,69 +538,44 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
483
538
 
484
539
  private async insertStageRows(
485
540
  sql: SQLClient,
486
- input: StageSourceRowsInput,
541
+ manifest: PgOntologySourceRow,
487
542
  rows: readonly SourceStageRow[]
488
543
  ): Promise<void> {
489
544
  if (rows.length === 0) return
490
- const payload = rows.map(({ entityKey, rootKey, row }) => {
545
+ const payload = rows.map(({ rootKey, row }) => {
491
546
  const entity = entityColumns(row.assertion)
492
- const root = entityColumns(row.root)
493
547
  return {
494
- entityKind: row.assertion.kind,
495
- entityKey: JSON.parse(entityKey) as unknown,
496
- entitySortKey: utf8SortKey(entityKey),
497
- rootKind: row.root.kind,
498
- rootKey: JSON.parse(rootKey) as unknown,
499
- rootSortKey: utf8SortKey(rootKey),
500
- stagingOrdinal: row.stagingOrdinal,
501
- root: row.root,
502
- assertion: row.assertion,
503
- objectTypeId: entity.objectTypeId,
504
- primaryId: entity.primaryId,
505
- sourceTypeId: entity.sourceTypeId,
506
- sourcePrimaryId: entity.sourcePrimaryId,
507
- linkId: entity.linkId,
508
- targetTypeId: entity.targetTypeId,
509
- targetPrimaryId: entity.targetPrimaryId,
510
- rootObjectTypeId: root.objectTypeId,
511
- rootPrimaryId: root.primaryId,
512
- rootSourceTypeId: root.sourceTypeId,
513
- rootSourcePrimaryId: root.sourcePrimaryId,
514
- rootLinkId: root.linkId,
515
- rootTargetTypeId: root.targetTypeId,
516
- rootTargetPrimaryId: root.targetPrimaryId,
548
+ root_key: rootKey,
549
+ entity_kind: row.assertion.kind,
550
+ object_type_id: entity.objectTypeId,
551
+ primary_id: entity.primaryId,
552
+ source_type_id: entity.sourceTypeId,
553
+ source_primary_id: entity.sourcePrimaryId,
554
+ link_id: entity.linkId,
555
+ target_type_id: entity.targetTypeId,
556
+ target_primary_id: entity.targetPrimaryId,
557
+ payload: sourceAssertionPayload(row.assertion),
517
558
  }
518
559
  })
519
- await sql`
520
- WITH staged AS (
521
- SELECT value
522
- FROM jsonb_array_elements(${jsonParameter(sql, payload)}::jsonb)
523
- )
560
+ const inserted = await sql`
524
561
  INSERT INTO ontology_source_rows (
525
- project_id, source_id, materialization_id,
526
- entity_kind, entity_key, entity_sort_key,
527
- root_kind, root_key, root_sort_key, staging_ordinal,
528
- root, assertion,
529
- object_type_id, primary_id,
530
- source_type_id, source_primary_id, link_id, target_type_id, target_primary_id,
531
- root_object_type_id, root_primary_id,
532
- root_source_type_id, root_source_primary_id, root_link_id,
533
- root_target_type_id, root_target_primary_id
562
+ root_id, entity_kind, object_type_id, primary_id,
563
+ source_type_id, source_primary_id, link_id, target_type_id, target_primary_id, payload
534
564
  )
535
- SELECT
536
- ${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
537
- value->>'entityKind', value->'entityKey', value->>'entitySortKey',
538
- value->>'rootKind', value->'rootKey', value->>'rootSortKey',
539
- (value->>'stagingOrdinal')::bigint,
540
- value->'root', value->'assertion',
541
- value->>'objectTypeId', value->>'primaryId',
542
- value->>'sourceTypeId', value->>'sourcePrimaryId', value->>'linkId',
543
- value->>'targetTypeId', value->>'targetPrimaryId',
544
- value->>'rootObjectTypeId', value->>'rootPrimaryId',
545
- value->>'rootSourceTypeId', value->>'rootSourcePrimaryId', value->>'rootLinkId',
546
- value->>'rootTargetTypeId', value->>'rootTargetPrimaryId'
547
- FROM staged
565
+ SELECT roots.id, staged.entity_kind, staged.object_type_id, staged.primary_id,
566
+ staged.source_type_id, staged.source_primary_id, staged.link_id,
567
+ staged.target_type_id, staged.target_primary_id, staged.payload
568
+ FROM jsonb_to_recordset(${jsonParameter(sql, payload)}) AS staged(
569
+ root_key TEXT, entity_kind TEXT, object_type_id TEXT, primary_id TEXT,
570
+ source_type_id TEXT, source_primary_id TEXT, link_id TEXT,
571
+ target_type_id TEXT, target_primary_id TEXT, payload JSONB
572
+ )
573
+ JOIN ontology_source_roots AS roots
574
+ ON roots.version_id = ${manifest.version_id} AND roots.root_key = staged.root_key
548
575
  `
576
+ if (inserted.count !== rows.length) {
577
+ throw sourceConflict("Staged source rows do not all belong to a staged root.")
578
+ }
549
579
  }
550
580
 
551
581
  private assertExecution(
package/src/pg-client.ts CHANGED
@@ -1,4 +1,7 @@
1
+ import { isIP } from "node:net"
2
+ import { connect as connectTls, type TLSSocket } from "node:tls"
1
3
  import postgres from "postgres"
4
+ import { pgErrorCode } from "./storage-errors"
2
5
 
3
6
  // Data-type policy (porsager defaults, deliberately not overridden):
4
7
  // - `timestamptz` -> JS `Date`. Row mappers normalize via `toIsoString(Date | string)`.
@@ -30,6 +33,9 @@ export type SQL = postgres.Sql<Record<string, never>>
30
33
  */
31
34
  export type SQLClient = postgres.ISql<Record<string, never>>
32
35
 
36
+ /** One pool connection held outside the pool until `release()` — see `withReservedPgConnection`. */
37
+ export type ReservedSQL = postgres.ReservedSql<Record<string, never>>
38
+
33
39
  /**
34
40
  * A value accepted as a positional parameter by {@link SQLClient.unsafe}. Dynamically built
35
41
  * parameter arrays (from the query-IR compiler / run-list helpers) are cast to this since
@@ -106,11 +112,11 @@ export function createPgClient(options: CreatePgClientOptions): SQL {
106
112
  // Match the previous (bun:sql) behavior of not surfacing routine PostgreSQL NOTICEs.
107
113
  onnotice: () => {},
108
114
  ...(options.prepare !== undefined ? { prepare: options.prepare } : {}),
109
- ...(options.ssl ? { ssl: options.ssl } : {}),
115
+ ...tlsOptions(options),
110
116
  }
111
117
 
112
118
  if (options.connectionString) {
113
- return postgres(options.connectionString, base)
119
+ return assertDirectTlsTarget(postgres(options.connectionString, base), base)
114
120
  }
115
121
 
116
122
  return postgres({
@@ -122,3 +128,127 @@ export function createPgClient(options: CreatePgClientOptions): SQL {
122
128
  ...(options.password !== undefined ? { password: options.password } : {}),
123
129
  })
124
130
  }
131
+
132
+ /**
133
+ * What the server's certificate is checked against when TLS is opened directly: nothing
134
+ * (`require`), the trusted CAs (`verify-ca`), or the CAs and the host name (`verify-full`).
135
+ */
136
+ type DirectTlsVerification = "none" | "ca" | "full"
137
+
138
+ // `sslnegotiation=direct` is read from the connection string, as libpq reads it. Without it,
139
+ // porsager negotiates TLS the classic way: an SSLRequest on a plain TCP socket, then an upgrade of
140
+ // that socket with `tls.connect({ socket })`. Bun 1.4.0–1.4.2 keeps every byte received on such an
141
+ // upgraded socket in memory for good, so a process that reads a lot grows until it is killed.
142
+ // Opening the TLS connection ourselves with `tls.connect({ host, port })` avoids that path, and
143
+ // saves the SSLRequest round trip on every connection.
144
+ function tlsOptions(options: CreatePgClientOptions): Record<string, unknown> {
145
+ const verification = directTlsVerification(options)
146
+ if (verification === null) {
147
+ return options.ssl ? { ssl: options.ssl } : {}
148
+ }
149
+ // porsager must not negotiate TLS again on the socket we hand it; explicit options also
150
+ // override the `sslmode` it would otherwise read from the connection string.
151
+ return { ssl: false, sslnegotiation: null, socket: directTlsSocket(verification) }
152
+ }
153
+
154
+ function directTlsVerification(options: CreatePgClientOptions): DirectTlsVerification | null {
155
+ const params = connectionStringParams(options.connectionString)
156
+ if (params.get("sslnegotiation") !== "direct") {
157
+ return null
158
+ }
159
+
160
+ const sslmode =
161
+ options.ssl === true
162
+ ? "verify-full"
163
+ : options.ssl ||
164
+ (params.get("sslrootcert") === "system"
165
+ ? "verify-full"
166
+ : (params.get("sslmode") ?? params.get("ssl")))
167
+ switch (sslmode) {
168
+ case "require":
169
+ return "none"
170
+ case "verify-ca":
171
+ return "ca"
172
+ case "verify-full":
173
+ case "true":
174
+ return "full"
175
+ default:
176
+ throw new Error(
177
+ `[SixbPg] sslnegotiation=direct requires sslmode=require, verify-ca or verify-full, got ${
178
+ sslmode ? `"${sslmode}"` : "no sslmode"
179
+ }: a direct TLS connection has no plaintext fallback.`
180
+ )
181
+ }
182
+ }
183
+
184
+ function connectionStringParams(connectionString: string | undefined): URLSearchParams {
185
+ return new URLSearchParams(connectionString?.split("?")[1] ?? "")
186
+ }
187
+
188
+ /**
189
+ * porsager's `socket` factory, called for every connection. It must return at once and never
190
+ * throw: a factory that fails leaves its pool slot stuck in porsager's connecting state for good,
191
+ * and one that returns an already-dead socket stalls the slot's next attempt (postgres.js 3.4.9).
192
+ * Failures are reported by the socket instead, through the `error` and `close` events porsager
193
+ * handles for its own sockets, and its connect timeout covers the handshake.
194
+ */
195
+ function directTlsSocket(
196
+ verification: DirectTlsVerification
197
+ ): (options: postgres.ParsedOptions) => TLSSocket {
198
+ let reportedRefusal = false
199
+
200
+ return ({ host: [host = "localhost"], port: [port = 5432] }) => {
201
+ const socket = connectTls({
202
+ host,
203
+ port,
204
+ servername: isIP(host) ? undefined : host,
205
+ // PostgreSQL 17+ accepts a direct TLS connection only for this ALPN protocol.
206
+ ALPNProtocols: ["postgresql"],
207
+ rejectUnauthorized: verification !== "none",
208
+ ...(verification === "ca" ? { checkServerIdentity: () => undefined } : {}),
209
+ })
210
+
211
+ let reachedServer = false
212
+ let secured = false
213
+ socket.once("connect", () => {
214
+ reachedServer = true
215
+ })
216
+ socket.once("secureConnect", () => {
217
+ secured = true
218
+ if (socket.alpnProtocol !== "postgresql") {
219
+ socket.destroy(
220
+ new Error(
221
+ `[SixbPg] ${host}:${port} accepted TLS without selecting the "postgresql" protocol` +
222
+ " (ALPN), so it is not a PostgreSQL server accepting sslnegotiation=direct."
223
+ )
224
+ )
225
+ }
226
+ })
227
+ // A server that expects an SSLRequest reads the TLS hello as a malformed startup packet and
228
+ // closes the connection, which porsager reports as a bare reset. Say what it means, once.
229
+ socket.once("error", (error: Error) => {
230
+ if (reportedRefusal || !reachedServer || secured) return
231
+ if (!["ECONNRESET", "EPIPE", "ERR_SOCKET_CLOSED"].includes(pgErrorCode(error) ?? "")) return
232
+ reportedRefusal = true
233
+ console.error(
234
+ `[SixbPg] ${host}:${port} closed the connection during a direct TLS handshake` +
235
+ " (sslnegotiation=direct). Direct TLS needs PostgreSQL 17 or later, reached without a" +
236
+ " proxy or pooler that expects an SSLRequest; remove sslnegotiation=direct otherwise."
237
+ )
238
+ })
239
+ return socket
240
+ }
241
+ }
242
+
243
+ function assertDirectTlsTarget(sql: SQL, options: Record<string, unknown>): SQL {
244
+ if (!("socket" in options)) {
245
+ return sql
246
+ }
247
+ if (sql.options.host.length !== 1 || sql.options.path) {
248
+ throw new Error(
249
+ "[SixbPg] sslnegotiation=direct supports one TCP host:port in the connection string, not" +
250
+ " several hosts or a Unix socket path."
251
+ )
252
+ }
253
+ return sql
254
+ }