@sixb/pg 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +8 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +743 -313
- package/dist/index.js.map +15 -15
- package/dist/latest-run-query.d.ts +2 -0
- package/dist/latest-run-query.d.ts.map +1 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +12 -9
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/source-cleanup.d.ts +7 -1
- package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
- package/dist/ontology-storage/source-roots.d.ts +8 -0
- package/dist/ontology-storage/source-roots.d.ts.map +1 -1
- package/dist/ontology-storage/sources.d.ts +8 -3
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/pg-client.d.ts +2 -0
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +3 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/storage-errors.d.ts +13 -3
- package/dist/storage-errors.d.ts.map +1 -1
- package/dist/transactions.d.ts +23 -1
- package/dist/transactions.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/index.ts +8 -1
- package/src/latest-run-query.ts +10 -2
- package/src/migrations/051-projection-run-supersession.sql +43 -0
- package/src/migrations/052-compact-source-storage.sql +118 -0
- package/src/migrations.ts +26 -23
- package/src/objects/query-compiler.ts +1 -1
- package/src/ontology-storage/materialization-state.ts +20 -46
- package/src/ontology-storage/shared.ts +18 -12
- package/src/ontology-storage/source-cleanup.ts +66 -22
- package/src/ontology-storage/source-roots.ts +164 -102
- package/src/ontology-storage/sources.ts +150 -120
- package/src/pg-client.ts +132 -2
- package/src/pg-projection-run-storage.ts +56 -2
- package/src/storage-errors.ts +101 -15
- package/src/transactions.ts +113 -20
|
@@ -9,15 +9,18 @@ import {
|
|
|
9
9
|
isExactStagingManifest,
|
|
10
10
|
reconcileSourceStageRows,
|
|
11
11
|
type SourceStageRow,
|
|
12
|
+
sourceAssertionPayload,
|
|
12
13
|
sourceConflict,
|
|
13
14
|
sourceMaterializationIdentity,
|
|
14
15
|
sourceStageRow,
|
|
15
16
|
sourceStageRows,
|
|
16
|
-
utf8SortKey,
|
|
17
17
|
} from "@sixb/core/internal/ontology-storage-provider"
|
|
18
18
|
import type {
|
|
19
|
+
AbandonRunSourceMaterializationInput,
|
|
19
20
|
AbandonSourceMaterializationCandidateInput,
|
|
20
21
|
AbandonSourceMaterializationInput,
|
|
22
|
+
AdoptedSourceMaterialization,
|
|
23
|
+
AdoptSourceMaterializationInput,
|
|
21
24
|
AssertSourceMaterializationExecutionInput,
|
|
22
25
|
BeginSourceMaterializationInput,
|
|
23
26
|
CleanupTerminalSourceMaterializationsInput,
|
|
@@ -26,7 +29,7 @@ import type {
|
|
|
26
29
|
MarkSourceMaterializationReadyInput,
|
|
27
30
|
OntologySourceRecord,
|
|
28
31
|
OntologySourceStorage,
|
|
29
|
-
|
|
32
|
+
PurgeAbandonedSourceMaterializationsInput,
|
|
30
33
|
StageSourceRowsInput,
|
|
31
34
|
StageSourceRowsResult,
|
|
32
35
|
SummarizeTerminalSourceMaterializationsInput,
|
|
@@ -49,8 +52,8 @@ import {
|
|
|
49
52
|
sourceRecord,
|
|
50
53
|
toIsoString,
|
|
51
54
|
} from "./shared"
|
|
52
|
-
import { cleanupSourceVersions } from "./source-cleanup"
|
|
53
|
-
import { assertSourceRootCoverage, stageSourceRoots } from "./source-roots"
|
|
55
|
+
import { cleanupSourceVersions, purgeAbandonedSourceVersions } from "./source-cleanup"
|
|
56
|
+
import { assertSourceRootCoverage, sourceAssertionColumns, stageSourceRoots } from "./source-roots"
|
|
54
57
|
|
|
55
58
|
export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
56
59
|
constructor(private readonly runRootOperation: PgRootOperation) {}
|
|
@@ -88,7 +91,7 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
88
91
|
const candidate = await this.getRunCandidate(sql, input)
|
|
89
92
|
if (candidate) {
|
|
90
93
|
throw sourceConflict(
|
|
91
|
-
`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization;
|
|
94
|
+
`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`
|
|
92
95
|
)
|
|
93
96
|
}
|
|
94
97
|
|
|
@@ -126,7 +129,7 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
126
129
|
}
|
|
127
130
|
if (await this.getRunCandidate(sql, input)) {
|
|
128
131
|
throw sourceConflict(
|
|
129
|
-
`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization;
|
|
132
|
+
`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`
|
|
130
133
|
)
|
|
131
134
|
}
|
|
132
135
|
throw sourceConflict(
|
|
@@ -154,10 +157,10 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
154
157
|
}
|
|
155
158
|
|
|
156
159
|
const rows = sourceStageRows(manifest.projection_kind, input.rows)
|
|
157
|
-
const existing = await this.findStageRows(sql,
|
|
160
|
+
const existing = await this.findStageRows(sql, manifest, rows)
|
|
158
161
|
const { pending, unchanged } = reconcileSourceStageRows(rows, existing)
|
|
159
162
|
await stageSourceRoots(sql, manifest, input)
|
|
160
|
-
await this.insertStageRows(sql,
|
|
163
|
+
await this.insertStageRows(sql, manifest, pending)
|
|
161
164
|
return { inserted: pending.length, unchanged }
|
|
162
165
|
})
|
|
163
166
|
}
|
|
@@ -236,17 +239,50 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
236
239
|
})
|
|
237
240
|
}
|
|
238
241
|
|
|
242
|
+
async adopt(
|
|
243
|
+
input: AdoptSourceMaterializationInput
|
|
244
|
+
): Promise<AdoptedSourceMaterialization | null> {
|
|
245
|
+
return this.runRootOperation(async (sql) => {
|
|
246
|
+
assertProjectAndSource(input)
|
|
247
|
+
assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken)
|
|
248
|
+
assertTimestamp(input.adoptedAt, "Source adoptedAt", true)
|
|
249
|
+
await this.assertExecution(sql, input)
|
|
250
|
+
const candidate = await this.lockRunCandidate(sql, input)
|
|
251
|
+
if (!candidate) return null
|
|
252
|
+
// Workers' clocks differ; adoption never moves the record back in time.
|
|
253
|
+
const [adopted] = await sql<PgOntologySourceRow[]>`
|
|
254
|
+
UPDATE ontology_sources
|
|
255
|
+
SET execution_token = ${input.execution.executionToken},
|
|
256
|
+
updated_at = GREATEST(updated_at, ${input.adoptedAt}::timestamptz)
|
|
257
|
+
WHERE project_id = ${candidate.project_id} AND source_id = ${candidate.source_id}
|
|
258
|
+
AND materialization_id = ${candidate.materialization_id}
|
|
259
|
+
AND status IN ('staging', 'ready')
|
|
260
|
+
RETURNING *
|
|
261
|
+
`
|
|
262
|
+
if (!adopted) {
|
|
263
|
+
throw sourceConflict(`Source materialization '${candidate.materialization_id}' changed.`)
|
|
264
|
+
}
|
|
265
|
+
return {
|
|
266
|
+
record: sourceRecord(adopted),
|
|
267
|
+
resumeStagingOrdinal:
|
|
268
|
+
adopted.status === "ready"
|
|
269
|
+
? Number(adopted.root_count)
|
|
270
|
+
: await this.lastStagedOrdinal(sql, adopted),
|
|
271
|
+
}
|
|
272
|
+
})
|
|
273
|
+
}
|
|
274
|
+
|
|
239
275
|
async abandon(input: AbandonSourceMaterializationCandidateInput): Promise<OntologySourceRecord>
|
|
240
|
-
async abandon(input:
|
|
276
|
+
async abandon(input: AbandonRunSourceMaterializationInput): Promise<OntologySourceRecord | null>
|
|
241
277
|
async abandon(input: AbandonSourceMaterializationInput): Promise<OntologySourceRecord | null> {
|
|
242
278
|
return this.runRootOperation(async (sql) => {
|
|
243
279
|
assertProjectAndSource(input)
|
|
244
280
|
assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken)
|
|
245
281
|
assertTimestamp(input.abandonedAt, "Source abandonedAt", true)
|
|
246
282
|
await this.assertExecution(sql, input)
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
283
|
+
if (input.kind === "candidate") return this.abandonCandidate(sql, input)
|
|
284
|
+
const candidate = await this.lockRunCandidate(sql, input)
|
|
285
|
+
return candidate ? this.transitionToAbandoned(sql, candidate, input.abandonedAt) : null
|
|
250
286
|
})
|
|
251
287
|
}
|
|
252
288
|
|
|
@@ -261,6 +297,16 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
261
297
|
})
|
|
262
298
|
}
|
|
263
299
|
|
|
300
|
+
async purgeAbandoned(
|
|
301
|
+
input: PurgeAbandonedSourceMaterializationsInput
|
|
302
|
+
): Promise<CleanupTerminalSourceMaterializationsResult> {
|
|
303
|
+
return this.runRootOperation(async (sql) => {
|
|
304
|
+
assertNonblank(input.projectId, "Abandoned source purge project id")
|
|
305
|
+
assertPositiveInteger(input.limit, "Abandoned source purge limit")
|
|
306
|
+
return purgeAbandonedSourceVersions(sql, input)
|
|
307
|
+
})
|
|
308
|
+
}
|
|
309
|
+
|
|
264
310
|
async summarizeTerminal(
|
|
265
311
|
input: SummarizeTerminalSourceMaterializationsInput
|
|
266
312
|
): Promise<TerminalSourceMaterializationSummary> {
|
|
@@ -311,10 +357,15 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
311
357
|
return this.transitionToAbandoned(sql, manifest, input.abandonedAt)
|
|
312
358
|
}
|
|
313
359
|
|
|
314
|
-
|
|
360
|
+
/** The run's one staging or ready candidate, locked, whichever execution staged it. */
|
|
361
|
+
private async lockRunCandidate(
|
|
315
362
|
sql: SQLClient,
|
|
316
|
-
input:
|
|
317
|
-
|
|
363
|
+
input: {
|
|
364
|
+
readonly projectId: string
|
|
365
|
+
readonly source: { readonly projectionId: string }
|
|
366
|
+
readonly execution: { readonly projectionRunId: string }
|
|
367
|
+
}
|
|
368
|
+
): Promise<PgOntologySourceRow | null> {
|
|
318
369
|
const candidates = await sql<PgOntologySourceRow[]>`
|
|
319
370
|
SELECT * FROM ontology_sources
|
|
320
371
|
WHERE project_id = ${input.projectId}
|
|
@@ -323,19 +374,20 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
323
374
|
AND status IN ('staging', 'ready')
|
|
324
375
|
FOR UPDATE
|
|
325
376
|
`
|
|
326
|
-
if (
|
|
327
|
-
candidates.some((candidate) => candidate.execution_token === input.execution.executionToken)
|
|
328
|
-
) {
|
|
329
|
-
throw sourceConflict(
|
|
330
|
-
`Projection run '${input.execution.projectionRunId}' already has a source materialization owned by the current execution.`
|
|
331
|
-
)
|
|
332
|
-
}
|
|
333
377
|
if (candidates.length > 1) {
|
|
334
378
|
throw sourceConflict(
|
|
335
379
|
`Projection run '${input.execution.projectionRunId}' has multiple nonterminal source materializations.`
|
|
336
380
|
)
|
|
337
381
|
}
|
|
338
|
-
return candidates[0]
|
|
382
|
+
return candidates[0] ?? null
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
private async lastStagedOrdinal(sql: SQLClient, manifest: PgOntologySourceRow): Promise<number> {
|
|
386
|
+
const [last] = await sql<{ readonly ordinal: number | string | null }[]>`
|
|
387
|
+
SELECT MAX(staging_ordinal) AS ordinal FROM ontology_source_roots
|
|
388
|
+
WHERE version_id = ${manifest.version_id}
|
|
389
|
+
`
|
|
390
|
+
return last?.ordinal == null ? 0 : Number(last.ordinal)
|
|
339
391
|
}
|
|
340
392
|
|
|
341
393
|
private async transitionToAbandoned(
|
|
@@ -366,9 +418,7 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
366
418
|
`Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`
|
|
367
419
|
)
|
|
368
420
|
}
|
|
369
|
-
|
|
370
|
-
WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
|
|
371
|
-
AND materialization_id = ${manifest.materialization_id} AND NOT active`
|
|
421
|
+
// Its roots were never live and stay unpublished; purging removes them with the version.
|
|
372
422
|
return sourceRecord(rows[0])
|
|
373
423
|
}
|
|
374
424
|
|
|
@@ -378,6 +428,7 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
378
428
|
rootCount: number,
|
|
379
429
|
assertionCount: number
|
|
380
430
|
): Promise<void> {
|
|
431
|
+
const version = manifest.version_id
|
|
381
432
|
const [counts] = await sql<
|
|
382
433
|
{
|
|
383
434
|
readonly assertions: number | string
|
|
@@ -387,15 +438,13 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
387
438
|
readonly max_ordinal: number | string | null
|
|
388
439
|
}[]
|
|
389
440
|
>`
|
|
390
|
-
SELECT (SELECT COUNT(*) FROM
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
COUNT(DISTINCT staging_ordinal) AS ordinals,
|
|
441
|
+
SELECT (SELECT COUNT(*) FROM ontology_source_roots AS roots
|
|
442
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
443
|
+
WHERE roots.version_id = ${version}) AS assertions,
|
|
444
|
+
COUNT(*) AS roots, COUNT(DISTINCT staging_ordinal) AS ordinals,
|
|
394
445
|
MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
|
|
395
446
|
FROM ontology_source_roots
|
|
396
|
-
WHERE
|
|
397
|
-
AND source_id = ${manifest.source_id}
|
|
398
|
-
AND materialization_id = ${manifest.materialization_id}
|
|
447
|
+
WHERE version_id = ${version}
|
|
399
448
|
`
|
|
400
449
|
const assertions = Number(counts?.assertions ?? 0)
|
|
401
450
|
const roots = Number(counts?.roots ?? 0)
|
|
@@ -414,39 +463,45 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
414
463
|
}
|
|
415
464
|
await assertSourceRootCoverage(sql, manifest)
|
|
416
465
|
|
|
466
|
+
// Roots store only their canonical key; compare its parts with each row's typed identity.
|
|
417
467
|
const [invalid] =
|
|
418
468
|
manifest.projection_kind === "link"
|
|
419
|
-
? await sql<{ readonly root_key:
|
|
420
|
-
SELECT root_key
|
|
421
|
-
FROM
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
GROUP BY root_key
|
|
469
|
+
? await sql<{ readonly root_key: string }[]>`
|
|
470
|
+
SELECT roots.root_key
|
|
471
|
+
FROM ontology_source_roots AS roots
|
|
472
|
+
CROSS JOIN LATERAL (SELECT roots.root_key::jsonb AS parts) AS root
|
|
473
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
474
|
+
WHERE roots.version_id = ${version}
|
|
475
|
+
GROUP BY roots.id, roots.root_key
|
|
426
476
|
HAVING COUNT(*) <> 1
|
|
427
|
-
OR BOOL_OR(
|
|
428
|
-
OR BOOL_OR(entity_kind <> 'link')
|
|
429
|
-
OR BOOL_OR(
|
|
477
|
+
OR BOOL_OR(root.parts->>0 <> 'link')
|
|
478
|
+
OR BOOL_OR(rows.entity_kind <> 'link')
|
|
479
|
+
OR BOOL_OR(rows.source_type_id <> root.parts->>1
|
|
480
|
+
OR rows.source_primary_id <> root.parts->>2 OR rows.link_id <> root.parts->>3
|
|
481
|
+
OR rows.target_type_id <> root.parts->>4 OR rows.target_primary_id <> root.parts->>5)
|
|
430
482
|
LIMIT 1
|
|
431
483
|
`
|
|
432
|
-
: await sql<{ readonly root_key:
|
|
433
|
-
SELECT root_key
|
|
434
|
-
FROM
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
GROUP BY root_key
|
|
439
|
-
HAVING BOOL_OR(
|
|
484
|
+
: await sql<{ readonly root_key: string }[]>`
|
|
485
|
+
SELECT roots.root_key
|
|
486
|
+
FROM ontology_source_roots AS roots
|
|
487
|
+
CROSS JOIN LATERAL (SELECT roots.root_key::jsonb AS parts) AS root
|
|
488
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
489
|
+
WHERE roots.version_id = ${version}
|
|
490
|
+
GROUP BY roots.id, roots.root_key
|
|
491
|
+
HAVING BOOL_OR(root.parts->>0 <> 'object')
|
|
440
492
|
OR COUNT(*) FILTER (
|
|
441
|
-
WHERE entity_kind = 'object' AND
|
|
493
|
+
WHERE rows.entity_kind = 'object' AND rows.object_type_id = root.parts->>1
|
|
494
|
+
AND rows.primary_id = root.parts->>2
|
|
442
495
|
) <> 1
|
|
443
496
|
OR COUNT(*) FILTER (
|
|
444
|
-
WHERE (entity_kind = 'object' AND
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
))
|
|
497
|
+
WHERE (rows.entity_kind = 'object' AND (rows.object_type_id <> root.parts->>1
|
|
498
|
+
OR rows.primary_id <> root.parts->>2))
|
|
499
|
+
OR (rows.entity_kind = 'link' AND (rows.source_type_id <> root.parts->>1
|
|
500
|
+
OR rows.source_primary_id <> root.parts->>2))
|
|
449
501
|
) <> 0
|
|
502
|
+
OR COUNT(*) FILTER (WHERE rows.entity_kind = 'link') <> COUNT(DISTINCT (
|
|
503
|
+
rows.link_id, rows.target_type_id, rows.target_primary_id
|
|
504
|
+
)) FILTER (WHERE rows.entity_kind = 'link')
|
|
450
505
|
LIMIT 1
|
|
451
506
|
`
|
|
452
507
|
if (invalid) {
|
|
@@ -458,24 +513,24 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
458
513
|
}
|
|
459
514
|
}
|
|
460
515
|
|
|
516
|
+
/** Already staged assertions sharing a root or an ordinal with `rows`. */
|
|
461
517
|
private async findStageRows(
|
|
462
518
|
sql: SQLClient,
|
|
463
|
-
|
|
519
|
+
manifest: PgOntologySourceRow,
|
|
464
520
|
rows: readonly SourceStageRow[]
|
|
465
521
|
): Promise<readonly SourceStageRow[]> {
|
|
466
522
|
if (rows.length === 0) return []
|
|
467
|
-
const
|
|
523
|
+
const rootKeys = [...new Set(rows.map((row) => row.rootKey))]
|
|
468
524
|
const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))]
|
|
469
|
-
const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey(row.entityKey)))]
|
|
470
525
|
const existing = await sql<PgOntologySourceAssertionRow[]>`
|
|
471
|
-
SELECT
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
526
|
+
SELECT ${sourceAssertionColumns(sql)}
|
|
527
|
+
FROM ontology_source_roots AS roots
|
|
528
|
+
JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
529
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
530
|
+
WHERE roots.version_id = ${manifest.version_id}
|
|
475
531
|
AND (
|
|
476
|
-
|
|
477
|
-
OR staging_ordinal = ANY(${sql.array(ordinals)}::bigint[])
|
|
478
|
-
OR entity_sort_key = ANY(${sql.array(entitySortKeys)}::text[])
|
|
532
|
+
roots.root_key = ANY(${sql.array(rootKeys)}::text[])
|
|
533
|
+
OR roots.staging_ordinal = ANY(${sql.array(ordinals)}::bigint[])
|
|
479
534
|
)
|
|
480
535
|
`
|
|
481
536
|
return existing.map((row) => sourceStageRow(sourceAssertion(row)))
|
|
@@ -483,69 +538,44 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
483
538
|
|
|
484
539
|
private async insertStageRows(
|
|
485
540
|
sql: SQLClient,
|
|
486
|
-
|
|
541
|
+
manifest: PgOntologySourceRow,
|
|
487
542
|
rows: readonly SourceStageRow[]
|
|
488
543
|
): Promise<void> {
|
|
489
544
|
if (rows.length === 0) return
|
|
490
|
-
const payload = rows.map(({
|
|
545
|
+
const payload = rows.map(({ rootKey, row }) => {
|
|
491
546
|
const entity = entityColumns(row.assertion)
|
|
492
|
-
const root = entityColumns(row.root)
|
|
493
547
|
return {
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
primaryId: entity.primaryId,
|
|
505
|
-
sourceTypeId: entity.sourceTypeId,
|
|
506
|
-
sourcePrimaryId: entity.sourcePrimaryId,
|
|
507
|
-
linkId: entity.linkId,
|
|
508
|
-
targetTypeId: entity.targetTypeId,
|
|
509
|
-
targetPrimaryId: entity.targetPrimaryId,
|
|
510
|
-
rootObjectTypeId: root.objectTypeId,
|
|
511
|
-
rootPrimaryId: root.primaryId,
|
|
512
|
-
rootSourceTypeId: root.sourceTypeId,
|
|
513
|
-
rootSourcePrimaryId: root.sourcePrimaryId,
|
|
514
|
-
rootLinkId: root.linkId,
|
|
515
|
-
rootTargetTypeId: root.targetTypeId,
|
|
516
|
-
rootTargetPrimaryId: root.targetPrimaryId,
|
|
548
|
+
root_key: rootKey,
|
|
549
|
+
entity_kind: row.assertion.kind,
|
|
550
|
+
object_type_id: entity.objectTypeId,
|
|
551
|
+
primary_id: entity.primaryId,
|
|
552
|
+
source_type_id: entity.sourceTypeId,
|
|
553
|
+
source_primary_id: entity.sourcePrimaryId,
|
|
554
|
+
link_id: entity.linkId,
|
|
555
|
+
target_type_id: entity.targetTypeId,
|
|
556
|
+
target_primary_id: entity.targetPrimaryId,
|
|
557
|
+
payload: sourceAssertionPayload(row.assertion),
|
|
517
558
|
}
|
|
518
559
|
})
|
|
519
|
-
await sql`
|
|
520
|
-
WITH staged AS (
|
|
521
|
-
SELECT value
|
|
522
|
-
FROM jsonb_array_elements(${jsonParameter(sql, payload)}::jsonb)
|
|
523
|
-
)
|
|
560
|
+
const inserted = await sql`
|
|
524
561
|
INSERT INTO ontology_source_rows (
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
root_kind, root_key, root_sort_key, staging_ordinal,
|
|
528
|
-
root, assertion,
|
|
529
|
-
object_type_id, primary_id,
|
|
530
|
-
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id,
|
|
531
|
-
root_object_type_id, root_primary_id,
|
|
532
|
-
root_source_type_id, root_source_primary_id, root_link_id,
|
|
533
|
-
root_target_type_id, root_target_primary_id
|
|
562
|
+
root_id, entity_kind, object_type_id, primary_id,
|
|
563
|
+
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id, payload
|
|
534
564
|
)
|
|
535
|
-
SELECT
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
value->>'rootSourceTypeId', value->>'rootSourcePrimaryId', value->>'rootLinkId',
|
|
546
|
-
value->>'rootTargetTypeId', value->>'rootTargetPrimaryId'
|
|
547
|
-
FROM staged
|
|
565
|
+
SELECT roots.id, staged.entity_kind, staged.object_type_id, staged.primary_id,
|
|
566
|
+
staged.source_type_id, staged.source_primary_id, staged.link_id,
|
|
567
|
+
staged.target_type_id, staged.target_primary_id, staged.payload
|
|
568
|
+
FROM jsonb_to_recordset(${jsonParameter(sql, payload)}) AS staged(
|
|
569
|
+
root_key TEXT, entity_kind TEXT, object_type_id TEXT, primary_id TEXT,
|
|
570
|
+
source_type_id TEXT, source_primary_id TEXT, link_id TEXT,
|
|
571
|
+
target_type_id TEXT, target_primary_id TEXT, payload JSONB
|
|
572
|
+
)
|
|
573
|
+
JOIN ontology_source_roots AS roots
|
|
574
|
+
ON roots.version_id = ${manifest.version_id} AND roots.root_key = staged.root_key
|
|
548
575
|
`
|
|
576
|
+
if (inserted.count !== rows.length) {
|
|
577
|
+
throw sourceConflict("Staged source rows do not all belong to a staged root.")
|
|
578
|
+
}
|
|
549
579
|
}
|
|
550
580
|
|
|
551
581
|
private assertExecution(
|
package/src/pg-client.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
import { isIP } from "node:net"
|
|
2
|
+
import { connect as connectTls, type TLSSocket } from "node:tls"
|
|
1
3
|
import postgres from "postgres"
|
|
4
|
+
import { pgErrorCode } from "./storage-errors"
|
|
2
5
|
|
|
3
6
|
// Data-type policy (porsager defaults, deliberately not overridden):
|
|
4
7
|
// - `timestamptz` -> JS `Date`. Row mappers normalize via `toIsoString(Date | string)`.
|
|
@@ -30,6 +33,9 @@ export type SQL = postgres.Sql<Record<string, never>>
|
|
|
30
33
|
*/
|
|
31
34
|
export type SQLClient = postgres.ISql<Record<string, never>>
|
|
32
35
|
|
|
36
|
+
/** One pool connection held outside the pool until `release()` — see `withReservedPgConnection`. */
|
|
37
|
+
export type ReservedSQL = postgres.ReservedSql<Record<string, never>>
|
|
38
|
+
|
|
33
39
|
/**
|
|
34
40
|
* A value accepted as a positional parameter by {@link SQLClient.unsafe}. Dynamically built
|
|
35
41
|
* parameter arrays (from the query-IR compiler / run-list helpers) are cast to this since
|
|
@@ -106,11 +112,11 @@ export function createPgClient(options: CreatePgClientOptions): SQL {
|
|
|
106
112
|
// Match the previous (bun:sql) behavior of not surfacing routine PostgreSQL NOTICEs.
|
|
107
113
|
onnotice: () => {},
|
|
108
114
|
...(options.prepare !== undefined ? { prepare: options.prepare } : {}),
|
|
109
|
-
...(options
|
|
115
|
+
...tlsOptions(options),
|
|
110
116
|
}
|
|
111
117
|
|
|
112
118
|
if (options.connectionString) {
|
|
113
|
-
return postgres(options.connectionString, base)
|
|
119
|
+
return assertDirectTlsTarget(postgres(options.connectionString, base), base)
|
|
114
120
|
}
|
|
115
121
|
|
|
116
122
|
return postgres({
|
|
@@ -122,3 +128,127 @@ export function createPgClient(options: CreatePgClientOptions): SQL {
|
|
|
122
128
|
...(options.password !== undefined ? { password: options.password } : {}),
|
|
123
129
|
})
|
|
124
130
|
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* What the server's certificate is checked against when TLS is opened directly: nothing
|
|
134
|
+
* (`require`), the trusted CAs (`verify-ca`), or the CAs and the host name (`verify-full`).
|
|
135
|
+
*/
|
|
136
|
+
type DirectTlsVerification = "none" | "ca" | "full"
|
|
137
|
+
|
|
138
|
+
// `sslnegotiation=direct` is read from the connection string, as libpq reads it. Without it,
|
|
139
|
+
// porsager negotiates TLS the classic way: an SSLRequest on a plain TCP socket, then an upgrade of
|
|
140
|
+
// that socket with `tls.connect({ socket })`. Bun 1.4.0–1.4.2 keeps every byte received on such an
|
|
141
|
+
// upgraded socket in memory for good, so a process that reads a lot grows until it is killed.
|
|
142
|
+
// Opening the TLS connection ourselves with `tls.connect({ host, port })` avoids that path, and
|
|
143
|
+
// saves the SSLRequest round trip on every connection.
|
|
144
|
+
function tlsOptions(options: CreatePgClientOptions): Record<string, unknown> {
|
|
145
|
+
const verification = directTlsVerification(options)
|
|
146
|
+
if (verification === null) {
|
|
147
|
+
return options.ssl ? { ssl: options.ssl } : {}
|
|
148
|
+
}
|
|
149
|
+
// porsager must not negotiate TLS again on the socket we hand it; explicit options also
|
|
150
|
+
// override the `sslmode` it would otherwise read from the connection string.
|
|
151
|
+
return { ssl: false, sslnegotiation: null, socket: directTlsSocket(verification) }
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function directTlsVerification(options: CreatePgClientOptions): DirectTlsVerification | null {
|
|
155
|
+
const params = connectionStringParams(options.connectionString)
|
|
156
|
+
if (params.get("sslnegotiation") !== "direct") {
|
|
157
|
+
return null
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const sslmode =
|
|
161
|
+
options.ssl === true
|
|
162
|
+
? "verify-full"
|
|
163
|
+
: options.ssl ||
|
|
164
|
+
(params.get("sslrootcert") === "system"
|
|
165
|
+
? "verify-full"
|
|
166
|
+
: (params.get("sslmode") ?? params.get("ssl")))
|
|
167
|
+
switch (sslmode) {
|
|
168
|
+
case "require":
|
|
169
|
+
return "none"
|
|
170
|
+
case "verify-ca":
|
|
171
|
+
return "ca"
|
|
172
|
+
case "verify-full":
|
|
173
|
+
case "true":
|
|
174
|
+
return "full"
|
|
175
|
+
default:
|
|
176
|
+
throw new Error(
|
|
177
|
+
`[SixbPg] sslnegotiation=direct requires sslmode=require, verify-ca or verify-full, got ${
|
|
178
|
+
sslmode ? `"${sslmode}"` : "no sslmode"
|
|
179
|
+
}: a direct TLS connection has no plaintext fallback.`
|
|
180
|
+
)
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function connectionStringParams(connectionString: string | undefined): URLSearchParams {
|
|
185
|
+
return new URLSearchParams(connectionString?.split("?")[1] ?? "")
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* porsager's `socket` factory, called for every connection. It must return at once and never
|
|
190
|
+
* throw: a factory that fails leaves its pool slot stuck in porsager's connecting state for good,
|
|
191
|
+
* and one that returns an already-dead socket stalls the slot's next attempt (postgres.js 3.4.9).
|
|
192
|
+
* Failures are reported by the socket instead, through the `error` and `close` events porsager
|
|
193
|
+
* handles for its own sockets, and its connect timeout covers the handshake.
|
|
194
|
+
*/
|
|
195
|
+
function directTlsSocket(
|
|
196
|
+
verification: DirectTlsVerification
|
|
197
|
+
): (options: postgres.ParsedOptions) => TLSSocket {
|
|
198
|
+
let reportedRefusal = false
|
|
199
|
+
|
|
200
|
+
return ({ host: [host = "localhost"], port: [port = 5432] }) => {
|
|
201
|
+
const socket = connectTls({
|
|
202
|
+
host,
|
|
203
|
+
port,
|
|
204
|
+
servername: isIP(host) ? undefined : host,
|
|
205
|
+
// PostgreSQL 17+ accepts a direct TLS connection only for this ALPN protocol.
|
|
206
|
+
ALPNProtocols: ["postgresql"],
|
|
207
|
+
rejectUnauthorized: verification !== "none",
|
|
208
|
+
...(verification === "ca" ? { checkServerIdentity: () => undefined } : {}),
|
|
209
|
+
})
|
|
210
|
+
|
|
211
|
+
let reachedServer = false
|
|
212
|
+
let secured = false
|
|
213
|
+
socket.once("connect", () => {
|
|
214
|
+
reachedServer = true
|
|
215
|
+
})
|
|
216
|
+
socket.once("secureConnect", () => {
|
|
217
|
+
secured = true
|
|
218
|
+
if (socket.alpnProtocol !== "postgresql") {
|
|
219
|
+
socket.destroy(
|
|
220
|
+
new Error(
|
|
221
|
+
`[SixbPg] ${host}:${port} accepted TLS without selecting the "postgresql" protocol` +
|
|
222
|
+
" (ALPN), so it is not a PostgreSQL server accepting sslnegotiation=direct."
|
|
223
|
+
)
|
|
224
|
+
)
|
|
225
|
+
}
|
|
226
|
+
})
|
|
227
|
+
// A server that expects an SSLRequest reads the TLS hello as a malformed startup packet and
|
|
228
|
+
// closes the connection, which porsager reports as a bare reset. Say what it means, once.
|
|
229
|
+
socket.once("error", (error: Error) => {
|
|
230
|
+
if (reportedRefusal || !reachedServer || secured) return
|
|
231
|
+
if (!["ECONNRESET", "EPIPE", "ERR_SOCKET_CLOSED"].includes(pgErrorCode(error) ?? "")) return
|
|
232
|
+
reportedRefusal = true
|
|
233
|
+
console.error(
|
|
234
|
+
`[SixbPg] ${host}:${port} closed the connection during a direct TLS handshake` +
|
|
235
|
+
" (sslnegotiation=direct). Direct TLS needs PostgreSQL 17 or later, reached without a" +
|
|
236
|
+
" proxy or pooler that expects an SSLRequest; remove sslnegotiation=direct otherwise."
|
|
237
|
+
)
|
|
238
|
+
})
|
|
239
|
+
return socket
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function assertDirectTlsTarget(sql: SQL, options: Record<string, unknown>): SQL {
|
|
244
|
+
if (!("socket" in options)) {
|
|
245
|
+
return sql
|
|
246
|
+
}
|
|
247
|
+
if (sql.options.host.length !== 1 || sql.options.path) {
|
|
248
|
+
throw new Error(
|
|
249
|
+
"[SixbPg] sslnegotiation=direct supports one TCP host:port in the connection string, not" +
|
|
250
|
+
" several hosts or a Unix socket path."
|
|
251
|
+
)
|
|
252
|
+
}
|
|
253
|
+
return sql
|
|
254
|
+
}
|