@sixb/sqlite 0.1.14 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +673 -241
- package/dist/index.js.map +10 -10
- package/dist/latest-run-query.d.ts +2 -0
- package/dist/latest-run-query.d.ts.map +1 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +11 -8
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/source-cleanup.d.ts +7 -1
- package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
- package/dist/ontology-storage/source-roots.d.ts +14 -2
- package/dist/ontology-storage/source-roots.d.ts.map +1 -1
- package/dist/ontology-storage/sources.d.ts +8 -3
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/projection-run-storage.d.ts +3 -1
- package/dist/projection-run-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/latest-run-query.ts +8 -2
- package/src/migrations/051-projection-run-supersession.sql +126 -0
- package/src/migrations/052-compact-source-storage.sql +181 -0
- package/src/migrations.ts +38 -0
- package/src/ontology-storage/materialization-state.ts +34 -44
- package/src/ontology-storage/shared.ts +20 -11
- package/src/ontology-storage/source-cleanup.ts +61 -24
- package/src/ontology-storage/source-roots.ts +166 -110
- package/src/ontology-storage/sources.ts +169 -118
- package/src/projection-run-storage.ts +59 -1
|
@@ -10,15 +10,18 @@ import {
|
|
|
10
10
|
isExactStagingManifest,
|
|
11
11
|
reconcileSourceStageRows,
|
|
12
12
|
type SourceStageRow,
|
|
13
|
+
sourceAssertionPayload,
|
|
13
14
|
sourceConflict,
|
|
14
15
|
sourceMaterializationIdentity,
|
|
15
16
|
sourceStageRow,
|
|
16
17
|
sourceStageRows,
|
|
17
|
-
utf8SortKey,
|
|
18
18
|
} from "@sixb/core/internal/ontology-storage-provider"
|
|
19
19
|
import type {
|
|
20
|
+
AbandonRunSourceMaterializationInput,
|
|
20
21
|
AbandonSourceMaterializationCandidateInput,
|
|
21
22
|
AbandonSourceMaterializationInput,
|
|
23
|
+
AdoptedSourceMaterialization,
|
|
24
|
+
AdoptSourceMaterializationInput,
|
|
22
25
|
AssertSourceMaterializationExecutionInput,
|
|
23
26
|
BeginSourceMaterializationInput,
|
|
24
27
|
CleanupTerminalSourceMaterializationsInput,
|
|
@@ -27,7 +30,7 @@ import type {
|
|
|
27
30
|
MarkSourceMaterializationReadyInput,
|
|
28
31
|
OntologySourceRecord,
|
|
29
32
|
OntologySourceStorage,
|
|
30
|
-
|
|
33
|
+
PurgeAbandonedSourceMaterializationsInput,
|
|
31
34
|
StageSourceRowsInput,
|
|
32
35
|
StageSourceRowsResult,
|
|
33
36
|
SummarizeTerminalSourceMaterializationsInput,
|
|
@@ -47,8 +50,12 @@ import {
|
|
|
47
50
|
sourceAssertion,
|
|
48
51
|
sourceRecord,
|
|
49
52
|
} from "./shared"
|
|
50
|
-
import { cleanupSourceVersions } from "./source-cleanup"
|
|
51
|
-
import {
|
|
53
|
+
import { cleanupSourceVersions, purgeAbandonedSourceVersions } from "./source-cleanup"
|
|
54
|
+
import {
|
|
55
|
+
assertSourceRootCoverage,
|
|
56
|
+
SOURCE_ASSERTION_COLUMNS,
|
|
57
|
+
stageSourceRoots,
|
|
58
|
+
} from "./source-roots"
|
|
52
59
|
|
|
53
60
|
export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
54
61
|
constructor(
|
|
@@ -87,7 +94,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
87
94
|
.get(input.projectId, input.execution.projectionRunId)
|
|
88
95
|
if (candidate) {
|
|
89
96
|
throw sourceConflict(
|
|
90
|
-
`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization;
|
|
97
|
+
`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`
|
|
91
98
|
)
|
|
92
99
|
}
|
|
93
100
|
|
|
@@ -146,10 +153,10 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
146
153
|
}
|
|
147
154
|
|
|
148
155
|
const rows = sourceStageRows(manifest.projection_kind, input.rows)
|
|
149
|
-
const existing = this.findStageRows(
|
|
156
|
+
const existing = this.findStageRows(manifest, rows)
|
|
150
157
|
const { pending, unchanged } = reconcileSourceStageRows(rows, existing)
|
|
151
158
|
stageSourceRoots(this.db, manifest, input)
|
|
152
|
-
this.insertStageRows(
|
|
159
|
+
this.insertStageRows(manifest, pending)
|
|
153
160
|
return { inserted: pending.length, unchanged }
|
|
154
161
|
})
|
|
155
162
|
await yieldSqliteEventLoop()
|
|
@@ -233,17 +240,59 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
233
240
|
})
|
|
234
241
|
}
|
|
235
242
|
|
|
243
|
+
async adopt(
|
|
244
|
+
input: AdoptSourceMaterializationInput
|
|
245
|
+
): Promise<AdoptedSourceMaterialization | null> {
|
|
246
|
+
return this.runRootOperation(() => {
|
|
247
|
+
assertProjectAndSource(input)
|
|
248
|
+
assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken)
|
|
249
|
+
assertTimestamp(input.adoptedAt, "Source adoptedAt", true)
|
|
250
|
+
this.assertExecution(input)
|
|
251
|
+
const candidate = this.findRunCandidate(input)
|
|
252
|
+
if (!candidate) return null
|
|
253
|
+
// Workers' clocks differ; adoption never moves the record back in time.
|
|
254
|
+
const changed = this.db
|
|
255
|
+
.query(
|
|
256
|
+
`
|
|
257
|
+
UPDATE ontology_sources SET execution_token = ?, updated_at = MAX(updated_at, ?)
|
|
258
|
+
WHERE project_id = ? AND source_id = ? AND materialization_id = ?
|
|
259
|
+
AND status IN ('staging', 'ready')
|
|
260
|
+
`
|
|
261
|
+
)
|
|
262
|
+
.run(
|
|
263
|
+
input.execution.executionToken,
|
|
264
|
+
input.adoptedAt,
|
|
265
|
+
candidate.project_id,
|
|
266
|
+
candidate.source_id,
|
|
267
|
+
candidate.materialization_id
|
|
268
|
+
).changes
|
|
269
|
+
if (changed !== 1) {
|
|
270
|
+
throw sourceConflict(`Source materialization '${candidate.materialization_id}' changed.`)
|
|
271
|
+
}
|
|
272
|
+
const adopted = this.requireManifest(
|
|
273
|
+
candidate.project_id,
|
|
274
|
+
candidate.source_id,
|
|
275
|
+
candidate.materialization_id
|
|
276
|
+
)
|
|
277
|
+
return {
|
|
278
|
+
record: sourceRecord(adopted),
|
|
279
|
+
resumeStagingOrdinal:
|
|
280
|
+
adopted.status === "ready" ? Number(adopted.root_count) : this.lastStagedOrdinal(adopted),
|
|
281
|
+
}
|
|
282
|
+
})
|
|
283
|
+
}
|
|
284
|
+
|
|
236
285
|
async abandon(input: AbandonSourceMaterializationCandidateInput): Promise<OntologySourceRecord>
|
|
237
|
-
async abandon(input:
|
|
286
|
+
async abandon(input: AbandonRunSourceMaterializationInput): Promise<OntologySourceRecord | null>
|
|
238
287
|
async abandon(input: AbandonSourceMaterializationInput): Promise<OntologySourceRecord | null> {
|
|
239
288
|
return this.runRootOperation(() => {
|
|
240
289
|
assertProjectAndSource(input)
|
|
241
290
|
assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken)
|
|
242
291
|
assertTimestamp(input.abandonedAt, "Source abandonedAt", true)
|
|
243
292
|
this.assertExecution(input)
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
293
|
+
if (input.kind === "candidate") return this.abandonCandidate(input)
|
|
294
|
+
const candidate = this.findRunCandidate(input)
|
|
295
|
+
return candidate ? this.transitionToAbandoned(candidate, input.abandonedAt) : null
|
|
247
296
|
})
|
|
248
297
|
}
|
|
249
298
|
|
|
@@ -258,6 +307,16 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
258
307
|
})
|
|
259
308
|
}
|
|
260
309
|
|
|
310
|
+
async purgeAbandoned(
|
|
311
|
+
input: PurgeAbandonedSourceMaterializationsInput
|
|
312
|
+
): Promise<CleanupTerminalSourceMaterializationsResult> {
|
|
313
|
+
return this.runRootOperation(() => {
|
|
314
|
+
assertNonblank(input.projectId, "Abandoned source purge project id")
|
|
315
|
+
assertPositiveInteger(input.limit, "Abandoned source purge limit")
|
|
316
|
+
return purgeAbandonedSourceVersions(this.db, input)
|
|
317
|
+
})
|
|
318
|
+
}
|
|
319
|
+
|
|
261
320
|
async summarizeTerminal(
|
|
262
321
|
input: SummarizeTerminalSourceMaterializationsInput
|
|
263
322
|
): Promise<TerminalSourceMaterializationSummary> {
|
|
@@ -305,7 +364,12 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
305
364
|
return this.transitionToAbandoned(manifest, input.abandonedAt)
|
|
306
365
|
}
|
|
307
366
|
|
|
308
|
-
|
|
367
|
+
/** The run's one staging or ready candidate, whichever execution staged it. */
|
|
368
|
+
private findRunCandidate(input: {
|
|
369
|
+
readonly projectId: string
|
|
370
|
+
readonly source: { readonly projectionId: string }
|
|
371
|
+
readonly execution: { readonly projectionRunId: string }
|
|
372
|
+
}): SqliteOntologySourceRow | null {
|
|
309
373
|
const candidates = this.db
|
|
310
374
|
.query(
|
|
311
375
|
`
|
|
@@ -319,19 +383,23 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
319
383
|
input.source.projectionId,
|
|
320
384
|
input.execution.projectionRunId
|
|
321
385
|
) as SqliteOntologySourceRow[]
|
|
322
|
-
if (
|
|
323
|
-
candidates.some((candidate) => candidate.execution_token === input.execution.executionToken)
|
|
324
|
-
) {
|
|
325
|
-
throw sourceConflict(
|
|
326
|
-
`Projection run '${input.execution.projectionRunId}' already has a source materialization owned by the current execution.`
|
|
327
|
-
)
|
|
328
|
-
}
|
|
329
386
|
if (candidates.length > 1) {
|
|
330
387
|
throw sourceConflict(
|
|
331
388
|
`Projection run '${input.execution.projectionRunId}' has multiple nonterminal source materializations.`
|
|
332
389
|
)
|
|
333
390
|
}
|
|
334
|
-
return candidates[0]
|
|
391
|
+
return candidates[0] ?? null
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
private lastStagedOrdinal(manifest: SqliteOntologySourceRow): number {
|
|
395
|
+
const last = this.db
|
|
396
|
+
.query(
|
|
397
|
+
"SELECT MAX(staging_ordinal) AS ordinal FROM ontology_source_roots WHERE version_id = ?"
|
|
398
|
+
)
|
|
399
|
+
.get(manifest.version_id) as {
|
|
400
|
+
readonly ordinal: number | null
|
|
401
|
+
} | null
|
|
402
|
+
return last?.ordinal ?? 0
|
|
335
403
|
}
|
|
336
404
|
|
|
337
405
|
private transitionToAbandoned(
|
|
@@ -367,10 +435,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
367
435
|
`Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`
|
|
368
436
|
)
|
|
369
437
|
}
|
|
370
|
-
|
|
371
|
-
.query(`UPDATE ontology_source_roots SET retired_at = ?
|
|
372
|
-
WHERE project_id = ? AND source_id = ? AND materialization_id = ? AND active = 0`)
|
|
373
|
-
.run(abandonedAt, manifest.project_id, manifest.source_id, manifest.materialization_id)
|
|
438
|
+
// Its roots were never live and stay unpublished; purging removes them with the version.
|
|
374
439
|
return sourceRecord(
|
|
375
440
|
this.requireManifest(manifest.project_id, manifest.source_id, manifest.materialization_id)
|
|
376
441
|
)
|
|
@@ -384,22 +449,16 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
384
449
|
const counts = this.db
|
|
385
450
|
.query(
|
|
386
451
|
`
|
|
387
|
-
SELECT (SELECT COUNT(*) FROM
|
|
388
|
-
|
|
389
|
-
|
|
452
|
+
SELECT (SELECT COUNT(*) FROM ontology_source_roots AS roots
|
|
453
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
454
|
+
WHERE roots.version_id = ?1) AS assertions,
|
|
455
|
+
COUNT(*) AS roots, COUNT(DISTINCT staging_ordinal) AS ordinals,
|
|
390
456
|
MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
|
|
391
457
|
FROM ontology_source_roots
|
|
392
|
-
WHERE
|
|
458
|
+
WHERE version_id = ?1
|
|
393
459
|
`
|
|
394
460
|
)
|
|
395
|
-
.get(
|
|
396
|
-
manifest.project_id,
|
|
397
|
-
manifest.source_id,
|
|
398
|
-
manifest.materialization_id,
|
|
399
|
-
manifest.project_id,
|
|
400
|
-
manifest.source_id,
|
|
401
|
-
manifest.materialization_id
|
|
402
|
-
) as {
|
|
461
|
+
.get(manifest.version_id) as {
|
|
403
462
|
assertions: number
|
|
404
463
|
roots: number
|
|
405
464
|
ordinals: number
|
|
@@ -417,36 +476,52 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
417
476
|
)
|
|
418
477
|
}
|
|
419
478
|
assertSourceRootCoverage(this.db, manifest)
|
|
479
|
+
// Roots store only their canonical key; compare its parts with each row's typed identity.
|
|
420
480
|
const invalid = this.db
|
|
421
481
|
.query(
|
|
422
482
|
manifest.projection_kind === "link"
|
|
423
483
|
? `
|
|
424
|
-
SELECT root_key
|
|
425
|
-
FROM
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
484
|
+
SELECT roots.root_key
|
|
485
|
+
FROM ontology_source_roots AS roots
|
|
486
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
487
|
+
WHERE roots.version_id = ?
|
|
488
|
+
GROUP BY roots.id
|
|
489
|
+
HAVING COUNT(*) <> 1
|
|
490
|
+
OR MAX(json_extract(roots.root_key, '$[0]') <> 'link')
|
|
491
|
+
OR MAX(rows.entity_kind <> 'link')
|
|
492
|
+
OR MAX(rows.source_type_id <> json_extract(roots.root_key, '$[1]')
|
|
493
|
+
OR rows.source_primary_id <> json_extract(roots.root_key, '$[2]')
|
|
494
|
+
OR rows.link_id <> json_extract(roots.root_key, '$[3]')
|
|
495
|
+
OR rows.target_type_id <> json_extract(roots.root_key, '$[4]')
|
|
496
|
+
OR rows.target_primary_id <> json_extract(roots.root_key, '$[5]'))
|
|
430
497
|
LIMIT 1
|
|
431
498
|
`
|
|
432
499
|
: `
|
|
433
|
-
SELECT root_key
|
|
434
|
-
FROM
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
OR SUM(
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
500
|
+
SELECT roots.root_key
|
|
501
|
+
FROM ontology_source_roots AS roots
|
|
502
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
503
|
+
WHERE roots.version_id = ?
|
|
504
|
+
GROUP BY roots.id
|
|
505
|
+
HAVING MAX(json_extract(roots.root_key, '$[0]') <> 'object')
|
|
506
|
+
OR SUM(rows.entity_kind = 'object'
|
|
507
|
+
AND rows.object_type_id = json_extract(roots.root_key, '$[1]')
|
|
508
|
+
AND rows.primary_id = json_extract(roots.root_key, '$[2]')) <> 1
|
|
509
|
+
OR SUM(
|
|
510
|
+
(rows.entity_kind = 'object' AND (
|
|
511
|
+
rows.object_type_id <> json_extract(roots.root_key, '$[1]')
|
|
512
|
+
OR rows.primary_id <> json_extract(roots.root_key, '$[2]')))
|
|
513
|
+
OR (rows.entity_kind = 'link' AND (
|
|
514
|
+
rows.source_type_id <> json_extract(roots.root_key, '$[1]')
|
|
515
|
+
OR rows.source_primary_id <> json_extract(roots.root_key, '$[2]')))
|
|
516
|
+
) <> 0
|
|
517
|
+
OR SUM(rows.entity_kind = 'link') <> COUNT(DISTINCT CASE
|
|
518
|
+
WHEN rows.entity_kind = 'link'
|
|
519
|
+
THEN json_array(rows.link_id, rows.target_type_id, rows.target_primary_id)
|
|
520
|
+
END)
|
|
446
521
|
LIMIT 1
|
|
447
522
|
`
|
|
448
523
|
)
|
|
449
|
-
.get(manifest.
|
|
524
|
+
.get(manifest.version_id)
|
|
450
525
|
if (invalid) {
|
|
451
526
|
throw new MaterializationValidationError(
|
|
452
527
|
manifest.projection_kind === "link"
|
|
@@ -456,52 +531,51 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
456
531
|
}
|
|
457
532
|
}
|
|
458
533
|
|
|
534
|
+
/** Already staged assertions sharing a root or an ordinal with `rows`. */
|
|
459
535
|
private findStageRows(
|
|
460
|
-
|
|
536
|
+
manifest: SqliteOntologySourceRow,
|
|
461
537
|
rows: readonly SourceStageRow[]
|
|
462
538
|
): readonly SourceStageRow[] {
|
|
463
539
|
if (rows.length === 0) return []
|
|
464
|
-
const
|
|
540
|
+
const rootKeys = [...new Set(rows.map((row) => row.rootKey))]
|
|
465
541
|
const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))]
|
|
466
|
-
|
|
542
|
+
// Two keyed lookups: an OR over both columns scans every root of the version per batch.
|
|
467
543
|
const existing = this.db
|
|
468
544
|
.query(
|
|
469
545
|
`
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
546
|
+
WITH selected AS (
|
|
547
|
+
SELECT roots.id FROM json_each(?2) AS requested
|
|
548
|
+
CROSS JOIN ontology_source_roots AS roots
|
|
549
|
+
ON roots.version_id = ?1 AND roots.root_key = CAST(requested.value AS TEXT)
|
|
550
|
+
UNION
|
|
551
|
+
SELECT roots.id FROM json_each(?3) AS requested
|
|
552
|
+
CROSS JOIN ontology_source_roots AS roots
|
|
553
|
+
ON roots.version_id = ?1 AND roots.staging_ordinal = CAST(requested.value AS INTEGER)
|
|
554
|
+
)
|
|
555
|
+
SELECT ${SOURCE_ASSERTION_COLUMNS} FROM selected
|
|
556
|
+
CROSS JOIN ontology_source_roots AS roots ON roots.id = selected.id
|
|
557
|
+
CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
558
|
+
CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
477
559
|
`
|
|
478
560
|
)
|
|
479
561
|
.all(
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
JSON.stringify(rootSortKeys),
|
|
484
|
-
JSON.stringify(ordinals),
|
|
485
|
-
JSON.stringify(entitySortKeys)
|
|
562
|
+
manifest.version_id,
|
|
563
|
+
JSON.stringify(rootKeys),
|
|
564
|
+
JSON.stringify(ordinals)
|
|
486
565
|
) as SqliteOntologySourceAssertionRow[]
|
|
487
566
|
return existing.map((row) => sourceStageRow(sourceAssertion(row)))
|
|
488
567
|
}
|
|
489
568
|
|
|
490
|
-
private insertStageRows(
|
|
569
|
+
private insertStageRows(
|
|
570
|
+
manifest: SqliteOntologySourceRow,
|
|
571
|
+
rows: readonly SourceStageRow[]
|
|
572
|
+
): void {
|
|
491
573
|
if (rows.length === 0) return
|
|
492
|
-
const payload = rows.map(({
|
|
574
|
+
const payload = rows.map(({ rootKey, row }) => {
|
|
493
575
|
const entity = entityColumns(row.assertion)
|
|
494
|
-
const root = entityColumns(row.root)
|
|
495
576
|
return {
|
|
496
|
-
entityKind: row.assertion.kind,
|
|
497
|
-
entityKey,
|
|
498
|
-
entitySortKey: utf8SortKey(entityKey),
|
|
499
|
-
rootKind: row.root.kind,
|
|
500
577
|
rootKey,
|
|
501
|
-
|
|
502
|
-
stagingOrdinal: row.stagingOrdinal,
|
|
503
|
-
root: row.root,
|
|
504
|
-
assertion: row.assertion,
|
|
578
|
+
entityKind: row.assertion.kind,
|
|
505
579
|
objectTypeId: entity.objectTypeId,
|
|
506
580
|
primaryId: entity.primaryId,
|
|
507
581
|
sourceTypeId: entity.sourceTypeId,
|
|
@@ -509,54 +583,31 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
|
|
|
509
583
|
linkId: entity.linkId,
|
|
510
584
|
targetTypeId: entity.targetTypeId,
|
|
511
585
|
targetPrimaryId: entity.targetPrimaryId,
|
|
512
|
-
|
|
513
|
-
rootPrimaryId: root.primaryId,
|
|
514
|
-
rootSourceTypeId: root.sourceTypeId,
|
|
515
|
-
rootSourcePrimaryId: root.sourcePrimaryId,
|
|
516
|
-
rootLinkId: root.linkId,
|
|
517
|
-
rootTargetTypeId: root.targetTypeId,
|
|
518
|
-
rootTargetPrimaryId: root.targetPrimaryId,
|
|
586
|
+
payload: sourceAssertionPayload(row.assertion),
|
|
519
587
|
}
|
|
520
588
|
})
|
|
521
|
-
this.db
|
|
589
|
+
const inserted = this.db
|
|
522
590
|
.query(
|
|
523
591
|
`
|
|
524
592
|
WITH staged(value) AS (SELECT value FROM json_each(?))
|
|
525
593
|
INSERT INTO ontology_source_rows (
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
root_kind, root_key, root_sort_key, staging_ordinal,
|
|
529
|
-
root, assertion,
|
|
530
|
-
object_type_id, primary_id,
|
|
531
|
-
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id,
|
|
532
|
-
root_object_type_id, root_primary_id,
|
|
533
|
-
root_source_type_id, root_source_primary_id, root_link_id,
|
|
534
|
-
root_target_type_id, root_target_primary_id
|
|
594
|
+
root_id, entity_kind, object_type_id, primary_id,
|
|
595
|
+
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id, payload
|
|
535
596
|
)
|
|
536
|
-
SELECT
|
|
537
|
-
?, ?, ?,
|
|
538
|
-
json_extract(value, '$.entityKind'), json_extract(value, '$.entityKey'),
|
|
539
|
-
json_extract(value, '$.entitySortKey'),
|
|
540
|
-
json_extract(value, '$.rootKind'), json_extract(value, '$.rootKey'),
|
|
541
|
-
json_extract(value, '$.rootSortKey'), json_extract(value, '$.stagingOrdinal'),
|
|
542
|
-
json(json_extract(value, '$.root')), json(json_extract(value, '$.assertion')),
|
|
597
|
+
SELECT roots.id, json_extract(value, '$.entityKind'),
|
|
543
598
|
json_extract(value, '$.objectTypeId'), json_extract(value, '$.primaryId'),
|
|
544
599
|
json_extract(value, '$.sourceTypeId'), json_extract(value, '$.sourcePrimaryId'),
|
|
545
600
|
json_extract(value, '$.linkId'), json_extract(value, '$.targetTypeId'),
|
|
546
|
-
json_extract(value, '$.targetPrimaryId'),
|
|
547
|
-
json_extract(value, '$.rootObjectTypeId'), json_extract(value, '$.rootPrimaryId'),
|
|
548
|
-
json_extract(value, '$.rootSourceTypeId'),
|
|
549
|
-
json_extract(value, '$.rootSourcePrimaryId'), json_extract(value, '$.rootLinkId'),
|
|
550
|
-
json_extract(value, '$.rootTargetTypeId'), json_extract(value, '$.rootTargetPrimaryId')
|
|
601
|
+
json_extract(value, '$.targetPrimaryId'), json_extract(value, '$.payload')
|
|
551
602
|
FROM staged
|
|
603
|
+
CROSS JOIN ontology_source_roots AS roots
|
|
604
|
+
ON roots.version_id = ? AND roots.root_key = json_extract(value, '$.rootKey')
|
|
552
605
|
`
|
|
553
606
|
)
|
|
554
|
-
.run(
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
input.materializationId
|
|
559
|
-
)
|
|
607
|
+
.run(canonicalJson(payload), manifest.version_id).changes
|
|
608
|
+
if (inserted !== rows.length) {
|
|
609
|
+
throw sourceConflict("Staged source rows do not all belong to a staged root.")
|
|
610
|
+
}
|
|
560
611
|
}
|
|
561
612
|
|
|
562
613
|
private assertExecution(
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
immutableDatasetVersionConflict,
|
|
20
20
|
mergeProjectionRunProgress,
|
|
21
21
|
type PersistedProjectionRunRecord,
|
|
22
|
+
parseProjectionAttemptFailure,
|
|
22
23
|
planProjectionRunFinish,
|
|
23
24
|
planProjectionRunReclaim,
|
|
24
25
|
projectionRunNotFound,
|
|
@@ -26,12 +27,14 @@ import {
|
|
|
26
27
|
requireTelemetryProjectionRun,
|
|
27
28
|
restoreProjectionRun,
|
|
28
29
|
type StoredProjectionRunRecord,
|
|
30
|
+
SUPERSEDING_PROJECTION_RUN_STATUSES,
|
|
29
31
|
staleProjectionRunExecution,
|
|
30
32
|
} from "@sixb/core/internal/projection-run-storage-provider"
|
|
31
33
|
import type {
|
|
32
34
|
AdvanceProjectionTelemetryCheckpointInput,
|
|
33
35
|
ExecutionStorage,
|
|
34
36
|
FailProjectionRunEnqueueInput,
|
|
37
|
+
FindSupersedingProjectionRunInput,
|
|
35
38
|
FinishProjectionRunInput,
|
|
36
39
|
ListLatestProjectionRunsInput,
|
|
37
40
|
ListLatestProjectionRunsResult,
|
|
@@ -45,6 +48,7 @@ import type {
|
|
|
45
48
|
ProjectionRunStatus,
|
|
46
49
|
ProjectionRunStorage,
|
|
47
50
|
QueueProjectionRunInput,
|
|
51
|
+
RecordProjectionAttemptFailureInput,
|
|
48
52
|
RecordProjectionMissingTargetInput,
|
|
49
53
|
StartOrReclaimProjectionRunInput,
|
|
50
54
|
TelemetryProjectionRunRecord,
|
|
@@ -177,7 +181,7 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
|
|
|
177
181
|
`
|
|
178
182
|
UPDATE projection_runs
|
|
179
183
|
SET status = 'running', started_at = COALESCE(started_at, ?), finished_at = NULL,
|
|
180
|
-
attempt = ?, execution_token =
|
|
184
|
+
attempt = ?, execution_token = ?
|
|
181
185
|
WHERE project_id = ? AND id = ?
|
|
182
186
|
AND (
|
|
183
187
|
(status = 'queued' AND execution_token IS NULL)
|
|
@@ -243,6 +247,31 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
|
|
|
243
247
|
})
|
|
244
248
|
}
|
|
245
249
|
|
|
250
|
+
async recordAttemptFailure(
|
|
251
|
+
input: RecordProjectionAttemptFailureInput
|
|
252
|
+
): Promise<ProjectionRunRecord> {
|
|
253
|
+
return runImmediateTransaction(this.db, () => {
|
|
254
|
+
const existingRow = this.requireMaterializationExecution(input)
|
|
255
|
+
const error = parseProjectionAttemptFailure(restoreProjectionRunRow(existingRow), input.error)
|
|
256
|
+
const result = this.db
|
|
257
|
+
.query(
|
|
258
|
+
`
|
|
259
|
+
UPDATE projection_runs
|
|
260
|
+
SET error = ?
|
|
261
|
+
WHERE project_id = ? AND id = ? AND status = 'running' AND execution_token = ?
|
|
262
|
+
`
|
|
263
|
+
)
|
|
264
|
+
.run(
|
|
265
|
+
serializeSixbFailure(error, PROJECTION_RUN_FAILURE_CODES),
|
|
266
|
+
input.projectId,
|
|
267
|
+
input.id,
|
|
268
|
+
input.executionToken
|
|
269
|
+
)
|
|
270
|
+
if (result.changes !== 1) throw staleProjectionRunExecution(input.id)
|
|
271
|
+
return rowToProjectionRunRecord(this.requireRow(input.projectId, input.id))
|
|
272
|
+
})
|
|
273
|
+
}
|
|
274
|
+
|
|
246
275
|
async finish(input: FinishProjectionRunInput): Promise<ProjectionRunRecord> {
|
|
247
276
|
return runImmediateTransaction(this.db, () => {
|
|
248
277
|
const existingRow = this.requireMaterializationExecution(input)
|
|
@@ -447,6 +476,34 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
|
|
|
447
476
|
return { runs, hasMore: offset + runs.length < totalRow.count, total: totalRow.count }
|
|
448
477
|
}
|
|
449
478
|
|
|
479
|
+
async findSupersedingRun(
|
|
480
|
+
input: FindSupersedingProjectionRunInput
|
|
481
|
+
): Promise<ProjectionRunRecord | null> {
|
|
482
|
+
const run = this.requireRow(input.projectId, input.id)
|
|
483
|
+
if (run.materialization_protocol !== "replacement") return null
|
|
484
|
+
// SQLite compares TEXT with the binary collation, under which canonical UTC ISO timestamps
|
|
485
|
+
// order chronologically.
|
|
486
|
+
const statuses = SUPERSEDING_PROJECTION_RUN_STATUSES
|
|
487
|
+
const newer = this.db
|
|
488
|
+
.query(
|
|
489
|
+
`
|
|
490
|
+
SELECT * FROM projection_runs
|
|
491
|
+
WHERE project_id = ? AND projection_id = ? AND materialization_protocol = 'replacement'
|
|
492
|
+
AND status IN (${statuses.map(() => "?").join(", ")})
|
|
493
|
+
AND dataset_version_created_at > ?
|
|
494
|
+
ORDER BY dataset_version_created_at DESC, id DESC
|
|
495
|
+
LIMIT 1
|
|
496
|
+
`
|
|
497
|
+
)
|
|
498
|
+
.get(
|
|
499
|
+
input.projectId,
|
|
500
|
+
run.projection_id,
|
|
501
|
+
...statuses,
|
|
502
|
+
run.dataset_version_created_at
|
|
503
|
+
) as DatabaseRow | null
|
|
504
|
+
return newer ? rowToProjectionRunRecord(newer) : null
|
|
505
|
+
}
|
|
506
|
+
|
|
450
507
|
async listLatestByProjectionIds(
|
|
451
508
|
input: ListLatestProjectionRunsInput
|
|
452
509
|
): Promise<ListLatestProjectionRunsResult> {
|
|
@@ -457,6 +514,7 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
|
|
|
457
514
|
ownerIds: input.projectionIds,
|
|
458
515
|
projectId: input.projectId,
|
|
459
516
|
ownerIdFor: (row) => row.projection_id,
|
|
517
|
+
excludedStatus: "superseded",
|
|
460
518
|
})
|
|
461
519
|
return { runs: rows.map(rowToProjectionRunRecord) }
|
|
462
520
|
}
|