@sixb/sqlite 0.1.13 → 0.1.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/auth-storage/auth-storage.d.ts +2 -1
  2. package/dist/auth-storage/auth-storage.d.ts.map +1 -1
  3. package/dist/auth-storage/rows.d.ts +1 -1
  4. package/dist/auth-storage/rows.d.ts.map +1 -1
  5. package/dist/index.js +726 -286
  6. package/dist/index.js.map +12 -12
  7. package/dist/latest-run-query.d.ts +2 -0
  8. package/dist/latest-run-query.d.ts.map +1 -1
  9. package/dist/migrations.d.ts.map +1 -1
  10. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  11. package/dist/ontology-storage/shared.d.ts +11 -8
  12. package/dist/ontology-storage/shared.d.ts.map +1 -1
  13. package/dist/ontology-storage/source-cleanup.d.ts +7 -1
  14. package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
  15. package/dist/ontology-storage/source-roots.d.ts +14 -2
  16. package/dist/ontology-storage/source-roots.d.ts.map +1 -1
  17. package/dist/ontology-storage/sources.d.ts +8 -3
  18. package/dist/ontology-storage/sources.d.ts.map +1 -1
  19. package/dist/projection-run-storage.d.ts +3 -1
  20. package/dist/projection-run-storage.d.ts.map +1 -1
  21. package/package.json +3 -3
  22. package/src/auth-storage/auth-storage.ts +44 -79
  23. package/src/auth-storage/rows.ts +1 -1
  24. package/src/latest-run-query.ts +8 -2
  25. package/src/migrations/050-directory-group-memberships.sql +21 -0
  26. package/src/migrations/051-projection-run-supersession.sql +126 -0
  27. package/src/migrations/052-compact-source-storage.sql +181 -0
  28. package/src/migrations.ts +42 -0
  29. package/src/ontology-storage/materialization-state.ts +34 -44
  30. package/src/ontology-storage/shared.ts +20 -11
  31. package/src/ontology-storage/source-cleanup.ts +61 -24
  32. package/src/ontology-storage/source-roots.ts +166 -110
  33. package/src/ontology-storage/sources.ts +169 -118
  34. package/src/projection-run-storage.ts +59 -1
@@ -10,15 +10,18 @@ import {
10
10
  isExactStagingManifest,
11
11
  reconcileSourceStageRows,
12
12
  type SourceStageRow,
13
+ sourceAssertionPayload,
13
14
  sourceConflict,
14
15
  sourceMaterializationIdentity,
15
16
  sourceStageRow,
16
17
  sourceStageRows,
17
- utf8SortKey,
18
18
  } from "@sixb/core/internal/ontology-storage-provider"
19
19
  import type {
20
+ AbandonRunSourceMaterializationInput,
20
21
  AbandonSourceMaterializationCandidateInput,
21
22
  AbandonSourceMaterializationInput,
23
+ AdoptedSourceMaterialization,
24
+ AdoptSourceMaterializationInput,
22
25
  AssertSourceMaterializationExecutionInput,
23
26
  BeginSourceMaterializationInput,
24
27
  CleanupTerminalSourceMaterializationsInput,
@@ -27,7 +30,7 @@ import type {
27
30
  MarkSourceMaterializationReadyInput,
28
31
  OntologySourceRecord,
29
32
  OntologySourceStorage,
30
- ReclaimSourceMaterializationInput,
33
+ PurgeAbandonedSourceMaterializationsInput,
31
34
  StageSourceRowsInput,
32
35
  StageSourceRowsResult,
33
36
  SummarizeTerminalSourceMaterializationsInput,
@@ -47,8 +50,12 @@ import {
47
50
  sourceAssertion,
48
51
  sourceRecord,
49
52
  } from "./shared"
50
- import { cleanupSourceVersions } from "./source-cleanup"
51
- import { assertSourceRootCoverage, stageSourceRoots } from "./source-roots"
53
+ import { cleanupSourceVersions, purgeAbandonedSourceVersions } from "./source-cleanup"
54
+ import {
55
+ assertSourceRootCoverage,
56
+ SOURCE_ASSERTION_COLUMNS,
57
+ stageSourceRoots,
58
+ } from "./source-roots"
52
59
 
53
60
  export class SqliteOntologySourceStorage implements OntologySourceStorage {
54
61
  constructor(
@@ -87,7 +94,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
87
94
  .get(input.projectId, input.execution.projectionRunId)
88
95
  if (candidate) {
89
96
  throw sourceConflict(
90
- `Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; reclaim it before beginning another.`
97
+ `Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`
91
98
  )
92
99
  }
93
100
 
@@ -146,10 +153,10 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
146
153
  }
147
154
 
148
155
  const rows = sourceStageRows(manifest.projection_kind, input.rows)
149
- const existing = this.findStageRows(input, rows)
156
+ const existing = this.findStageRows(manifest, rows)
150
157
  const { pending, unchanged } = reconcileSourceStageRows(rows, existing)
151
158
  stageSourceRoots(this.db, manifest, input)
152
- this.insertStageRows(input, pending)
159
+ this.insertStageRows(manifest, pending)
153
160
  return { inserted: pending.length, unchanged }
154
161
  })
155
162
  await yieldSqliteEventLoop()
@@ -233,17 +240,59 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
233
240
  })
234
241
  }
235
242
 
243
+ async adopt(
244
+ input: AdoptSourceMaterializationInput
245
+ ): Promise<AdoptedSourceMaterialization | null> {
246
+ return this.runRootOperation(() => {
247
+ assertProjectAndSource(input)
248
+ assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken)
249
+ assertTimestamp(input.adoptedAt, "Source adoptedAt", true)
250
+ this.assertExecution(input)
251
+ const candidate = this.findRunCandidate(input)
252
+ if (!candidate) return null
253
+ // Workers' clocks differ; adoption never moves the record back in time.
254
+ const changed = this.db
255
+ .query(
256
+ `
257
+ UPDATE ontology_sources SET execution_token = ?, updated_at = MAX(updated_at, ?)
258
+ WHERE project_id = ? AND source_id = ? AND materialization_id = ?
259
+ AND status IN ('staging', 'ready')
260
+ `
261
+ )
262
+ .run(
263
+ input.execution.executionToken,
264
+ input.adoptedAt,
265
+ candidate.project_id,
266
+ candidate.source_id,
267
+ candidate.materialization_id
268
+ ).changes
269
+ if (changed !== 1) {
270
+ throw sourceConflict(`Source materialization '${candidate.materialization_id}' changed.`)
271
+ }
272
+ const adopted = this.requireManifest(
273
+ candidate.project_id,
274
+ candidate.source_id,
275
+ candidate.materialization_id
276
+ )
277
+ return {
278
+ record: sourceRecord(adopted),
279
+ resumeStagingOrdinal:
280
+ adopted.status === "ready" ? Number(adopted.root_count) : this.lastStagedOrdinal(adopted),
281
+ }
282
+ })
283
+ }
284
+
236
285
  async abandon(input: AbandonSourceMaterializationCandidateInput): Promise<OntologySourceRecord>
237
- async abandon(input: ReclaimSourceMaterializationInput): Promise<OntologySourceRecord | null>
286
+ async abandon(input: AbandonRunSourceMaterializationInput): Promise<OntologySourceRecord | null>
238
287
  async abandon(input: AbandonSourceMaterializationInput): Promise<OntologySourceRecord | null> {
239
288
  return this.runRootOperation(() => {
240
289
  assertProjectAndSource(input)
241
290
  assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken)
242
291
  assertTimestamp(input.abandonedAt, "Source abandonedAt", true)
243
292
  this.assertExecution(input)
244
- return input.kind === "candidate"
245
- ? this.abandonCandidate(input)
246
- : this.abandonForReclaim(input)
293
+ if (input.kind === "candidate") return this.abandonCandidate(input)
294
+ const candidate = this.findRunCandidate(input)
295
+ return candidate ? this.transitionToAbandoned(candidate, input.abandonedAt) : null
247
296
  })
248
297
  }
249
298
 
@@ -258,6 +307,16 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
258
307
  })
259
308
  }
260
309
 
310
+ async purgeAbandoned(
311
+ input: PurgeAbandonedSourceMaterializationsInput
312
+ ): Promise<CleanupTerminalSourceMaterializationsResult> {
313
+ return this.runRootOperation(() => {
314
+ assertNonblank(input.projectId, "Abandoned source purge project id")
315
+ assertPositiveInteger(input.limit, "Abandoned source purge limit")
316
+ return purgeAbandonedSourceVersions(this.db, input)
317
+ })
318
+ }
319
+
261
320
  async summarizeTerminal(
262
321
  input: SummarizeTerminalSourceMaterializationsInput
263
322
  ): Promise<TerminalSourceMaterializationSummary> {
@@ -305,7 +364,12 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
305
364
  return this.transitionToAbandoned(manifest, input.abandonedAt)
306
365
  }
307
366
 
308
- private abandonForReclaim(input: ReclaimSourceMaterializationInput): OntologySourceRecord | null {
367
+ /** The run's one staging or ready candidate, whichever execution staged it. */
368
+ private findRunCandidate(input: {
369
+ readonly projectId: string
370
+ readonly source: { readonly projectionId: string }
371
+ readonly execution: { readonly projectionRunId: string }
372
+ }): SqliteOntologySourceRow | null {
309
373
  const candidates = this.db
310
374
  .query(
311
375
  `
@@ -319,19 +383,23 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
319
383
  input.source.projectionId,
320
384
  input.execution.projectionRunId
321
385
  ) as SqliteOntologySourceRow[]
322
- if (
323
- candidates.some((candidate) => candidate.execution_token === input.execution.executionToken)
324
- ) {
325
- throw sourceConflict(
326
- `Projection run '${input.execution.projectionRunId}' already has a source materialization owned by the current execution.`
327
- )
328
- }
329
386
  if (candidates.length > 1) {
330
387
  throw sourceConflict(
331
388
  `Projection run '${input.execution.projectionRunId}' has multiple nonterminal source materializations.`
332
389
  )
333
390
  }
334
- return candidates[0] ? this.transitionToAbandoned(candidates[0], input.abandonedAt) : null
391
+ return candidates[0] ?? null
392
+ }
393
+
394
+ private lastStagedOrdinal(manifest: SqliteOntologySourceRow): number {
395
+ const last = this.db
396
+ .query(
397
+ "SELECT MAX(staging_ordinal) AS ordinal FROM ontology_source_roots WHERE version_id = ?"
398
+ )
399
+ .get(manifest.version_id) as {
400
+ readonly ordinal: number | null
401
+ } | null
402
+ return last?.ordinal ?? 0
335
403
  }
336
404
 
337
405
  private transitionToAbandoned(
@@ -367,10 +435,7 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
367
435
  `Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`
368
436
  )
369
437
  }
370
- this.db
371
- .query(`UPDATE ontology_source_roots SET retired_at = ?
372
- WHERE project_id = ? AND source_id = ? AND materialization_id = ? AND active = 0`)
373
- .run(abandonedAt, manifest.project_id, manifest.source_id, manifest.materialization_id)
438
+ // Its roots were never live and stay unpublished; purging removes them with the version.
374
439
  return sourceRecord(
375
440
  this.requireManifest(manifest.project_id, manifest.source_id, manifest.materialization_id)
376
441
  )
@@ -384,22 +449,16 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
384
449
  const counts = this.db
385
450
  .query(
386
451
  `
387
- SELECT (SELECT COUNT(*) FROM ontology_source_rows
388
- WHERE project_id = ? AND source_id = ? AND materialization_id = ?) AS assertions, COUNT(*) AS roots,
389
- COUNT(DISTINCT staging_ordinal) AS ordinals,
452
+ SELECT (SELECT COUNT(*) FROM ontology_source_roots AS roots
453
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
454
+ WHERE roots.version_id = ?1) AS assertions,
455
+ COUNT(*) AS roots, COUNT(DISTINCT staging_ordinal) AS ordinals,
390
456
  MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
391
457
  FROM ontology_source_roots
392
- WHERE project_id = ? AND source_id = ? AND materialization_id = ?
458
+ WHERE version_id = ?1
393
459
  `
394
460
  )
395
- .get(
396
- manifest.project_id,
397
- manifest.source_id,
398
- manifest.materialization_id,
399
- manifest.project_id,
400
- manifest.source_id,
401
- manifest.materialization_id
402
- ) as {
461
+ .get(manifest.version_id) as {
403
462
  assertions: number
404
463
  roots: number
405
464
  ordinals: number
@@ -417,36 +476,52 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
417
476
  )
418
477
  }
419
478
  assertSourceRootCoverage(this.db, manifest)
479
+ // Roots store only their canonical key; compare its parts with each row's typed identity.
420
480
  const invalid = this.db
421
481
  .query(
422
482
  manifest.projection_kind === "link"
423
483
  ? `
424
- SELECT root_key
425
- FROM ontology_source_rows
426
- WHERE project_id = ? AND source_id = ? AND materialization_id = ?
427
- GROUP BY root_key
428
- HAVING COUNT(*) <> 1 OR MIN(root_kind) <> 'link' OR MIN(entity_kind) <> 'link'
429
- OR MIN(root_key) <> MIN(entity_key)
484
+ SELECT roots.root_key
485
+ FROM ontology_source_roots AS roots
486
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
487
+ WHERE roots.version_id = ?
488
+ GROUP BY roots.id
489
+ HAVING COUNT(*) <> 1
490
+ OR MAX(json_extract(roots.root_key, '$[0]') <> 'link')
491
+ OR MAX(rows.entity_kind <> 'link')
492
+ OR MAX(rows.source_type_id <> json_extract(roots.root_key, '$[1]')
493
+ OR rows.source_primary_id <> json_extract(roots.root_key, '$[2]')
494
+ OR rows.link_id <> json_extract(roots.root_key, '$[3]')
495
+ OR rows.target_type_id <> json_extract(roots.root_key, '$[4]')
496
+ OR rows.target_primary_id <> json_extract(roots.root_key, '$[5]'))
430
497
  LIMIT 1
431
498
  `
432
499
  : `
433
- SELECT root_key
434
- FROM ontology_source_rows
435
- WHERE project_id = ? AND source_id = ? AND materialization_id = ?
436
- GROUP BY root_key
437
- HAVING MIN(root_kind) <> 'object'
438
- OR SUM(CASE WHEN entity_kind = 'object' AND entity_key = root_key THEN 1 ELSE 0 END) <> 1
439
- OR SUM(CASE
440
- WHEN entity_kind = 'object' AND entity_key <> root_key THEN 1
441
- WHEN entity_kind = 'link' AND (
442
- source_type_id <> root_object_type_id OR source_primary_id <> root_primary_id
443
- ) THEN 1
444
- ELSE 0
445
- END) <> 0
500
+ SELECT roots.root_key
501
+ FROM ontology_source_roots AS roots
502
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
503
+ WHERE roots.version_id = ?
504
+ GROUP BY roots.id
505
+ HAVING MAX(json_extract(roots.root_key, '$[0]') <> 'object')
506
+ OR SUM(rows.entity_kind = 'object'
507
+ AND rows.object_type_id = json_extract(roots.root_key, '$[1]')
508
+ AND rows.primary_id = json_extract(roots.root_key, '$[2]')) <> 1
509
+ OR SUM(
510
+ (rows.entity_kind = 'object' AND (
511
+ rows.object_type_id <> json_extract(roots.root_key, '$[1]')
512
+ OR rows.primary_id <> json_extract(roots.root_key, '$[2]')))
513
+ OR (rows.entity_kind = 'link' AND (
514
+ rows.source_type_id <> json_extract(roots.root_key, '$[1]')
515
+ OR rows.source_primary_id <> json_extract(roots.root_key, '$[2]')))
516
+ ) <> 0
517
+ OR SUM(rows.entity_kind = 'link') <> COUNT(DISTINCT CASE
518
+ WHEN rows.entity_kind = 'link'
519
+ THEN json_array(rows.link_id, rows.target_type_id, rows.target_primary_id)
520
+ END)
446
521
  LIMIT 1
447
522
  `
448
523
  )
449
- .get(manifest.project_id, manifest.source_id, manifest.materialization_id)
524
+ .get(manifest.version_id)
450
525
  if (invalid) {
451
526
  throw new MaterializationValidationError(
452
527
  manifest.projection_kind === "link"
@@ -456,52 +531,51 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
456
531
  }
457
532
  }
458
533
 
534
+ /** Already staged assertions sharing a root or an ordinal with `rows`. */
459
535
  private findStageRows(
460
- input: StageSourceRowsInput,
536
+ manifest: SqliteOntologySourceRow,
461
537
  rows: readonly SourceStageRow[]
462
538
  ): readonly SourceStageRow[] {
463
539
  if (rows.length === 0) return []
464
- const rootSortKeys = [...new Set(rows.map((row) => utf8SortKey(row.rootKey)))]
540
+ const rootKeys = [...new Set(rows.map((row) => row.rootKey))]
465
541
  const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))]
466
- const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey(row.entityKey)))]
542
+ // Two keyed lookups: an OR over both columns scans every root of the version per batch.
467
543
  const existing = this.db
468
544
  .query(
469
545
  `
470
- SELECT * FROM ontology_source_rows
471
- WHERE project_id = ? AND source_id = ? AND materialization_id = ?
472
- AND (
473
- root_sort_key IN (SELECT CAST(value AS TEXT) FROM json_each(?))
474
- OR staging_ordinal IN (SELECT CAST(value AS INTEGER) FROM json_each(?))
475
- OR entity_sort_key IN (SELECT CAST(value AS TEXT) FROM json_each(?))
476
- )
546
+ WITH selected AS (
547
+ SELECT roots.id FROM json_each(?2) AS requested
548
+ CROSS JOIN ontology_source_roots AS roots
549
+ ON roots.version_id = ?1 AND roots.root_key = CAST(requested.value AS TEXT)
550
+ UNION
551
+ SELECT roots.id FROM json_each(?3) AS requested
552
+ CROSS JOIN ontology_source_roots AS roots
553
+ ON roots.version_id = ?1 AND roots.staging_ordinal = CAST(requested.value AS INTEGER)
554
+ )
555
+ SELECT ${SOURCE_ASSERTION_COLUMNS} FROM selected
556
+ CROSS JOIN ontology_source_roots AS roots ON roots.id = selected.id
557
+ CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
558
+ CROSS JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
477
559
  `
478
560
  )
479
561
  .all(
480
- input.projectId,
481
- input.source.projectionId,
482
- input.materializationId,
483
- JSON.stringify(rootSortKeys),
484
- JSON.stringify(ordinals),
485
- JSON.stringify(entitySortKeys)
562
+ manifest.version_id,
563
+ JSON.stringify(rootKeys),
564
+ JSON.stringify(ordinals)
486
565
  ) as SqliteOntologySourceAssertionRow[]
487
566
  return existing.map((row) => sourceStageRow(sourceAssertion(row)))
488
567
  }
489
568
 
490
- private insertStageRows(input: StageSourceRowsInput, rows: readonly SourceStageRow[]): void {
569
+ private insertStageRows(
570
+ manifest: SqliteOntologySourceRow,
571
+ rows: readonly SourceStageRow[]
572
+ ): void {
491
573
  if (rows.length === 0) return
492
- const payload = rows.map(({ entityKey, rootKey, row }) => {
574
+ const payload = rows.map(({ rootKey, row }) => {
493
575
  const entity = entityColumns(row.assertion)
494
- const root = entityColumns(row.root)
495
576
  return {
496
- entityKind: row.assertion.kind,
497
- entityKey,
498
- entitySortKey: utf8SortKey(entityKey),
499
- rootKind: row.root.kind,
500
577
  rootKey,
501
- rootSortKey: utf8SortKey(rootKey),
502
- stagingOrdinal: row.stagingOrdinal,
503
- root: row.root,
504
- assertion: row.assertion,
578
+ entityKind: row.assertion.kind,
505
579
  objectTypeId: entity.objectTypeId,
506
580
  primaryId: entity.primaryId,
507
581
  sourceTypeId: entity.sourceTypeId,
@@ -509,54 +583,31 @@ export class SqliteOntologySourceStorage implements OntologySourceStorage {
509
583
  linkId: entity.linkId,
510
584
  targetTypeId: entity.targetTypeId,
511
585
  targetPrimaryId: entity.targetPrimaryId,
512
- rootObjectTypeId: root.objectTypeId,
513
- rootPrimaryId: root.primaryId,
514
- rootSourceTypeId: root.sourceTypeId,
515
- rootSourcePrimaryId: root.sourcePrimaryId,
516
- rootLinkId: root.linkId,
517
- rootTargetTypeId: root.targetTypeId,
518
- rootTargetPrimaryId: root.targetPrimaryId,
586
+ payload: sourceAssertionPayload(row.assertion),
519
587
  }
520
588
  })
521
- this.db
589
+ const inserted = this.db
522
590
  .query(
523
591
  `
524
592
  WITH staged(value) AS (SELECT value FROM json_each(?))
525
593
  INSERT INTO ontology_source_rows (
526
- project_id, source_id, materialization_id,
527
- entity_kind, entity_key, entity_sort_key,
528
- root_kind, root_key, root_sort_key, staging_ordinal,
529
- root, assertion,
530
- object_type_id, primary_id,
531
- source_type_id, source_primary_id, link_id, target_type_id, target_primary_id,
532
- root_object_type_id, root_primary_id,
533
- root_source_type_id, root_source_primary_id, root_link_id,
534
- root_target_type_id, root_target_primary_id
594
+ root_id, entity_kind, object_type_id, primary_id,
595
+ source_type_id, source_primary_id, link_id, target_type_id, target_primary_id, payload
535
596
  )
536
- SELECT
537
- ?, ?, ?,
538
- json_extract(value, '$.entityKind'), json_extract(value, '$.entityKey'),
539
- json_extract(value, '$.entitySortKey'),
540
- json_extract(value, '$.rootKind'), json_extract(value, '$.rootKey'),
541
- json_extract(value, '$.rootSortKey'), json_extract(value, '$.stagingOrdinal'),
542
- json(json_extract(value, '$.root')), json(json_extract(value, '$.assertion')),
597
+ SELECT roots.id, json_extract(value, '$.entityKind'),
543
598
  json_extract(value, '$.objectTypeId'), json_extract(value, '$.primaryId'),
544
599
  json_extract(value, '$.sourceTypeId'), json_extract(value, '$.sourcePrimaryId'),
545
600
  json_extract(value, '$.linkId'), json_extract(value, '$.targetTypeId'),
546
- json_extract(value, '$.targetPrimaryId'),
547
- json_extract(value, '$.rootObjectTypeId'), json_extract(value, '$.rootPrimaryId'),
548
- json_extract(value, '$.rootSourceTypeId'),
549
- json_extract(value, '$.rootSourcePrimaryId'), json_extract(value, '$.rootLinkId'),
550
- json_extract(value, '$.rootTargetTypeId'), json_extract(value, '$.rootTargetPrimaryId')
601
+ json_extract(value, '$.targetPrimaryId'), json_extract(value, '$.payload')
551
602
  FROM staged
603
+ CROSS JOIN ontology_source_roots AS roots
604
+ ON roots.version_id = ? AND roots.root_key = json_extract(value, '$.rootKey')
552
605
  `
553
606
  )
554
- .run(
555
- canonicalJson(payload),
556
- input.projectId,
557
- input.source.projectionId,
558
- input.materializationId
559
- )
607
+ .run(canonicalJson(payload), manifest.version_id).changes
608
+ if (inserted !== rows.length) {
609
+ throw sourceConflict("Staged source rows do not all belong to a staged root.")
610
+ }
560
611
  }
561
612
 
562
613
  private assertExecution(
@@ -19,6 +19,7 @@ import {
19
19
  immutableDatasetVersionConflict,
20
20
  mergeProjectionRunProgress,
21
21
  type PersistedProjectionRunRecord,
22
+ parseProjectionAttemptFailure,
22
23
  planProjectionRunFinish,
23
24
  planProjectionRunReclaim,
24
25
  projectionRunNotFound,
@@ -26,12 +27,14 @@ import {
26
27
  requireTelemetryProjectionRun,
27
28
  restoreProjectionRun,
28
29
  type StoredProjectionRunRecord,
30
+ SUPERSEDING_PROJECTION_RUN_STATUSES,
29
31
  staleProjectionRunExecution,
30
32
  } from "@sixb/core/internal/projection-run-storage-provider"
31
33
  import type {
32
34
  AdvanceProjectionTelemetryCheckpointInput,
33
35
  ExecutionStorage,
34
36
  FailProjectionRunEnqueueInput,
37
+ FindSupersedingProjectionRunInput,
35
38
  FinishProjectionRunInput,
36
39
  ListLatestProjectionRunsInput,
37
40
  ListLatestProjectionRunsResult,
@@ -45,6 +48,7 @@ import type {
45
48
  ProjectionRunStatus,
46
49
  ProjectionRunStorage,
47
50
  QueueProjectionRunInput,
51
+ RecordProjectionAttemptFailureInput,
48
52
  RecordProjectionMissingTargetInput,
49
53
  StartOrReclaimProjectionRunInput,
50
54
  TelemetryProjectionRunRecord,
@@ -177,7 +181,7 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
177
181
  `
178
182
  UPDATE projection_runs
179
183
  SET status = 'running', started_at = COALESCE(started_at, ?), finished_at = NULL,
180
- attempt = ?, execution_token = ?, error = NULL
184
+ attempt = ?, execution_token = ?
181
185
  WHERE project_id = ? AND id = ?
182
186
  AND (
183
187
  (status = 'queued' AND execution_token IS NULL)
@@ -243,6 +247,31 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
243
247
  })
244
248
  }
245
249
 
250
+ async recordAttemptFailure(
251
+ input: RecordProjectionAttemptFailureInput
252
+ ): Promise<ProjectionRunRecord> {
253
+ return runImmediateTransaction(this.db, () => {
254
+ const existingRow = this.requireMaterializationExecution(input)
255
+ const error = parseProjectionAttemptFailure(restoreProjectionRunRow(existingRow), input.error)
256
+ const result = this.db
257
+ .query(
258
+ `
259
+ UPDATE projection_runs
260
+ SET error = ?
261
+ WHERE project_id = ? AND id = ? AND status = 'running' AND execution_token = ?
262
+ `
263
+ )
264
+ .run(
265
+ serializeSixbFailure(error, PROJECTION_RUN_FAILURE_CODES),
266
+ input.projectId,
267
+ input.id,
268
+ input.executionToken
269
+ )
270
+ if (result.changes !== 1) throw staleProjectionRunExecution(input.id)
271
+ return rowToProjectionRunRecord(this.requireRow(input.projectId, input.id))
272
+ })
273
+ }
274
+
246
275
  async finish(input: FinishProjectionRunInput): Promise<ProjectionRunRecord> {
247
276
  return runImmediateTransaction(this.db, () => {
248
277
  const existingRow = this.requireMaterializationExecution(input)
@@ -447,6 +476,34 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
447
476
  return { runs, hasMore: offset + runs.length < totalRow.count, total: totalRow.count }
448
477
  }
449
478
 
479
+ async findSupersedingRun(
480
+ input: FindSupersedingProjectionRunInput
481
+ ): Promise<ProjectionRunRecord | null> {
482
+ const run = this.requireRow(input.projectId, input.id)
483
+ if (run.materialization_protocol !== "replacement") return null
484
+ // SQLite compares TEXT with the binary collation, under which canonical UTC ISO timestamps
485
+ // order chronologically.
486
+ const statuses = SUPERSEDING_PROJECTION_RUN_STATUSES
487
+ const newer = this.db
488
+ .query(
489
+ `
490
+ SELECT * FROM projection_runs
491
+ WHERE project_id = ? AND projection_id = ? AND materialization_protocol = 'replacement'
492
+ AND status IN (${statuses.map(() => "?").join(", ")})
493
+ AND dataset_version_created_at > ?
494
+ ORDER BY dataset_version_created_at DESC, id DESC
495
+ LIMIT 1
496
+ `
497
+ )
498
+ .get(
499
+ input.projectId,
500
+ run.projection_id,
501
+ ...statuses,
502
+ run.dataset_version_created_at
503
+ ) as DatabaseRow | null
504
+ return newer ? rowToProjectionRunRecord(newer) : null
505
+ }
506
+
450
507
  async listLatestByProjectionIds(
451
508
  input: ListLatestProjectionRunsInput
452
509
  ): Promise<ListLatestProjectionRunsResult> {
@@ -457,6 +514,7 @@ export class SqliteProjectionRunStorage implements ProjectionRunStorage {
457
514
  ownerIds: input.projectionIds,
458
515
  projectId: input.projectId,
459
516
  ownerIdFor: (row) => row.projection_id,
517
+ excludedStatus: "superseded",
460
518
  })
461
519
  return { runs: rows.map(rowToProjectionRunRecord) }
462
520
  }