@sixb/pg 0.1.20 → 0.1.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/README.md +1 -0
  2. package/dist/agents/agent-storage.d.ts +8 -2
  3. package/dist/agents/agent-storage.d.ts.map +1 -1
  4. package/dist/agents/approvals.d.ts +48 -0
  5. package/dist/agents/approvals.d.ts.map +1 -0
  6. package/dist/agents/checkpoints.d.ts +1 -0
  7. package/dist/agents/checkpoints.d.ts.map +1 -1
  8. package/dist/agents/index.d.ts +1 -0
  9. package/dist/agents/index.d.ts.map +1 -1
  10. package/dist/agents/memory.d.ts +19 -0
  11. package/dist/agents/memory.d.ts.map +1 -0
  12. package/dist/agents/messages.d.ts.map +1 -1
  13. package/dist/agents/rows.d.ts +15 -1
  14. package/dist/agents/rows.d.ts.map +1 -1
  15. package/dist/agents/runs.d.ts +3 -1
  16. package/dist/agents/runs.d.ts.map +1 -1
  17. package/dist/agents/settings.d.ts +29 -0
  18. package/dist/agents/settings.d.ts.map +1 -0
  19. package/dist/agents/threads.d.ts +10 -1
  20. package/dist/agents/threads.d.ts.map +1 -1
  21. package/dist/ai-cost-groups.d.ts.map +1 -1
  22. package/dist/auth-storage/auth-storage.d.ts +3 -2
  23. package/dist/auth-storage/auth-storage.d.ts.map +1 -1
  24. package/dist/auth-storage/sessions.d.ts +2 -1
  25. package/dist/auth-storage/sessions.d.ts.map +1 -1
  26. package/dist/auth-storage/users.d.ts +8 -4
  27. package/dist/auth-storage/users.d.ts.map +1 -1
  28. package/dist/index.d.ts +4 -0
  29. package/dist/index.d.ts.map +1 -1
  30. package/dist/index.js +6001 -2923
  31. package/dist/index.js.map +45 -37
  32. package/dist/migrations.d.ts.map +1 -1
  33. package/dist/objects/conditional-reader.d.ts +17 -0
  34. package/dist/objects/conditional-reader.d.ts.map +1 -0
  35. package/dist/objects/query-compiler.d.ts +27 -1
  36. package/dist/objects/query-compiler.d.ts.map +1 -1
  37. package/dist/objects/query-indexes.d.ts.map +1 -1
  38. package/dist/objects/read-scope.d.ts +10 -1
  39. package/dist/objects/read-scope.d.ts.map +1 -1
  40. package/dist/objects/reader.d.ts +2 -0
  41. package/dist/objects/reader.d.ts.map +1 -1
  42. package/dist/objects/scoped-reader.d.ts +12 -0
  43. package/dist/objects/scoped-reader.d.ts.map +1 -0
  44. package/dist/objects/selected-reader.d.ts +1 -0
  45. package/dist/objects/selected-reader.d.ts.map +1 -1
  46. package/dist/objects/storage.d.ts +7 -1
  47. package/dist/objects/storage.d.ts.map +1 -1
  48. package/dist/ontology-storage/index.d.ts +2 -0
  49. package/dist/ontology-storage/index.d.ts.map +1 -1
  50. package/dist/ontology-storage/materialization-session.d.ts +6 -0
  51. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  52. package/dist/ontology-storage/materialization-writer.d.ts +15 -0
  53. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  54. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  55. package/dist/ontology-storage/replacement-plans.d.ts.map +1 -1
  56. package/dist/ontology-storage/sources.d.ts +2 -1
  57. package/dist/ontology-storage/sources.d.ts.map +1 -1
  58. package/dist/ontology-storage/unique-values.d.ts +60 -0
  59. package/dist/ontology-storage/unique-values.d.ts.map +1 -0
  60. package/dist/pg-action-run-storage.d.ts +4 -0
  61. package/dist/pg-action-run-storage.d.ts.map +1 -1
  62. package/dist/pg-activity-storage.d.ts +22 -0
  63. package/dist/pg-activity-storage.d.ts.map +1 -0
  64. package/dist/pg-ai-cost-storage.d.ts.map +1 -1
  65. package/dist/pg-ai-usage-storage.d.ts +8 -1
  66. package/dist/pg-ai-usage-storage.d.ts.map +1 -1
  67. package/dist/pg-artifact-storage.d.ts +26 -0
  68. package/dist/pg-artifact-storage.d.ts.map +1 -0
  69. package/dist/pg-execution-storage.d.ts +3 -1
  70. package/dist/pg-execution-storage.d.ts.map +1 -1
  71. package/dist/pg-workflow-run-storage.d.ts +4 -0
  72. package/dist/pg-workflow-run-storage.d.ts.map +1 -1
  73. package/dist/share-grant-storage.d.ts +2 -1
  74. package/dist/share-grant-storage.d.ts.map +1 -1
  75. package/dist/storage-errors.d.ts +2 -0
  76. package/dist/storage-errors.d.ts.map +1 -1
  77. package/package.json +3 -3
  78. package/src/agents/agent-storage.ts +11 -2
  79. package/src/agents/approvals.ts +400 -0
  80. package/src/agents/checkpoints.ts +8 -4
  81. package/src/agents/index.ts +1 -0
  82. package/src/agents/memory.ts +175 -0
  83. package/src/agents/messages.ts +14 -7
  84. package/src/agents/rows.ts +46 -1
  85. package/src/agents/runs.ts +154 -22
  86. package/src/agents/settings.ts +105 -0
  87. package/src/agents/threads.ts +379 -3
  88. package/src/ai-cost-groups.ts +11 -1
  89. package/src/auth-storage/auth-storage.ts +45 -13
  90. package/src/auth-storage/sessions.ts +20 -0
  91. package/src/auth-storage/users.ts +105 -43
  92. package/src/index.ts +10 -0
  93. package/src/migrations/063-artifacts.sql +67 -0
  94. package/src/migrations/064-drop-output-diagnostics.sql +14 -0
  95. package/src/migrations/065-plan-work-payload-statistics.sql +5 -0
  96. package/src/migrations/066-activity.sql +61 -0
  97. package/src/migrations/067-activity-timeline.sql +15 -0
  98. package/src/migrations/068-activity-interactions.sql +20 -0
  99. package/src/migrations/069-artifact-kinds.sql +4 -0
  100. package/src/migrations/070-artifact-state.sql +16 -0
  101. package/src/migrations/071-agent-thread-participants.sql +16 -0
  102. package/src/migrations/072-forked-context-checkpoints.sql +23 -0
  103. package/src/migrations/073-unique-values.sql +25 -0
  104. package/src/migrations/074-generated-media-usage.sql +21 -0
  105. package/src/migrations/075-model-call-files.sql +11 -0
  106. package/src/migrations/076-agent-memory.sql +33 -0
  107. package/src/migrations/077-artifact-threads.sql +21 -0
  108. package/src/migrations/078-agent-approvals.sql +58 -0
  109. package/src/migrations/079-agent-approval-resume.sql +20 -0
  110. package/src/migrations/080-subagent-threads.sql +145 -0
  111. package/src/migrations/081-subagent-questions.sql +6 -0
  112. package/src/migrations.ts +52 -0
  113. package/src/objects/conditional-reader.ts +44 -0
  114. package/src/objects/prepare-queries.ts +1 -1
  115. package/src/objects/query-compiler.ts +96 -33
  116. package/src/objects/query-indexes.ts +29 -10
  117. package/src/objects/read-scope.ts +140 -25
  118. package/src/objects/reader.ts +50 -6
  119. package/src/objects/scoped-reader.ts +101 -0
  120. package/src/objects/selected-reader.ts +16 -76
  121. package/src/objects/storage.ts +24 -5
  122. package/src/objects/text-count-index.ts +1 -0
  123. package/src/ontology-storage/index.ts +3 -0
  124. package/src/ontology-storage/materialization-session.ts +8 -0
  125. package/src/ontology-storage/materialization-writer.ts +76 -1
  126. package/src/ontology-storage/materializations.ts +20 -1
  127. package/src/ontology-storage/replacement-plans.ts +25 -7
  128. package/src/ontology-storage/sources.ts +37 -0
  129. package/src/ontology-storage/unique-values.ts +274 -0
  130. package/src/pg-action-run-storage.ts +23 -0
  131. package/src/pg-activity-storage.ts +486 -0
  132. package/src/pg-ai-cost-storage.ts +62 -12
  133. package/src/pg-ai-usage-storage.ts +90 -1
  134. package/src/pg-artifact-storage.ts +521 -0
  135. package/src/pg-execution-storage.ts +83 -0
  136. package/src/pg-workflow-run-storage.ts +23 -0
  137. package/src/share-grant-storage.ts +21 -0
  138. package/src/storage-errors.ts +7 -0
  139. package/src/transactions.ts +1 -1
@@ -16,6 +16,7 @@ import {
16
16
  import type { MaterializationPlanWorkItem } from "@sixb/core/storage"
17
17
  import type postgres from "postgres"
18
18
  import type { SQL, SQLClient } from "../pg-client"
19
+ import { claimPgUniqueValues, heldValue } from "./unique-values"
19
20
 
20
21
  export interface PgStagedPlanTarget {
21
22
  /** The work table and the id its rows carry: a session's temp table, or a durable plan. */
@@ -24,11 +25,19 @@ export interface PgStagedPlanTarget {
24
25
  readonly projectId: string
25
26
  readonly commitId: string
26
27
  readonly committedAt: string
28
+ /** Whether unique value items were staged; probed in the work table when unknown. */
29
+ readonly uniqueWork?: PgUniqueWork
30
+ }
31
+
32
+ export interface PgUniqueWork {
33
+ readonly releases: boolean
34
+ readonly claims: boolean
27
35
  }
28
36
 
29
37
  export interface PgAppliedPlan {
30
38
  readonly objectWrites: number
31
39
  readonly linkWrites: number
40
+ readonly uniqueValueWrites: number
32
41
  readonly eventCount: number
33
42
  }
34
43
 
@@ -55,13 +64,32 @@ export class PgMaterializationWriter {
55
64
  await this.applyObjectOverrides(target)
56
65
  await this.applyLinkOverrides(target)
57
66
  await this.applyPoints(target)
67
+ const uniqueWork = target.uniqueWork ?? (await this.stagedUniqueWork(target))
68
+ if (uniqueWork.releases) await this.releaseUniqueValues(target)
58
69
  let linkWrites = await this.deleteLinks(target)
59
70
  let objectWrites = await this.deleteObjects(target)
60
71
  objectWrites += await this.upsertObjects(target)
61
72
  linkWrites += await this.upsertLinks(target)
73
+ const uniqueValueWrites = uniqueWork.claims ? await this.claimUniqueValues(target) : 0
62
74
  const eventCount = await this.writeOutbox(target)
63
75
  await this.writeTouches(target)
64
- return { objectWrites, linkWrites, eventCount }
76
+ return { objectWrites, linkWrites, uniqueValueWrites, eventCount }
77
+ }
78
+
79
+ /** Two probes of the work table's lane index: a commit with no unique values runs nothing else. */
80
+ private async stagedUniqueWork(target: PgStagedPlanTarget): Promise<PgUniqueWork> {
81
+ const probe = (kind: "unique-value-release" | "unique-value-claim") => this.sql`
82
+ EXISTS (
83
+ SELECT 1 FROM ${this.sql(target.workTable)}
84
+ WHERE work_id = ${target.workId} AND lane = 'apply'
85
+ AND major_order = ${materializationApplyPhase(kind)}
86
+ AND minor_order = ${materializationPlanKindRank(kind)}
87
+ )
88
+ `
89
+ const [row] = await this.sql<{ readonly releases: boolean; readonly claims: boolean }[]>`
90
+ SELECT ${probe("unique-value-release")} AS releases, ${probe("unique-value-claim")} AS claims
91
+ `
92
+ return { releases: row?.releases ?? false, claims: row?.claims ?? false }
65
93
  }
66
94
 
67
95
  /** The staged values of one plan item kind, as `value`. */
@@ -330,6 +358,51 @@ export class PgMaterializationWriter {
330
358
  )
331
359
  }
332
360
 
361
+ /** The staged unique values of one item kind, as the columns their table keys them by. */
362
+ private stagedUniqueValues(
363
+ target: PgStagedPlanTarget,
364
+ kind: "unique-value-release" | "unique-value-claim"
365
+ ) {
366
+ return this.sql`
367
+ SELECT value->>'objectTypeId' AS object_type_id, value->>'propertyId' AS property_id,
368
+ value->>'valueKey' AS value_key, value->>'primaryId' AS primary_id,
369
+ value->>'keyKind' AS key_kind, (value->>'keyVersion')::integer AS key_version
370
+ FROM (${this.staged(target, kind)}) AS staged
371
+ `
372
+ }
373
+
374
+ /** A value whose row already names another object, or none, is simply no longer this one's. */
375
+ private async releaseUniqueValues(target: PgStagedPlanTarget): Promise<void> {
376
+ await this.sql`
377
+ WITH candidates AS MATERIALIZED (
378
+ ${this.stagedUniqueValues(target, "unique-value-release")}
379
+ ), released AS (
380
+ SELECT candidates.object_type_id, candidates.property_id, held.value_hash
381
+ FROM candidates
382
+ CROSS JOIN LATERAL (
383
+ ${heldValue(this.sql, target.projectId)}
384
+ AND stored.primary_id = candidates.primary_id
385
+ LIMIT 1
386
+ ) AS held
387
+ )
388
+ DELETE FROM object_unique_values AS stored USING released
389
+ WHERE stored.project_id = ${target.projectId}
390
+ AND stored.object_type_id = released.object_type_id
391
+ AND stored.property_id = released.property_id
392
+ AND stored.value_hash = released.value_hash
393
+ `
394
+ }
395
+
396
+ /** Runs once every release of the commit has applied, so values can move between objects. */
397
+ private async claimUniqueValues(target: PgStagedPlanTarget): Promise<number> {
398
+ return claimPgUniqueValues(
399
+ this.sql,
400
+ target.projectId,
401
+ this.stagedUniqueValues(target, "unique-value-claim"),
402
+ { requirePrepared: true }
403
+ )
404
+ }
405
+
333
406
  private async deleteLinks(target: PgStagedPlanTarget): Promise<number> {
334
407
  const candidates = this.staged(target, "link-delete")
335
408
  return this.write(
@@ -586,6 +659,8 @@ export class PgMaterializationWriter {
586
659
  END AS ref
587
660
  ) AS entity
588
661
  WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
662
+ -- Unique values are no plan input: a stale claim fails when it applies.
663
+ AND entity.kind IS NOT NULL
589
664
  `
590
665
  }
591
666
 
@@ -63,6 +63,7 @@ import {
63
63
  toIsoString,
64
64
  } from "./shared"
65
65
  import { activateSourceRoots } from "./source-roots"
66
+ import { pgUniqueHolders } from "./unique-values"
66
67
 
67
68
  interface TelemetrySummaryRow {
68
69
  readonly classified_points: number | string
@@ -203,6 +204,17 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
203
204
  yield { objects: [], links: [], linkScopes: [], points: stored }
204
205
  }
205
206
  }
207
+ const uniqueValues = request.uniqueValues ?? []
208
+ for (let offset = 0; offset < uniqueValues.length; offset += input.pageRows) {
209
+ const uniqueHolders = await pgUniqueHolders(
210
+ this.sql,
211
+ session.header.commit.projectId,
212
+ uniqueValues.slice(offset, offset + input.pageRows)
213
+ )
214
+ if (uniqueHolders.length > 0) {
215
+ yield { objects: [], links: [], linkScopes: [], points: [], uniqueHolders }
216
+ }
217
+ }
206
218
  }
207
219
  }
208
220
 
@@ -238,9 +250,11 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
238
250
  projectId: commit.projectId,
239
251
  commitId: commit.id,
240
252
  committedAt: commit.committedAt,
253
+ ...(session.plan ? {} : { uniqueWork: session.uniqueWork }),
241
254
  })
242
255
  session.changedObjects = applied.objectWrites
243
256
  session.changedLinks = applied.linkWrites
257
+ session.changedUniqueValues = applied.uniqueValueWrites
244
258
  session.appliedEventCount = applied.eventCount
245
259
  return { eventCount: applied.eventCount }
246
260
  }
@@ -256,6 +270,11 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
256
270
  // With empty/stale statistics, a link lookup can scan every edge sharing its target.
257
271
  await refreshMaterializedStatistics(this.sql, "objects", session.changedObjects)
258
272
  await refreshMaterializedStatistics(this.sql, "links", session.changedLinks)
273
+ await refreshMaterializedStatistics(
274
+ this.sql,
275
+ "object_unique_values",
276
+ session.changedUniqueValues
277
+ )
259
278
  const record = await this.insertCommit(session.header, input)
260
279
  await this.sessions.release(session)
261
280
  return { commit: record }
@@ -743,7 +762,7 @@ function databaseCount(value: number | string | undefined): number {
743
762
  * count can still include a previous transaction's manually analyzed writes at this point. */
744
763
  async function refreshMaterializedStatistics(
745
764
  sql: SQLClient,
746
- table: "objects" | "links",
765
+ table: "objects" | "links" | "object_unique_values",
747
766
  changed: number
748
767
  ): Promise<void> {
749
768
  if (changed < 1_000) return
@@ -50,11 +50,13 @@ const IDENTITIES = "ontology_replacement_plan_identities"
50
50
  const SORT_KEY = "encode(convert_to(identity_key, 'UTF8'), 'hex')"
51
51
 
52
52
  /**
53
- * Rows a plan writes at once before its tables are analyzed again. Autovacuum lags behind a bulk
54
- * write, and with the statistics of near-empty tables the planner scans a plan's whole work for
55
- * each keyed lookup instead of seeking it.
53
+ * Rows a plan writes before its tables are analyzed again: identities when it opens or expands,
54
+ * work once a stream has staged it. Every plan fills and empties the same tables, so their
55
+ * statistics describe the plans that were there when they were last analyzed. Until they describe
56
+ * this one, the planner expects none of its rows and reads its whole work for each key it looks
57
+ * up. Below this many rows, those reads cost less than an ANALYZE.
56
58
  */
57
- const ANALYZE_ROWS = 10_000
59
+ const ANALYZE_ROWS = 1_000
58
60
 
59
61
  /** The plan a session applies: its work rows carry the candidate's source version as `work_id`. */
60
62
  export interface PgReplacementPlan {
@@ -240,7 +242,7 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
240
242
  input: StreamReplacementPlanStateInput
241
243
  ): AsyncIterable<ReplacementPlanStatePage> {
242
244
  assertPageRows(input.pageRows)
243
- await runPgTransaction(this.sql, async (sql) => {
245
+ const versionId = await runPgTransaction(this.sql, async (sql) => {
244
246
  const { candidate } = await requirePlan(sql, input)
245
247
  if (input.entityKind === "object" && candidate.projection_kind !== "object") {
246
248
  throw new MaterializationConflictError(
@@ -258,6 +260,7 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
258
260
  const expanded = await expandLinks(sql, candidate.version_id, input.projectId, "pending")
259
261
  if (expanded >= ANALYZE_ROWS && !isWithinPgTransaction(this.sql)) await analyzePlans(sql)
260
262
  }
263
+ return candidate.version_id
261
264
  })
262
265
  let after = ""
263
266
  let streamed = 0
@@ -296,8 +299,8 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
296
299
  if (!page) {
297
300
  // Every page is staged by now: later lookups and the commit join this work. Never in the
298
301
  // commit transaction, where ANALYZE would hold a lock other commits queue behind.
299
- if (streamed >= ANALYZE_ROWS && !isWithinPgTransaction(this.sql)) {
300
- await analyzePlans(this.sql)
302
+ if (streamed > 0 && !isWithinPgTransaction(this.sql)) {
303
+ await analyzeStagedWork(this.sql, versionId)
301
304
  }
302
305
  return
303
306
  }
@@ -994,6 +997,21 @@ async function analyzePlans(sql: SQLClient): Promise<void> {
994
997
  await sql`ANALYZE ${sql(IDENTITIES)}, ${sql(PG_PLAN_WORK_TABLE)}`
995
998
  }
996
999
 
1000
+ /**
1001
+ * Analyzes the plan tables once the plan holds `ANALYZE_ROWS` rows of work. Counted in work rows,
1002
+ * not identities: an object stages its classification, its existence, its change and its event.
1003
+ */
1004
+ async function analyzeStagedWork(sql: SQLClient, versionId: string): Promise<void> {
1005
+ const [staged] = await sql<{ readonly rows: number | string }[]>`
1006
+ SELECT COUNT(*) AS rows FROM (
1007
+ SELECT 1 FROM ${sql(PG_PLAN_WORK_TABLE)} WHERE work_id = ${versionId} LIMIT ${ANALYZE_ROWS}
1008
+ ) AS work
1009
+ `
1010
+
1011
+ if (Number(staged?.rows ?? 0) < ANALYZE_ROWS) return
1012
+ await analyzePlans(sql)
1013
+ }
1014
+
997
1015
  /**
998
1016
  * Deletes the work of the identities of `kind` left to plan: an identity to plan holds none. The
999
1017
  * keys come first, so the work is sought by key whatever the plan tables' statistics.
@@ -19,6 +19,7 @@ import type {
19
19
  AbandonRunSourceMaterializationInput,
20
20
  AbandonSourceMaterializationCandidateInput,
21
21
  AbandonSourceMaterializationInput,
22
+ ActiveOntologySource,
22
23
  AdoptedSourceMaterialization,
23
24
  AdoptSourceMaterializationInput,
24
25
  AssertSourceMaterializationExecutionInput,
@@ -26,6 +27,7 @@ import type {
26
27
  CleanupTerminalSourceMaterializationsInput,
27
28
  CleanupTerminalSourceMaterializationsResult,
28
29
  GetActiveOntologySourceInput,
30
+ ListActiveOntologySourcesInput,
29
31
  MarkSourceMaterializationReadyInput,
30
32
  OntologySourceRecord,
31
33
  OntologySourceStorage,
@@ -237,6 +239,32 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
237
239
  })
238
240
  }
239
241
 
242
+ async listActive(
243
+ input: ListActiveOntologySourcesInput
244
+ ): Promise<readonly ActiveOntologySource[]> {
245
+ return this.runRootOperation(async (sql) => {
246
+ assertNonblank(input.projectId, "Source project id")
247
+
248
+ // Served by the partial index on active sources.
249
+ const rows = await sql<ActiveSourceRow[]>`
250
+ SELECT source_id, dataset_id, dataset_version_id, dataset_version_created_at,
251
+ projection_revision
252
+ FROM ontology_sources
253
+ WHERE project_id = ${input.projectId} AND status = 'active'
254
+ `
255
+
256
+ return rows.map((row) => ({
257
+ source: { projectionId: row.source_id },
258
+ datasetVersion: {
259
+ datasetId: row.dataset_id,
260
+ versionId: row.dataset_version_id,
261
+ createdAt: toIsoString(row.dataset_version_created_at),
262
+ },
263
+ projectionRevision: row.projection_revision,
264
+ }))
265
+ })
266
+ }
267
+
240
268
  async adopt(
241
269
  input: AdoptSourceMaterializationInput
242
270
  ): Promise<AdoptedSourceMaterialization | null> {
@@ -620,3 +648,12 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
620
648
  function numberOrNull(value: number | string | null): number | null {
621
649
  return value === null ? null : Number(value)
622
650
  }
651
+
652
+ type ActiveSourceRow = Pick<
653
+ PgOntologySourceRow,
654
+ | "source_id"
655
+ | "dataset_id"
656
+ | "dataset_version_id"
657
+ | "dataset_version_created_at"
658
+ | "projection_revision"
659
+ >
@@ -0,0 +1,274 @@
1
+ import {
2
+ uniquePropertyNotPrepared,
3
+ uniqueValueViolation,
4
+ } from "@sixb/core/internal/ontology-storage-provider"
5
+ import type {
6
+ ExactUniqueValueWrite,
7
+ OntologyUniqueValueStorage,
8
+ PreparedUniqueProperty,
9
+ UniqueValueEntry,
10
+ UniqueValueRef,
11
+ } from "@sixb/core/storage"
12
+ import type postgres from "postgres"
13
+ import type { SQLClient } from "../pg-client"
14
+ import { jsonParameter, type PgRootOperation } from "./shared"
15
+
16
+ type Query = postgres.PendingQuery<postgres.Row[]>
17
+
18
+ interface PgClaimCounts {
19
+ readonly staged: number | string
20
+ readonly written: number | string
21
+ readonly unprepared_type: string | null
22
+ readonly unprepared_property: string | null
23
+ }
24
+
25
+ /**
26
+ * Gives each candidate value to its object, set-based. `candidates` selects `object_type_id`,
27
+ * `property_id`, `value_key` and `primary_id`, and with `requirePrepared` also the claim's
28
+ * `key_kind` and `key_version`, which must match how the property was prepared.
29
+ *
30
+ * The insert goes first and conflicts do nothing, as the plan writer's statements do: only when
31
+ * it writes fewer rows than it was given does a second query look for the holder to name. A value
32
+ * another object holds, or that two candidates give to different objects, throws
33
+ * `UniqueValueViolationError`; a value already held by its own object is no conflict. A value a
34
+ * concurrent transaction claimed after this one's snapshot fails serialization instead, so the
35
+ * commit retries and then finds it held.
36
+ */
37
+ export async function claimPgUniqueValues(
38
+ sql: SQLClient,
39
+ projectId: string,
40
+ candidates: Query,
41
+ options: { readonly requirePrepared: boolean }
42
+ ): Promise<number> {
43
+ const unprepared = options.requirePrepared
44
+ ? sql`
45
+ SELECT claimed.object_type_id, claimed.property_id
46
+ FROM (
47
+ SELECT DISTINCT object_type_id, property_id, key_kind, key_version FROM candidates
48
+ ) AS claimed
49
+ WHERE NOT EXISTS (
50
+ SELECT 1 FROM object_unique_properties AS prepared
51
+ WHERE prepared.project_id = ${projectId}
52
+ AND prepared.object_type_id = claimed.object_type_id
53
+ AND prepared.property_id = claimed.property_id
54
+ AND prepared.key_kind = claimed.key_kind
55
+ AND prepared.key_version = claimed.key_version
56
+ )
57
+ LIMIT 1
58
+ `
59
+ : sql`SELECT NULL::text AS object_type_id, NULL::text AS property_id WHERE false`
60
+
61
+ const [counts] = await sql<PgClaimCounts[]>`
62
+ WITH candidates AS MATERIALIZED (${candidates}),
63
+ unprepared AS (${unprepared}),
64
+ written AS (
65
+ INSERT INTO object_unique_values (
66
+ project_id, object_type_id, property_id, value_hash, value_key, primary_id
67
+ )
68
+ SELECT ${projectId}, object_type_id, property_id,
69
+ sha256(convert_to(value_key, 'UTF8')), value_key, primary_id
70
+ FROM candidates
71
+ WHERE NOT EXISTS (SELECT 1 FROM unprepared)
72
+ ON CONFLICT DO NOTHING
73
+ RETURNING 1
74
+ )
75
+ SELECT (SELECT COUNT(*) FROM candidates) AS staged, (SELECT COUNT(*) FROM written) AS written,
76
+ (SELECT object_type_id FROM unprepared) AS unprepared_type,
77
+ (SELECT property_id FROM unprepared) AS unprepared_property
78
+ `
79
+
80
+ if (counts?.unprepared_type && counts.unprepared_property) {
81
+ throw uniquePropertyNotPrepared({
82
+ objectTypeId: counts.unprepared_type,
83
+ propertyId: counts.unprepared_property,
84
+ })
85
+ }
86
+
87
+ const written = Number(counts?.written ?? 0)
88
+ if (written !== Number(counts?.staged ?? 0)) await assertNoOtherHolder(sql, projectId, candidates)
89
+ return written
90
+ }
91
+
92
+ /** Names a candidate whose value another object holds, now that every candidate was inserted. */
93
+ async function assertNoOtherHolder(
94
+ sql: SQLClient,
95
+ projectId: string,
96
+ candidates: Query
97
+ ): Promise<void> {
98
+ const [conflict] = await sql<
99
+ {
100
+ readonly object_type_id: string
101
+ readonly property_id: string
102
+ readonly holder: string
103
+ readonly claimant: string
104
+ }[]
105
+ >`
106
+ WITH candidates AS MATERIALIZED (${candidates})
107
+ SELECT candidates.object_type_id, candidates.property_id,
108
+ held.primary_id AS holder, candidates.primary_id AS claimant
109
+ FROM candidates
110
+ CROSS JOIN LATERAL (${heldValue(sql, projectId)} LIMIT 1) AS held
111
+ WHERE held.primary_id <> candidates.primary_id
112
+ LIMIT 1
113
+ `
114
+ if (!conflict) return
115
+
116
+ throw uniqueValueViolation(
117
+ { objectTypeId: conflict.object_type_id, propertyId: conflict.property_id },
118
+ conflict.holder,
119
+ conflict.claimant
120
+ )
121
+ }
122
+
123
+ /**
124
+ * The stored row of `candidates`' value, by its whole primary key: a lateral lookup per candidate
125
+ * keeps the planner on the index whatever it guesses of the table.
126
+ */
127
+ export function heldValue(sql: SQLClient, projectId: string) {
128
+ return sql`
129
+ SELECT stored.primary_id, stored.value_hash
130
+ FROM object_unique_values AS stored
131
+ WHERE stored.project_id = ${projectId}
132
+ AND stored.object_type_id = candidates.object_type_id
133
+ AND stored.property_id = candidates.property_id
134
+ AND stored.value_hash = sha256(convert_to(candidates.value_key, 'UTF8'))
135
+ AND stored.value_key = candidates.value_key
136
+ `
137
+ }
138
+
139
+ /** The objects holding these unique values; a value no object holds is left out. */
140
+ export async function pgUniqueHolders(
141
+ sql: SQLClient,
142
+ projectId: string,
143
+ values: readonly UniqueValueRef[]
144
+ ): Promise<ExactUniqueValueWrite[]> {
145
+ if (values.length === 0) return []
146
+ const requested = values.map((value) => ({
147
+ object_type_id: value.objectTypeId,
148
+ property_id: value.propertyId,
149
+ value_key: value.valueKey,
150
+ }))
151
+ const rows = await sql<
152
+ {
153
+ readonly object_type_id: string
154
+ readonly property_id: string
155
+ readonly value_key: string
156
+ readonly primary_id: string
157
+ }[]
158
+ >`
159
+ WITH candidates AS MATERIALIZED (
160
+ SELECT * FROM jsonb_to_recordset(${jsonParameter(sql, requested)}::jsonb)
161
+ AS requested(object_type_id text, property_id text, value_key text)
162
+ )
163
+ SELECT candidates.object_type_id, candidates.property_id, candidates.value_key,
164
+ held.primary_id
165
+ FROM candidates
166
+ CROSS JOIN LATERAL (${heldValue(sql, projectId)} LIMIT 1) AS held
167
+ `
168
+ return rows.map((row) => ({
169
+ objectTypeId: row.object_type_id,
170
+ propertyId: row.property_id,
171
+ valueKey: row.value_key,
172
+ primaryId: row.primary_id,
173
+ }))
174
+ }
175
+
176
+ export class PgOntologyUniqueValueStorage implements OntologyUniqueValueStorage {
177
+ constructor(private readonly runRootOperation: PgRootOperation) {}
178
+
179
+ /**
180
+ * `LOCK TABLE` takes no snapshot, so as the transaction's first statement it waits before the
181
+ * serializable snapshot is taken, and a preparer that waited sees the values the previous one
182
+ * built instead of building them again. An advisory lock would be taken by a `SELECT`, after
183
+ * the snapshot. Only preparation writes this table, so claims, which read it, never wait.
184
+ */
185
+ async lockPreparation(_input: { readonly projectId: string }): Promise<void> {
186
+ await this.runRootOperation(async (sql) => {
187
+ await sql`LOCK TABLE object_unique_properties IN SHARE ROW EXCLUSIVE MODE`
188
+ })
189
+ }
190
+
191
+ async listPrepared(input: {
192
+ readonly projectId: string
193
+ }): Promise<readonly PreparedUniqueProperty[]> {
194
+ return this.runRootOperation(async (sql) => {
195
+ const rows = await sql<
196
+ {
197
+ readonly object_type_id: string
198
+ readonly property_id: string
199
+ readonly key_kind: PreparedUniqueProperty["keyKind"]
200
+ readonly key_version: number
201
+ }[]
202
+ >`
203
+ SELECT object_type_id, property_id, key_kind, key_version FROM object_unique_properties
204
+ WHERE project_id = ${input.projectId}
205
+ ORDER BY object_type_id, property_id
206
+ `
207
+ return rows.map((row) => ({
208
+ objectTypeId: row.object_type_id,
209
+ propertyId: row.property_id,
210
+ keyKind: row.key_kind,
211
+ keyVersion: row.key_version,
212
+ }))
213
+ })
214
+ }
215
+
216
+ async clear(input: {
217
+ readonly projectId: string
218
+ readonly objectTypeId: string
219
+ readonly propertyId: string
220
+ }): Promise<void> {
221
+ await this.runRootOperation(async (sql) => {
222
+ await sql`
223
+ DELETE FROM object_unique_values
224
+ WHERE project_id = ${input.projectId} AND object_type_id = ${input.objectTypeId}
225
+ AND property_id = ${input.propertyId}
226
+ `
227
+ await sql`
228
+ DELETE FROM object_unique_properties
229
+ WHERE project_id = ${input.projectId} AND object_type_id = ${input.objectTypeId}
230
+ AND property_id = ${input.propertyId}
231
+ `
232
+ })
233
+ }
234
+
235
+ async insert(input: {
236
+ readonly projectId: string
237
+ readonly objectTypeId: string
238
+ readonly propertyId: string
239
+ readonly entries: readonly UniqueValueEntry[]
240
+ }): Promise<void> {
241
+ await this.runRootOperation((sql) =>
242
+ claimPgUniqueValues(
243
+ sql,
244
+ input.projectId,
245
+ sql`
246
+ SELECT ${input.objectTypeId}::text AS object_type_id,
247
+ ${input.propertyId}::text AS property_id,
248
+ entry->>'valueKey' AS value_key, entry->>'primaryId' AS primary_id
249
+ FROM jsonb_array_elements(${jsonParameter(sql, input.entries)}::jsonb) AS entry
250
+ `,
251
+ // The values are being prepared: nothing to require yet.
252
+ { requirePrepared: false }
253
+ )
254
+ )
255
+ }
256
+
257
+ async markPrepared(
258
+ input: PreparedUniqueProperty & { readonly projectId: string; readonly preparedAt: string }
259
+ ): Promise<void> {
260
+ await this.runRootOperation(async (sql) => {
261
+ await sql`
262
+ INSERT INTO object_unique_properties (
263
+ project_id, object_type_id, property_id, key_kind, key_version, prepared_at
264
+ ) VALUES (
265
+ ${input.projectId}, ${input.objectTypeId}, ${input.propertyId}, ${input.keyKind},
266
+ ${input.keyVersion}, ${input.preparedAt}::timestamptz
267
+ )
268
+ ON CONFLICT (project_id, object_type_id, property_id)
269
+ DO UPDATE SET key_kind = EXCLUDED.key_kind, key_version = EXCLUDED.key_version,
270
+ prepared_at = EXCLUDED.prepared_at
271
+ `
272
+ })
273
+ }
274
+ }
@@ -136,6 +136,29 @@ export class PgActionRunStorage implements ActionRunStorage {
136
136
  return row ? rowToActionRunRecord(row) : null
137
137
  }
138
138
 
139
+ async getByExecutionIds(params: {
140
+ readonly projectId: string
141
+ readonly executionIds: readonly string[]
142
+ }): Promise<readonly ActionRunRecord[]> {
143
+ const executionIds = [...new Set(params.executionIds)]
144
+ if (executionIds.length === 0) {
145
+ return []
146
+ }
147
+
148
+ const rows = await this.sql<DatabaseRow[]>`
149
+ SELECT *
150
+ FROM action_runs
151
+ WHERE project_id = ${params.projectId}
152
+ AND execution_id = ANY(${this.sql.array(executionIds)}::text[])
153
+ `
154
+
155
+ const byExecution = new Map(rows.map((row) => [row.execution_id, row]))
156
+ return executionIds.flatMap((executionId) => {
157
+ const row = byExecution.get(executionId)
158
+ return row ? [rowToActionRunRecord(row)] : []
159
+ })
160
+ }
161
+
139
162
  async list(input: ListActionRunsInput): Promise<ListActionRunsResult> {
140
163
  if ((input.statuses && input.statuses.length === 0) || input.actionIds?.length === 0) {
141
164
  return {