@sixb/pg 0.1.20 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/agents/agent-storage.d.ts +8 -2
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/approvals.d.ts +48 -0
- package/dist/agents/approvals.d.ts.map +1 -0
- package/dist/agents/checkpoints.d.ts +1 -0
- package/dist/agents/checkpoints.d.ts.map +1 -1
- package/dist/agents/index.d.ts +1 -0
- package/dist/agents/index.d.ts.map +1 -1
- package/dist/agents/memory.d.ts +19 -0
- package/dist/agents/memory.d.ts.map +1 -0
- package/dist/agents/messages.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +15 -1
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts +3 -1
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/agents/settings.d.ts +29 -0
- package/dist/agents/settings.d.ts.map +1 -0
- package/dist/agents/threads.d.ts +10 -1
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/ai-cost-groups.d.ts.map +1 -1
- package/dist/auth-storage/auth-storage.d.ts +3 -2
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/sessions.d.ts +2 -1
- package/dist/auth-storage/sessions.d.ts.map +1 -1
- package/dist/auth-storage/users.d.ts +8 -4
- package/dist/auth-storage/users.d.ts.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6001 -2923
- package/dist/index.js.map +45 -37
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/conditional-reader.d.ts +17 -0
- package/dist/objects/conditional-reader.d.ts.map +1 -0
- package/dist/objects/query-compiler.d.ts +27 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts +10 -1
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +2 -0
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/scoped-reader.d.ts +12 -0
- package/dist/objects/scoped-reader.d.ts.map +1 -0
- package/dist/objects/selected-reader.d.ts +1 -0
- package/dist/objects/selected-reader.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +7 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/ontology-storage/index.d.ts +2 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +6 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +15 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/replacement-plans.d.ts.map +1 -1
- package/dist/ontology-storage/sources.d.ts +2 -1
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/ontology-storage/unique-values.d.ts +60 -0
- package/dist/ontology-storage/unique-values.d.ts.map +1 -0
- package/dist/pg-action-run-storage.d.ts +4 -0
- package/dist/pg-action-run-storage.d.ts.map +1 -1
- package/dist/pg-activity-storage.d.ts +22 -0
- package/dist/pg-activity-storage.d.ts.map +1 -0
- package/dist/pg-ai-cost-storage.d.ts.map +1 -1
- package/dist/pg-ai-usage-storage.d.ts +8 -1
- package/dist/pg-ai-usage-storage.d.ts.map +1 -1
- package/dist/pg-artifact-storage.d.ts +26 -0
- package/dist/pg-artifact-storage.d.ts.map +1 -0
- package/dist/pg-execution-storage.d.ts +3 -1
- package/dist/pg-execution-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +4 -0
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/share-grant-storage.d.ts +2 -1
- package/dist/share-grant-storage.d.ts.map +1 -1
- package/dist/storage-errors.d.ts +2 -0
- package/dist/storage-errors.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/agent-storage.ts +11 -2
- package/src/agents/approvals.ts +400 -0
- package/src/agents/checkpoints.ts +8 -4
- package/src/agents/index.ts +1 -0
- package/src/agents/memory.ts +175 -0
- package/src/agents/messages.ts +14 -7
- package/src/agents/rows.ts +46 -1
- package/src/agents/runs.ts +154 -22
- package/src/agents/settings.ts +105 -0
- package/src/agents/threads.ts +379 -3
- package/src/ai-cost-groups.ts +11 -1
- package/src/auth-storage/auth-storage.ts +45 -13
- package/src/auth-storage/sessions.ts +20 -0
- package/src/auth-storage/users.ts +105 -43
- package/src/index.ts +10 -0
- package/src/migrations/063-artifacts.sql +67 -0
- package/src/migrations/064-drop-output-diagnostics.sql +14 -0
- package/src/migrations/065-plan-work-payload-statistics.sql +5 -0
- package/src/migrations/066-activity.sql +61 -0
- package/src/migrations/067-activity-timeline.sql +15 -0
- package/src/migrations/068-activity-interactions.sql +20 -0
- package/src/migrations/069-artifact-kinds.sql +4 -0
- package/src/migrations/070-artifact-state.sql +16 -0
- package/src/migrations/071-agent-thread-participants.sql +16 -0
- package/src/migrations/072-forked-context-checkpoints.sql +23 -0
- package/src/migrations/073-unique-values.sql +25 -0
- package/src/migrations/074-generated-media-usage.sql +21 -0
- package/src/migrations/075-model-call-files.sql +11 -0
- package/src/migrations/076-agent-memory.sql +33 -0
- package/src/migrations/077-artifact-threads.sql +21 -0
- package/src/migrations/078-agent-approvals.sql +58 -0
- package/src/migrations/079-agent-approval-resume.sql +20 -0
- package/src/migrations/080-subagent-threads.sql +145 -0
- package/src/migrations/081-subagent-questions.sql +6 -0
- package/src/migrations.ts +52 -0
- package/src/objects/conditional-reader.ts +44 -0
- package/src/objects/prepare-queries.ts +1 -1
- package/src/objects/query-compiler.ts +96 -33
- package/src/objects/query-indexes.ts +29 -10
- package/src/objects/read-scope.ts +140 -25
- package/src/objects/reader.ts +50 -6
- package/src/objects/scoped-reader.ts +101 -0
- package/src/objects/selected-reader.ts +16 -76
- package/src/objects/storage.ts +24 -5
- package/src/objects/text-count-index.ts +1 -0
- package/src/ontology-storage/index.ts +3 -0
- package/src/ontology-storage/materialization-session.ts +8 -0
- package/src/ontology-storage/materialization-writer.ts +76 -1
- package/src/ontology-storage/materializations.ts +20 -1
- package/src/ontology-storage/replacement-plans.ts +25 -7
- package/src/ontology-storage/sources.ts +37 -0
- package/src/ontology-storage/unique-values.ts +274 -0
- package/src/pg-action-run-storage.ts +23 -0
- package/src/pg-activity-storage.ts +486 -0
- package/src/pg-ai-cost-storage.ts +62 -12
- package/src/pg-ai-usage-storage.ts +90 -1
- package/src/pg-artifact-storage.ts +521 -0
- package/src/pg-execution-storage.ts +83 -0
- package/src/pg-workflow-run-storage.ts +23 -0
- package/src/share-grant-storage.ts +21 -0
- package/src/storage-errors.ts +7 -0
- package/src/transactions.ts +1 -1
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
import type { MaterializationPlanWorkItem } from "@sixb/core/storage"
|
|
17
17
|
import type postgres from "postgres"
|
|
18
18
|
import type { SQL, SQLClient } from "../pg-client"
|
|
19
|
+
import { claimPgUniqueValues, heldValue } from "./unique-values"
|
|
19
20
|
|
|
20
21
|
export interface PgStagedPlanTarget {
|
|
21
22
|
/** The work table and the id its rows carry: a session's temp table, or a durable plan. */
|
|
@@ -24,11 +25,19 @@ export interface PgStagedPlanTarget {
|
|
|
24
25
|
readonly projectId: string
|
|
25
26
|
readonly commitId: string
|
|
26
27
|
readonly committedAt: string
|
|
28
|
+
/** Whether unique value items were staged; probed in the work table when unknown. */
|
|
29
|
+
readonly uniqueWork?: PgUniqueWork
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface PgUniqueWork {
|
|
33
|
+
readonly releases: boolean
|
|
34
|
+
readonly claims: boolean
|
|
27
35
|
}
|
|
28
36
|
|
|
29
37
|
export interface PgAppliedPlan {
|
|
30
38
|
readonly objectWrites: number
|
|
31
39
|
readonly linkWrites: number
|
|
40
|
+
readonly uniqueValueWrites: number
|
|
32
41
|
readonly eventCount: number
|
|
33
42
|
}
|
|
34
43
|
|
|
@@ -55,13 +64,32 @@ export class PgMaterializationWriter {
|
|
|
55
64
|
await this.applyObjectOverrides(target)
|
|
56
65
|
await this.applyLinkOverrides(target)
|
|
57
66
|
await this.applyPoints(target)
|
|
67
|
+
const uniqueWork = target.uniqueWork ?? (await this.stagedUniqueWork(target))
|
|
68
|
+
if (uniqueWork.releases) await this.releaseUniqueValues(target)
|
|
58
69
|
let linkWrites = await this.deleteLinks(target)
|
|
59
70
|
let objectWrites = await this.deleteObjects(target)
|
|
60
71
|
objectWrites += await this.upsertObjects(target)
|
|
61
72
|
linkWrites += await this.upsertLinks(target)
|
|
73
|
+
const uniqueValueWrites = uniqueWork.claims ? await this.claimUniqueValues(target) : 0
|
|
62
74
|
const eventCount = await this.writeOutbox(target)
|
|
63
75
|
await this.writeTouches(target)
|
|
64
|
-
return { objectWrites, linkWrites, eventCount }
|
|
76
|
+
return { objectWrites, linkWrites, uniqueValueWrites, eventCount }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Two probes of the work table's lane index: a commit with no unique values runs nothing else. */
|
|
80
|
+
private async stagedUniqueWork(target: PgStagedPlanTarget): Promise<PgUniqueWork> {
|
|
81
|
+
const probe = (kind: "unique-value-release" | "unique-value-claim") => this.sql`
|
|
82
|
+
EXISTS (
|
|
83
|
+
SELECT 1 FROM ${this.sql(target.workTable)}
|
|
84
|
+
WHERE work_id = ${target.workId} AND lane = 'apply'
|
|
85
|
+
AND major_order = ${materializationApplyPhase(kind)}
|
|
86
|
+
AND minor_order = ${materializationPlanKindRank(kind)}
|
|
87
|
+
)
|
|
88
|
+
`
|
|
89
|
+
const [row] = await this.sql<{ readonly releases: boolean; readonly claims: boolean }[]>`
|
|
90
|
+
SELECT ${probe("unique-value-release")} AS releases, ${probe("unique-value-claim")} AS claims
|
|
91
|
+
`
|
|
92
|
+
return { releases: row?.releases ?? false, claims: row?.claims ?? false }
|
|
65
93
|
}
|
|
66
94
|
|
|
67
95
|
/** The staged values of one plan item kind, as `value`. */
|
|
@@ -330,6 +358,51 @@ export class PgMaterializationWriter {
|
|
|
330
358
|
)
|
|
331
359
|
}
|
|
332
360
|
|
|
361
|
+
/** The staged unique values of one item kind, as the columns their table keys them by. */
|
|
362
|
+
private stagedUniqueValues(
|
|
363
|
+
target: PgStagedPlanTarget,
|
|
364
|
+
kind: "unique-value-release" | "unique-value-claim"
|
|
365
|
+
) {
|
|
366
|
+
return this.sql`
|
|
367
|
+
SELECT value->>'objectTypeId' AS object_type_id, value->>'propertyId' AS property_id,
|
|
368
|
+
value->>'valueKey' AS value_key, value->>'primaryId' AS primary_id,
|
|
369
|
+
value->>'keyKind' AS key_kind, (value->>'keyVersion')::integer AS key_version
|
|
370
|
+
FROM (${this.staged(target, kind)}) AS staged
|
|
371
|
+
`
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/** A value whose row already names another object, or none, is simply no longer this one's. */
|
|
375
|
+
private async releaseUniqueValues(target: PgStagedPlanTarget): Promise<void> {
|
|
376
|
+
await this.sql`
|
|
377
|
+
WITH candidates AS MATERIALIZED (
|
|
378
|
+
${this.stagedUniqueValues(target, "unique-value-release")}
|
|
379
|
+
), released AS (
|
|
380
|
+
SELECT candidates.object_type_id, candidates.property_id, held.value_hash
|
|
381
|
+
FROM candidates
|
|
382
|
+
CROSS JOIN LATERAL (
|
|
383
|
+
${heldValue(this.sql, target.projectId)}
|
|
384
|
+
AND stored.primary_id = candidates.primary_id
|
|
385
|
+
LIMIT 1
|
|
386
|
+
) AS held
|
|
387
|
+
)
|
|
388
|
+
DELETE FROM object_unique_values AS stored USING released
|
|
389
|
+
WHERE stored.project_id = ${target.projectId}
|
|
390
|
+
AND stored.object_type_id = released.object_type_id
|
|
391
|
+
AND stored.property_id = released.property_id
|
|
392
|
+
AND stored.value_hash = released.value_hash
|
|
393
|
+
`
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/** Runs once every release of the commit has applied, so values can move between objects. */
|
|
397
|
+
private async claimUniqueValues(target: PgStagedPlanTarget): Promise<number> {
|
|
398
|
+
return claimPgUniqueValues(
|
|
399
|
+
this.sql,
|
|
400
|
+
target.projectId,
|
|
401
|
+
this.stagedUniqueValues(target, "unique-value-claim"),
|
|
402
|
+
{ requirePrepared: true }
|
|
403
|
+
)
|
|
404
|
+
}
|
|
405
|
+
|
|
333
406
|
private async deleteLinks(target: PgStagedPlanTarget): Promise<number> {
|
|
334
407
|
const candidates = this.staged(target, "link-delete")
|
|
335
408
|
return this.write(
|
|
@@ -586,6 +659,8 @@ export class PgMaterializationWriter {
|
|
|
586
659
|
END AS ref
|
|
587
660
|
) AS entity
|
|
588
661
|
WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
|
|
662
|
+
-- Unique values are no plan input: a stale claim fails when it applies.
|
|
663
|
+
AND entity.kind IS NOT NULL
|
|
589
664
|
`
|
|
590
665
|
}
|
|
591
666
|
|
|
@@ -63,6 +63,7 @@ import {
|
|
|
63
63
|
toIsoString,
|
|
64
64
|
} from "./shared"
|
|
65
65
|
import { activateSourceRoots } from "./source-roots"
|
|
66
|
+
import { pgUniqueHolders } from "./unique-values"
|
|
66
67
|
|
|
67
68
|
interface TelemetrySummaryRow {
|
|
68
69
|
readonly classified_points: number | string
|
|
@@ -203,6 +204,17 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
203
204
|
yield { objects: [], links: [], linkScopes: [], points: stored }
|
|
204
205
|
}
|
|
205
206
|
}
|
|
207
|
+
const uniqueValues = request.uniqueValues ?? []
|
|
208
|
+
for (let offset = 0; offset < uniqueValues.length; offset += input.pageRows) {
|
|
209
|
+
const uniqueHolders = await pgUniqueHolders(
|
|
210
|
+
this.sql,
|
|
211
|
+
session.header.commit.projectId,
|
|
212
|
+
uniqueValues.slice(offset, offset + input.pageRows)
|
|
213
|
+
)
|
|
214
|
+
if (uniqueHolders.length > 0) {
|
|
215
|
+
yield { objects: [], links: [], linkScopes: [], points: [], uniqueHolders }
|
|
216
|
+
}
|
|
217
|
+
}
|
|
206
218
|
}
|
|
207
219
|
}
|
|
208
220
|
|
|
@@ -238,9 +250,11 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
238
250
|
projectId: commit.projectId,
|
|
239
251
|
commitId: commit.id,
|
|
240
252
|
committedAt: commit.committedAt,
|
|
253
|
+
...(session.plan ? {} : { uniqueWork: session.uniqueWork }),
|
|
241
254
|
})
|
|
242
255
|
session.changedObjects = applied.objectWrites
|
|
243
256
|
session.changedLinks = applied.linkWrites
|
|
257
|
+
session.changedUniqueValues = applied.uniqueValueWrites
|
|
244
258
|
session.appliedEventCount = applied.eventCount
|
|
245
259
|
return { eventCount: applied.eventCount }
|
|
246
260
|
}
|
|
@@ -256,6 +270,11 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
256
270
|
// With empty/stale statistics, a link lookup can scan every edge sharing its target.
|
|
257
271
|
await refreshMaterializedStatistics(this.sql, "objects", session.changedObjects)
|
|
258
272
|
await refreshMaterializedStatistics(this.sql, "links", session.changedLinks)
|
|
273
|
+
await refreshMaterializedStatistics(
|
|
274
|
+
this.sql,
|
|
275
|
+
"object_unique_values",
|
|
276
|
+
session.changedUniqueValues
|
|
277
|
+
)
|
|
259
278
|
const record = await this.insertCommit(session.header, input)
|
|
260
279
|
await this.sessions.release(session)
|
|
261
280
|
return { commit: record }
|
|
@@ -743,7 +762,7 @@ function databaseCount(value: number | string | undefined): number {
|
|
|
743
762
|
* count can still include a previous transaction's manually analyzed writes at this point. */
|
|
744
763
|
async function refreshMaterializedStatistics(
|
|
745
764
|
sql: SQLClient,
|
|
746
|
-
table: "objects" | "links",
|
|
765
|
+
table: "objects" | "links" | "object_unique_values",
|
|
747
766
|
changed: number
|
|
748
767
|
): Promise<void> {
|
|
749
768
|
if (changed < 1_000) return
|
|
@@ -50,11 +50,13 @@ const IDENTITIES = "ontology_replacement_plan_identities"
|
|
|
50
50
|
const SORT_KEY = "encode(convert_to(identity_key, 'UTF8'), 'hex')"
|
|
51
51
|
|
|
52
52
|
/**
|
|
53
|
-
* Rows a plan writes
|
|
54
|
-
*
|
|
55
|
-
*
|
|
53
|
+
* Rows a plan writes before its tables are analyzed again: identities when it opens or expands,
|
|
54
|
+
* work once a stream has staged it. Every plan fills and empties the same tables, so their
|
|
55
|
+
* statistics describe the plans that were there when they were last analyzed. Until they describe
|
|
56
|
+
* this one, the planner expects none of its rows and reads its whole work for each key it looks
|
|
57
|
+
* up. Below this many rows, those reads cost less than an ANALYZE.
|
|
56
58
|
*/
|
|
57
|
-
const ANALYZE_ROWS =
|
|
59
|
+
const ANALYZE_ROWS = 1_000
|
|
58
60
|
|
|
59
61
|
/** The plan a session applies: its work rows carry the candidate's source version as `work_id`. */
|
|
60
62
|
export interface PgReplacementPlan {
|
|
@@ -240,7 +242,7 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
|
|
|
240
242
|
input: StreamReplacementPlanStateInput
|
|
241
243
|
): AsyncIterable<ReplacementPlanStatePage> {
|
|
242
244
|
assertPageRows(input.pageRows)
|
|
243
|
-
await runPgTransaction(this.sql, async (sql) => {
|
|
245
|
+
const versionId = await runPgTransaction(this.sql, async (sql) => {
|
|
244
246
|
const { candidate } = await requirePlan(sql, input)
|
|
245
247
|
if (input.entityKind === "object" && candidate.projection_kind !== "object") {
|
|
246
248
|
throw new MaterializationConflictError(
|
|
@@ -258,6 +260,7 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
|
|
|
258
260
|
const expanded = await expandLinks(sql, candidate.version_id, input.projectId, "pending")
|
|
259
261
|
if (expanded >= ANALYZE_ROWS && !isWithinPgTransaction(this.sql)) await analyzePlans(sql)
|
|
260
262
|
}
|
|
263
|
+
return candidate.version_id
|
|
261
264
|
})
|
|
262
265
|
let after = ""
|
|
263
266
|
let streamed = 0
|
|
@@ -296,8 +299,8 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
|
|
|
296
299
|
if (!page) {
|
|
297
300
|
// Every page is staged by now: later lookups and the commit join this work. Never in the
|
|
298
301
|
// commit transaction, where ANALYZE would hold a lock other commits queue behind.
|
|
299
|
-
if (streamed
|
|
300
|
-
await
|
|
302
|
+
if (streamed > 0 && !isWithinPgTransaction(this.sql)) {
|
|
303
|
+
await analyzeStagedWork(this.sql, versionId)
|
|
301
304
|
}
|
|
302
305
|
return
|
|
303
306
|
}
|
|
@@ -994,6 +997,21 @@ async function analyzePlans(sql: SQLClient): Promise<void> {
|
|
|
994
997
|
await sql`ANALYZE ${sql(IDENTITIES)}, ${sql(PG_PLAN_WORK_TABLE)}`
|
|
995
998
|
}
|
|
996
999
|
|
|
1000
|
+
/**
|
|
1001
|
+
* Analyzes the plan tables once the plan holds `ANALYZE_ROWS` rows of work. Counted in work rows,
|
|
1002
|
+
* not identities: an object stages its classification, its existence, its change and its event.
|
|
1003
|
+
*/
|
|
1004
|
+
async function analyzeStagedWork(sql: SQLClient, versionId: string): Promise<void> {
|
|
1005
|
+
const [staged] = await sql<{ readonly rows: number | string }[]>`
|
|
1006
|
+
SELECT COUNT(*) AS rows FROM (
|
|
1007
|
+
SELECT 1 FROM ${sql(PG_PLAN_WORK_TABLE)} WHERE work_id = ${versionId} LIMIT ${ANALYZE_ROWS}
|
|
1008
|
+
) AS work
|
|
1009
|
+
`
|
|
1010
|
+
|
|
1011
|
+
if (Number(staged?.rows ?? 0) < ANALYZE_ROWS) return
|
|
1012
|
+
await analyzePlans(sql)
|
|
1013
|
+
}
|
|
1014
|
+
|
|
997
1015
|
/**
|
|
998
1016
|
* Deletes the work of the identities of `kind` left to plan: an identity to plan holds none. The
|
|
999
1017
|
* keys come first, so the work is sought by key whatever the plan tables' statistics.
|
|
@@ -19,6 +19,7 @@ import type {
|
|
|
19
19
|
AbandonRunSourceMaterializationInput,
|
|
20
20
|
AbandonSourceMaterializationCandidateInput,
|
|
21
21
|
AbandonSourceMaterializationInput,
|
|
22
|
+
ActiveOntologySource,
|
|
22
23
|
AdoptedSourceMaterialization,
|
|
23
24
|
AdoptSourceMaterializationInput,
|
|
24
25
|
AssertSourceMaterializationExecutionInput,
|
|
@@ -26,6 +27,7 @@ import type {
|
|
|
26
27
|
CleanupTerminalSourceMaterializationsInput,
|
|
27
28
|
CleanupTerminalSourceMaterializationsResult,
|
|
28
29
|
GetActiveOntologySourceInput,
|
|
30
|
+
ListActiveOntologySourcesInput,
|
|
29
31
|
MarkSourceMaterializationReadyInput,
|
|
30
32
|
OntologySourceRecord,
|
|
31
33
|
OntologySourceStorage,
|
|
@@ -237,6 +239,32 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
237
239
|
})
|
|
238
240
|
}
|
|
239
241
|
|
|
242
|
+
async listActive(
|
|
243
|
+
input: ListActiveOntologySourcesInput
|
|
244
|
+
): Promise<readonly ActiveOntologySource[]> {
|
|
245
|
+
return this.runRootOperation(async (sql) => {
|
|
246
|
+
assertNonblank(input.projectId, "Source project id")
|
|
247
|
+
|
|
248
|
+
// Served by the partial index on active sources.
|
|
249
|
+
const rows = await sql<ActiveSourceRow[]>`
|
|
250
|
+
SELECT source_id, dataset_id, dataset_version_id, dataset_version_created_at,
|
|
251
|
+
projection_revision
|
|
252
|
+
FROM ontology_sources
|
|
253
|
+
WHERE project_id = ${input.projectId} AND status = 'active'
|
|
254
|
+
`
|
|
255
|
+
|
|
256
|
+
return rows.map((row) => ({
|
|
257
|
+
source: { projectionId: row.source_id },
|
|
258
|
+
datasetVersion: {
|
|
259
|
+
datasetId: row.dataset_id,
|
|
260
|
+
versionId: row.dataset_version_id,
|
|
261
|
+
createdAt: toIsoString(row.dataset_version_created_at),
|
|
262
|
+
},
|
|
263
|
+
projectionRevision: row.projection_revision,
|
|
264
|
+
}))
|
|
265
|
+
})
|
|
266
|
+
}
|
|
267
|
+
|
|
240
268
|
async adopt(
|
|
241
269
|
input: AdoptSourceMaterializationInput
|
|
242
270
|
): Promise<AdoptedSourceMaterialization | null> {
|
|
@@ -620,3 +648,12 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
620
648
|
function numberOrNull(value: number | string | null): number | null {
|
|
621
649
|
return value === null ? null : Number(value)
|
|
622
650
|
}
|
|
651
|
+
|
|
652
|
+
type ActiveSourceRow = Pick<
|
|
653
|
+
PgOntologySourceRow,
|
|
654
|
+
| "source_id"
|
|
655
|
+
| "dataset_id"
|
|
656
|
+
| "dataset_version_id"
|
|
657
|
+
| "dataset_version_created_at"
|
|
658
|
+
| "projection_revision"
|
|
659
|
+
>
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import {
|
|
2
|
+
uniquePropertyNotPrepared,
|
|
3
|
+
uniqueValueViolation,
|
|
4
|
+
} from "@sixb/core/internal/ontology-storage-provider"
|
|
5
|
+
import type {
|
|
6
|
+
ExactUniqueValueWrite,
|
|
7
|
+
OntologyUniqueValueStorage,
|
|
8
|
+
PreparedUniqueProperty,
|
|
9
|
+
UniqueValueEntry,
|
|
10
|
+
UniqueValueRef,
|
|
11
|
+
} from "@sixb/core/storage"
|
|
12
|
+
import type postgres from "postgres"
|
|
13
|
+
import type { SQLClient } from "../pg-client"
|
|
14
|
+
import { jsonParameter, type PgRootOperation } from "./shared"
|
|
15
|
+
|
|
16
|
+
type Query = postgres.PendingQuery<postgres.Row[]>
|
|
17
|
+
|
|
18
|
+
interface PgClaimCounts {
|
|
19
|
+
readonly staged: number | string
|
|
20
|
+
readonly written: number | string
|
|
21
|
+
readonly unprepared_type: string | null
|
|
22
|
+
readonly unprepared_property: string | null
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Gives each candidate value to its object, set-based. `candidates` selects `object_type_id`,
|
|
27
|
+
* `property_id`, `value_key` and `primary_id`, and with `requirePrepared` also the claim's
|
|
28
|
+
* `key_kind` and `key_version`, which must match how the property was prepared.
|
|
29
|
+
*
|
|
30
|
+
* The insert goes first and conflicts do nothing, as the plan writer's statements do: only when
|
|
31
|
+
* it writes fewer rows than it was given does a second query look for the holder to name. A value
|
|
32
|
+
* another object holds, or that two candidates give to different objects, throws
|
|
33
|
+
* `UniqueValueViolationError`; a value already held by its own object is no conflict. A value a
|
|
34
|
+
* concurrent transaction claimed after this one's snapshot fails serialization instead, so the
|
|
35
|
+
* commit retries and then finds it held.
|
|
36
|
+
*/
|
|
37
|
+
export async function claimPgUniqueValues(
|
|
38
|
+
sql: SQLClient,
|
|
39
|
+
projectId: string,
|
|
40
|
+
candidates: Query,
|
|
41
|
+
options: { readonly requirePrepared: boolean }
|
|
42
|
+
): Promise<number> {
|
|
43
|
+
const unprepared = options.requirePrepared
|
|
44
|
+
? sql`
|
|
45
|
+
SELECT claimed.object_type_id, claimed.property_id
|
|
46
|
+
FROM (
|
|
47
|
+
SELECT DISTINCT object_type_id, property_id, key_kind, key_version FROM candidates
|
|
48
|
+
) AS claimed
|
|
49
|
+
WHERE NOT EXISTS (
|
|
50
|
+
SELECT 1 FROM object_unique_properties AS prepared
|
|
51
|
+
WHERE prepared.project_id = ${projectId}
|
|
52
|
+
AND prepared.object_type_id = claimed.object_type_id
|
|
53
|
+
AND prepared.property_id = claimed.property_id
|
|
54
|
+
AND prepared.key_kind = claimed.key_kind
|
|
55
|
+
AND prepared.key_version = claimed.key_version
|
|
56
|
+
)
|
|
57
|
+
LIMIT 1
|
|
58
|
+
`
|
|
59
|
+
: sql`SELECT NULL::text AS object_type_id, NULL::text AS property_id WHERE false`
|
|
60
|
+
|
|
61
|
+
const [counts] = await sql<PgClaimCounts[]>`
|
|
62
|
+
WITH candidates AS MATERIALIZED (${candidates}),
|
|
63
|
+
unprepared AS (${unprepared}),
|
|
64
|
+
written AS (
|
|
65
|
+
INSERT INTO object_unique_values (
|
|
66
|
+
project_id, object_type_id, property_id, value_hash, value_key, primary_id
|
|
67
|
+
)
|
|
68
|
+
SELECT ${projectId}, object_type_id, property_id,
|
|
69
|
+
sha256(convert_to(value_key, 'UTF8')), value_key, primary_id
|
|
70
|
+
FROM candidates
|
|
71
|
+
WHERE NOT EXISTS (SELECT 1 FROM unprepared)
|
|
72
|
+
ON CONFLICT DO NOTHING
|
|
73
|
+
RETURNING 1
|
|
74
|
+
)
|
|
75
|
+
SELECT (SELECT COUNT(*) FROM candidates) AS staged, (SELECT COUNT(*) FROM written) AS written,
|
|
76
|
+
(SELECT object_type_id FROM unprepared) AS unprepared_type,
|
|
77
|
+
(SELECT property_id FROM unprepared) AS unprepared_property
|
|
78
|
+
`
|
|
79
|
+
|
|
80
|
+
if (counts?.unprepared_type && counts.unprepared_property) {
|
|
81
|
+
throw uniquePropertyNotPrepared({
|
|
82
|
+
objectTypeId: counts.unprepared_type,
|
|
83
|
+
propertyId: counts.unprepared_property,
|
|
84
|
+
})
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const written = Number(counts?.written ?? 0)
|
|
88
|
+
if (written !== Number(counts?.staged ?? 0)) await assertNoOtherHolder(sql, projectId, candidates)
|
|
89
|
+
return written
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Names a candidate whose value another object holds, now that every candidate was inserted. */
|
|
93
|
+
async function assertNoOtherHolder(
|
|
94
|
+
sql: SQLClient,
|
|
95
|
+
projectId: string,
|
|
96
|
+
candidates: Query
|
|
97
|
+
): Promise<void> {
|
|
98
|
+
const [conflict] = await sql<
|
|
99
|
+
{
|
|
100
|
+
readonly object_type_id: string
|
|
101
|
+
readonly property_id: string
|
|
102
|
+
readonly holder: string
|
|
103
|
+
readonly claimant: string
|
|
104
|
+
}[]
|
|
105
|
+
>`
|
|
106
|
+
WITH candidates AS MATERIALIZED (${candidates})
|
|
107
|
+
SELECT candidates.object_type_id, candidates.property_id,
|
|
108
|
+
held.primary_id AS holder, candidates.primary_id AS claimant
|
|
109
|
+
FROM candidates
|
|
110
|
+
CROSS JOIN LATERAL (${heldValue(sql, projectId)} LIMIT 1) AS held
|
|
111
|
+
WHERE held.primary_id <> candidates.primary_id
|
|
112
|
+
LIMIT 1
|
|
113
|
+
`
|
|
114
|
+
if (!conflict) return
|
|
115
|
+
|
|
116
|
+
throw uniqueValueViolation(
|
|
117
|
+
{ objectTypeId: conflict.object_type_id, propertyId: conflict.property_id },
|
|
118
|
+
conflict.holder,
|
|
119
|
+
conflict.claimant
|
|
120
|
+
)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* The stored row of `candidates`' value, by its whole primary key: a lateral lookup per candidate
|
|
125
|
+
* keeps the planner on the index whatever it guesses of the table.
|
|
126
|
+
*/
|
|
127
|
+
export function heldValue(sql: SQLClient, projectId: string) {
|
|
128
|
+
return sql`
|
|
129
|
+
SELECT stored.primary_id, stored.value_hash
|
|
130
|
+
FROM object_unique_values AS stored
|
|
131
|
+
WHERE stored.project_id = ${projectId}
|
|
132
|
+
AND stored.object_type_id = candidates.object_type_id
|
|
133
|
+
AND stored.property_id = candidates.property_id
|
|
134
|
+
AND stored.value_hash = sha256(convert_to(candidates.value_key, 'UTF8'))
|
|
135
|
+
AND stored.value_key = candidates.value_key
|
|
136
|
+
`
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** The objects holding these unique values; a value no object holds is left out. */
|
|
140
|
+
export async function pgUniqueHolders(
|
|
141
|
+
sql: SQLClient,
|
|
142
|
+
projectId: string,
|
|
143
|
+
values: readonly UniqueValueRef[]
|
|
144
|
+
): Promise<ExactUniqueValueWrite[]> {
|
|
145
|
+
if (values.length === 0) return []
|
|
146
|
+
const requested = values.map((value) => ({
|
|
147
|
+
object_type_id: value.objectTypeId,
|
|
148
|
+
property_id: value.propertyId,
|
|
149
|
+
value_key: value.valueKey,
|
|
150
|
+
}))
|
|
151
|
+
const rows = await sql<
|
|
152
|
+
{
|
|
153
|
+
readonly object_type_id: string
|
|
154
|
+
readonly property_id: string
|
|
155
|
+
readonly value_key: string
|
|
156
|
+
readonly primary_id: string
|
|
157
|
+
}[]
|
|
158
|
+
>`
|
|
159
|
+
WITH candidates AS MATERIALIZED (
|
|
160
|
+
SELECT * FROM jsonb_to_recordset(${jsonParameter(sql, requested)}::jsonb)
|
|
161
|
+
AS requested(object_type_id text, property_id text, value_key text)
|
|
162
|
+
)
|
|
163
|
+
SELECT candidates.object_type_id, candidates.property_id, candidates.value_key,
|
|
164
|
+
held.primary_id
|
|
165
|
+
FROM candidates
|
|
166
|
+
CROSS JOIN LATERAL (${heldValue(sql, projectId)} LIMIT 1) AS held
|
|
167
|
+
`
|
|
168
|
+
return rows.map((row) => ({
|
|
169
|
+
objectTypeId: row.object_type_id,
|
|
170
|
+
propertyId: row.property_id,
|
|
171
|
+
valueKey: row.value_key,
|
|
172
|
+
primaryId: row.primary_id,
|
|
173
|
+
}))
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export class PgOntologyUniqueValueStorage implements OntologyUniqueValueStorage {
|
|
177
|
+
constructor(private readonly runRootOperation: PgRootOperation) {}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* `LOCK TABLE` takes no snapshot, so as the transaction's first statement it waits before the
|
|
181
|
+
* serializable snapshot is taken, and a preparer that waited sees the values the previous one
|
|
182
|
+
* built instead of building them again. An advisory lock would be taken by a `SELECT`, after
|
|
183
|
+
* the snapshot. Only preparation writes this table, so claims, which read it, never wait.
|
|
184
|
+
*/
|
|
185
|
+
async lockPreparation(_input: { readonly projectId: string }): Promise<void> {
|
|
186
|
+
await this.runRootOperation(async (sql) => {
|
|
187
|
+
await sql`LOCK TABLE object_unique_properties IN SHARE ROW EXCLUSIVE MODE`
|
|
188
|
+
})
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async listPrepared(input: {
|
|
192
|
+
readonly projectId: string
|
|
193
|
+
}): Promise<readonly PreparedUniqueProperty[]> {
|
|
194
|
+
return this.runRootOperation(async (sql) => {
|
|
195
|
+
const rows = await sql<
|
|
196
|
+
{
|
|
197
|
+
readonly object_type_id: string
|
|
198
|
+
readonly property_id: string
|
|
199
|
+
readonly key_kind: PreparedUniqueProperty["keyKind"]
|
|
200
|
+
readonly key_version: number
|
|
201
|
+
}[]
|
|
202
|
+
>`
|
|
203
|
+
SELECT object_type_id, property_id, key_kind, key_version FROM object_unique_properties
|
|
204
|
+
WHERE project_id = ${input.projectId}
|
|
205
|
+
ORDER BY object_type_id, property_id
|
|
206
|
+
`
|
|
207
|
+
return rows.map((row) => ({
|
|
208
|
+
objectTypeId: row.object_type_id,
|
|
209
|
+
propertyId: row.property_id,
|
|
210
|
+
keyKind: row.key_kind,
|
|
211
|
+
keyVersion: row.key_version,
|
|
212
|
+
}))
|
|
213
|
+
})
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async clear(input: {
|
|
217
|
+
readonly projectId: string
|
|
218
|
+
readonly objectTypeId: string
|
|
219
|
+
readonly propertyId: string
|
|
220
|
+
}): Promise<void> {
|
|
221
|
+
await this.runRootOperation(async (sql) => {
|
|
222
|
+
await sql`
|
|
223
|
+
DELETE FROM object_unique_values
|
|
224
|
+
WHERE project_id = ${input.projectId} AND object_type_id = ${input.objectTypeId}
|
|
225
|
+
AND property_id = ${input.propertyId}
|
|
226
|
+
`
|
|
227
|
+
await sql`
|
|
228
|
+
DELETE FROM object_unique_properties
|
|
229
|
+
WHERE project_id = ${input.projectId} AND object_type_id = ${input.objectTypeId}
|
|
230
|
+
AND property_id = ${input.propertyId}
|
|
231
|
+
`
|
|
232
|
+
})
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
async insert(input: {
|
|
236
|
+
readonly projectId: string
|
|
237
|
+
readonly objectTypeId: string
|
|
238
|
+
readonly propertyId: string
|
|
239
|
+
readonly entries: readonly UniqueValueEntry[]
|
|
240
|
+
}): Promise<void> {
|
|
241
|
+
await this.runRootOperation((sql) =>
|
|
242
|
+
claimPgUniqueValues(
|
|
243
|
+
sql,
|
|
244
|
+
input.projectId,
|
|
245
|
+
sql`
|
|
246
|
+
SELECT ${input.objectTypeId}::text AS object_type_id,
|
|
247
|
+
${input.propertyId}::text AS property_id,
|
|
248
|
+
entry->>'valueKey' AS value_key, entry->>'primaryId' AS primary_id
|
|
249
|
+
FROM jsonb_array_elements(${jsonParameter(sql, input.entries)}::jsonb) AS entry
|
|
250
|
+
`,
|
|
251
|
+
// The values are being prepared: nothing to require yet.
|
|
252
|
+
{ requirePrepared: false }
|
|
253
|
+
)
|
|
254
|
+
)
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
async markPrepared(
|
|
258
|
+
input: PreparedUniqueProperty & { readonly projectId: string; readonly preparedAt: string }
|
|
259
|
+
): Promise<void> {
|
|
260
|
+
await this.runRootOperation(async (sql) => {
|
|
261
|
+
await sql`
|
|
262
|
+
INSERT INTO object_unique_properties (
|
|
263
|
+
project_id, object_type_id, property_id, key_kind, key_version, prepared_at
|
|
264
|
+
) VALUES (
|
|
265
|
+
${input.projectId}, ${input.objectTypeId}, ${input.propertyId}, ${input.keyKind},
|
|
266
|
+
${input.keyVersion}, ${input.preparedAt}::timestamptz
|
|
267
|
+
)
|
|
268
|
+
ON CONFLICT (project_id, object_type_id, property_id)
|
|
269
|
+
DO UPDATE SET key_kind = EXCLUDED.key_kind, key_version = EXCLUDED.key_version,
|
|
270
|
+
prepared_at = EXCLUDED.prepared_at
|
|
271
|
+
`
|
|
272
|
+
})
|
|
273
|
+
}
|
|
274
|
+
}
|
|
@@ -136,6 +136,29 @@ export class PgActionRunStorage implements ActionRunStorage {
|
|
|
136
136
|
return row ? rowToActionRunRecord(row) : null
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
+
async getByExecutionIds(params: {
|
|
140
|
+
readonly projectId: string
|
|
141
|
+
readonly executionIds: readonly string[]
|
|
142
|
+
}): Promise<readonly ActionRunRecord[]> {
|
|
143
|
+
const executionIds = [...new Set(params.executionIds)]
|
|
144
|
+
if (executionIds.length === 0) {
|
|
145
|
+
return []
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const rows = await this.sql<DatabaseRow[]>`
|
|
149
|
+
SELECT *
|
|
150
|
+
FROM action_runs
|
|
151
|
+
WHERE project_id = ${params.projectId}
|
|
152
|
+
AND execution_id = ANY(${this.sql.array(executionIds)}::text[])
|
|
153
|
+
`
|
|
154
|
+
|
|
155
|
+
const byExecution = new Map(rows.map((row) => [row.execution_id, row]))
|
|
156
|
+
return executionIds.flatMap((executionId) => {
|
|
157
|
+
const row = byExecution.get(executionId)
|
|
158
|
+
return row ? [rowToActionRunRecord(row)] : []
|
|
159
|
+
})
|
|
160
|
+
}
|
|
161
|
+
|
|
139
162
|
async list(input: ListActionRunsInput): Promise<ListActionRunsResult> {
|
|
140
163
|
if ((input.statuses && input.statuses.length === 0) || input.actionIds?.length === 0) {
|
|
141
164
|
return {
|