@sixb/pg 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/dist/agents/agent-storage.d.ts +2 -1
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +4 -11
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts +3 -2
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/auth-storage/service-accounts.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts +0 -4
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts +9 -0
- package/dist/connector-connection-storage.d.ts.map +1 -0
- package/dist/index.d.ts +8 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3086 -730
- package/dist/index.js.map +28 -25
- package/dist/latest-run-query.d.ts.map +1 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +0 -2
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/outbox.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +3 -6
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/pg-action-run-storage.d.ts +3 -2
- package/dist/pg-action-run-storage.d.ts.map +1 -1
- package/dist/pg-ai-usage-storage.d.ts +16 -0
- package/dist/pg-ai-usage-storage.d.ts.map +1 -0
- package/dist/pg-execution-storage.d.ts +14 -0
- package/dist/pg-execution-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -2
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +5 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +5 -2
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +5 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-list-query.d.ts +1 -1
- package/dist/run-list-query.d.ts.map +1 -1
- package/dist/testing.d.ts +6 -0
- package/dist/testing.d.ts.map +1 -0
- package/package.json +3 -3
- package/src/agents/agent-storage.ts +3 -2
- package/src/agents/rows.ts +12 -43
- package/src/agents/runs.ts +29 -15
- package/src/auth-storage/service-accounts.ts +14 -15
- package/src/auth-storage/shared.ts +0 -8
- package/src/connector-connection-storage.ts +760 -0
- package/src/index.ts +32 -15
- package/src/latest-run-query.ts +7 -1
- package/src/migrations/004-executions.sql +140 -0
- package/src/migrations/005-workflow-executions.sql +10 -0
- package/src/migrations/006-narrow-ontology-source-root-index.sql +4 -0
- package/src/migrations/007-split-overrides.sql +153 -0
- package/src/migrations/008-action-executions.sql +7 -0
- package/src/migrations/009-agent-executions.sql +21 -0
- package/src/migrations/010-ai-usage-accounting-foundation.sql +76 -0
- package/src/migrations/011-sync-failure-record.sql +23 -0
- package/src/migrations/012-pipeline-failure-record.sql +47 -0
- package/src/migrations/013-workflow-failure-record.sql +45 -0
- package/src/migrations/014-agent-failure-record.sql +56 -0
- package/src/migrations/015-projection-failure-record.sql +20 -0
- package/src/migrations/016-webhook-run-failure-record.sql +19 -0
- package/src/migrations/017-action-failure-record.sql +65 -0
- package/src/migrations/018-ontology-outbox-failure-record.sql +29 -0
- package/src/migrations/019-webhook-delivery-failure-record.sql +27 -0
- package/src/migrations/020-drop-run-usage-projections.sql +8 -0
- package/src/migrations/021-sync-pipeline-executions.sql +50 -0
- package/src/migrations/022-projection-executions.sql +115 -0
- package/src/migrations/023-webhook-executions.sql +30 -0
- package/src/migrations/024-ontology-commit-executions.sql +12 -0
- package/src/migrations/025-connector-connections.sql +231 -0
- package/src/migrations.ts +72 -0
- package/src/ontology-storage/materialization-session.ts +25 -28
- package/src/ontology-storage/materialization-state.ts +159 -43
- package/src/ontology-storage/materialization-writer.ts +146 -75
- package/src/ontology-storage/materializations.ts +3 -1
- package/src/ontology-storage/outbox.ts +9 -2
- package/src/ontology-storage/shared.ts +10 -20
- package/src/pg-action-run-storage.ts +43 -67
- package/src/pg-ai-usage-storage.ts +290 -0
- package/src/pg-execution-storage.ts +147 -0
- package/src/pg-pipeline-run-storage.ts +107 -41
- package/src/pg-projection-run-storage.ts +159 -91
- package/src/pg-sync-run-storage.ts +100 -28
- package/src/pg-webhook-run-storage.ts +89 -57
- package/src/pg-workflow-run-storage.ts +97 -130
- package/src/run-list-query.ts +9 -4
- package/src/testing.ts +18 -0
- package/dist/pg-webhook-delivery-storage.d.ts +0 -17
- package/dist/pg-webhook-delivery-storage.d.ts.map +0 -1
- package/src/pg-webhook-delivery-storage.ts +0 -180
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
import {
|
|
19
19
|
appendScopeSnapshot,
|
|
20
20
|
finishScopeAccumulator,
|
|
21
|
+
linkSlotOverrideValue,
|
|
21
22
|
startScopeAccumulator,
|
|
22
23
|
} from "@sixb/core/internal/ontology-storage-provider"
|
|
23
24
|
import type {
|
|
@@ -27,6 +28,7 @@ import type {
|
|
|
27
28
|
SourceReplacementLinkState,
|
|
28
29
|
SourceReplacementObjectState,
|
|
29
30
|
StoredLinkOverride,
|
|
31
|
+
StoredLinkSlotOverride,
|
|
30
32
|
StoredObjectOverride,
|
|
31
33
|
StoredSourceAssertion,
|
|
32
34
|
StoredSourceLinkAssertion,
|
|
@@ -36,12 +38,11 @@ import type {
|
|
|
36
38
|
import type { SQLClient } from "../pg-client"
|
|
37
39
|
import {
|
|
38
40
|
databaseSafeInteger,
|
|
39
|
-
jsonKeyParameter,
|
|
40
41
|
jsonParameter,
|
|
41
42
|
linkRefFromColumns,
|
|
42
43
|
objectRefFromColumns,
|
|
43
|
-
type PgOntologyOverrideRow,
|
|
44
44
|
type PgOntologySourceAssertionRow,
|
|
45
|
+
type PgStoredOverrideRow,
|
|
45
46
|
toIsoString,
|
|
46
47
|
} from "./shared"
|
|
47
48
|
|
|
@@ -94,12 +95,12 @@ interface LinkScopeRow extends EffectiveLinkRow {
|
|
|
94
95
|
readonly link_sort_key: string
|
|
95
96
|
}
|
|
96
97
|
|
|
97
|
-
interface ObjectOverrideRow extends
|
|
98
|
+
interface ObjectOverrideRow extends PgStoredOverrideRow {
|
|
98
99
|
readonly object_type_id: string
|
|
99
100
|
readonly primary_id: string
|
|
100
101
|
}
|
|
101
102
|
|
|
102
|
-
interface LinkOverrideRow extends
|
|
103
|
+
interface LinkOverrideRow extends PgStoredOverrideRow {
|
|
103
104
|
readonly source_type_id: string
|
|
104
105
|
readonly source_primary_id: string
|
|
105
106
|
readonly link_id: string
|
|
@@ -107,6 +108,17 @@ interface LinkOverrideRow extends PgOntologyOverrideRow {
|
|
|
107
108
|
readonly target_primary_id: string
|
|
108
109
|
}
|
|
109
110
|
|
|
111
|
+
interface LinkSlotOverrideRow {
|
|
112
|
+
readonly source_type_id: string
|
|
113
|
+
readonly source_primary_id: string
|
|
114
|
+
readonly link_id: string
|
|
115
|
+
readonly target_type_id: string
|
|
116
|
+
readonly target_primary_id: string
|
|
117
|
+
readonly value: unknown
|
|
118
|
+
readonly last_commit_id: string
|
|
119
|
+
readonly updated_at: Date | string
|
|
120
|
+
}
|
|
121
|
+
|
|
110
122
|
export type ReplacementIdentity =
|
|
111
123
|
| {
|
|
112
124
|
readonly kind: "object"
|
|
@@ -193,10 +205,9 @@ export class PgMaterializationStateReader {
|
|
|
193
205
|
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
|
|
194
206
|
AS requested_values(object_type_id TEXT, primary_id TEXT)
|
|
195
207
|
)
|
|
196
|
-
SELECT overrides.* FROM
|
|
208
|
+
SELECT overrides.* FROM ontology_object_overrides AS overrides
|
|
197
209
|
JOIN requested USING (object_type_id, primary_id)
|
|
198
210
|
WHERE overrides.project_id = ${this.projectId}
|
|
199
|
-
AND overrides.entity_kind = 'object'
|
|
200
211
|
`,
|
|
201
212
|
this.sql<TelemetryRow[]>`
|
|
202
213
|
WITH requested AS (
|
|
@@ -266,7 +277,7 @@ export class PgMaterializationStateReader {
|
|
|
266
277
|
target_primary_id: ref.target.primaryId,
|
|
267
278
|
}))
|
|
268
279
|
const requestedParameter = jsonParameter(this.sql, requested)
|
|
269
|
-
const [effectiveRows, sourceRows, overrideRows] = await Promise.all([
|
|
280
|
+
const [effectiveRows, sourceRows, overrideRows, slotOverrideRows] = await Promise.all([
|
|
270
281
|
this.sql<EffectiveLinkRow[]>`
|
|
271
282
|
WITH requested AS (
|
|
272
283
|
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
|
|
@@ -305,12 +316,23 @@ export class PgMaterializationStateReader {
|
|
|
305
316
|
target_type_id TEXT, target_primary_id TEXT
|
|
306
317
|
)
|
|
307
318
|
)
|
|
308
|
-
SELECT overrides.* FROM
|
|
319
|
+
SELECT overrides.* FROM ontology_link_overrides AS overrides
|
|
309
320
|
JOIN requested USING (
|
|
310
321
|
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
|
|
311
322
|
)
|
|
312
323
|
WHERE overrides.project_id = ${this.projectId}
|
|
313
|
-
AND overrides.
|
|
324
|
+
AND overrides.identity_kind = 'edge'
|
|
325
|
+
`,
|
|
326
|
+
this.sql<LinkSlotOverrideRow[]>`
|
|
327
|
+
WITH requested AS (
|
|
328
|
+
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
|
|
329
|
+
source_type_id TEXT, source_primary_id TEXT, link_id TEXT
|
|
330
|
+
)
|
|
331
|
+
)
|
|
332
|
+
SELECT overrides.* FROM ontology_link_overrides AS overrides
|
|
333
|
+
JOIN requested USING (source_type_id, source_primary_id, link_id)
|
|
334
|
+
WHERE overrides.project_id = ${this.projectId}
|
|
335
|
+
AND overrides.identity_kind = 'slot'
|
|
314
336
|
`,
|
|
315
337
|
])
|
|
316
338
|
const effective = new Map(
|
|
@@ -322,6 +344,12 @@ export class PgMaterializationStateReader {
|
|
|
322
344
|
(row) => [linkRefKey(linkRefFromOverrideColumns(row)), storedLinkOverride(row)] as const
|
|
323
345
|
)
|
|
324
346
|
)
|
|
347
|
+
const slotOverrides = new Map(
|
|
348
|
+
slotOverrideRows.map((row) => {
|
|
349
|
+
const stored = storedLinkSlotOverride(row)
|
|
350
|
+
return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
|
|
351
|
+
})
|
|
352
|
+
)
|
|
325
353
|
return refs.map((ref) => {
|
|
326
354
|
const key = linkRefKey(ref)
|
|
327
355
|
const effectiveRow = effective.get(key)
|
|
@@ -329,6 +357,7 @@ export class PgMaterializationStateReader {
|
|
|
329
357
|
ref: structuredClone(ref),
|
|
330
358
|
source: sourceLink(sources.get(projectionEntityKey({ kind: "link", ref }))),
|
|
331
359
|
override: overrides.get(key) ?? null,
|
|
360
|
+
slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
|
|
332
361
|
effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null,
|
|
333
362
|
}
|
|
334
363
|
})
|
|
@@ -390,15 +419,63 @@ export class PgMaterializationStateReader {
|
|
|
390
419
|
const accumulators = new Map(
|
|
391
420
|
scopes.map(({ source, linkId }) => {
|
|
392
421
|
const key = linkScopeSortKey(source, linkId)
|
|
393
|
-
return [key, startScopeAccumulator(source, linkId
|
|
422
|
+
return [key, startScopeAccumulator(source, linkId)] as const
|
|
394
423
|
})
|
|
395
424
|
)
|
|
425
|
+
const requestedParameter = jsonParameter(this.sql, requested)
|
|
426
|
+
const [sourceRows, slotOverrideRows] = await Promise.all([
|
|
427
|
+
this.sql<PgOntologySourceAssertionRow[]>`
|
|
428
|
+
WITH requested AS (
|
|
429
|
+
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
|
|
430
|
+
AS requested_values(
|
|
431
|
+
scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
|
|
432
|
+
)
|
|
433
|
+
)
|
|
434
|
+
SELECT rows.*
|
|
435
|
+
FROM ontology_source_rows AS rows
|
|
436
|
+
JOIN requested
|
|
437
|
+
ON requested.source_type_id = rows.source_type_id
|
|
438
|
+
AND requested.source_id = rows.source_primary_id
|
|
439
|
+
AND requested.link_id = rows.link_id
|
|
440
|
+
JOIN ontology_sources AS sources
|
|
441
|
+
ON sources.project_id = rows.project_id
|
|
442
|
+
AND sources.source_id = rows.source_id
|
|
443
|
+
AND sources.materialization_id = rows.materialization_id
|
|
444
|
+
WHERE rows.project_id = ${this.projectId}
|
|
445
|
+
AND rows.entity_kind = 'link'
|
|
446
|
+
AND sources.status = 'active'
|
|
447
|
+
`,
|
|
448
|
+
this.sql<LinkSlotOverrideRow[]>`
|
|
449
|
+
WITH requested AS (
|
|
450
|
+
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
|
|
451
|
+
AS requested_values(
|
|
452
|
+
scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
|
|
453
|
+
)
|
|
454
|
+
)
|
|
455
|
+
SELECT overrides.*
|
|
456
|
+
FROM ontology_link_overrides AS overrides
|
|
457
|
+
JOIN requested
|
|
458
|
+
ON requested.source_type_id = overrides.source_type_id
|
|
459
|
+
AND requested.source_id = overrides.source_primary_id
|
|
460
|
+
AND requested.link_id = overrides.link_id
|
|
461
|
+
WHERE overrides.project_id = ${this.projectId}
|
|
462
|
+
AND overrides.identity_kind = 'slot'
|
|
463
|
+
`,
|
|
464
|
+
])
|
|
465
|
+
const sources = activeScopeSourceMap(sourceRows)
|
|
466
|
+
const overrides = new Map(
|
|
467
|
+
slotOverrideRows.map((row) => {
|
|
468
|
+
const stored = storedLinkSlotOverride(row)
|
|
469
|
+
return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
|
|
470
|
+
})
|
|
471
|
+
)
|
|
472
|
+
const effective = new Map<string, EffectiveLinkSnapshot | null>()
|
|
396
473
|
let scopeCursor: string | null = null
|
|
397
474
|
let linkCursor: string | null = null
|
|
398
475
|
while (true) {
|
|
399
476
|
const rows: LinkScopeRow[] = await this.sql`
|
|
400
477
|
WITH requested AS (
|
|
401
|
-
SELECT DISTINCT * FROM jsonb_to_recordset(${
|
|
478
|
+
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
|
|
402
479
|
AS requested_values(
|
|
403
480
|
scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
|
|
404
481
|
)
|
|
@@ -423,16 +500,24 @@ export class PgMaterializationStateReader {
|
|
|
423
500
|
`Unexpected link scope '${row.scope_sort_key}' returned by storage.`
|
|
424
501
|
)
|
|
425
502
|
}
|
|
426
|
-
|
|
503
|
+
const snapshot = effectiveLinkSnapshot(row)
|
|
504
|
+
appendScopeSnapshot(accumulator, snapshot)
|
|
505
|
+
effective.set(row.scope_sort_key, accumulator.effectiveCount === 1 ? snapshot : null)
|
|
427
506
|
}
|
|
428
507
|
if (rows.length < 500) break
|
|
429
508
|
const last: LinkScopeRow = rows[rows.length - 1]!
|
|
430
509
|
scopeCursor = last.scope_sort_key
|
|
431
510
|
linkCursor = last.link_sort_key
|
|
432
511
|
}
|
|
433
|
-
return scopes.map(({ source, linkId }) =>
|
|
434
|
-
|
|
435
|
-
|
|
512
|
+
return scopes.map(({ source, linkId }) => {
|
|
513
|
+
const key = linkScopeSortKey(source, linkId)
|
|
514
|
+
return {
|
|
515
|
+
...finishScopeAccumulator(accumulators.get(key)!),
|
|
516
|
+
sourceAssertion: sources.get(key) ?? null,
|
|
517
|
+
override: overrides.get(key) ?? null,
|
|
518
|
+
effective: effective.get(key) ?? null,
|
|
519
|
+
}
|
|
520
|
+
})
|
|
436
521
|
}
|
|
437
522
|
|
|
438
523
|
async linkScope(
|
|
@@ -485,20 +570,20 @@ export class PgMaterializationStateReader {
|
|
|
485
570
|
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
|
|
486
571
|
overrides.link_id, overrides.target_type_id,
|
|
487
572
|
overrides.target_primary_id AS target_id
|
|
488
|
-
FROM
|
|
573
|
+
FROM ontology_link_overrides AS overrides
|
|
489
574
|
JOIN requested
|
|
490
575
|
ON requested.object_type_id = overrides.source_type_id
|
|
491
576
|
AND requested.primary_id = overrides.source_primary_id
|
|
492
|
-
WHERE overrides.project_id = ${this.projectId}
|
|
577
|
+
WHERE overrides.project_id = ${this.projectId}
|
|
493
578
|
UNION
|
|
494
579
|
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
|
|
495
580
|
overrides.link_id, overrides.target_type_id,
|
|
496
581
|
overrides.target_primary_id AS target_id
|
|
497
|
-
FROM
|
|
582
|
+
FROM ontology_link_overrides AS overrides
|
|
498
583
|
JOIN requested
|
|
499
584
|
ON requested.object_type_id = overrides.target_type_id
|
|
500
585
|
AND requested.primary_id = overrides.target_primary_id
|
|
501
|
-
WHERE overrides.project_id = ${this.projectId}
|
|
586
|
+
WHERE overrides.project_id = ${this.projectId}
|
|
502
587
|
UNION
|
|
503
588
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
504
589
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
@@ -609,6 +694,7 @@ export class PgMaterializationStateReader {
|
|
|
609
694
|
? sourceLink(candidate?.get(key))
|
|
610
695
|
: state.source,
|
|
611
696
|
override: state.override,
|
|
697
|
+
slotOverride: state.slotOverride,
|
|
612
698
|
effective: state.effective,
|
|
613
699
|
diffRequired: identities[index]!.diffRequired,
|
|
614
700
|
}
|
|
@@ -701,16 +787,6 @@ export class PgMaterializationStateReader {
|
|
|
701
787
|
)
|
|
702
788
|
}
|
|
703
789
|
|
|
704
|
-
async overrideLastCommit(kind: "object" | "link", entityKey: string): Promise<string | null> {
|
|
705
|
-
const [row] = await this.sql<{ readonly last_commit_id: string }[]>`
|
|
706
|
-
SELECT last_commit_id FROM ontology_overrides
|
|
707
|
-
WHERE project_id = ${this.projectId}
|
|
708
|
-
AND entity_kind = ${kind}
|
|
709
|
-
AND entity_key = ${jsonKeyParameter(this.sql, entityKey)}
|
|
710
|
-
`
|
|
711
|
-
return row?.last_commit_id ?? null
|
|
712
|
-
}
|
|
713
|
-
|
|
714
790
|
private async replacementSources(
|
|
715
791
|
sourceId: string,
|
|
716
792
|
materializationIds: readonly string[],
|
|
@@ -824,20 +900,20 @@ export class PgMaterializationStateReader {
|
|
|
824
900
|
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
|
|
825
901
|
overrides.link_id, overrides.target_type_id,
|
|
826
902
|
overrides.target_primary_id AS target_id
|
|
827
|
-
FROM
|
|
903
|
+
FROM ontology_link_overrides AS overrides
|
|
828
904
|
JOIN incident_objects
|
|
829
905
|
ON incident_objects.object_type_id = overrides.source_type_id
|
|
830
906
|
AND incident_objects.primary_id = overrides.source_primary_id
|
|
831
|
-
WHERE overrides.project_id = ${this.projectId}
|
|
907
|
+
WHERE overrides.project_id = ${this.projectId}
|
|
832
908
|
UNION
|
|
833
909
|
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
|
|
834
910
|
overrides.link_id, overrides.target_type_id,
|
|
835
911
|
overrides.target_primary_id AS target_id
|
|
836
|
-
FROM
|
|
912
|
+
FROM ontology_link_overrides AS overrides
|
|
837
913
|
JOIN incident_objects
|
|
838
914
|
ON incident_objects.object_type_id = overrides.target_type_id
|
|
839
915
|
AND incident_objects.primary_id = overrides.target_primary_id
|
|
840
|
-
WHERE overrides.project_id = ${this.projectId}
|
|
916
|
+
WHERE overrides.project_id = ${this.projectId}
|
|
841
917
|
UNION
|
|
842
918
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
843
919
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
@@ -879,6 +955,16 @@ export class PgMaterializationStateReader {
|
|
|
879
955
|
FROM links
|
|
880
956
|
JOIN affected_scopes USING (source_type_id, source_id, link_id)
|
|
881
957
|
WHERE links.project_id = ${this.projectId}
|
|
958
|
+
UNION ALL
|
|
959
|
+
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
|
|
960
|
+
overrides.link_id, overrides.target_type_id,
|
|
961
|
+
overrides.target_primary_id AS target_id, FALSE AS diff_required
|
|
962
|
+
FROM ontology_link_overrides AS overrides
|
|
963
|
+
JOIN affected_scopes
|
|
964
|
+
ON affected_scopes.source_type_id = overrides.source_type_id
|
|
965
|
+
AND affected_scopes.source_id = overrides.source_primary_id
|
|
966
|
+
AND affected_scopes.link_id = overrides.link_id
|
|
967
|
+
WHERE overrides.project_id = ${this.projectId} AND overrides.identity_kind = 'slot'
|
|
882
968
|
), selected AS (
|
|
883
969
|
SELECT source_type_id, source_id, link_id, target_type_id, target_id,
|
|
884
970
|
${this.sql.unsafe(linkKeyExpression("all_links"))} AS identity_key,
|
|
@@ -939,6 +1025,26 @@ function activeSourceMap(
|
|
|
939
1025
|
return result
|
|
940
1026
|
}
|
|
941
1027
|
|
|
1028
|
+
function activeScopeSourceMap(
|
|
1029
|
+
rows: readonly PgOntologySourceAssertionRow[]
|
|
1030
|
+
): ReadonlyMap<string, StoredSourceLinkAssertion> {
|
|
1031
|
+
const result = new Map<string, StoredSourceLinkAssertion>()
|
|
1032
|
+
for (const row of rows) {
|
|
1033
|
+
const source = sourceLink(storedSource(row))
|
|
1034
|
+
if (!source) continue
|
|
1035
|
+
const ref = source.assertion.ref
|
|
1036
|
+
const key = linkScopeSortKey(ref.source, ref.linkId)
|
|
1037
|
+
if (result.has(key)) {
|
|
1038
|
+
throw new MaterializationConflictError(
|
|
1039
|
+
"source-materialization",
|
|
1040
|
+
`Multiple active sources assert cardinality-one link slot ${key}.`
|
|
1041
|
+
)
|
|
1042
|
+
}
|
|
1043
|
+
result.set(key, source)
|
|
1044
|
+
}
|
|
1045
|
+
return result
|
|
1046
|
+
}
|
|
1047
|
+
|
|
942
1048
|
function sourceObject(
|
|
943
1049
|
source: StoredSourceAssertion | undefined
|
|
944
1050
|
): StoredSourceObjectAssertion | null {
|
|
@@ -969,6 +1075,19 @@ function storedLinkOverride(row: LinkOverrideRow): StoredLinkOverride {
|
|
|
969
1075
|
}
|
|
970
1076
|
}
|
|
971
1077
|
|
|
1078
|
+
function storedLinkSlotOverride(row: LinkSlotOverrideRow): StoredLinkSlotOverride {
|
|
1079
|
+
const ref = {
|
|
1080
|
+
source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
|
|
1081
|
+
linkId: row.link_id,
|
|
1082
|
+
}
|
|
1083
|
+
return {
|
|
1084
|
+
ref,
|
|
1085
|
+
value: linkSlotOverrideValue(row.value, linkScopeLabel(ref)),
|
|
1086
|
+
lastCommitId: row.last_commit_id,
|
|
1087
|
+
updatedAt: toIsoString(row.updated_at),
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
|
|
972
1091
|
function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
|
|
973
1092
|
return {
|
|
974
1093
|
source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
|
|
@@ -977,6 +1096,13 @@ function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
|
|
|
977
1096
|
}
|
|
978
1097
|
}
|
|
979
1098
|
|
|
1099
|
+
function linkScopeLabel(scope: {
|
|
1100
|
+
readonly source: OntologyObjectRef
|
|
1101
|
+
readonly linkId: string
|
|
1102
|
+
}): string {
|
|
1103
|
+
return JSON.stringify([scope.source.objectTypeId, scope.source.primaryId, scope.linkId])
|
|
1104
|
+
}
|
|
1105
|
+
|
|
980
1106
|
function storedSource(row: PgOntologySourceAssertionRow): StoredSourceAssertion {
|
|
981
1107
|
return {
|
|
982
1108
|
source: { projectionId: row.source_id },
|
|
@@ -1090,16 +1216,6 @@ function linkKeyExpression(alias?: string): string {
|
|
|
1090
1216
|
])
|
|
1091
1217
|
}
|
|
1092
1218
|
|
|
1093
|
-
export function pointSortExpression(alias?: string): string {
|
|
1094
|
-
const prefix = alias ? `${alias}.` : ""
|
|
1095
|
-
return utf8SortExpression([
|
|
1096
|
-
`to_jsonb(${prefix}object_type_id)::text`,
|
|
1097
|
-
`to_jsonb(${prefix}object_id)::text`,
|
|
1098
|
-
`to_jsonb(${prefix}property_id)::text`,
|
|
1099
|
-
`to_jsonb(to_char(${prefix}at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))::text`,
|
|
1100
|
-
])
|
|
1101
|
-
}
|
|
1102
|
-
|
|
1103
1219
|
function utf8SortExpression(parts: readonly string[]): string {
|
|
1104
1220
|
return `encode(convert_to(${jsonTupleExpression(parts)}, 'UTF8'), 'hex')`
|
|
1105
1221
|
}
|
|
@@ -4,11 +4,7 @@ import {
|
|
|
4
4
|
objectRefKey,
|
|
5
5
|
telemetryPointKey,
|
|
6
6
|
} from "@sixb/core/internal/materialization"
|
|
7
|
-
import {
|
|
8
|
-
effectiveConflict,
|
|
9
|
-
type OverrideEntity,
|
|
10
|
-
overrideEntityColumns as overrideColumns,
|
|
11
|
-
} from "@sixb/core/internal/ontology-storage-provider"
|
|
7
|
+
import { effectiveConflict } from "@sixb/core/internal/ontology-storage-provider"
|
|
12
8
|
import type { MaterializationPlanChunk } from "@sixb/core/storage"
|
|
13
9
|
import type { SQLClient } from "../pg-client"
|
|
14
10
|
import { assertTimestamp, jsonParameter } from "./shared"
|
|
@@ -25,82 +21,171 @@ export class PgMaterializationWriter {
|
|
|
25
21
|
}
|
|
26
22
|
|
|
27
23
|
private async applyOverrides(projectId: string, chunk: MaterializationPlanChunk): Promise<void> {
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const
|
|
37
|
-
if (
|
|
38
|
-
const rows = await this.sql<{ readonly
|
|
24
|
+
const objectUpserts = chunk.overrides.objects.upserts.map((item) => ({
|
|
25
|
+
objectTypeId: item.ref.objectTypeId,
|
|
26
|
+
primaryId: item.ref.primaryId,
|
|
27
|
+
value: item.value,
|
|
28
|
+
lastCommitId: item.lastCommitId,
|
|
29
|
+
updatedAt: item.updatedAt,
|
|
30
|
+
expectedLastCommitId: item.expectedLastCommitId,
|
|
31
|
+
}))
|
|
32
|
+
const objectInserts = objectUpserts.filter((item) => item.expectedLastCommitId === null)
|
|
33
|
+
if (objectInserts.length > 0) {
|
|
34
|
+
const rows = await this.sql<{ readonly object_type_id: string }[]>`
|
|
39
35
|
WITH staged AS (
|
|
40
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql,
|
|
36
|
+
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectInserts)}::jsonb)
|
|
41
37
|
)
|
|
42
|
-
INSERT INTO
|
|
43
|
-
project_id,
|
|
44
|
-
object_type_id, primary_id, source_type_id, source_primary_id,
|
|
45
|
-
link_id, target_type_id, target_primary_id,
|
|
46
|
-
value, last_commit_id, updated_at
|
|
38
|
+
INSERT INTO ontology_object_overrides (
|
|
39
|
+
project_id, object_type_id, primary_id, value, last_commit_id, updated_at
|
|
47
40
|
)
|
|
48
|
-
SELECT ${projectId}, value->>'
|
|
49
|
-
value->>'
|
|
50
|
-
value->>'sourcePrimaryId', value->>'linkId', value->>'targetTypeId',
|
|
51
|
-
value->>'targetPrimaryId', value->'value', value->>'lastCommitId',
|
|
41
|
+
SELECT ${projectId}, value->>'objectTypeId', value->>'primaryId', value->'value',
|
|
42
|
+
value->>'lastCommitId',
|
|
52
43
|
(value->>'updatedAt')::timestamptz
|
|
53
44
|
FROM staged
|
|
54
45
|
ON CONFLICT DO NOTHING
|
|
55
|
-
RETURNING
|
|
46
|
+
RETURNING object_type_id
|
|
56
47
|
`
|
|
57
|
-
if (rows.length !==
|
|
48
|
+
if (rows.length !== objectInserts.length) throw objectOverrideConflict()
|
|
58
49
|
}
|
|
59
50
|
|
|
60
|
-
const
|
|
61
|
-
if (
|
|
62
|
-
const rows = await this.sql<{ readonly
|
|
51
|
+
const objectUpdates = objectUpserts.filter((item) => item.expectedLastCommitId !== null)
|
|
52
|
+
if (objectUpdates.length > 0) {
|
|
53
|
+
const rows = await this.sql<{ readonly object_type_id: string }[]>`
|
|
63
54
|
WITH staged AS (
|
|
64
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql,
|
|
55
|
+
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectUpdates)}::jsonb)
|
|
65
56
|
)
|
|
66
|
-
UPDATE
|
|
57
|
+
UPDATE ontology_object_overrides AS overrides
|
|
67
58
|
SET value = staged.value->'value', last_commit_id = staged.value->>'lastCommitId',
|
|
68
59
|
updated_at = (staged.value->>'updatedAt')::timestamptz
|
|
69
60
|
FROM staged
|
|
70
61
|
WHERE overrides.project_id = ${projectId}
|
|
71
|
-
AND overrides.
|
|
72
|
-
AND overrides.
|
|
62
|
+
AND overrides.object_type_id = staged.value->>'objectTypeId'
|
|
63
|
+
AND overrides.primary_id = staged.value->>'primaryId'
|
|
64
|
+
AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
|
|
65
|
+
RETURNING overrides.object_type_id
|
|
66
|
+
`
|
|
67
|
+
if (rows.length !== objectUpdates.length) throw objectOverrideConflict()
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const objectDeletes = chunk.overrides.objects.deletes.map((item) => ({
|
|
71
|
+
objectTypeId: item.ref.objectTypeId,
|
|
72
|
+
primaryId: item.ref.primaryId,
|
|
73
|
+
expectedLastCommitId: item.expectedLastCommitId,
|
|
74
|
+
}))
|
|
75
|
+
if (objectDeletes.length > 0) {
|
|
76
|
+
const rows = await this.sql<{ readonly object_type_id: string }[]>`
|
|
77
|
+
WITH staged AS (
|
|
78
|
+
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectDeletes)}::jsonb)
|
|
79
|
+
)
|
|
80
|
+
DELETE FROM ontology_object_overrides AS overrides USING staged
|
|
81
|
+
WHERE overrides.project_id = ${projectId}
|
|
82
|
+
AND overrides.object_type_id = staged.value->>'objectTypeId'
|
|
83
|
+
AND overrides.primary_id = staged.value->>'primaryId'
|
|
84
|
+
AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
|
|
85
|
+
RETURNING overrides.object_type_id
|
|
86
|
+
`
|
|
87
|
+
if (rows.length !== objectDeletes.length) throw objectOverrideConflict()
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const linkUpserts = [
|
|
91
|
+
...chunk.overrides.links.edges.upserts.map((item) => ({
|
|
92
|
+
identityKind: "edge" as const,
|
|
93
|
+
identityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
|
|
94
|
+
sourceTypeId: item.ref.source.objectTypeId,
|
|
95
|
+
sourcePrimaryId: item.ref.source.primaryId,
|
|
96
|
+
linkId: item.ref.linkId,
|
|
97
|
+
targetTypeId: item.ref.target.objectTypeId,
|
|
98
|
+
targetPrimaryId: item.ref.target.primaryId,
|
|
99
|
+
value: item.value,
|
|
100
|
+
lastCommitId: item.lastCommitId,
|
|
101
|
+
updatedAt: item.updatedAt,
|
|
102
|
+
expectedLastCommitId: item.expectedLastCommitId,
|
|
103
|
+
})),
|
|
104
|
+
...chunk.overrides.links.slots.upserts.map((item) => ({
|
|
105
|
+
identityKind: "slot" as const,
|
|
106
|
+
identityKey: [item.ref.source.objectTypeId, item.ref.source.primaryId, item.ref.linkId],
|
|
107
|
+
sourceTypeId: item.ref.source.objectTypeId,
|
|
108
|
+
sourcePrimaryId: item.ref.source.primaryId,
|
|
109
|
+
linkId: item.ref.linkId,
|
|
110
|
+
targetTypeId: item.value.target.objectTypeId,
|
|
111
|
+
targetPrimaryId: item.value.target.primaryId,
|
|
112
|
+
value: item.value,
|
|
113
|
+
lastCommitId: item.lastCommitId,
|
|
114
|
+
updatedAt: item.updatedAt,
|
|
115
|
+
expectedLastCommitId: item.expectedLastCommitId,
|
|
116
|
+
})),
|
|
117
|
+
]
|
|
118
|
+
const linkInserts = linkUpserts.filter((item) => item.expectedLastCommitId === null)
|
|
119
|
+
if (linkInserts.length > 0) {
|
|
120
|
+
const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
|
|
121
|
+
WITH staged AS (
|
|
122
|
+
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkInserts)}::jsonb)
|
|
123
|
+
)
|
|
124
|
+
INSERT INTO ontology_link_overrides (
|
|
125
|
+
project_id, identity_kind, identity_key, source_type_id, source_primary_id, link_id,
|
|
126
|
+
target_type_id, target_primary_id, value, last_commit_id, updated_at
|
|
127
|
+
)
|
|
128
|
+
SELECT ${projectId}, value->>'identityKind', value->'identityKey',
|
|
129
|
+
value->>'sourceTypeId', value->>'sourcePrimaryId', value->>'linkId',
|
|
130
|
+
value->>'targetTypeId', value->>'targetPrimaryId', value->'value',
|
|
131
|
+
value->>'lastCommitId', (value->>'updatedAt')::timestamptz
|
|
132
|
+
FROM staged
|
|
133
|
+
ON CONFLICT DO NOTHING
|
|
134
|
+
RETURNING identity_kind
|
|
135
|
+
`
|
|
136
|
+
if (rows.length !== linkInserts.length)
|
|
137
|
+
throw linkOverrideConflict(linkInserts[0]!.identityKind)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const linkUpdates = linkUpserts.filter((item) => item.expectedLastCommitId !== null)
|
|
141
|
+
if (linkUpdates.length > 0) {
|
|
142
|
+
const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
|
|
143
|
+
WITH staged AS (
|
|
144
|
+
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkUpdates)}::jsonb)
|
|
145
|
+
)
|
|
146
|
+
UPDATE ontology_link_overrides AS overrides
|
|
147
|
+
SET target_type_id = staged.value->>'targetTypeId',
|
|
148
|
+
target_primary_id = staged.value->>'targetPrimaryId',
|
|
149
|
+
value = staged.value->'value',
|
|
150
|
+
last_commit_id = staged.value->>'lastCommitId',
|
|
151
|
+
updated_at = (staged.value->>'updatedAt')::timestamptz
|
|
152
|
+
FROM staged
|
|
153
|
+
WHERE overrides.project_id = ${projectId}
|
|
154
|
+
AND overrides.identity_kind = staged.value->>'identityKind'
|
|
155
|
+
AND overrides.identity_key = staged.value->'identityKey'
|
|
73
156
|
AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
|
|
74
|
-
RETURNING overrides.
|
|
157
|
+
RETURNING overrides.identity_kind
|
|
75
158
|
`
|
|
76
|
-
if (rows.length !==
|
|
159
|
+
if (rows.length !== linkUpdates.length)
|
|
160
|
+
throw linkOverrideConflict(linkUpdates[0]!.identityKind)
|
|
77
161
|
}
|
|
78
162
|
|
|
79
|
-
const
|
|
80
|
-
...chunk.overrides.
|
|
81
|
-
|
|
82
|
-
|
|
163
|
+
const linkDeletes = [
|
|
164
|
+
...chunk.overrides.links.edges.deletes.map((item) => ({
|
|
165
|
+
identityKind: "edge" as const,
|
|
166
|
+
identityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
|
|
83
167
|
expectedLastCommitId: item.expectedLastCommitId,
|
|
84
168
|
})),
|
|
85
|
-
...chunk.overrides.
|
|
86
|
-
|
|
87
|
-
|
|
169
|
+
...chunk.overrides.links.slots.deletes.map((item) => ({
|
|
170
|
+
identityKind: "slot" as const,
|
|
171
|
+
identityKey: [item.ref.source.objectTypeId, item.ref.source.primaryId, item.ref.linkId],
|
|
88
172
|
expectedLastCommitId: item.expectedLastCommitId,
|
|
89
173
|
})),
|
|
90
174
|
]
|
|
91
|
-
if (
|
|
92
|
-
const rows = await this.sql<{ readonly
|
|
175
|
+
if (linkDeletes.length > 0) {
|
|
176
|
+
const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
|
|
93
177
|
WITH staged AS (
|
|
94
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql,
|
|
178
|
+
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkDeletes)}::jsonb)
|
|
95
179
|
)
|
|
96
|
-
DELETE FROM
|
|
180
|
+
DELETE FROM ontology_link_overrides AS overrides USING staged
|
|
97
181
|
WHERE overrides.project_id = ${projectId}
|
|
98
|
-
AND overrides.
|
|
99
|
-
AND overrides.
|
|
182
|
+
AND overrides.identity_kind = staged.value->>'identityKind'
|
|
183
|
+
AND overrides.identity_key = staged.value->'identityKey'
|
|
100
184
|
AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
|
|
101
|
-
RETURNING overrides.
|
|
185
|
+
RETURNING overrides.identity_kind
|
|
102
186
|
`
|
|
103
|
-
if (rows.length !==
|
|
187
|
+
if (rows.length !== linkDeletes.length)
|
|
188
|
+
throw linkOverrideConflict(linkDeletes[0]!.identityKind)
|
|
104
189
|
}
|
|
105
190
|
}
|
|
106
191
|
|
|
@@ -408,7 +493,7 @@ export class PgMaterializationWriter {
|
|
|
408
493
|
INSERT INTO ontology_outbox (
|
|
409
494
|
project_id, id, commit_id, commit_ordinal, envelope,
|
|
410
495
|
available_at, attempts, lease_id, lease_expires_at,
|
|
411
|
-
published_at,
|
|
496
|
+
published_at, last_failure, created_at
|
|
412
497
|
)
|
|
413
498
|
SELECT ${projectId}, value->>'id', ${commitId},
|
|
414
499
|
(value->>'commitOrdinal')::bigint, value->'envelope',
|
|
@@ -426,30 +511,16 @@ export class PgMaterializationWriter {
|
|
|
426
511
|
}
|
|
427
512
|
}
|
|
428
513
|
|
|
429
|
-
function
|
|
430
|
-
|
|
431
|
-
item: {
|
|
432
|
-
readonly value: unknown
|
|
433
|
-
readonly lastCommitId: string
|
|
434
|
-
readonly updatedAt: string
|
|
435
|
-
readonly expectedLastCommitId: string | null
|
|
436
|
-
}
|
|
437
|
-
) {
|
|
438
|
-
const key = entity.kind === "object" ? objectRefKey(entity.ref) : linkRefKey(entity.ref)
|
|
439
|
-
const columns = overrideColumns(entity)
|
|
440
|
-
return {
|
|
441
|
-
kind: entity.kind,
|
|
442
|
-
entityKey: JSON.parse(key) as unknown,
|
|
443
|
-
...columns,
|
|
444
|
-
value: item.value,
|
|
445
|
-
lastCommitId: item.lastCommitId,
|
|
446
|
-
updatedAt: item.updatedAt,
|
|
447
|
-
expectedLastCommitId: item.expectedLastCommitId,
|
|
448
|
-
}
|
|
514
|
+
function objectOverrideConflict(): MaterializationConflictError {
|
|
515
|
+
return effectiveConflict("Expected object override changed.")
|
|
449
516
|
}
|
|
450
517
|
|
|
451
|
-
function
|
|
452
|
-
return effectiveConflict(
|
|
518
|
+
function linkOverrideConflict(identityKind: "edge" | "slot"): MaterializationConflictError {
|
|
519
|
+
return effectiveConflict(
|
|
520
|
+
identityKind === "edge"
|
|
521
|
+
? "Expected link edge override changed."
|
|
522
|
+
: "Expected link slot override changed."
|
|
523
|
+
)
|
|
453
524
|
}
|
|
454
525
|
|
|
455
526
|
function objectIdentityKey(item: {
|