@sixb/pg 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-storage/auth-storage.d.ts +2 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/identities.d.ts +4 -0
- package/dist/auth-storage/identities.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +2 -1
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/users.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1807 -279
- package/dist/index.js.map +25 -21
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +10 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +5 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +14 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-artifact-storage.d.ts +14 -2
- package/dist/pg-artifact-storage.d.ts.map +1 -1
- package/dist/pg-dataset-ingest-storage.d.ts +6 -0
- package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
- package/dist/pg-lake-retention-run-storage.d.ts +11 -0
- package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +6 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +2 -1
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +4 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-claim-sql.d.ts +37 -0
- package/dist/run-claim-sql.d.ts.map +1 -0
- package/dist/share-session-storage.d.ts +2 -1
- package/dist/share-session-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/auth-storage/auth-storage.ts +66 -0
- package/src/auth-storage/identities.ts +15 -0
- package/src/auth-storage/rows.ts +3 -0
- package/src/auth-storage/users.ts +3 -0
- package/src/connector-connection-storage.ts +33 -16
- package/src/index.ts +15 -0
- package/src/migrations/089-maintenance-heartbeats.sql +10 -0
- package/src/migrations/090-lake-retention-runs.sql +8 -0
- package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
- package/src/migrations/092-run-recovery.sql +23 -0
- package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
- package/src/migrations/094-artifact-board-previews.sql +14 -0
- package/src/migrations/095-artifact-open-versions.sql +101 -0
- package/src/migrations/096-artifact-documents.sql +11 -0
- package/src/migrations/097-artifact-frame-files.sql +8 -0
- package/src/migrations/098-artifact-comments.sql +36 -0
- package/src/migrations/099-object-artifact-refs.sql +12 -0
- package/src/migrations/100-shared-link-guests.sql +17 -0
- package/src/migrations/101-connector-connection-owners.sql +38 -0
- package/src/migrations/102-user-suspension-source.sql +12 -0
- package/src/migrations.ts +40 -0
- package/src/objects/storage.ts +49 -0
- package/src/ontology-storage/materialization-session.ts +4 -0
- package/src/ontology-storage/materialization-writer.ts +68 -1
- package/src/ontology-storage/materializations.ts +3 -1
- package/src/pg-artifact-storage.ts +798 -137
- package/src/pg-dataset-ingest-storage.ts +305 -0
- package/src/pg-lake-retention-run-storage.ts +40 -0
- package/src/pg-maintenance-heartbeat-storage.ts +53 -0
- package/src/pg-pipeline-run-storage.ts +85 -42
- package/src/pg-projection-run-storage.ts +61 -2
- package/src/pg-sync-run-storage.ts +100 -37
- package/src/pg-webhook-run-storage.ts +37 -11
- package/src/pg-workflow-run-storage.ts +39 -100
- package/src/run-claim-sql.ts +173 -0
- package/src/share-session-storage.ts +20 -0
package/src/migrations.ts
CHANGED
|
@@ -459,6 +459,20 @@ export const postgresStorageMigrations = defineMigrations<PostgresMigrationConte
|
|
|
459
459
|
pgSql("086-partition-objects", partitionObjectsSql),
|
|
460
460
|
pgSql("087-agent-computer-background-commands", agentComputerBackgroundCommandsSql),
|
|
461
461
|
pgSql("088-web-search-model-kind", webSearchModelKindSql),
|
|
462
|
+
pgSql("089-maintenance-heartbeats", maintenanceHeartbeatsSql),
|
|
463
|
+
pgSql("090-lake-retention-runs", lakeRetentionRunsSql),
|
|
464
|
+
pgSql("091-dataset-ingest-inbox", datasetIngestInboxSql),
|
|
465
|
+
pgSql("092-run-recovery", runRecoverySql),
|
|
466
|
+
pgSql("093-webhook-and-projection-leases", webhookAndProjectionLeasesSql),
|
|
467
|
+
pgSql("094-artifact-board-previews", artifactBoardPreviewsSql),
|
|
468
|
+
pgSql("095-artifact-open-versions", artifactOpenVersionsSql),
|
|
469
|
+
pgSql("096-artifact-documents", artifactDocumentsSql),
|
|
470
|
+
pgSql("097-artifact-frame-files", artifactFrameFilesSql),
|
|
471
|
+
pgSql("098-artifact-comments", artifactCommentsSql),
|
|
472
|
+
pgSql("099-object-artifact-refs", objectArtifactRefsSql),
|
|
473
|
+
pgSql("100-shared-link-guests", sharedLinkGuestsSql),
|
|
474
|
+
pgSql("101-connector-connection-owners", connectorConnectionOwnersSql),
|
|
475
|
+
pgSql("102-user-suspension-source", userSuspensionSourceSql),
|
|
462
476
|
],
|
|
463
477
|
})
|
|
464
478
|
|
|
@@ -535,3 +549,29 @@ import agentComputerBackgroundCommandsSql from "./migrations/087-agent-computer-
|
|
|
535
549
|
type: "text",
|
|
536
550
|
}
|
|
537
551
|
import webSearchModelKindSql from "./migrations/088-web-search-model-kind.sql" with { type: "text" }
|
|
552
|
+
import maintenanceHeartbeatsSql from "./migrations/089-maintenance-heartbeats.sql" with {
|
|
553
|
+
type: "text",
|
|
554
|
+
}
|
|
555
|
+
import lakeRetentionRunsSql from "./migrations/090-lake-retention-runs.sql" with { type: "text" }
|
|
556
|
+
import datasetIngestInboxSql from "./migrations/091-dataset-ingest-inbox.sql" with { type: "text" }
|
|
557
|
+
import runRecoverySql from "./migrations/092-run-recovery.sql" with { type: "text" }
|
|
558
|
+
import webhookAndProjectionLeasesSql from "./migrations/093-webhook-and-projection-leases.sql" with {
|
|
559
|
+
type: "text",
|
|
560
|
+
}
|
|
561
|
+
import artifactBoardPreviewsSql from "./migrations/094-artifact-board-previews.sql" with {
|
|
562
|
+
type: "text",
|
|
563
|
+
}
|
|
564
|
+
import artifactOpenVersionsSql from "./migrations/095-artifact-open-versions.sql" with {
|
|
565
|
+
type: "text",
|
|
566
|
+
}
|
|
567
|
+
import artifactDocumentsSql from "./migrations/096-artifact-documents.sql" with { type: "text" }
|
|
568
|
+
import artifactFrameFilesSql from "./migrations/097-artifact-frame-files.sql" with { type: "text" }
|
|
569
|
+
import artifactCommentsSql from "./migrations/098-artifact-comments.sql" with { type: "text" }
|
|
570
|
+
import objectArtifactRefsSql from "./migrations/099-object-artifact-refs.sql" with { type: "text" }
|
|
571
|
+
import sharedLinkGuestsSql from "./migrations/100-shared-link-guests.sql" with { type: "text" }
|
|
572
|
+
import connectorConnectionOwnersSql from "./migrations/101-connector-connection-owners.sql" with {
|
|
573
|
+
type: "text",
|
|
574
|
+
}
|
|
575
|
+
import userSuspensionSourceSql from "./migrations/102-user-suspension-source.sql" with {
|
|
576
|
+
type: "text",
|
|
577
|
+
}
|
package/src/objects/storage.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { hasVectorProfile } from "@sixb/core/internal/query"
|
|
2
2
|
import type {
|
|
3
|
+
ArtifactHolder,
|
|
3
4
|
CompiledConditionalObjectReadScope,
|
|
4
5
|
CompiledSelectedObjectReadScope,
|
|
5
6
|
CountObjectsInput,
|
|
@@ -100,6 +101,7 @@ const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
|
|
|
100
101
|
timestamp: { equality: true, ordering: true },
|
|
101
102
|
// jsonb equality and containment compare both fields of `{ type, id }`, whatever key order.
|
|
102
103
|
userRef: { equality: true },
|
|
104
|
+
artifactRef: { equality: true },
|
|
103
105
|
},
|
|
104
106
|
limits: {
|
|
105
107
|
totalCount: true,
|
|
@@ -307,6 +309,33 @@ export class PgObjectStorage implements ObjectStorage {
|
|
|
307
309
|
}
|
|
308
310
|
}
|
|
309
311
|
|
|
312
|
+
async listArtifactHolders(params: {
|
|
313
|
+
projectId: string
|
|
314
|
+
artifactId: string
|
|
315
|
+
after?: ArtifactHolder
|
|
316
|
+
limit: number
|
|
317
|
+
}): Promise<{ holders: readonly ArtifactHolder[]; next?: ArtifactHolder }> {
|
|
318
|
+
assertArtifactHolderPageLimit(params.limit)
|
|
319
|
+
|
|
320
|
+
const { after } = params
|
|
321
|
+
const rows = await this.sql<ArtifactHolderRow[]>`
|
|
322
|
+
SELECT object_type_id, primary_id, property_id FROM object_artifact_refs
|
|
323
|
+
WHERE project_id = ${params.projectId} AND artifact_id = ${params.artifactId}
|
|
324
|
+
${
|
|
325
|
+
after
|
|
326
|
+
? this.sql`AND (object_type_id, primary_id, property_id)
|
|
327
|
+
> (${after.objectTypeId}, ${after.primaryId}, ${after.propertyId})`
|
|
328
|
+
: this.sql``
|
|
329
|
+
}
|
|
330
|
+
ORDER BY object_type_id, primary_id, property_id
|
|
331
|
+
LIMIT ${params.limit + 1}
|
|
332
|
+
`
|
|
333
|
+
|
|
334
|
+
const holders = rows.slice(0, params.limit).map(artifactHolderFromRow)
|
|
335
|
+
const last = holders.at(-1)
|
|
336
|
+
return { holders, ...(rows.length > params.limit && last ? { next: last } : {}) }
|
|
337
|
+
}
|
|
338
|
+
|
|
310
339
|
async list(params: {
|
|
311
340
|
projectId: string
|
|
312
341
|
objectTypeId?: string | readonly string[]
|
|
@@ -370,3 +399,23 @@ function assertReconciliationPageLimit(limit: number): void {
|
|
|
370
399
|
throw new Error("Object reconciliation page limit must be a positive safe integer.")
|
|
371
400
|
}
|
|
372
401
|
}
|
|
402
|
+
|
|
403
|
+
interface ArtifactHolderRow {
|
|
404
|
+
readonly object_type_id: string
|
|
405
|
+
readonly primary_id: string
|
|
406
|
+
readonly property_id: string
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
function artifactHolderFromRow(row: ArtifactHolderRow): ArtifactHolder {
|
|
410
|
+
return {
|
|
411
|
+
objectTypeId: row.object_type_id,
|
|
412
|
+
primaryId: row.primary_id,
|
|
413
|
+
propertyId: row.property_id,
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
function assertArtifactHolderPageLimit(limit: number): void {
|
|
418
|
+
if (!Number.isSafeInteger(limit) || limit <= 0) {
|
|
419
|
+
throw new Error("Artifact holder page limit must be a positive safe integer.")
|
|
420
|
+
}
|
|
421
|
+
}
|
|
@@ -38,6 +38,8 @@ export class PgMaterializationSessionState extends ProviderMaterializationSessio
|
|
|
38
38
|
changedUniqueValues = 0
|
|
39
39
|
/** What unique value items this session staged, known without reading its work back. */
|
|
40
40
|
readonly uniqueWork = { releases: false, claims: false }
|
|
41
|
+
/** What artifact holder items this session staged, likewise. */
|
|
42
|
+
readonly artifactHolderWork = { deletes: false, inserts: false }
|
|
41
43
|
workAnalyzed = false
|
|
42
44
|
/** Work staged on the session lives in a temp table; a plan-bound session reads its plan's. */
|
|
43
45
|
readonly workTable: string
|
|
@@ -172,6 +174,8 @@ export class PgMaterializationSessions {
|
|
|
172
174
|
if (record.kind !== "plan") continue
|
|
173
175
|
if (record.item.kind === "unique-value-release") session.uniqueWork.releases = true
|
|
174
176
|
if (record.item.kind === "unique-value-claim") session.uniqueWork.claims = true
|
|
177
|
+
if (record.item.kind === "artifact-holder-delete") session.artifactHolderWork.deletes = true
|
|
178
|
+
if (record.item.kind === "artifact-holder-insert") session.artifactHolderWork.inserts = true
|
|
175
179
|
}
|
|
176
180
|
}
|
|
177
181
|
|
|
@@ -27,6 +27,8 @@ export interface PgStagedPlanTarget {
|
|
|
27
27
|
readonly committedAt: string
|
|
28
28
|
/** Whether unique value items were staged; probed in the work table when unknown. */
|
|
29
29
|
readonly uniqueWork?: PgUniqueWork
|
|
30
|
+
/** Whether artifact holder items were staged; probed in the work table when unknown. */
|
|
31
|
+
readonly artifactHolderWork?: PgArtifactHolderWork
|
|
30
32
|
}
|
|
31
33
|
|
|
32
34
|
export interface PgUniqueWork {
|
|
@@ -34,6 +36,11 @@ export interface PgUniqueWork {
|
|
|
34
36
|
readonly claims: boolean
|
|
35
37
|
}
|
|
36
38
|
|
|
39
|
+
export interface PgArtifactHolderWork {
|
|
40
|
+
readonly deletes: boolean
|
|
41
|
+
readonly inserts: boolean
|
|
42
|
+
}
|
|
43
|
+
|
|
37
44
|
export interface PgAppliedPlan {
|
|
38
45
|
readonly objectWrites: number
|
|
39
46
|
readonly linkWrites: number
|
|
@@ -65,12 +72,15 @@ export class PgMaterializationWriter {
|
|
|
65
72
|
await this.applyLinkOverrides(target)
|
|
66
73
|
await this.applyPoints(target)
|
|
67
74
|
const uniqueWork = target.uniqueWork ?? (await this.stagedUniqueWork(target))
|
|
75
|
+
const holderWork = target.artifactHolderWork ?? (await this.stagedArtifactHolderWork(target))
|
|
68
76
|
if (uniqueWork.releases) await this.releaseUniqueValues(target)
|
|
77
|
+
if (holderWork.deletes) await this.deleteArtifactHolders(target)
|
|
69
78
|
let linkWrites = await this.deleteLinks(target)
|
|
70
79
|
let objectWrites = await this.deleteObjects(target)
|
|
71
80
|
objectWrites += await this.upsertObjects(target)
|
|
72
81
|
linkWrites += await this.upsertLinks(target)
|
|
73
82
|
const uniqueValueWrites = uniqueWork.claims ? await this.claimUniqueValues(target) : 0
|
|
83
|
+
if (holderWork.inserts) await this.insertArtifactHolders(target)
|
|
74
84
|
const eventCount = await this.writeOutbox(target)
|
|
75
85
|
await this.writeTouches(target)
|
|
76
86
|
return { objectWrites, linkWrites, uniqueValueWrites, eventCount }
|
|
@@ -106,6 +116,25 @@ export class PgMaterializationWriter {
|
|
|
106
116
|
return { releases: row?.releases ?? false, claims: row?.claims ?? false }
|
|
107
117
|
}
|
|
108
118
|
|
|
119
|
+
/** The same probes for the artifact holder index: most commits hold no artifact. */
|
|
120
|
+
private async stagedArtifactHolderWork(
|
|
121
|
+
target: PgStagedPlanTarget
|
|
122
|
+
): Promise<PgArtifactHolderWork> {
|
|
123
|
+
const probe = (kind: "artifact-holder-delete" | "artifact-holder-insert") => this.sql`
|
|
124
|
+
EXISTS (
|
|
125
|
+
SELECT 1 FROM ${this.sql(target.workTable)}
|
|
126
|
+
WHERE work_id = ${target.workId} AND lane = 'apply'
|
|
127
|
+
AND major_order = ${materializationApplyPhase(kind)}
|
|
128
|
+
AND minor_order = ${materializationPlanKindRank(kind)}
|
|
129
|
+
)
|
|
130
|
+
`
|
|
131
|
+
const [row] = await this.sql<{ readonly deletes: boolean; readonly inserts: boolean }[]>`
|
|
132
|
+
SELECT ${probe("artifact-holder-delete")} AS deletes,
|
|
133
|
+
${probe("artifact-holder-insert")} AS inserts
|
|
134
|
+
`
|
|
135
|
+
return { deletes: row?.deletes ?? false, inserts: row?.inserts ?? false }
|
|
136
|
+
}
|
|
137
|
+
|
|
109
138
|
/** The staged values of one plan item kind, as `value`. */
|
|
110
139
|
private staged(target: PgStagedPlanTarget, kind: MaterializationPlanWorkItem["kind"]) {
|
|
111
140
|
return this.sql`
|
|
@@ -417,6 +446,43 @@ export class PgMaterializationWriter {
|
|
|
417
446
|
)
|
|
418
447
|
}
|
|
419
448
|
|
|
449
|
+
/** The staged rows of the artifact holder index of one item kind, as its columns. */
|
|
450
|
+
private stagedArtifactHolders(
|
|
451
|
+
target: PgStagedPlanTarget,
|
|
452
|
+
kind: "artifact-holder-delete" | "artifact-holder-insert"
|
|
453
|
+
) {
|
|
454
|
+
return this.sql`
|
|
455
|
+
SELECT value->>'artifactId' AS artifact_id, value->>'objectTypeId' AS object_type_id,
|
|
456
|
+
value->>'primaryId' AS primary_id, value->>'propertyId' AS property_id
|
|
457
|
+
FROM (${this.staged(target, kind)}) AS staged
|
|
458
|
+
`
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/** A row already gone is no conflict: the index only follows what objects hold. */
|
|
462
|
+
private async deleteArtifactHolders(target: PgStagedPlanTarget): Promise<void> {
|
|
463
|
+
await this.sql`
|
|
464
|
+
WITH candidates AS (${this.stagedArtifactHolders(target, "artifact-holder-delete")})
|
|
465
|
+
DELETE FROM object_artifact_refs AS stored USING candidates
|
|
466
|
+
WHERE stored.project_id = ${target.projectId}
|
|
467
|
+
AND stored.artifact_id = candidates.artifact_id
|
|
468
|
+
AND stored.object_type_id = candidates.object_type_id
|
|
469
|
+
AND stored.primary_id = candidates.primary_id
|
|
470
|
+
AND stored.property_id = candidates.property_id
|
|
471
|
+
`
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/** A row already there is no conflict either. */
|
|
475
|
+
private async insertArtifactHolders(target: PgStagedPlanTarget): Promise<void> {
|
|
476
|
+
await this.sql`
|
|
477
|
+
INSERT INTO object_artifact_refs (
|
|
478
|
+
project_id, artifact_id, object_type_id, primary_id, property_id
|
|
479
|
+
)
|
|
480
|
+
SELECT ${target.projectId}, artifact_id, object_type_id, primary_id, property_id
|
|
481
|
+
FROM (${this.stagedArtifactHolders(target, "artifact-holder-insert")}) AS candidates
|
|
482
|
+
ON CONFLICT DO NOTHING
|
|
483
|
+
`
|
|
484
|
+
}
|
|
485
|
+
|
|
420
486
|
private async deleteLinks(target: PgStagedPlanTarget): Promise<number> {
|
|
421
487
|
const candidates = this.staged(target, "link-delete")
|
|
422
488
|
return this.write(
|
|
@@ -673,7 +739,8 @@ export class PgMaterializationWriter {
|
|
|
673
739
|
END AS ref
|
|
674
740
|
) AS entity
|
|
675
741
|
WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
|
|
676
|
-
-- Unique values are no plan input: a stale claim fails when it applies.
|
|
742
|
+
-- Unique values are no plan input: a stale claim fails when it applies. Nor are the
|
|
743
|
+
-- artifacts objects hold, which follow the objects.
|
|
677
744
|
AND entity.kind IS NOT NULL
|
|
678
745
|
`
|
|
679
746
|
}
|
|
@@ -268,7 +268,9 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
268
268
|
projectId: commit.projectId,
|
|
269
269
|
commitId: commit.id,
|
|
270
270
|
committedAt: commit.committedAt,
|
|
271
|
-
...(session.plan
|
|
271
|
+
...(session.plan
|
|
272
|
+
? {}
|
|
273
|
+
: { uniqueWork: session.uniqueWork, artifactHolderWork: session.artifactHolderWork }),
|
|
272
274
|
})
|
|
273
275
|
// Below this no type can reach the statistics threshold, so the work table stays unread.
|
|
274
276
|
if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {
|