@sixb/pg 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/auth-storage/auth-storage.d.ts +2 -1
  2. package/dist/auth-storage/auth-storage.d.ts.map +1 -1
  3. package/dist/auth-storage/identities.d.ts +4 -0
  4. package/dist/auth-storage/identities.d.ts.map +1 -1
  5. package/dist/auth-storage/rows.d.ts +2 -1
  6. package/dist/auth-storage/rows.d.ts.map +1 -1
  7. package/dist/auth-storage/users.d.ts.map +1 -1
  8. package/dist/connector-connection-storage.d.ts.map +1 -1
  9. package/dist/index.d.ts +6 -0
  10. package/dist/index.d.ts.map +1 -1
  11. package/dist/index.js +1807 -279
  12. package/dist/index.js.map +25 -21
  13. package/dist/migrations.d.ts.map +1 -1
  14. package/dist/objects/storage.d.ts +10 -1
  15. package/dist/objects/storage.d.ts.map +1 -1
  16. package/dist/ontology-storage/materialization-session.d.ts +5 -0
  17. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  18. package/dist/ontology-storage/materialization-writer.d.ts +14 -0
  19. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  20. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  21. package/dist/pg-artifact-storage.d.ts +14 -2
  22. package/dist/pg-artifact-storage.d.ts.map +1 -1
  23. package/dist/pg-dataset-ingest-storage.d.ts +6 -0
  24. package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
  25. package/dist/pg-lake-retention-run-storage.d.ts +11 -0
  26. package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
  27. package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
  28. package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
  29. package/dist/pg-pipeline-run-storage.d.ts +5 -2
  30. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  31. package/dist/pg-projection-run-storage.d.ts +5 -1
  32. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  33. package/dist/pg-sync-run-storage.d.ts +6 -2
  34. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  35. package/dist/pg-webhook-run-storage.d.ts +2 -1
  36. package/dist/pg-webhook-run-storage.d.ts.map +1 -1
  37. package/dist/pg-workflow-run-storage.d.ts +4 -3
  38. package/dist/pg-workflow-run-storage.d.ts.map +1 -1
  39. package/dist/run-claim-sql.d.ts +37 -0
  40. package/dist/run-claim-sql.d.ts.map +1 -0
  41. package/dist/share-session-storage.d.ts +2 -1
  42. package/dist/share-session-storage.d.ts.map +1 -1
  43. package/package.json +3 -3
  44. package/src/auth-storage/auth-storage.ts +66 -0
  45. package/src/auth-storage/identities.ts +15 -0
  46. package/src/auth-storage/rows.ts +3 -0
  47. package/src/auth-storage/users.ts +3 -0
  48. package/src/connector-connection-storage.ts +33 -16
  49. package/src/index.ts +15 -0
  50. package/src/migrations/089-maintenance-heartbeats.sql +10 -0
  51. package/src/migrations/090-lake-retention-runs.sql +8 -0
  52. package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
  53. package/src/migrations/092-run-recovery.sql +23 -0
  54. package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
  55. package/src/migrations/094-artifact-board-previews.sql +14 -0
  56. package/src/migrations/095-artifact-open-versions.sql +101 -0
  57. package/src/migrations/096-artifact-documents.sql +11 -0
  58. package/src/migrations/097-artifact-frame-files.sql +8 -0
  59. package/src/migrations/098-artifact-comments.sql +36 -0
  60. package/src/migrations/099-object-artifact-refs.sql +12 -0
  61. package/src/migrations/100-shared-link-guests.sql +17 -0
  62. package/src/migrations/101-connector-connection-owners.sql +38 -0
  63. package/src/migrations/102-user-suspension-source.sql +12 -0
  64. package/src/migrations.ts +40 -0
  65. package/src/objects/storage.ts +49 -0
  66. package/src/ontology-storage/materialization-session.ts +4 -0
  67. package/src/ontology-storage/materialization-writer.ts +68 -1
  68. package/src/ontology-storage/materializations.ts +3 -1
  69. package/src/pg-artifact-storage.ts +798 -137
  70. package/src/pg-dataset-ingest-storage.ts +305 -0
  71. package/src/pg-lake-retention-run-storage.ts +40 -0
  72. package/src/pg-maintenance-heartbeat-storage.ts +53 -0
  73. package/src/pg-pipeline-run-storage.ts +85 -42
  74. package/src/pg-projection-run-storage.ts +61 -2
  75. package/src/pg-sync-run-storage.ts +100 -37
  76. package/src/pg-webhook-run-storage.ts +37 -11
  77. package/src/pg-workflow-run-storage.ts +39 -100
  78. package/src/run-claim-sql.ts +173 -0
  79. package/src/share-session-storage.ts +20 -0
package/src/migrations.ts CHANGED
@@ -459,6 +459,20 @@ export const postgresStorageMigrations = defineMigrations<PostgresMigrationConte
459
459
  pgSql("086-partition-objects", partitionObjectsSql),
460
460
  pgSql("087-agent-computer-background-commands", agentComputerBackgroundCommandsSql),
461
461
  pgSql("088-web-search-model-kind", webSearchModelKindSql),
462
+ pgSql("089-maintenance-heartbeats", maintenanceHeartbeatsSql),
463
+ pgSql("090-lake-retention-runs", lakeRetentionRunsSql),
464
+ pgSql("091-dataset-ingest-inbox", datasetIngestInboxSql),
465
+ pgSql("092-run-recovery", runRecoverySql),
466
+ pgSql("093-webhook-and-projection-leases", webhookAndProjectionLeasesSql),
467
+ pgSql("094-artifact-board-previews", artifactBoardPreviewsSql),
468
+ pgSql("095-artifact-open-versions", artifactOpenVersionsSql),
469
+ pgSql("096-artifact-documents", artifactDocumentsSql),
470
+ pgSql("097-artifact-frame-files", artifactFrameFilesSql),
471
+ pgSql("098-artifact-comments", artifactCommentsSql),
472
+ pgSql("099-object-artifact-refs", objectArtifactRefsSql),
473
+ pgSql("100-shared-link-guests", sharedLinkGuestsSql),
474
+ pgSql("101-connector-connection-owners", connectorConnectionOwnersSql),
475
+ pgSql("102-user-suspension-source", userSuspensionSourceSql),
462
476
  ],
463
477
  })
464
478
 
@@ -535,3 +549,29 @@ import agentComputerBackgroundCommandsSql from "./migrations/087-agent-computer-
535
549
  type: "text",
536
550
  }
537
551
  import webSearchModelKindSql from "./migrations/088-web-search-model-kind.sql" with { type: "text" }
552
+ import maintenanceHeartbeatsSql from "./migrations/089-maintenance-heartbeats.sql" with {
553
+ type: "text",
554
+ }
555
+ import lakeRetentionRunsSql from "./migrations/090-lake-retention-runs.sql" with { type: "text" }
556
+ import datasetIngestInboxSql from "./migrations/091-dataset-ingest-inbox.sql" with { type: "text" }
557
+ import runRecoverySql from "./migrations/092-run-recovery.sql" with { type: "text" }
558
+ import webhookAndProjectionLeasesSql from "./migrations/093-webhook-and-projection-leases.sql" with {
559
+ type: "text",
560
+ }
561
+ import artifactBoardPreviewsSql from "./migrations/094-artifact-board-previews.sql" with {
562
+ type: "text",
563
+ }
564
+ import artifactOpenVersionsSql from "./migrations/095-artifact-open-versions.sql" with {
565
+ type: "text",
566
+ }
567
+ import artifactDocumentsSql from "./migrations/096-artifact-documents.sql" with { type: "text" }
568
+ import artifactFrameFilesSql from "./migrations/097-artifact-frame-files.sql" with { type: "text" }
569
+ import artifactCommentsSql from "./migrations/098-artifact-comments.sql" with { type: "text" }
570
+ import objectArtifactRefsSql from "./migrations/099-object-artifact-refs.sql" with { type: "text" }
571
+ import sharedLinkGuestsSql from "./migrations/100-shared-link-guests.sql" with { type: "text" }
572
+ import connectorConnectionOwnersSql from "./migrations/101-connector-connection-owners.sql" with {
573
+ type: "text",
574
+ }
575
+ import userSuspensionSourceSql from "./migrations/102-user-suspension-source.sql" with {
576
+ type: "text",
577
+ }
@@ -1,5 +1,6 @@
1
1
  import { hasVectorProfile } from "@sixb/core/internal/query"
2
2
  import type {
3
+ ArtifactHolder,
3
4
  CompiledConditionalObjectReadScope,
4
5
  CompiledSelectedObjectReadScope,
5
6
  CountObjectsInput,
@@ -100,6 +101,7 @@ const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
100
101
  timestamp: { equality: true, ordering: true },
101
102
  // jsonb equality and containment compare both fields of `{ type, id }`, whatever key order.
102
103
  userRef: { equality: true },
104
+ artifactRef: { equality: true },
103
105
  },
104
106
  limits: {
105
107
  totalCount: true,
@@ -307,6 +309,33 @@ export class PgObjectStorage implements ObjectStorage {
307
309
  }
308
310
  }
309
311
 
312
+ async listArtifactHolders(params: {
313
+ projectId: string
314
+ artifactId: string
315
+ after?: ArtifactHolder
316
+ limit: number
317
+ }): Promise<{ holders: readonly ArtifactHolder[]; next?: ArtifactHolder }> {
318
+ assertArtifactHolderPageLimit(params.limit)
319
+
320
+ const { after } = params
321
+ const rows = await this.sql<ArtifactHolderRow[]>`
322
+ SELECT object_type_id, primary_id, property_id FROM object_artifact_refs
323
+ WHERE project_id = ${params.projectId} AND artifact_id = ${params.artifactId}
324
+ ${
325
+ after
326
+ ? this.sql`AND (object_type_id, primary_id, property_id)
327
+ > (${after.objectTypeId}, ${after.primaryId}, ${after.propertyId})`
328
+ : this.sql``
329
+ }
330
+ ORDER BY object_type_id, primary_id, property_id
331
+ LIMIT ${params.limit + 1}
332
+ `
333
+
334
+ const holders = rows.slice(0, params.limit).map(artifactHolderFromRow)
335
+ const last = holders.at(-1)
336
+ return { holders, ...(rows.length > params.limit && last ? { next: last } : {}) }
337
+ }
338
+
310
339
  async list(params: {
311
340
  projectId: string
312
341
  objectTypeId?: string | readonly string[]
@@ -370,3 +399,23 @@ function assertReconciliationPageLimit(limit: number): void {
370
399
  throw new Error("Object reconciliation page limit must be a positive safe integer.")
371
400
  }
372
401
  }
402
+
403
+ interface ArtifactHolderRow {
404
+ readonly object_type_id: string
405
+ readonly primary_id: string
406
+ readonly property_id: string
407
+ }
408
+
409
+ function artifactHolderFromRow(row: ArtifactHolderRow): ArtifactHolder {
410
+ return {
411
+ objectTypeId: row.object_type_id,
412
+ primaryId: row.primary_id,
413
+ propertyId: row.property_id,
414
+ }
415
+ }
416
+
417
+ function assertArtifactHolderPageLimit(limit: number): void {
418
+ if (!Number.isSafeInteger(limit) || limit <= 0) {
419
+ throw new Error("Artifact holder page limit must be a positive safe integer.")
420
+ }
421
+ }
@@ -38,6 +38,8 @@ export class PgMaterializationSessionState extends ProviderMaterializationSessio
38
38
  changedUniqueValues = 0
39
39
  /** What unique value items this session staged, known without reading its work back. */
40
40
  readonly uniqueWork = { releases: false, claims: false }
41
+ /** What artifact holder items this session staged, likewise. */
42
+ readonly artifactHolderWork = { deletes: false, inserts: false }
41
43
  workAnalyzed = false
42
44
  /** Work staged on the session lives in a temp table; a plan-bound session reads its plan's. */
43
45
  readonly workTable: string
@@ -172,6 +174,8 @@ export class PgMaterializationSessions {
172
174
  if (record.kind !== "plan") continue
173
175
  if (record.item.kind === "unique-value-release") session.uniqueWork.releases = true
174
176
  if (record.item.kind === "unique-value-claim") session.uniqueWork.claims = true
177
+ if (record.item.kind === "artifact-holder-delete") session.artifactHolderWork.deletes = true
178
+ if (record.item.kind === "artifact-holder-insert") session.artifactHolderWork.inserts = true
175
179
  }
176
180
  }
177
181
 
@@ -27,6 +27,8 @@ export interface PgStagedPlanTarget {
27
27
  readonly committedAt: string
28
28
  /** Whether unique value items were staged; probed in the work table when unknown. */
29
29
  readonly uniqueWork?: PgUniqueWork
30
+ /** Whether artifact holder items were staged; probed in the work table when unknown. */
31
+ readonly artifactHolderWork?: PgArtifactHolderWork
30
32
  }
31
33
 
32
34
  export interface PgUniqueWork {
@@ -34,6 +36,11 @@ export interface PgUniqueWork {
34
36
  readonly claims: boolean
35
37
  }
36
38
 
39
+ export interface PgArtifactHolderWork {
40
+ readonly deletes: boolean
41
+ readonly inserts: boolean
42
+ }
43
+
37
44
  export interface PgAppliedPlan {
38
45
  readonly objectWrites: number
39
46
  readonly linkWrites: number
@@ -65,12 +72,15 @@ export class PgMaterializationWriter {
65
72
  await this.applyLinkOverrides(target)
66
73
  await this.applyPoints(target)
67
74
  const uniqueWork = target.uniqueWork ?? (await this.stagedUniqueWork(target))
75
+ const holderWork = target.artifactHolderWork ?? (await this.stagedArtifactHolderWork(target))
68
76
  if (uniqueWork.releases) await this.releaseUniqueValues(target)
77
+ if (holderWork.deletes) await this.deleteArtifactHolders(target)
69
78
  let linkWrites = await this.deleteLinks(target)
70
79
  let objectWrites = await this.deleteObjects(target)
71
80
  objectWrites += await this.upsertObjects(target)
72
81
  linkWrites += await this.upsertLinks(target)
73
82
  const uniqueValueWrites = uniqueWork.claims ? await this.claimUniqueValues(target) : 0
83
+ if (holderWork.inserts) await this.insertArtifactHolders(target)
74
84
  const eventCount = await this.writeOutbox(target)
75
85
  await this.writeTouches(target)
76
86
  return { objectWrites, linkWrites, uniqueValueWrites, eventCount }
@@ -106,6 +116,25 @@ export class PgMaterializationWriter {
106
116
  return { releases: row?.releases ?? false, claims: row?.claims ?? false }
107
117
  }
108
118
 
119
+ /** The same probes for the artifact holder index: most commits hold no artifact. */
120
+ private async stagedArtifactHolderWork(
121
+ target: PgStagedPlanTarget
122
+ ): Promise<PgArtifactHolderWork> {
123
+ const probe = (kind: "artifact-holder-delete" | "artifact-holder-insert") => this.sql`
124
+ EXISTS (
125
+ SELECT 1 FROM ${this.sql(target.workTable)}
126
+ WHERE work_id = ${target.workId} AND lane = 'apply'
127
+ AND major_order = ${materializationApplyPhase(kind)}
128
+ AND minor_order = ${materializationPlanKindRank(kind)}
129
+ )
130
+ `
131
+ const [row] = await this.sql<{ readonly deletes: boolean; readonly inserts: boolean }[]>`
132
+ SELECT ${probe("artifact-holder-delete")} AS deletes,
133
+ ${probe("artifact-holder-insert")} AS inserts
134
+ `
135
+ return { deletes: row?.deletes ?? false, inserts: row?.inserts ?? false }
136
+ }
137
+
109
138
  /** The staged values of one plan item kind, as `value`. */
110
139
  private staged(target: PgStagedPlanTarget, kind: MaterializationPlanWorkItem["kind"]) {
111
140
  return this.sql`
@@ -417,6 +446,43 @@ export class PgMaterializationWriter {
417
446
  )
418
447
  }
419
448
 
449
+ /** The staged rows of the artifact holder index of one item kind, as its columns. */
450
+ private stagedArtifactHolders(
451
+ target: PgStagedPlanTarget,
452
+ kind: "artifact-holder-delete" | "artifact-holder-insert"
453
+ ) {
454
+ return this.sql`
455
+ SELECT value->>'artifactId' AS artifact_id, value->>'objectTypeId' AS object_type_id,
456
+ value->>'primaryId' AS primary_id, value->>'propertyId' AS property_id
457
+ FROM (${this.staged(target, kind)}) AS staged
458
+ `
459
+ }
460
+
461
+ /** A row already gone is no conflict: the index only follows what objects hold. */
462
+ private async deleteArtifactHolders(target: PgStagedPlanTarget): Promise<void> {
463
+ await this.sql`
464
+ WITH candidates AS (${this.stagedArtifactHolders(target, "artifact-holder-delete")})
465
+ DELETE FROM object_artifact_refs AS stored USING candidates
466
+ WHERE stored.project_id = ${target.projectId}
467
+ AND stored.artifact_id = candidates.artifact_id
468
+ AND stored.object_type_id = candidates.object_type_id
469
+ AND stored.primary_id = candidates.primary_id
470
+ AND stored.property_id = candidates.property_id
471
+ `
472
+ }
473
+
474
+ /** A row already there is no conflict either. */
475
+ private async insertArtifactHolders(target: PgStagedPlanTarget): Promise<void> {
476
+ await this.sql`
477
+ INSERT INTO object_artifact_refs (
478
+ project_id, artifact_id, object_type_id, primary_id, property_id
479
+ )
480
+ SELECT ${target.projectId}, artifact_id, object_type_id, primary_id, property_id
481
+ FROM (${this.stagedArtifactHolders(target, "artifact-holder-insert")}) AS candidates
482
+ ON CONFLICT DO NOTHING
483
+ `
484
+ }
485
+
420
486
  private async deleteLinks(target: PgStagedPlanTarget): Promise<number> {
421
487
  const candidates = this.staged(target, "link-delete")
422
488
  return this.write(
@@ -673,7 +739,8 @@ export class PgMaterializationWriter {
673
739
  END AS ref
674
740
  ) AS entity
675
741
  WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
676
- -- Unique values are no plan input: a stale claim fails when it applies.
742
+ -- Unique values are no plan input: a stale claim fails when it applies. Nor are the
743
+ -- artifacts objects hold, which follow the objects.
677
744
  AND entity.kind IS NOT NULL
678
745
  `
679
746
  }
@@ -268,7 +268,9 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
268
268
  projectId: commit.projectId,
269
269
  commitId: commit.id,
270
270
  committedAt: commit.committedAt,
271
- ...(session.plan ? {} : { uniqueWork: session.uniqueWork }),
271
+ ...(session.plan
272
+ ? {}
273
+ : { uniqueWork: session.uniqueWork, artifactHolderWork: session.artifactHolderWork }),
272
274
  })
273
275
  // Below this no type can reach the statistics threshold, so the work table stays unread.
274
276
  if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {