@sixb/pg 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/auth-storage/auth-storage.d.ts +2 -1
  2. package/dist/auth-storage/auth-storage.d.ts.map +1 -1
  3. package/dist/auth-storage/identities.d.ts +4 -0
  4. package/dist/auth-storage/identities.d.ts.map +1 -1
  5. package/dist/auth-storage/rows.d.ts +2 -1
  6. package/dist/auth-storage/rows.d.ts.map +1 -1
  7. package/dist/auth-storage/users.d.ts.map +1 -1
  8. package/dist/connector-connection-storage.d.ts.map +1 -1
  9. package/dist/index.d.ts +6 -0
  10. package/dist/index.d.ts.map +1 -1
  11. package/dist/index.js +1807 -279
  12. package/dist/index.js.map +25 -21
  13. package/dist/migrations.d.ts.map +1 -1
  14. package/dist/objects/storage.d.ts +10 -1
  15. package/dist/objects/storage.d.ts.map +1 -1
  16. package/dist/ontology-storage/materialization-session.d.ts +5 -0
  17. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  18. package/dist/ontology-storage/materialization-writer.d.ts +14 -0
  19. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  20. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  21. package/dist/pg-artifact-storage.d.ts +14 -2
  22. package/dist/pg-artifact-storage.d.ts.map +1 -1
  23. package/dist/pg-dataset-ingest-storage.d.ts +6 -0
  24. package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
  25. package/dist/pg-lake-retention-run-storage.d.ts +11 -0
  26. package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
  27. package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
  28. package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
  29. package/dist/pg-pipeline-run-storage.d.ts +5 -2
  30. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  31. package/dist/pg-projection-run-storage.d.ts +5 -1
  32. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  33. package/dist/pg-sync-run-storage.d.ts +6 -2
  34. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  35. package/dist/pg-webhook-run-storage.d.ts +2 -1
  36. package/dist/pg-webhook-run-storage.d.ts.map +1 -1
  37. package/dist/pg-workflow-run-storage.d.ts +4 -3
  38. package/dist/pg-workflow-run-storage.d.ts.map +1 -1
  39. package/dist/run-claim-sql.d.ts +37 -0
  40. package/dist/run-claim-sql.d.ts.map +1 -0
  41. package/dist/share-session-storage.d.ts +2 -1
  42. package/dist/share-session-storage.d.ts.map +1 -1
  43. package/package.json +3 -3
  44. package/src/auth-storage/auth-storage.ts +66 -0
  45. package/src/auth-storage/identities.ts +15 -0
  46. package/src/auth-storage/rows.ts +3 -0
  47. package/src/auth-storage/users.ts +3 -0
  48. package/src/connector-connection-storage.ts +33 -16
  49. package/src/index.ts +15 -0
  50. package/src/migrations/089-maintenance-heartbeats.sql +10 -0
  51. package/src/migrations/090-lake-retention-runs.sql +8 -0
  52. package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
  53. package/src/migrations/092-run-recovery.sql +23 -0
  54. package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
  55. package/src/migrations/094-artifact-board-previews.sql +14 -0
  56. package/src/migrations/095-artifact-open-versions.sql +101 -0
  57. package/src/migrations/096-artifact-documents.sql +11 -0
  58. package/src/migrations/097-artifact-frame-files.sql +8 -0
  59. package/src/migrations/098-artifact-comments.sql +36 -0
  60. package/src/migrations/099-object-artifact-refs.sql +12 -0
  61. package/src/migrations/100-shared-link-guests.sql +17 -0
  62. package/src/migrations/101-connector-connection-owners.sql +38 -0
  63. package/src/migrations/102-user-suspension-source.sql +12 -0
  64. package/src/migrations.ts +40 -0
  65. package/src/objects/storage.ts +49 -0
  66. package/src/ontology-storage/materialization-session.ts +4 -0
  67. package/src/ontology-storage/materialization-writer.ts +68 -1
  68. package/src/ontology-storage/materializations.ts +3 -1
  69. package/src/pg-artifact-storage.ts +798 -137
  70. package/src/pg-dataset-ingest-storage.ts +305 -0
  71. package/src/pg-lake-retention-run-storage.ts +40 -0
  72. package/src/pg-maintenance-heartbeat-storage.ts +53 -0
  73. package/src/pg-pipeline-run-storage.ts +85 -42
  74. package/src/pg-projection-run-storage.ts +61 -2
  75. package/src/pg-sync-run-storage.ts +100 -37
  76. package/src/pg-webhook-run-storage.ts +37 -11
  77. package/src/pg-workflow-run-storage.ts +39 -100
  78. package/src/run-claim-sql.ts +173 -0
  79. package/src/share-session-storage.ts +20 -0
@@ -15,6 +15,7 @@ import type {
15
15
  UserIdentityRecord,
16
16
  UserRecord,
17
17
  UserStatus,
18
+ UserSuspensionSource,
18
19
  } from "@sixb/core/storage"
19
20
 
20
21
  type PgDate = Date | string
@@ -26,6 +27,7 @@ export interface PgAuthUserRow {
26
27
  readonly display_name: string | null
27
28
  readonly avatar_url: string | null
28
29
  readonly status: UserStatus
30
+ readonly suspension_source: UserSuspensionSource | null
29
31
  readonly created_at: PgDate
30
32
  readonly updated_at: PgDate
31
33
  }
@@ -177,6 +179,7 @@ export function rowToUserRecord(row: PgAuthUserRow): UserRecord {
177
179
  displayName: row.display_name ?? undefined,
178
180
  avatarUrl: row.avatar_url ?? undefined,
179
181
  status: row.status,
182
+ ...(row.suspension_source === null ? {} : { suspensionSource: row.suspension_source }),
180
183
  createdAt: toDate(row.created_at),
181
184
  updatedAt: toDate(row.updated_at),
182
185
  }
@@ -58,6 +58,7 @@ export class PgAuthUserStore implements AuthUserStore {
58
58
  display_name,
59
59
  avatar_url,
60
60
  status,
61
+ suspension_source,
61
62
  created_at,
62
63
  updated_at
63
64
  ) VALUES (
@@ -67,6 +68,7 @@ export class PgAuthUserStore implements AuthUserStore {
67
68
  ${input.displayName ?? null},
68
69
  ${input.avatarUrl ?? null},
69
70
  ${input.status ?? "active"},
71
+ ${input.status === "suspended" ? "manual" : null},
70
72
  ${createdAt},
71
73
  ${updatedAt}
72
74
  )
@@ -185,6 +187,7 @@ export class PgAuthUserStore implements AuthUserStore {
185
187
  const [row] = await tx<PgAuthUserRow[]>`
186
188
  UPDATE auth_users
187
189
  SET status = ${input.status},
190
+ suspension_source = ${input.status === "suspended" ? "manual" : null},
188
191
  updated_at = ${updatedAt}
189
192
  WHERE project_id = ${input.projectId}
190
193
  AND id = ${input.id}
@@ -2,13 +2,16 @@ import {
2
2
  type ConnectorConnectionPersistence,
3
3
  type ConnectorConnectionPersistenceBackend,
4
4
  type ConnectorConnectionPersistenceScope,
5
+ connectorConnectionOwnerFromColumns,
5
6
  DurableConnectorConnectionStorage,
7
+ connectorConnectionOwnerColumns as ownerColumns,
6
8
  } from "@sixb/core/internal/connector-connection-storage-provider"
7
9
  import type {
8
10
  ConnectorAuthorizationAttemptRecord,
9
11
  ConnectorAuthorizationRecord,
10
12
  ConnectorConnectionRecord,
11
13
  ConnectorConnectionRunRecord,
14
+ ConnectorConnectionSelector,
12
15
  ConnectorCredentialMutation,
13
16
  ConnectorStagedCredentials,
14
17
  } from "@sixb/core/storage"
@@ -76,15 +79,17 @@ class PgConnectorConnectionPersistence implements ConnectorConnectionPersistence
76
79
  }
77
80
 
78
81
  async insertAuthorizationAttempt(record: ConnectorAuthorizationAttemptRecord): Promise<boolean> {
82
+ const owner = ownerColumns(record.owner)
79
83
  try {
80
84
  await this.sql`
81
85
  INSERT INTO connector_authorization_attempts (
82
- project_id, connector_id, id, slot, initiated_by_execution_id,
86
+ project_id, connector_id, id, owner_type, owner_id, slot, initiated_by_execution_id,
83
87
  state_hash, code_verifier, redirect_uri, connection_run_id, return_to,
84
88
  callback_binding_hash, reauthorization_id, reauthorization_revision,
85
89
  reauthorization_connection_ids, created_at, expires_at
86
90
  ) VALUES (
87
- ${record.projectId}, ${record.connectorId}, ${record.id}, ${record.slot},
91
+ ${record.projectId}, ${record.connectorId}, ${record.id},
92
+ ${owner.ownerType}, ${owner.ownerId}, ${record.slot},
88
93
  ${record.initiatedByExecutionId}, ${record.stateHash},
89
94
  ${record.codeVerifier === undefined ? null : jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
90
95
  ${record.connectionRunId ?? null}, ${record.returnTo ?? null},
@@ -158,13 +163,14 @@ class PgConnectorConnectionPersistence implements ConnectorConnectionPersistence
158
163
  const values = runValues(record)
159
164
  await this.sql`
160
165
  INSERT INTO connector_connection_runs (
161
- project_id, connector_id, id, kind, slot, initiated_by_execution_id,
166
+ project_id, connector_id, id, kind, owner_type, owner_id, slot, initiated_by_execution_id,
162
167
  status, waiting_for, processing_id, callback_started_at,
163
168
  expires_at, authorization_id, cleanup_authorization_id, connections, error,
164
169
  created_at, updated_at, finished_at
165
170
  ) VALUES (
166
171
  ${record.projectId}, ${record.connectorId}, ${record.id}, ${values.kind},
167
- ${values.slot}, ${values.initiatedByExecutionId}, ${values.status},
172
+ ${values.ownerType}, ${values.ownerId}, ${values.slot},
173
+ ${values.initiatedByExecutionId}, ${values.status},
168
174
  ${values.waitingFor}, ${values.processingId},
169
175
  ${values.callbackStartedAt}, ${values.expiresAt}, ${values.authorizationId},
170
176
  ${values.cleanupAuthorizationId},
@@ -199,8 +205,8 @@ class PgConnectorConnectionPersistence implements ConnectorConnectionPersistence
199
205
  const values = runValues(record)
200
206
  const rows = await this.sql<{ readonly id: string }[]>`
201
207
  UPDATE connector_connection_runs
202
- SET kind = ${values.kind}, slot = ${values.slot},
203
- initiated_by_execution_id = ${values.initiatedByExecutionId}, status = ${values.status},
208
+ SET kind = ${values.kind}, owner_type = ${values.ownerType}, owner_id = ${values.ownerId},
209
+ slot = ${values.slot}, initiated_by_execution_id = ${values.initiatedByExecutionId}, status = ${values.status},
204
210
  waiting_for = ${values.waitingFor},
205
211
  processing_id = ${values.processingId}, callback_started_at = ${values.callbackStartedAt},
206
212
  expires_at = ${values.expiresAt}, authorization_id = ${values.authorizationId},
@@ -292,14 +298,15 @@ class PgConnectorConnectionPersistence implements ConnectorConnectionPersistence
292
298
  }
293
299
 
294
300
  async insertConnection(record: ConnectorConnectionRecord): Promise<boolean> {
301
+ const owner = ownerColumns(record.owner)
295
302
  try {
296
303
  await this.sql`
297
304
  INSERT INTO connector_connections (
298
- project_id, connector_id, id, authorization_id, slot, account,
305
+ project_id, connector_id, id, authorization_id, owner_type, owner_id, slot, account,
299
306
  account_id, status, disconnected_at, created_at, updated_at
300
307
  ) VALUES (
301
308
  ${record.projectId}, ${record.connectorId}, ${record.id}, ${record.authorizationId},
302
- ${record.slot}, ${jsonParameter(this.sql, record.account)},
309
+ ${owner.ownerType}, ${owner.ownerId}, ${record.slot}, ${jsonParameter(this.sql, record.account)},
303
310
  ${record.account.id}, ${record.status}, ${record.disconnectedAt ?? null},
304
311
  ${record.createdAt}, ${record.updatedAt}
305
312
  )
@@ -316,14 +323,15 @@ class PgConnectorConnectionPersistence implements ConnectorConnectionPersistence
316
323
  return rows[0] ? connectionFromRow(rows[0]) : null
317
324
  }
318
325
 
319
- async getConnectionBySelector(input: {
320
- readonly owner: { readonly type: "project" }
321
- readonly slot: string
322
- }): Promise<ConnectorConnectionRecord | null> {
326
+ async getConnectionBySelector(
327
+ input: ConnectorConnectionSelector
328
+ ): Promise<ConnectorConnectionRecord | null> {
323
329
  if (!this.scope.connectorId) return null
330
+ const owner = ownerColumns(input.owner)
324
331
  const rows = await this.sql<PgConnectionRow[]>`
325
332
  SELECT * FROM connector_connections
326
333
  WHERE project_id = ${this.scope.projectId} AND connector_id = ${this.scope.connectorId}
334
+ AND owner_type = ${owner.ownerType} AND owner_id = ${owner.ownerId}
327
335
  AND slot = ${input.slot}
328
336
  ${this.lockClause()}
329
337
  `
@@ -365,9 +373,11 @@ class PgConnectorConnectionPersistence implements ConnectorConnectionPersistence
365
373
  }
366
374
 
367
375
  async updateConnection(record: ConnectorConnectionRecord): Promise<void> {
376
+ const owner = ownerColumns(record.owner)
368
377
  const rows = await this.sql<{ readonly id: string }[]>`
369
378
  UPDATE connector_connections
370
- SET authorization_id = ${record.authorizationId}, slot = ${record.slot},
379
+ SET authorization_id = ${record.authorizationId}, owner_type = ${owner.ownerType},
380
+ owner_id = ${owner.ownerId}, slot = ${record.slot},
371
381
  account = ${jsonParameter(this.sql, record.account)},
372
382
  account_id = ${record.account.id}, status = ${record.status},
373
383
  disconnected_at = ${record.disconnectedAt ?? null}, updated_at = ${record.updatedAt}
@@ -435,7 +445,7 @@ function authorizationAttemptFromRow(
435
445
  id: row.id,
436
446
  projectId: row.project_id,
437
447
  connectorId: row.connector_id,
438
- owner: { type: "project" },
448
+ owner: connectorConnectionOwnerFromColumns(row.owner_type, row.owner_id),
439
449
  slot: row.slot,
440
450
  initiatedByExecutionId: row.initiated_by_execution_id,
441
451
  stateHash: row.state_hash,
@@ -461,6 +471,7 @@ function authorizationAttemptFromRow(
461
471
  function runValues(record: ConnectorConnectionRunRecord) {
462
472
  return {
463
473
  kind: record.kind,
474
+ ...ownerColumns(record.owner),
464
475
  slot: record.slot,
465
476
  initiatedByExecutionId: record.initiatedByExecutionId,
466
477
  status: record.status,
@@ -489,7 +500,7 @@ function connectionRunFromRow(row: PgConnectionRunRow): ConnectorConnectionRunRe
489
500
  projectId: row.project_id,
490
501
  connectorId: row.connector_id,
491
502
  kind: row.kind,
492
- owner: { type: "project" as const },
503
+ owner: connectorConnectionOwnerFromColumns(row.owner_type, row.owner_id),
493
504
  slot: row.slot,
494
505
  initiatedByExecutionId: row.initiated_by_execution_id,
495
506
  createdAt: date(row.created_at),
@@ -635,7 +646,7 @@ function connectionFromRow(row: PgConnectionRow): ConnectorConnectionRecord {
635
646
  projectId: row.project_id,
636
647
  connectorId: row.connector_id,
637
648
  authorizationId: row.authorization_id,
638
- owner: { type: "project" },
649
+ owner: connectorConnectionOwnerFromColumns(row.owner_type, row.owner_id),
639
650
  slot: row.slot,
640
651
  account: row.account,
641
652
  status: row.status,
@@ -694,6 +705,8 @@ interface PgAuthorizationAttemptRow {
694
705
  readonly project_id: string
695
706
  readonly connector_id: string
696
707
  readonly id: string
708
+ readonly owner_type: string
709
+ readonly owner_id: string
697
710
  readonly slot: string
698
711
  readonly initiated_by_execution_id: string
699
712
  readonly state_hash: string
@@ -714,6 +727,8 @@ interface PgConnectionRunRow {
714
727
  readonly connector_id: string
715
728
  readonly id: string
716
729
  readonly kind: ConnectorConnectionRunRecord["kind"]
730
+ readonly owner_type: string
731
+ readonly owner_id: string
717
732
  readonly slot: string
718
733
  readonly initiated_by_execution_id: string
719
734
  readonly status: ConnectorConnectionRunRecord["status"]
@@ -761,6 +776,8 @@ interface PgConnectionRow {
761
776
  readonly connector_id: string
762
777
  readonly id: string
763
778
  readonly authorization_id: string
779
+ readonly owner_type: string
780
+ readonly owner_id: string
764
781
  readonly slot: string
765
782
  readonly account: ConnectorConnectionRecord["account"]
766
783
  readonly status: ConnectorConnectionRecord["status"]
package/src/index.ts CHANGED
@@ -55,9 +55,12 @@ import { PgAiLimitStorage } from "./pg-ai-limit-storage"
55
55
  import { PgAiUsageStorage } from "./pg-ai-usage-storage"
56
56
  import { PgArtifactStorage } from "./pg-artifact-storage"
57
57
  import { createPgClient, type SQL, type SQLClient } from "./pg-client"
58
+ import { PgDatasetIngestStorage } from "./pg-dataset-ingest-storage"
58
59
  import { PgExecutionStorage } from "./pg-execution-storage"
59
60
  import { PgFileDownloadGrantStorage } from "./pg-file-download-grant-storage"
60
61
  import { PgFileUploadSessionStorage } from "./pg-file-upload-session-storage"
62
+ import { PgLakeRetentionRunStorage } from "./pg-lake-retention-run-storage"
63
+ import { PgMaintenanceHeartbeatStorage } from "./pg-maintenance-heartbeat-storage"
61
64
  import { PgPipelineRunStorage } from "./pg-pipeline-run-storage"
62
65
  import { PgProjectionRunStorage } from "./pg-projection-run-storage"
63
66
  import { PgRulesStorage } from "./pg-rules-storage"
@@ -185,8 +188,11 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
185
188
  readonly shareGrants: PgShareGrantStorage
186
189
  readonly shareSessions: PgShareSessionStorage
187
190
  readonly fileUploadSessions: PgFileUploadSessionStorage
191
+ readonly datasetIngest: PgDatasetIngestStorage
188
192
  readonly fileDownloadGrants: PgFileDownloadGrantStorage
189
193
  readonly activity: PgActivityStorage
194
+ readonly maintenanceHeartbeats: PgMaintenanceHeartbeatStorage
195
+ readonly lakeRetentionRuns: PgLakeRetentionRunStorage
190
196
  readonly migrators: readonly StorageMigrator[]
191
197
 
192
198
  private readonly sql: SQL
@@ -275,8 +281,11 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
275
281
  this.shareGrants = createOperationScopedFacade(stores.shareGrants, scope)
276
282
  this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope)
277
283
  this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope)
284
+ this.datasetIngest = createOperationScopedFacade(stores.datasetIngest, scope)
278
285
  this.fileDownloadGrants = createOperationScopedFacade(stores.fileDownloadGrants, scope)
279
286
  this.activity = createOperationScopedFacade(stores.activity, scope)
287
+ this.maintenanceHeartbeats = createOperationScopedFacade(stores.maintenanceHeartbeats, scope)
288
+ this.lakeRetentionRuns = createOperationScopedFacade(stores.lakeRetentionRuns, scope)
280
289
  registerPostgresStorageTestingAdapter(this, (durationMs) =>
281
290
  stores.connectorConnections.advanceTimeForTesting(durationMs)
282
291
  )
@@ -450,8 +459,11 @@ function createPostgresStores(
450
459
  shareGrants,
451
460
  shareSessions,
452
461
  fileUploadSessions: new PgFileUploadSessionStorage(sql),
462
+ datasetIngest: new PgDatasetIngestStorage(sql),
453
463
  fileDownloadGrants: new PgFileDownloadGrantStorage(sql),
454
464
  activity: new PgActivityStorage(sql),
465
+ maintenanceHeartbeats: new PgMaintenanceHeartbeatStorage(sql),
466
+ lakeRetentionRuns: new PgLakeRetentionRunStorage(sql),
455
467
  }
456
468
  }
457
469
 
@@ -478,8 +490,11 @@ interface PostgresStoreSet {
478
490
  readonly shareGrants: PgShareGrantStorage
479
491
  readonly shareSessions: PgShareSessionStorage
480
492
  readonly fileUploadSessions: PgFileUploadSessionStorage
493
+ readonly datasetIngest: PgDatasetIngestStorage
481
494
  readonly fileDownloadGrants: PgFileDownloadGrantStorage
482
495
  readonly activity: PgActivityStorage
496
+ readonly maintenanceHeartbeats: PgMaintenanceHeartbeatStorage
497
+ readonly lakeRetentionRuns: PgLakeRetentionRunStorage
483
498
  }
484
499
 
485
500
  function resolveTimeoutMillis(value: number | undefined, label: string): number | undefined {
@@ -0,0 +1,10 @@
1
+ -- One row per project and owner, rewritten by that owner's maintenance loop as it runs. Another
2
+ -- process reads it to tell a stopped loop from a quiet one.
3
+ CREATE TABLE maintenance_heartbeats (
4
+ project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
5
+ owner TEXT NOT NULL CHECK (owner IN ('orchestrator', 'embedded')),
6
+ beat_at TIMESTAMPTZ NOT NULL,
7
+ every_ms INTEGER NOT NULL CHECK (every_ms > 0),
8
+ passes JSONB NOT NULL CHECK (jsonb_typeof(passes) = 'array'),
9
+ PRIMARY KEY (project_id, owner)
10
+ );
@@ -0,0 +1,8 @@
1
+ -- One row per project, rewritten by each automatic lake retention run. The API reads it to tell a
2
+ -- request nobody consumes from one a worker ran.
3
+ CREATE TABLE lake_retention_runs (
4
+ project_id TEXT PRIMARY KEY CHECK (length(trim(project_id)) > 0),
5
+ started_at TIMESTAMPTZ NOT NULL,
6
+ finished_at TIMESTAMPTZ,
7
+ failed BOOLEAN NOT NULL
8
+ );
@@ -0,0 +1,38 @@
1
+ -- Changes `sixb.datasets.ingest` stored and no flush has written to the lake yet. A flush tags the
2
+ -- deliveries it takes with its id, writes them as one dataset version, then deletes them. One
3
+ -- refused when written stays, marked, until the orchestrator purges it.
4
+ CREATE TABLE dataset_ingest_deliveries (
5
+ id BIGSERIAL,
6
+ project_id TEXT NOT NULL,
7
+ dataset_id TEXT NOT NULL,
8
+ execution_id TEXT NOT NULL,
9
+ correlation_id TEXT NOT NULL,
10
+ received_at TIMESTAMPTZ NOT NULL,
11
+ change_count INTEGER NOT NULL CHECK (change_count > 0),
12
+ definition_hash TEXT NOT NULL,
13
+ -- Text, not JSONB: nothing reads inside it, and JSONB refuses strings JSON allows.
14
+ changes TEXT NOT NULL,
15
+ flush_id TEXT,
16
+ rejected_at TIMESTAMPTZ,
17
+ rejection TEXT,
18
+ -- Led by the dataset, so no plan for one dataset walks the deliveries of another.
19
+ PRIMARY KEY (project_id, dataset_id, id)
20
+ );
21
+ CREATE INDEX dataset_ingest_deliveries_by_flush
22
+ ON dataset_ingest_deliveries (project_id, dataset_id, flush_id, id);
23
+
24
+ -- One row per dataset: its open flush request, the lease of the flush that holds it, the batch
25
+ -- that flush took, and when a batch last completed. Locking this row orders requests against batches.
26
+ CREATE TABLE dataset_ingest_state (
27
+ project_id TEXT NOT NULL,
28
+ dataset_id TEXT NOT NULL,
29
+ request BIGINT NOT NULL DEFAULT 0,
30
+ requested_at TIMESTAMPTZ,
31
+ lease_owner TEXT,
32
+ lease_expires_at TIMESTAMPTZ,
33
+ flush_id TEXT,
34
+ flush_started_at TIMESTAMPTZ,
35
+ failure_streak INTEGER NOT NULL DEFAULT 0,
36
+ last_flushed_at TIMESTAMPTZ,
37
+ PRIMARY KEY (project_id, dataset_id)
38
+ );
@@ -0,0 +1,23 @@
1
+ -- A sync or pipeline run now survives its worker: each execution holds the run under a token that
2
+ -- fences its writes, and projects its queue lease here so another delivery can tell a live holder
3
+ -- from a dead one. A run with neither was started by an older version: nothing backfills it, and the
4
+ -- recovery pass settles it once its old worker cannot still be alive.
5
+ --
6
+ -- Every column is added without touching a row. Constraints are `NOT VALID`: rows that exist cannot
7
+ -- break them, and validating would scan the table under this lock.
8
+ ALTER TABLE sync_runs
9
+ ADD COLUMN attempt BIGINT NOT NULL DEFAULT 0,
10
+ ADD COLUMN execution_token TEXT,
11
+ ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ,
12
+ ADD COLUMN commit_prepared_at TIMESTAMPTZ,
13
+ ADD CONSTRAINT sync_runs_attempt_check CHECK (attempt >= 0) NOT VALID,
14
+ ADD CONSTRAINT sync_runs_execution_lease_check
15
+ CHECK ((execution_token IS NULL) = (execution_queue_lease_expires_at IS NULL)) NOT VALID;
16
+
17
+ ALTER TABLE pipeline_runs
18
+ ADD COLUMN attempt BIGINT NOT NULL DEFAULT 0,
19
+ ADD COLUMN execution_token TEXT,
20
+ ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ,
21
+ ADD CONSTRAINT pipeline_runs_attempt_check CHECK (attempt >= 0) NOT VALID,
22
+ ADD CONSTRAINT pipeline_runs_execution_lease_check
23
+ CHECK ((execution_token IS NULL) = (execution_queue_lease_expires_at IS NULL)) NOT VALID;
@@ -0,0 +1,10 @@
1
+ -- A webhook handler runs in the request, with no queue behind it, so the process running it keeps
2
+ -- its own lease. A run with none was started by an older API, and is judged from when it started.
3
+ -- A projection run holds a lease as other runs do, so a duplicate delivery leaves a live run alone.
4
+ --
5
+ -- These add nullable columns, which touches no row.
6
+ ALTER TABLE webhook_runs
7
+ ADD COLUMN execution_token TEXT,
8
+ ADD COLUMN execution_lease_expires_at TIMESTAMPTZ;
9
+
10
+ ALTER TABLE projection_runs ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ;
@@ -0,0 +1,14 @@
1
+ -- A picture of a canvas's board in one mode and size, kept under its render key: a hash of what the board
2
+ -- shows, which its version's outline names. Versions that keep a board as it was share its
3
+ -- picture, so no artifact owns one and deleting an artifact leaves them. The first picture kept
4
+ -- under a key stays.
5
+ CREATE TABLE artifact_board_previews (
6
+ project_id TEXT NOT NULL,
7
+ render_key TEXT NOT NULL CHECK (length(render_key) > 0),
8
+ theme TEXT NOT NULL CHECK (theme IN ('light', 'dark')),
9
+ -- `full`, a PNG of the board at its own size; `small`, a WebP at most 480 pixels wide.
10
+ size TEXT NOT NULL CHECK (size IN ('full', 'small')),
11
+ file JSONB NOT NULL,
12
+ created_at TIMESTAMPTZ NOT NULL,
13
+ PRIMARY KEY (project_id, render_key, theme, size)
14
+ );
@@ -0,0 +1,101 @@
1
+ -- An artifact's latest version is open until someone creates a version, which freezes it: a write
2
+ -- replaces an open version's content, and the first write after a freeze opens the next version.
3
+ -- `revision` counts every change to an artifact and only grows; a version's is the artifact's
4
+ -- when it last changed. Every existing version was published to stay as it is, so each is frozen
5
+ -- when it was published, and an address pinned to one keeps naming what it named.
6
+ ALTER TABLE artifacts ADD COLUMN revision INTEGER;
7
+ UPDATE artifacts SET revision = latest_version;
8
+ ALTER TABLE artifacts
9
+ ALTER COLUMN revision SET NOT NULL,
10
+ ADD CONSTRAINT artifacts_revision_check CHECK (revision > 0);
11
+
12
+ -- `created_at` is when a version opened, `updated_at` when its content was last written, and
13
+ -- `frozen_at` when someone created it, `frozen_by_*`: an existing version by whoever published it.
14
+ ALTER TABLE artifact_versions
15
+ ADD COLUMN revision INTEGER,
16
+ ADD COLUMN updated_at TIMESTAMPTZ,
17
+ ADD COLUMN frozen_at TIMESTAMPTZ,
18
+ ADD COLUMN frozen_by_principal_type TEXT
19
+ CHECK (frozen_by_principal_type IN ('user', 'serviceAccount', 'system')),
20
+ ADD COLUMN frozen_by_principal_id TEXT CHECK (length(frozen_by_principal_id) > 0);
21
+ UPDATE artifact_versions
22
+ SET revision = version, updated_at = created_at, frozen_at = created_at,
23
+ frozen_by_principal_type = author_principal_type, frozen_by_principal_id = author_principal_id;
24
+ ALTER TABLE artifact_versions
25
+ ALTER COLUMN revision SET NOT NULL,
26
+ ALTER COLUMN updated_at SET NOT NULL,
27
+ ADD CONSTRAINT artifact_versions_revision_check CHECK (revision > 0),
28
+ ADD CONSTRAINT artifact_versions_frozen_by_check CHECK (
29
+ (frozen_at IS NULL) = (frozen_by_principal_type IS NULL)
30
+ AND (frozen_by_principal_type IS NULL) = (frozen_by_principal_id IS NULL)
31
+ );
32
+
33
+ -- At most one open version: the latest, which storage keeps so.
34
+ CREATE UNIQUE INDEX idx_artifact_versions_open
35
+ ON artifact_versions (project_id, artifact_id) WHERE frozen_at IS NULL;
36
+
37
+ -- A retry no longer finds what a turn wrote by its versions' origins, which the next write
38
+ -- replaces: `artifact_turn_writes` keeps it.
39
+ DROP INDEX idx_artifact_versions_origin_run;
40
+ DROP INDEX idx_artifact_versions_origin_thread;
41
+
42
+ -- A frame link serves one revision of its version, and of each file it lends. The links issued
43
+ -- before revisions name none, and expire within the hour: they go now, and the app asks again.
44
+ DELETE FROM artifact_frame_grants;
45
+ ALTER TABLE artifact_frame_grants
46
+ ADD COLUMN revision INTEGER NOT NULL CHECK (revision > 0);
47
+
48
+ -- The Agent turn whose write shared an artifact in a thread: a retry of the turn takes the link
49
+ -- back. Links made before turns were kept stay for as long as the artifact lasts.
50
+ ALTER TABLE artifact_threads ADD COLUMN linked_by_run_id TEXT;
51
+ CREATE INDEX idx_artifact_threads_linked_by_run
52
+ ON artifact_threads (project_id, linked_by_run_id) WHERE linked_by_run_id IS NOT NULL;
53
+
54
+ -- What an artifact's latest version held before a change that can be undone: an Agent turn's
55
+ -- first write to it, one per turn, or a restore. Kept by the revision whose content it holds; each
56
+ -- artifact keeps its newest 20. `refs`, as in `artifact_versions`.
57
+ CREATE TABLE artifact_restore_points (
58
+ project_id TEXT NOT NULL,
59
+ artifact_id TEXT NOT NULL,
60
+ revision INTEGER NOT NULL CHECK (revision > 0),
61
+ version INTEGER NOT NULL CHECK (version > 0),
62
+ frozen BOOLEAN NOT NULL,
63
+ title TEXT NOT NULL CHECK (length(trim(title)) > 0),
64
+ entry TEXT NOT NULL CHECK (length(entry) > 0),
65
+ files JSONB NOT NULL CHECK (jsonb_typeof(files) = 'array'),
66
+ outline JSONB NOT NULL CHECK (jsonb_typeof(outline) = 'object'),
67
+ refs JSONB NOT NULL CHECK (jsonb_typeof(refs) = 'array'),
68
+ author_principal_type TEXT NOT NULL
69
+ CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
70
+ author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
71
+ origin_thread_id TEXT,
72
+ origin_run_id TEXT,
73
+ updated_at TIMESTAMPTZ NOT NULL,
74
+ reason TEXT NOT NULL CHECK (reason IN ('turn', 'restore')),
75
+ turn_run_id TEXT,
76
+ created_at TIMESTAMPTZ NOT NULL,
77
+ PRIMARY KEY (project_id, artifact_id, revision),
78
+ FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
79
+ CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL)),
80
+ CHECK ((reason = 'turn') = (turn_run_id IS NOT NULL))
81
+ );
82
+
83
+ CREATE UNIQUE INDEX idx_artifact_restore_points_turn
84
+ ON artifact_restore_points (project_id, artifact_id, turn_run_id) WHERE turn_run_id IS NOT NULL;
85
+
86
+ -- The artifacts each Agent turn wrote, its sub-agents' writes among them, the revision its last
87
+ -- write left and how many changes it made. Every change moves an artifact's revision by one, so a
88
+ -- retry undoes an artifact only while those changes are all its revision moved by since the turn
89
+ -- began: anyone's change in between keeps it. Each artifact keeps its newest 20 turns.
90
+ CREATE TABLE artifact_turn_writes (
91
+ project_id TEXT NOT NULL,
92
+ artifact_id TEXT NOT NULL,
93
+ turn_run_id TEXT NOT NULL CHECK (length(turn_run_id) > 0),
94
+ created BOOLEAN NOT NULL,
95
+ revision INTEGER NOT NULL CHECK (revision > 0),
96
+ writes INTEGER NOT NULL CHECK (writes > 0),
97
+ PRIMARY KEY (project_id, artifact_id, turn_run_id),
98
+ FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE
99
+ );
100
+
101
+ CREATE INDEX idx_artifact_turn_writes_turn ON artifact_turn_writes (project_id, turn_run_id);
@@ -0,0 +1,11 @@
1
+ -- A canvas's content as the Yjs document everyone editing it shares: its whole encoded state, in a
2
+ -- blob, replaced by each write that changes the content, and the revision whose content it holds.
3
+ -- It is not a journal of changes, only the document's own encoding, so a write merges into the
4
+ -- same history wherever it is made. One whose revision is not the artifact's was left behind by a
5
+ -- write that stored none: it is brought up to the content before it is used. Null until a write
6
+ -- stores one, and for kinds that are not edited live.
7
+ ALTER TABLE artifacts
8
+ ADD COLUMN document JSONB CHECK (document IS NULL OR jsonb_typeof(document) = 'object'),
9
+ ADD COLUMN document_revision INTEGER CHECK (document_revision > 0),
10
+ ADD CONSTRAINT artifacts_document_pair_check
11
+ CHECK ((document IS NULL) = (document_revision IS NULL));
@@ -0,0 +1,8 @@
1
+ -- A frame link names the files it serves, as a SHA-256 hex of their paths and digests, in place of
2
+ -- the revision it was issued at: a canvas edited live is written every few seconds, and a link
3
+ -- stays good while its boards' files are the same. Links last an hour, so those issued before are
4
+ -- dropped.
5
+ DELETE FROM artifact_frame_grants;
6
+ ALTER TABLE artifact_frame_grants
7
+ DROP COLUMN revision,
8
+ ADD COLUMN files_digest TEXT NOT NULL CHECK (length(files_digest) = 64);
@@ -0,0 +1,36 @@
1
+ -- Comments on an artifact, and replies to them (`parent_id`), each about a board or an element of
2
+ -- a canvas, or the artifact itself, at the revision its author saw (`anchor`). They go with the
3
+ -- artifact, and a reply with what it replies to; a copy of the artifact keeps none.
4
+ -- `seq` orders comments made in the same instant as they were made.
5
+ CREATE TABLE artifact_comments (
6
+ seq BIGINT GENERATED ALWAYS AS IDENTITY,
7
+ project_id TEXT NOT NULL,
8
+ artifact_id TEXT NOT NULL,
9
+ id TEXT NOT NULL CHECK (length(id) > 0),
10
+ parent_id TEXT,
11
+ anchor JSONB NOT NULL CHECK (jsonb_typeof(anchor) = 'object'),
12
+ body TEXT NOT NULL CHECK (length(body) > 0),
13
+ author_principal_type TEXT NOT NULL
14
+ CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
15
+ author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
16
+ created_at TIMESTAMPTZ NOT NULL,
17
+ resolved_at TIMESTAMPTZ,
18
+ resolved_by_principal_type TEXT
19
+ CHECK (resolved_by_principal_type IN ('user', 'serviceAccount', 'system')),
20
+ resolved_by_principal_id TEXT CHECK (length(resolved_by_principal_id) > 0),
21
+ -- The Agent's run that wrote it for its author, and that run's conversation.
22
+ origin_thread_id TEXT,
23
+ origin_run_id TEXT,
24
+ PRIMARY KEY (project_id, artifact_id, id),
25
+ FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
26
+ FOREIGN KEY (project_id, artifact_id, parent_id)
27
+ REFERENCES artifact_comments (project_id, artifact_id, id) ON DELETE CASCADE,
28
+ CHECK (
29
+ (resolved_at IS NULL) = (resolved_by_principal_type IS NULL)
30
+ AND (resolved_by_principal_type IS NULL) = (resolved_by_principal_id IS NULL)
31
+ ),
32
+ CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL))
33
+ );
34
+
35
+ CREATE INDEX idx_artifact_comments_created
36
+ ON artifact_comments (project_id, artifact_id, created_at, seq);
@@ -0,0 +1,12 @@
1
+ -- The artifacts objects hold: one row per artifact an object property holds, wherever it sits in
2
+ -- the property's value. Every object write keeps them current in its own transaction, so whoever
3
+ -- reaches an artifact through the objects that hold it finds them here. Holders list by code
4
+ -- point, which the "C" collation keeps in the key's order.
5
+ CREATE TABLE object_artifact_refs (
6
+ project_id TEXT NOT NULL,
7
+ artifact_id TEXT NOT NULL,
8
+ object_type_id TEXT COLLATE "C" NOT NULL,
9
+ primary_id TEXT COLLATE "C" NOT NULL,
10
+ property_id TEXT COLLATE "C" NOT NULL,
11
+ PRIMARY KEY (project_id, artifact_id, object_type_id, primary_id, property_id)
12
+ );
@@ -0,0 +1,17 @@
1
+ -- A shared link's visitor has no account: they give a name once, which their session keeps.
2
+ ALTER TABLE share_sessions
3
+ ADD COLUMN guest_name TEXT CHECK (guest_name IS NULL OR length(guest_name) > 0);
4
+
5
+ -- They comment as a guest: the author's id is their session, and the comment keeps the name they
6
+ -- gave, whatever becomes of the session.
7
+ ALTER TABLE artifact_comments
8
+ DROP CONSTRAINT artifact_comments_author_principal_type_check;
9
+
10
+ ALTER TABLE artifact_comments
11
+ ADD COLUMN author_name TEXT,
12
+ ADD CONSTRAINT artifact_comments_author_principal_type_check
13
+ CHECK (author_principal_type IN ('user', 'serviceAccount', 'system', 'guest')),
14
+ ADD CONSTRAINT artifact_comments_author_name_check CHECK (
15
+ (author_principal_type = 'guest') = (author_name IS NOT NULL)
16
+ AND (author_name IS NULL OR length(author_name) > 0)
17
+ );
@@ -0,0 +1,38 @@
1
+ -- A connection belongs to the project or to one person. The project's owner_id is empty, so the
2
+ -- unique key over (owner_type, owner_id, slot) never meets a NULL. Existing rows become the
3
+ -- project's; the defaults are dropped afterwards so every write has to name its owner.
4
+ ALTER TABLE connector_connections
5
+ ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
6
+ ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
7
+
8
+ ALTER TABLE connector_connections
9
+ ALTER COLUMN owner_type DROP DEFAULT,
10
+ ALTER COLUMN owner_id DROP DEFAULT,
11
+ ADD CONSTRAINT connector_connections_owner_check CHECK (
12
+ (owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
13
+ ),
14
+ DROP CONSTRAINT connector_connections_project_id_connector_id_slot_key,
15
+ ADD CONSTRAINT connector_connections_owner_slot_key
16
+ UNIQUE (project_id, connector_id, owner_type, owner_id, slot);
17
+
18
+ ALTER TABLE connector_connection_runs
19
+ ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
20
+ ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
21
+
22
+ ALTER TABLE connector_connection_runs
23
+ ALTER COLUMN owner_type DROP DEFAULT,
24
+ ALTER COLUMN owner_id DROP DEFAULT,
25
+ ADD CONSTRAINT connector_connection_runs_owner_check CHECK (
26
+ (owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
27
+ );
28
+
29
+ ALTER TABLE connector_authorization_attempts
30
+ ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
31
+ ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
32
+
33
+ ALTER TABLE connector_authorization_attempts
34
+ ALTER COLUMN owner_type DROP DEFAULT,
35
+ ALTER COLUMN owner_id DROP DEFAULT,
36
+ ADD CONSTRAINT connector_authorization_attempts_owner_check CHECK (
37
+ (owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
38
+ );
@@ -0,0 +1,12 @@
1
+ -- Who suspended a user: an administrator or server code ('manual'), or the identity provider's
2
+ -- directory ('directory'), which may lift only its own suspensions. Only a suspended user has one,
3
+ -- and every suspension made before this column existed was manual.
4
+ ALTER TABLE auth_users ADD COLUMN suspension_source TEXT;
5
+
6
+ UPDATE auth_users SET suspension_source = 'manual' WHERE status = 'suspended';
7
+
8
+ ALTER TABLE auth_users
9
+ ADD CONSTRAINT auth_users_suspension_source_check CHECK (
10
+ (status = 'suspended' AND suspension_source IN ('manual', 'directory'))
11
+ OR (status = 'active' AND suspension_source IS NULL)
12
+ );