@sixb/pg 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +9 -8
  2. package/dist/agents/computers.d.ts +8 -2
  3. package/dist/agents/computers.d.ts.map +1 -1
  4. package/dist/auth-storage/auth-storage.d.ts +2 -1
  5. package/dist/auth-storage/auth-storage.d.ts.map +1 -1
  6. package/dist/auth-storage/identities.d.ts +4 -0
  7. package/dist/auth-storage/identities.d.ts.map +1 -1
  8. package/dist/auth-storage/rows.d.ts +2 -1
  9. package/dist/auth-storage/rows.d.ts.map +1 -1
  10. package/dist/auth-storage/users.d.ts.map +1 -1
  11. package/dist/connector-connection-storage.d.ts.map +1 -1
  12. package/dist/index.d.ts +6 -0
  13. package/dist/index.d.ts.map +1 -1
  14. package/dist/index.js +2377 -428
  15. package/dist/index.js.map +37 -30
  16. package/dist/migrations.d.ts.map +1 -1
  17. package/dist/objects/conditional-reader.d.ts +7 -2
  18. package/dist/objects/conditional-reader.d.ts.map +1 -1
  19. package/dist/objects/partitions.d.ts.map +1 -1
  20. package/dist/objects/prepare-queries.d.ts.map +1 -1
  21. package/dist/objects/query-compiler.d.ts +14 -1
  22. package/dist/objects/query-compiler.d.ts.map +1 -1
  23. package/dist/objects/query-indexes.d.ts +13 -1
  24. package/dist/objects/query-indexes.d.ts.map +1 -1
  25. package/dist/objects/read-scope.d.ts.map +1 -1
  26. package/dist/objects/reader.d.ts +14 -1
  27. package/dist/objects/reader.d.ts.map +1 -1
  28. package/dist/objects/storage.d.ts +11 -1
  29. package/dist/objects/storage.d.ts.map +1 -1
  30. package/dist/objects/visibility-indexes.d.ts +22 -0
  31. package/dist/objects/visibility-indexes.d.ts.map +1 -0
  32. package/dist/objects/visibility-statistics.d.ts +26 -0
  33. package/dist/objects/visibility-statistics.d.ts.map +1 -0
  34. package/dist/objects/visibility.d.ts +34 -0
  35. package/dist/objects/visibility.d.ts.map +1 -0
  36. package/dist/ontology-storage/materialization-session.d.ts +5 -0
  37. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  38. package/dist/ontology-storage/materialization-writer.d.ts +14 -0
  39. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  40. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  41. package/dist/pg-artifact-storage.d.ts +14 -2
  42. package/dist/pg-artifact-storage.d.ts.map +1 -1
  43. package/dist/pg-client.d.ts.map +1 -1
  44. package/dist/pg-dataset-ingest-storage.d.ts +6 -0
  45. package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
  46. package/dist/pg-lake-retention-run-storage.d.ts +11 -0
  47. package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
  48. package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
  49. package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
  50. package/dist/pg-pipeline-run-storage.d.ts +5 -2
  51. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  52. package/dist/pg-projection-run-storage.d.ts +5 -1
  53. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  54. package/dist/pg-sync-run-storage.d.ts +6 -2
  55. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  56. package/dist/pg-webhook-run-storage.d.ts +2 -1
  57. package/dist/pg-webhook-run-storage.d.ts.map +1 -1
  58. package/dist/pg-workflow-run-storage.d.ts +4 -3
  59. package/dist/pg-workflow-run-storage.d.ts.map +1 -1
  60. package/dist/run-claim-sql.d.ts +37 -0
  61. package/dist/run-claim-sql.d.ts.map +1 -0
  62. package/dist/share-session-storage.d.ts +2 -1
  63. package/dist/share-session-storage.d.ts.map +1 -1
  64. package/package.json +3 -3
  65. package/src/agents/computers.ts +54 -5
  66. package/src/auth-storage/auth-storage.ts +66 -0
  67. package/src/auth-storage/identities.ts +15 -0
  68. package/src/auth-storage/rows.ts +3 -0
  69. package/src/auth-storage/users.ts +3 -0
  70. package/src/connector-connection-storage.ts +33 -16
  71. package/src/index.ts +15 -0
  72. package/src/migrations/087-agent-computer-background-commands.sql +4 -0
  73. package/src/migrations/088-web-search-model-kind.sql +12 -0
  74. package/src/migrations/089-maintenance-heartbeats.sql +10 -0
  75. package/src/migrations/090-lake-retention-runs.sql +8 -0
  76. package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
  77. package/src/migrations/092-run-recovery.sql +23 -0
  78. package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
  79. package/src/migrations/094-artifact-board-previews.sql +14 -0
  80. package/src/migrations/095-artifact-open-versions.sql +101 -0
  81. package/src/migrations/096-artifact-documents.sql +11 -0
  82. package/src/migrations/097-artifact-frame-files.sql +8 -0
  83. package/src/migrations/098-artifact-comments.sql +36 -0
  84. package/src/migrations/099-object-artifact-refs.sql +12 -0
  85. package/src/migrations/100-shared-link-guests.sql +17 -0
  86. package/src/migrations/101-connector-connection-owners.sql +38 -0
  87. package/src/migrations/102-user-suspension-source.sql +12 -0
  88. package/src/migrations.ts +46 -0
  89. package/src/objects/conditional-reader.ts +26 -4
  90. package/src/objects/partitions.ts +17 -4
  91. package/src/objects/prepare-queries.ts +8 -3
  92. package/src/objects/query-compiler.ts +64 -23
  93. package/src/objects/query-indexes.ts +93 -27
  94. package/src/objects/read-scope.ts +147 -77
  95. package/src/objects/reader.ts +46 -18
  96. package/src/objects/storage.ts +53 -0
  97. package/src/objects/visibility-indexes.ts +222 -0
  98. package/src/objects/visibility-statistics.ts +73 -0
  99. package/src/objects/visibility.ts +135 -0
  100. package/src/ontology-storage/materialization-session.ts +4 -0
  101. package/src/ontology-storage/materialization-writer.ts +68 -1
  102. package/src/ontology-storage/materializations.ts +6 -2
  103. package/src/pg-artifact-storage.ts +798 -137
  104. package/src/pg-client.ts +4 -0
  105. package/src/pg-dataset-ingest-storage.ts +305 -0
  106. package/src/pg-lake-retention-run-storage.ts +40 -0
  107. package/src/pg-maintenance-heartbeat-storage.ts +53 -0
  108. package/src/pg-pipeline-run-storage.ts +85 -42
  109. package/src/pg-projection-run-storage.ts +61 -2
  110. package/src/pg-sync-run-storage.ts +100 -37
  111. package/src/pg-webhook-run-storage.ts +37 -11
  112. package/src/pg-workflow-run-storage.ts +39 -100
  113. package/src/run-claim-sql.ts +173 -0
  114. package/src/share-session-storage.ts +20 -0
package/src/index.ts CHANGED
@@ -55,9 +55,12 @@ import { PgAiLimitStorage } from "./pg-ai-limit-storage"
55
55
  import { PgAiUsageStorage } from "./pg-ai-usage-storage"
56
56
  import { PgArtifactStorage } from "./pg-artifact-storage"
57
57
  import { createPgClient, type SQL, type SQLClient } from "./pg-client"
58
+ import { PgDatasetIngestStorage } from "./pg-dataset-ingest-storage"
58
59
  import { PgExecutionStorage } from "./pg-execution-storage"
59
60
  import { PgFileDownloadGrantStorage } from "./pg-file-download-grant-storage"
60
61
  import { PgFileUploadSessionStorage } from "./pg-file-upload-session-storage"
62
+ import { PgLakeRetentionRunStorage } from "./pg-lake-retention-run-storage"
63
+ import { PgMaintenanceHeartbeatStorage } from "./pg-maintenance-heartbeat-storage"
61
64
  import { PgPipelineRunStorage } from "./pg-pipeline-run-storage"
62
65
  import { PgProjectionRunStorage } from "./pg-projection-run-storage"
63
66
  import { PgRulesStorage } from "./pg-rules-storage"
@@ -185,8 +188,11 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
185
188
  readonly shareGrants: PgShareGrantStorage
186
189
  readonly shareSessions: PgShareSessionStorage
187
190
  readonly fileUploadSessions: PgFileUploadSessionStorage
191
+ readonly datasetIngest: PgDatasetIngestStorage
188
192
  readonly fileDownloadGrants: PgFileDownloadGrantStorage
189
193
  readonly activity: PgActivityStorage
194
+ readonly maintenanceHeartbeats: PgMaintenanceHeartbeatStorage
195
+ readonly lakeRetentionRuns: PgLakeRetentionRunStorage
190
196
  readonly migrators: readonly StorageMigrator[]
191
197
 
192
198
  private readonly sql: SQL
@@ -275,8 +281,11 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
275
281
  this.shareGrants = createOperationScopedFacade(stores.shareGrants, scope)
276
282
  this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope)
277
283
  this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope)
284
+ this.datasetIngest = createOperationScopedFacade(stores.datasetIngest, scope)
278
285
  this.fileDownloadGrants = createOperationScopedFacade(stores.fileDownloadGrants, scope)
279
286
  this.activity = createOperationScopedFacade(stores.activity, scope)
287
+ this.maintenanceHeartbeats = createOperationScopedFacade(stores.maintenanceHeartbeats, scope)
288
+ this.lakeRetentionRuns = createOperationScopedFacade(stores.lakeRetentionRuns, scope)
280
289
  registerPostgresStorageTestingAdapter(this, (durationMs) =>
281
290
  stores.connectorConnections.advanceTimeForTesting(durationMs)
282
291
  )
@@ -450,8 +459,11 @@ function createPostgresStores(
450
459
  shareGrants,
451
460
  shareSessions,
452
461
  fileUploadSessions: new PgFileUploadSessionStorage(sql),
462
+ datasetIngest: new PgDatasetIngestStorage(sql),
453
463
  fileDownloadGrants: new PgFileDownloadGrantStorage(sql),
454
464
  activity: new PgActivityStorage(sql),
465
+ maintenanceHeartbeats: new PgMaintenanceHeartbeatStorage(sql),
466
+ lakeRetentionRuns: new PgLakeRetentionRunStorage(sql),
455
467
  }
456
468
  }
457
469
 
@@ -478,8 +490,11 @@ interface PostgresStoreSet {
478
490
  readonly shareGrants: PgShareGrantStorage
479
491
  readonly shareSessions: PgShareSessionStorage
480
492
  readonly fileUploadSessions: PgFileUploadSessionStorage
493
+ readonly datasetIngest: PgDatasetIngestStorage
481
494
  readonly fileDownloadGrants: PgFileDownloadGrantStorage
482
495
  readonly activity: PgActivityStorage
496
+ readonly maintenanceHeartbeats: PgMaintenanceHeartbeatStorage
497
+ readonly lakeRetentionRuns: PgLakeRetentionRunStorage
483
498
  }
484
499
 
485
500
  function resolveTimeoutMillis(value: number | undefined, label: string): number | undefined {
@@ -0,0 +1,4 @@
1
+ -- What each open sixb reports of its threads' background commands, which outlive the turn that
2
+ -- started them, and the ones someone asked to stop from the web: its renewals hand those back.
3
+ ALTER TABLE agent_computer_sessions ADD COLUMN background_commands JSONB NOT NULL DEFAULT '[]';
4
+ ALTER TABLE agent_computer_sessions ADD COLUMN stop_requests JSONB NOT NULL DEFAULT '[]';
@@ -0,0 +1,12 @@
1
+ ALTER TABLE ai_model_call_usage
2
+ DROP CONSTRAINT ai_model_call_usage_model_kind_check;
3
+
4
+ ALTER TABLE ai_model_call_usage
5
+ ADD CONSTRAINT ai_model_call_usage_model_kind_check
6
+ CHECK (
7
+ model_kind IS NULL
8
+ OR model_kind IN (
9
+ 'language', 'image', 'video', 'embedding', 'decision', 'transcription', 'speech', 'reranking',
10
+ 'web-search'
11
+ )
12
+ );
@@ -0,0 +1,10 @@
1
+ -- One row per project and owner, rewritten by that owner's maintenance loop as it runs. Another
2
+ -- process reads it to tell a stopped loop from a quiet one.
3
+ CREATE TABLE maintenance_heartbeats (
4
+ project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
5
+ owner TEXT NOT NULL CHECK (owner IN ('orchestrator', 'embedded')),
6
+ beat_at TIMESTAMPTZ NOT NULL,
7
+ every_ms INTEGER NOT NULL CHECK (every_ms > 0),
8
+ passes JSONB NOT NULL CHECK (jsonb_typeof(passes) = 'array'),
9
+ PRIMARY KEY (project_id, owner)
10
+ );
@@ -0,0 +1,8 @@
1
+ -- One row per project, rewritten by each automatic lake retention run. The API reads it to tell a
2
+ -- request nobody consumes from one a worker ran.
3
+ CREATE TABLE lake_retention_runs (
4
+ project_id TEXT PRIMARY KEY CHECK (length(trim(project_id)) > 0),
5
+ started_at TIMESTAMPTZ NOT NULL,
6
+ finished_at TIMESTAMPTZ,
7
+ failed BOOLEAN NOT NULL
8
+ );
@@ -0,0 +1,38 @@
1
+ -- Changes `sixb.datasets.ingest` stored and no flush has written to the lake yet. A flush tags the
2
+ -- deliveries it takes with its id, writes them as one dataset version, then deletes them. One
3
+ -- refused when written stays, marked, until the orchestrator purges it.
4
+ CREATE TABLE dataset_ingest_deliveries (
5
+ id BIGSERIAL,
6
+ project_id TEXT NOT NULL,
7
+ dataset_id TEXT NOT NULL,
8
+ execution_id TEXT NOT NULL,
9
+ correlation_id TEXT NOT NULL,
10
+ received_at TIMESTAMPTZ NOT NULL,
11
+ change_count INTEGER NOT NULL CHECK (change_count > 0),
12
+ definition_hash TEXT NOT NULL,
13
+ -- Text, not JSONB: nothing reads inside it, and JSONB refuses strings JSON allows.
14
+ changes TEXT NOT NULL,
15
+ flush_id TEXT,
16
+ rejected_at TIMESTAMPTZ,
17
+ rejection TEXT,
18
+ -- Led by the dataset, so no plan for one dataset walks the deliveries of another.
19
+ PRIMARY KEY (project_id, dataset_id, id)
20
+ );
21
+ CREATE INDEX dataset_ingest_deliveries_by_flush
22
+ ON dataset_ingest_deliveries (project_id, dataset_id, flush_id, id);
23
+
24
+ -- One row per dataset: its open flush request, the lease of the flush that holds it, the batch
25
+ -- that flush took, and when a batch last completed. Locking this row orders requests against batches.
26
+ CREATE TABLE dataset_ingest_state (
27
+ project_id TEXT NOT NULL,
28
+ dataset_id TEXT NOT NULL,
29
+ request BIGINT NOT NULL DEFAULT 0,
30
+ requested_at TIMESTAMPTZ,
31
+ lease_owner TEXT,
32
+ lease_expires_at TIMESTAMPTZ,
33
+ flush_id TEXT,
34
+ flush_started_at TIMESTAMPTZ,
35
+ failure_streak INTEGER NOT NULL DEFAULT 0,
36
+ last_flushed_at TIMESTAMPTZ,
37
+ PRIMARY KEY (project_id, dataset_id)
38
+ );
@@ -0,0 +1,23 @@
1
+ -- A sync or pipeline run now survives its worker: each execution holds the run under a token that
2
+ -- fences its writes, and projects its queue lease here so another delivery can tell a live holder
3
+ -- from a dead one. A run with neither was started by an older version: nothing backfills it, and the
4
+ -- recovery pass settles it once its old worker cannot still be alive.
5
+ --
6
+ -- Every column is added without touching a row. Constraints are `NOT VALID`: rows that exist cannot
7
+ -- break them, and validating would scan the table under this lock.
8
+ ALTER TABLE sync_runs
9
+ ADD COLUMN attempt BIGINT NOT NULL DEFAULT 0,
10
+ ADD COLUMN execution_token TEXT,
11
+ ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ,
12
+ ADD COLUMN commit_prepared_at TIMESTAMPTZ,
13
+ ADD CONSTRAINT sync_runs_attempt_check CHECK (attempt >= 0) NOT VALID,
14
+ ADD CONSTRAINT sync_runs_execution_lease_check
15
+ CHECK ((execution_token IS NULL) = (execution_queue_lease_expires_at IS NULL)) NOT VALID;
16
+
17
+ ALTER TABLE pipeline_runs
18
+ ADD COLUMN attempt BIGINT NOT NULL DEFAULT 0,
19
+ ADD COLUMN execution_token TEXT,
20
+ ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ,
21
+ ADD CONSTRAINT pipeline_runs_attempt_check CHECK (attempt >= 0) NOT VALID,
22
+ ADD CONSTRAINT pipeline_runs_execution_lease_check
23
+ CHECK ((execution_token IS NULL) = (execution_queue_lease_expires_at IS NULL)) NOT VALID;
@@ -0,0 +1,10 @@
1
+ -- A webhook handler runs in the request, with no queue behind it, so the process running it keeps
2
+ -- its own lease. A run with none was started by an older API, and is judged from when it started.
3
+ -- A projection run holds a lease as other runs do, so a duplicate delivery leaves a live run alone.
4
+ --
5
+ -- These add nullable columns, which touches no row.
6
+ ALTER TABLE webhook_runs
7
+ ADD COLUMN execution_token TEXT,
8
+ ADD COLUMN execution_lease_expires_at TIMESTAMPTZ;
9
+
10
+ ALTER TABLE projection_runs ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ;
@@ -0,0 +1,14 @@
1
+ -- A picture of a canvas's board in one mode and size, kept under its render key: a hash of what the board
2
+ -- shows, which its version's outline names. Versions that keep a board as it was share its
3
+ -- picture, so no artifact owns one and deleting an artifact leaves them. The first picture kept
4
+ -- under a key stays.
5
+ CREATE TABLE artifact_board_previews (
6
+ project_id TEXT NOT NULL,
7
+ render_key TEXT NOT NULL CHECK (length(render_key) > 0),
8
+ theme TEXT NOT NULL CHECK (theme IN ('light', 'dark')),
9
+ -- `full`, a PNG of the board at its own size; `small`, a WebP at most 480 pixels wide.
10
+ size TEXT NOT NULL CHECK (size IN ('full', 'small')),
11
+ file JSONB NOT NULL,
12
+ created_at TIMESTAMPTZ NOT NULL,
13
+ PRIMARY KEY (project_id, render_key, theme, size)
14
+ );
@@ -0,0 +1,101 @@
1
+ -- An artifact's latest version is open until someone creates a version, which freezes it: a write
2
+ -- replaces an open version's content, and the first write after a freeze opens the next version.
3
+ -- `revision` counts every change to an artifact and only grows; a version's is the artifact's
4
+ -- when it last changed. Every existing version was published to stay as it is, so each is frozen
5
+ -- when it was published, and an address pinned to one keeps naming what it named.
6
+ ALTER TABLE artifacts ADD COLUMN revision INTEGER;
7
+ UPDATE artifacts SET revision = latest_version;
8
+ ALTER TABLE artifacts
9
+ ALTER COLUMN revision SET NOT NULL,
10
+ ADD CONSTRAINT artifacts_revision_check CHECK (revision > 0);
11
+
12
+ -- `created_at` is when a version opened, `updated_at` when its content was last written, and
13
+ -- `frozen_at` when someone created it, `frozen_by_*`: an existing version by whoever published it.
14
+ ALTER TABLE artifact_versions
15
+ ADD COLUMN revision INTEGER,
16
+ ADD COLUMN updated_at TIMESTAMPTZ,
17
+ ADD COLUMN frozen_at TIMESTAMPTZ,
18
+ ADD COLUMN frozen_by_principal_type TEXT
19
+ CHECK (frozen_by_principal_type IN ('user', 'serviceAccount', 'system')),
20
+ ADD COLUMN frozen_by_principal_id TEXT CHECK (length(frozen_by_principal_id) > 0);
21
+ UPDATE artifact_versions
22
+ SET revision = version, updated_at = created_at, frozen_at = created_at,
23
+ frozen_by_principal_type = author_principal_type, frozen_by_principal_id = author_principal_id;
24
+ ALTER TABLE artifact_versions
25
+ ALTER COLUMN revision SET NOT NULL,
26
+ ALTER COLUMN updated_at SET NOT NULL,
27
+ ADD CONSTRAINT artifact_versions_revision_check CHECK (revision > 0),
28
+ ADD CONSTRAINT artifact_versions_frozen_by_check CHECK (
29
+ (frozen_at IS NULL) = (frozen_by_principal_type IS NULL)
30
+ AND (frozen_by_principal_type IS NULL) = (frozen_by_principal_id IS NULL)
31
+ );
32
+
33
+ -- At most one open version: the latest, which storage keeps so.
34
+ CREATE UNIQUE INDEX idx_artifact_versions_open
35
+ ON artifact_versions (project_id, artifact_id) WHERE frozen_at IS NULL;
36
+
37
+ -- A retry no longer finds what a turn wrote by its versions' origins, which the next write
38
+ -- replaces: `artifact_turn_writes` keeps it.
39
+ DROP INDEX idx_artifact_versions_origin_run;
40
+ DROP INDEX idx_artifact_versions_origin_thread;
41
+
42
+ -- A frame link serves one revision of its version, and of each file it lends. The links issued
43
+ -- before revisions name none, and expire within the hour: they go now, and the app asks again.
44
+ DELETE FROM artifact_frame_grants;
45
+ ALTER TABLE artifact_frame_grants
46
+ ADD COLUMN revision INTEGER NOT NULL CHECK (revision > 0);
47
+
48
+ -- The Agent turn whose write shared an artifact in a thread: a retry of the turn takes the link
49
+ -- back. Links made before turns were kept stay for as long as the artifact lasts.
50
+ ALTER TABLE artifact_threads ADD COLUMN linked_by_run_id TEXT;
51
+ CREATE INDEX idx_artifact_threads_linked_by_run
52
+ ON artifact_threads (project_id, linked_by_run_id) WHERE linked_by_run_id IS NOT NULL;
53
+
54
+ -- What an artifact's latest version held before a change that can be undone: an Agent turn's
55
+ -- first write to it, one per turn, or a restore. Kept by the revision whose content it holds; each
56
+ -- artifact keeps its newest 20. `refs`, as in `artifact_versions`.
57
+ CREATE TABLE artifact_restore_points (
58
+ project_id TEXT NOT NULL,
59
+ artifact_id TEXT NOT NULL,
60
+ revision INTEGER NOT NULL CHECK (revision > 0),
61
+ version INTEGER NOT NULL CHECK (version > 0),
62
+ frozen BOOLEAN NOT NULL,
63
+ title TEXT NOT NULL CHECK (length(trim(title)) > 0),
64
+ entry TEXT NOT NULL CHECK (length(entry) > 0),
65
+ files JSONB NOT NULL CHECK (jsonb_typeof(files) = 'array'),
66
+ outline JSONB NOT NULL CHECK (jsonb_typeof(outline) = 'object'),
67
+ refs JSONB NOT NULL CHECK (jsonb_typeof(refs) = 'array'),
68
+ author_principal_type TEXT NOT NULL
69
+ CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
70
+ author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
71
+ origin_thread_id TEXT,
72
+ origin_run_id TEXT,
73
+ updated_at TIMESTAMPTZ NOT NULL,
74
+ reason TEXT NOT NULL CHECK (reason IN ('turn', 'restore')),
75
+ turn_run_id TEXT,
76
+ created_at TIMESTAMPTZ NOT NULL,
77
+ PRIMARY KEY (project_id, artifact_id, revision),
78
+ FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
79
+ CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL)),
80
+ CHECK ((reason = 'turn') = (turn_run_id IS NOT NULL))
81
+ );
82
+
83
+ CREATE UNIQUE INDEX idx_artifact_restore_points_turn
84
+ ON artifact_restore_points (project_id, artifact_id, turn_run_id) WHERE turn_run_id IS NOT NULL;
85
+
86
+ -- The artifacts each Agent turn wrote, its sub-agents' writes among them, the revision its last
87
+ -- write left and how many changes it made. Every change moves an artifact's revision by one, so a
88
+ -- retry undoes an artifact only while those changes are all its revision moved by since the turn
89
+ -- began: anyone's change in between keeps it. Each artifact keeps its newest 20 turns.
90
+ CREATE TABLE artifact_turn_writes (
91
+ project_id TEXT NOT NULL,
92
+ artifact_id TEXT NOT NULL,
93
+ turn_run_id TEXT NOT NULL CHECK (length(turn_run_id) > 0),
94
+ created BOOLEAN NOT NULL,
95
+ revision INTEGER NOT NULL CHECK (revision > 0),
96
+ writes INTEGER NOT NULL CHECK (writes > 0),
97
+ PRIMARY KEY (project_id, artifact_id, turn_run_id),
98
+ FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE
99
+ );
100
+
101
+ CREATE INDEX idx_artifact_turn_writes_turn ON artifact_turn_writes (project_id, turn_run_id);
@@ -0,0 +1,11 @@
1
+ -- A canvas's content as the Yjs document everyone editing it shares: its whole encoded state, in a
2
+ -- blob, replaced by each write that changes the content, and the revision whose content it holds.
3
+ -- It is not a journal of changes, only the document's own encoding, so a write merges into the
4
+ -- same history wherever it is made. One whose revision is not the artifact's was left behind by a
5
+ -- write that stored none: it is brought up to the content before it is used. Null until a write
6
+ -- stores one, and for kinds that are not edited live.
7
+ ALTER TABLE artifacts
8
+ ADD COLUMN document JSONB CHECK (document IS NULL OR jsonb_typeof(document) = 'object'),
9
+ ADD COLUMN document_revision INTEGER CHECK (document_revision > 0),
10
+ ADD CONSTRAINT artifacts_document_pair_check
11
+ CHECK ((document IS NULL) = (document_revision IS NULL));
@@ -0,0 +1,8 @@
1
+ -- A frame link names the files it serves, as a SHA-256 hex of their paths and digests, in place of
2
+ -- the revision it was issued at: a canvas edited live is written every few seconds, and a link
3
+ -- stays good while its boards' files are the same. Links last an hour, so those issued before are
4
+ -- dropped.
5
+ DELETE FROM artifact_frame_grants;
6
+ ALTER TABLE artifact_frame_grants
7
+ DROP COLUMN revision,
8
+ ADD COLUMN files_digest TEXT NOT NULL CHECK (length(files_digest) = 64);
@@ -0,0 +1,36 @@
1
+ -- Comments on an artifact, and replies to them (`parent_id`), each about a board or an element of
2
+ -- a canvas, or the artifact itself, at the revision its author saw (`anchor`). They go with the
3
+ -- artifact, and a reply with what it replies to; a copy of the artifact keeps none.
4
+ -- `seq` orders comments made in the same instant as they were made.
5
+ CREATE TABLE artifact_comments (
6
+ seq BIGINT GENERATED ALWAYS AS IDENTITY,
7
+ project_id TEXT NOT NULL,
8
+ artifact_id TEXT NOT NULL,
9
+ id TEXT NOT NULL CHECK (length(id) > 0),
10
+ parent_id TEXT,
11
+ anchor JSONB NOT NULL CHECK (jsonb_typeof(anchor) = 'object'),
12
+ body TEXT NOT NULL CHECK (length(body) > 0),
13
+ author_principal_type TEXT NOT NULL
14
+ CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
15
+ author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
16
+ created_at TIMESTAMPTZ NOT NULL,
17
+ resolved_at TIMESTAMPTZ,
18
+ resolved_by_principal_type TEXT
19
+ CHECK (resolved_by_principal_type IN ('user', 'serviceAccount', 'system')),
20
+ resolved_by_principal_id TEXT CHECK (length(resolved_by_principal_id) > 0),
21
+ -- The Agent's run that wrote it for its author, and that run's conversation.
22
+ origin_thread_id TEXT,
23
+ origin_run_id TEXT,
24
+ PRIMARY KEY (project_id, artifact_id, id),
25
+ FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
26
+ FOREIGN KEY (project_id, artifact_id, parent_id)
27
+ REFERENCES artifact_comments (project_id, artifact_id, id) ON DELETE CASCADE,
28
+ CHECK (
29
+ (resolved_at IS NULL) = (resolved_by_principal_type IS NULL)
30
+ AND (resolved_by_principal_type IS NULL) = (resolved_by_principal_id IS NULL)
31
+ ),
32
+ CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL))
33
+ );
34
+
35
+ CREATE INDEX idx_artifact_comments_created
36
+ ON artifact_comments (project_id, artifact_id, created_at, seq);
@@ -0,0 +1,12 @@
1
+ -- The artifacts objects hold: one row per artifact an object property holds, wherever it sits in
2
+ -- the property's value. Every object write keeps them current in its own transaction, so whoever
3
+ -- reaches an artifact through the objects that hold it finds them here. Holders list by code
4
+ -- point, which the "C" collation keeps in the key's order.
5
+ CREATE TABLE object_artifact_refs (
6
+ project_id TEXT NOT NULL,
7
+ artifact_id TEXT NOT NULL,
8
+ object_type_id TEXT COLLATE "C" NOT NULL,
9
+ primary_id TEXT COLLATE "C" NOT NULL,
10
+ property_id TEXT COLLATE "C" NOT NULL,
11
+ PRIMARY KEY (project_id, artifact_id, object_type_id, primary_id, property_id)
12
+ );
@@ -0,0 +1,17 @@
1
+ -- A shared link's visitor has no account: they give a name once, which their session keeps.
2
+ ALTER TABLE share_sessions
3
+ ADD COLUMN guest_name TEXT CHECK (guest_name IS NULL OR length(guest_name) > 0);
4
+
5
+ -- They comment as a guest: the author's id is their session, and the comment keeps the name they
6
+ -- gave, whatever becomes of the session.
7
+ ALTER TABLE artifact_comments
8
+ DROP CONSTRAINT artifact_comments_author_principal_type_check;
9
+
10
+ ALTER TABLE artifact_comments
11
+ ADD COLUMN author_name TEXT,
12
+ ADD CONSTRAINT artifact_comments_author_principal_type_check
13
+ CHECK (author_principal_type IN ('user', 'serviceAccount', 'system', 'guest')),
14
+ ADD CONSTRAINT artifact_comments_author_name_check CHECK (
15
+ (author_principal_type = 'guest') = (author_name IS NOT NULL)
16
+ AND (author_name IS NULL OR length(author_name) > 0)
17
+ );
@@ -0,0 +1,38 @@
1
+ -- A connection belongs to the project or to one person. The project's owner_id is empty, so the
2
+ -- unique key over (owner_type, owner_id, slot) never meets a NULL. Existing rows become the
3
+ -- project's; the defaults are dropped afterwards so every write has to name its owner.
4
+ ALTER TABLE connector_connections
5
+ ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
6
+ ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
7
+
8
+ ALTER TABLE connector_connections
9
+ ALTER COLUMN owner_type DROP DEFAULT,
10
+ ALTER COLUMN owner_id DROP DEFAULT,
11
+ ADD CONSTRAINT connector_connections_owner_check CHECK (
12
+ (owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
13
+ ),
14
+ DROP CONSTRAINT connector_connections_project_id_connector_id_slot_key,
15
+ ADD CONSTRAINT connector_connections_owner_slot_key
16
+ UNIQUE (project_id, connector_id, owner_type, owner_id, slot);
17
+
18
+ ALTER TABLE connector_connection_runs
19
+ ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
20
+ ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
21
+
22
+ ALTER TABLE connector_connection_runs
23
+ ALTER COLUMN owner_type DROP DEFAULT,
24
+ ALTER COLUMN owner_id DROP DEFAULT,
25
+ ADD CONSTRAINT connector_connection_runs_owner_check CHECK (
26
+ (owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
27
+ );
28
+
29
+ ALTER TABLE connector_authorization_attempts
30
+ ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
31
+ ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
32
+
33
+ ALTER TABLE connector_authorization_attempts
34
+ ALTER COLUMN owner_type DROP DEFAULT,
35
+ ALTER COLUMN owner_id DROP DEFAULT,
36
+ ADD CONSTRAINT connector_authorization_attempts_owner_check CHECK (
37
+ (owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
38
+ );
@@ -0,0 +1,12 @@
1
+ -- Who suspended a user: an administrator or server code ('manual'), or the identity provider's
2
+ -- directory ('directory'), which may lift only its own suspensions. Only a suspended user has one,
3
+ -- and every suspension made before this column existed was manual.
4
+ ALTER TABLE auth_users ADD COLUMN suspension_source TEXT;
5
+
6
+ UPDATE auth_users SET suspension_source = 'manual' WHERE status = 'suspended';
7
+
8
+ ALTER TABLE auth_users
9
+ ADD CONSTRAINT auth_users_suspension_source_check CHECK (
10
+ (status = 'suspended' AND suspension_source IN ('manual', 'directory'))
11
+ OR (status = 'active' AND suspension_source IS NULL)
12
+ );
package/src/migrations.ts CHANGED
@@ -457,6 +457,22 @@ export const postgresStorageMigrations = defineMigrations<PostgresMigrationConte
457
457
  pgSql("084-agent-token-rates", agentTokenRatesSql),
458
458
  pgSql("085-agent-computer-sessions", agentComputerSessionsSql),
459
459
  pgSql("086-partition-objects", partitionObjectsSql),
460
+ pgSql("087-agent-computer-background-commands", agentComputerBackgroundCommandsSql),
461
+ pgSql("088-web-search-model-kind", webSearchModelKindSql),
462
+ pgSql("089-maintenance-heartbeats", maintenanceHeartbeatsSql),
463
+ pgSql("090-lake-retention-runs", lakeRetentionRunsSql),
464
+ pgSql("091-dataset-ingest-inbox", datasetIngestInboxSql),
465
+ pgSql("092-run-recovery", runRecoverySql),
466
+ pgSql("093-webhook-and-projection-leases", webhookAndProjectionLeasesSql),
467
+ pgSql("094-artifact-board-previews", artifactBoardPreviewsSql),
468
+ pgSql("095-artifact-open-versions", artifactOpenVersionsSql),
469
+ pgSql("096-artifact-documents", artifactDocumentsSql),
470
+ pgSql("097-artifact-frame-files", artifactFrameFilesSql),
471
+ pgSql("098-artifact-comments", artifactCommentsSql),
472
+ pgSql("099-object-artifact-refs", objectArtifactRefsSql),
473
+ pgSql("100-shared-link-guests", sharedLinkGuestsSql),
474
+ pgSql("101-connector-connection-owners", connectorConnectionOwnersSql),
475
+ pgSql("102-user-suspension-source", userSuspensionSourceSql),
460
476
  ],
461
477
  })
462
478
 
@@ -529,3 +545,33 @@ import agentComputerSessionsSql from "./migrations/085-agent-computer-sessions.s
529
545
  type: "text",
530
546
  }
531
547
  import partitionObjectsSql from "./migrations/086-partition-objects.sql" with { type: "text" }
548
+ import agentComputerBackgroundCommandsSql from "./migrations/087-agent-computer-background-commands.sql" with {
549
+ type: "text",
550
+ }
551
+ import webSearchModelKindSql from "./migrations/088-web-search-model-kind.sql" with { type: "text" }
552
+ import maintenanceHeartbeatsSql from "./migrations/089-maintenance-heartbeats.sql" with {
553
+ type: "text",
554
+ }
555
+ import lakeRetentionRunsSql from "./migrations/090-lake-retention-runs.sql" with { type: "text" }
556
+ import datasetIngestInboxSql from "./migrations/091-dataset-ingest-inbox.sql" with { type: "text" }
557
+ import runRecoverySql from "./migrations/092-run-recovery.sql" with { type: "text" }
558
+ import webhookAndProjectionLeasesSql from "./migrations/093-webhook-and-projection-leases.sql" with {
559
+ type: "text",
560
+ }
561
+ import artifactBoardPreviewsSql from "./migrations/094-artifact-board-previews.sql" with {
562
+ type: "text",
563
+ }
564
+ import artifactOpenVersionsSql from "./migrations/095-artifact-open-versions.sql" with {
565
+ type: "text",
566
+ }
567
+ import artifactDocumentsSql from "./migrations/096-artifact-documents.sql" with { type: "text" }
568
+ import artifactFrameFilesSql from "./migrations/097-artifact-frame-files.sql" with { type: "text" }
569
+ import artifactCommentsSql from "./migrations/098-artifact-comments.sql" with { type: "text" }
570
+ import objectArtifactRefsSql from "./migrations/099-object-artifact-refs.sql" with { type: "text" }
571
+ import sharedLinkGuestsSql from "./migrations/100-shared-link-guests.sql" with { type: "text" }
572
+ import connectorConnectionOwnersSql from "./migrations/101-connector-connection-owners.sql" with {
573
+ type: "text",
574
+ }
575
+ import userSuspensionSourceSql from "./migrations/102-user-suspension-source.sql" with {
576
+ type: "text",
577
+ }
@@ -5,17 +5,21 @@ import type {
5
5
  ObjectQueryCapabilities,
6
6
  ObjectReadStorage,
7
7
  } from "@sixb/core/storage"
8
+ import type { SQLClient } from "../pg-client"
8
9
  import { type PgStoreClient, runPgRepeatableReadTransaction } from "../transactions"
9
10
  import type { PgAggregateAdmission } from "./aggregate-admission"
11
+ import type { PgSortIndexCatalog } from "./query-indexes"
10
12
  import { compilePgConditionalObjectReadSource } from "./read-scope"
11
13
  import { PgObjectReader } from "./reader"
12
14
  import { createPgScopedReader } from "./scoped-reader"
15
+ import { type PgVisibilityStatisticsCatalog, readsFewObjects } from "./visibility-statistics"
13
16
 
14
17
  /**
15
18
  * Read objects and links through relations that apply the scope's conditions as they are read.
16
19
  *
17
- * Reads run as unscoped ones do, under the same aggregate admission: on the pool, and a vector
18
- * query on one repeatable-read snapshot for its candidate probe and its rows.
20
+ * Reads run as unscoped ones do, under the same aggregate admission and over the same sort
21
+ * indexes: on the pool, and a vector query on one repeatable-read snapshot for its candidate probe
22
+ * and its rows.
19
23
  */
20
24
  export function createPgConditionalReader(
21
25
  sql: PgStoreClient,
@@ -25,18 +29,36 @@ export function createPgConditionalReader(
25
29
  },
26
30
  options: {
27
31
  aggregateAdmission?: PgAggregateAdmission
32
+ sortIndexes: PgSortIndexCatalog
33
+ visibilityStatistics: PgVisibilityStatisticsCatalog
28
34
  queryCapabilities: () => ObjectQueryCapabilities
29
35
  }
30
36
  ): ObjectReadStorage {
31
37
  const source = compilePgConditionalObjectReadSource(params.projectId, params.scope)
32
- const reader = new PgObjectReader(sql, source, options.aggregateAdmission)
38
+
39
+ const fewObjects = async (client: SQLClient, objectTypeId: string) => {
40
+ const condition = params.scope.conditions.find((item) => item.objectTypeId === objectTypeId)
41
+ if (!condition) return false
42
+
43
+ return readsFewObjects(condition, await options.visibilityStatistics.commonKeys(client))
44
+ }
45
+
46
+ const reader = new PgObjectReader(
47
+ sql,
48
+ source,
49
+ options.aggregateAdmission,
50
+ options.sortIndexes,
51
+ fewObjects
52
+ )
33
53
 
34
54
  return createPgScopedReader(
35
55
  {
36
56
  projectId: params.projectId,
37
57
  read: (run, query?: ObjectQuery) =>
38
58
  query && hasVectorProfile(query)
39
- ? runPgRepeatableReadTransaction(sql, (tx) => run(new PgObjectReader(tx, source)))
59
+ ? runPgRepeatableReadTransaction(sql, (tx) =>
60
+ run(new PgObjectReader(tx, source, undefined, options.sortIndexes, fewObjects))
61
+ )
40
62
  : run(reader),
41
63
  },
42
64
  options.queryCapabilities
@@ -11,6 +11,12 @@ const LOCK_TIMEOUT = "1s"
11
11
  const LOCK_ATTEMPTS = 120
12
12
  const LOCK_RETRY_MS = 1_000
13
13
 
14
+ // Autovacuum analyzes a partition once 50 rows plus 2% of it changed, instead of 10%. A value the
15
+ // statistics have never seen is estimated at a row or two: once 5% of a 30,000-row type held one,
16
+ // a read planned on that took 6 s instead of 75 ms (at 4% it was still fine). ANALYZE samples at
17
+ // most 30,000 rows, so on any large partition it takes about half a second to a second.
18
+ const ANALYZE_SCALE_FACTOR = "autovacuum_analyze_scale_factor=0.02"
19
+
14
20
  /** Migration 086 derives the same name in SQL: `'objects_' || left(md5(type), 24)`. */
15
21
  export function objectPartitionName(objectTypeId: string): string {
16
22
  return `objects_${createHash("md5").update(objectTypeId).digest("hex").slice(0, 24)}`
@@ -28,9 +34,16 @@ export async function ensureObjectPartition(
28
34
  ): Promise<string> {
29
35
  const name = objectPartitionName(objectTypeId)
30
36
  const partition = `${identifier(schema)}.${identifier(name)}`
31
- const [existing] = await connection<{ found: string | null }[]>`
32
- SELECT to_regclass(${partition})::text AS found`
33
- if (existing?.found) return name
37
+ const [existing] = await connection<{ tuned: boolean }[]>`
38
+ SELECT coalesce(reloptions, '{}') @> ARRAY[${ANALYZE_SCALE_FACTOR}] AS tuned
39
+ FROM pg_class WHERE oid = to_regclass(${partition})`
40
+ if (existing) {
41
+ // Partitions created without it, such as migration 086's, take it on their next preparation.
42
+ if (!existing.tuned) {
43
+ await connection.unsafe(`ALTER TABLE ${partition} SET (${ANALYZE_SCALE_FACTOR})`)
44
+ }
45
+ return name
46
+ }
34
47
 
35
48
  for (let attempt = 1; ; attempt++) {
36
49
  try {
@@ -100,7 +113,7 @@ async function createObjectPartition(
100
113
  }
101
114
 
102
115
  await connection.unsafe(
103
- `CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE)`
116
+ `CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE) WITH (${ANALYZE_SCALE_FACTOR})`
104
117
  )
105
118
  if (moving) await moveRowsFromDefault(connection, parent, fallback, partition, type, references)
106
119
  await connection.unsafe(
@@ -8,6 +8,8 @@ import { ensureObjectPartition } from "./partitions"
8
8
  import { lockPgPreparation } from "./preparation-lock"
9
9
  import { ensurePgObjectQueryIndexes, type PgObjectQueryIndex } from "./query-indexes"
10
10
  import { type PgObjectTextCountIndex, preparePgObjectTextCounts } from "./text-count-index"
11
+ import { compilePgVisibilityKeys } from "./visibility"
12
+ import { preparePgVisibilityIndexes } from "./visibility-indexes"
11
13
 
12
14
  /** Reconcile ontology indexes and text columns once per plan, coordinating concurrent startups. */
13
15
  export async function preparePgObjectQueries(
@@ -61,11 +63,13 @@ function preparationFingerprint(plan: ObjectQueryPreparationPlan): string {
61
63
  .map(normalizeIndex)
62
64
  .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
63
65
  links: [...type.links].sort((a, b) => a.linkId.localeCompare(b.linkId)),
66
+ // The SQL each condition's index is built on, so a change to its compilation rebuilds it.
67
+ ...(type.visibility ? { visibility: type.visibility.map(compilePgVisibilityKeys) } : {}),
64
68
  }))
65
69
  .sort((a, b) => a.objectTypeId.localeCompare(b.objectTypeId))
66
70
 
67
71
  // Bump the version when physical generation changes, even if the ontology stays the same.
68
- const definition = { version: 3, projectId: plan.projectId, objectTypes }
72
+ const definition = { version: 5, projectId: plan.projectId, objectTypes }
69
73
  return createHash("sha256").update(JSON.stringify(definition)).digest("hex")
70
74
  }
71
75
 
@@ -117,11 +121,12 @@ async function prepareStructures(
117
121
 
118
122
  const names = await ensurePgObjectQueryIndexes(sql, schema, indexes)
119
123
  const texts = await preparePgObjectTextCounts(sql, schema, textCounts)
124
+ const visibility = await preparePgVisibilityIndexes(sql, schema, plan)
120
125
  return {
121
126
  status: "prepared",
122
127
  objectTypes: plan.objectTypes.length,
123
- indexes: new Set([...names, ...texts]).size,
124
- warnings,
128
+ indexes: new Set([...names, ...texts, ...visibility.names]).size,
129
+ warnings: [...warnings, ...visibility.warnings],
125
130
  }
126
131
  }
127
132