@sixb/pg 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/agents/computers.d.ts +8 -2
- package/dist/agents/computers.d.ts.map +1 -1
- package/dist/auth-storage/auth-storage.d.ts +2 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/identities.d.ts +4 -0
- package/dist/auth-storage/identities.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +2 -1
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/users.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2377 -428
- package/dist/index.js.map +37 -30
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/conditional-reader.d.ts +7 -2
- package/dist/objects/conditional-reader.d.ts.map +1 -1
- package/dist/objects/partitions.d.ts.map +1 -1
- package/dist/objects/prepare-queries.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +14 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts +13 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +14 -1
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +11 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/visibility-indexes.d.ts +22 -0
- package/dist/objects/visibility-indexes.d.ts.map +1 -0
- package/dist/objects/visibility-statistics.d.ts +26 -0
- package/dist/objects/visibility-statistics.d.ts.map +1 -0
- package/dist/objects/visibility.d.ts +34 -0
- package/dist/objects/visibility.d.ts.map +1 -0
- package/dist/ontology-storage/materialization-session.d.ts +5 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +14 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-artifact-storage.d.ts +14 -2
- package/dist/pg-artifact-storage.d.ts.map +1 -1
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/pg-dataset-ingest-storage.d.ts +6 -0
- package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
- package/dist/pg-lake-retention-run-storage.d.ts +11 -0
- package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +6 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +2 -1
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +4 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-claim-sql.d.ts +37 -0
- package/dist/run-claim-sql.d.ts.map +1 -0
- package/dist/share-session-storage.d.ts +2 -1
- package/dist/share-session-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/computers.ts +54 -5
- package/src/auth-storage/auth-storage.ts +66 -0
- package/src/auth-storage/identities.ts +15 -0
- package/src/auth-storage/rows.ts +3 -0
- package/src/auth-storage/users.ts +3 -0
- package/src/connector-connection-storage.ts +33 -16
- package/src/index.ts +15 -0
- package/src/migrations/087-agent-computer-background-commands.sql +4 -0
- package/src/migrations/088-web-search-model-kind.sql +12 -0
- package/src/migrations/089-maintenance-heartbeats.sql +10 -0
- package/src/migrations/090-lake-retention-runs.sql +8 -0
- package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
- package/src/migrations/092-run-recovery.sql +23 -0
- package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
- package/src/migrations/094-artifact-board-previews.sql +14 -0
- package/src/migrations/095-artifact-open-versions.sql +101 -0
- package/src/migrations/096-artifact-documents.sql +11 -0
- package/src/migrations/097-artifact-frame-files.sql +8 -0
- package/src/migrations/098-artifact-comments.sql +36 -0
- package/src/migrations/099-object-artifact-refs.sql +12 -0
- package/src/migrations/100-shared-link-guests.sql +17 -0
- package/src/migrations/101-connector-connection-owners.sql +38 -0
- package/src/migrations/102-user-suspension-source.sql +12 -0
- package/src/migrations.ts +46 -0
- package/src/objects/conditional-reader.ts +26 -4
- package/src/objects/partitions.ts +17 -4
- package/src/objects/prepare-queries.ts +8 -3
- package/src/objects/query-compiler.ts +64 -23
- package/src/objects/query-indexes.ts +93 -27
- package/src/objects/read-scope.ts +147 -77
- package/src/objects/reader.ts +46 -18
- package/src/objects/storage.ts +53 -0
- package/src/objects/visibility-indexes.ts +222 -0
- package/src/objects/visibility-statistics.ts +73 -0
- package/src/objects/visibility.ts +135 -0
- package/src/ontology-storage/materialization-session.ts +4 -0
- package/src/ontology-storage/materialization-writer.ts +68 -1
- package/src/ontology-storage/materializations.ts +6 -2
- package/src/pg-artifact-storage.ts +798 -137
- package/src/pg-client.ts +4 -0
- package/src/pg-dataset-ingest-storage.ts +305 -0
- package/src/pg-lake-retention-run-storage.ts +40 -0
- package/src/pg-maintenance-heartbeat-storage.ts +53 -0
- package/src/pg-pipeline-run-storage.ts +85 -42
- package/src/pg-projection-run-storage.ts +61 -2
- package/src/pg-sync-run-storage.ts +100 -37
- package/src/pg-webhook-run-storage.ts +37 -11
- package/src/pg-workflow-run-storage.ts +39 -100
- package/src/run-claim-sql.ts +173 -0
- package/src/share-session-storage.ts +20 -0
package/src/index.ts
CHANGED
|
@@ -55,9 +55,12 @@ import { PgAiLimitStorage } from "./pg-ai-limit-storage"
|
|
|
55
55
|
import { PgAiUsageStorage } from "./pg-ai-usage-storage"
|
|
56
56
|
import { PgArtifactStorage } from "./pg-artifact-storage"
|
|
57
57
|
import { createPgClient, type SQL, type SQLClient } from "./pg-client"
|
|
58
|
+
import { PgDatasetIngestStorage } from "./pg-dataset-ingest-storage"
|
|
58
59
|
import { PgExecutionStorage } from "./pg-execution-storage"
|
|
59
60
|
import { PgFileDownloadGrantStorage } from "./pg-file-download-grant-storage"
|
|
60
61
|
import { PgFileUploadSessionStorage } from "./pg-file-upload-session-storage"
|
|
62
|
+
import { PgLakeRetentionRunStorage } from "./pg-lake-retention-run-storage"
|
|
63
|
+
import { PgMaintenanceHeartbeatStorage } from "./pg-maintenance-heartbeat-storage"
|
|
61
64
|
import { PgPipelineRunStorage } from "./pg-pipeline-run-storage"
|
|
62
65
|
import { PgProjectionRunStorage } from "./pg-projection-run-storage"
|
|
63
66
|
import { PgRulesStorage } from "./pg-rules-storage"
|
|
@@ -185,8 +188,11 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
|
|
|
185
188
|
readonly shareGrants: PgShareGrantStorage
|
|
186
189
|
readonly shareSessions: PgShareSessionStorage
|
|
187
190
|
readonly fileUploadSessions: PgFileUploadSessionStorage
|
|
191
|
+
readonly datasetIngest: PgDatasetIngestStorage
|
|
188
192
|
readonly fileDownloadGrants: PgFileDownloadGrantStorage
|
|
189
193
|
readonly activity: PgActivityStorage
|
|
194
|
+
readonly maintenanceHeartbeats: PgMaintenanceHeartbeatStorage
|
|
195
|
+
readonly lakeRetentionRuns: PgLakeRetentionRunStorage
|
|
190
196
|
readonly migrators: readonly StorageMigrator[]
|
|
191
197
|
|
|
192
198
|
private readonly sql: SQL
|
|
@@ -275,8 +281,11 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
|
|
|
275
281
|
this.shareGrants = createOperationScopedFacade(stores.shareGrants, scope)
|
|
276
282
|
this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope)
|
|
277
283
|
this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope)
|
|
284
|
+
this.datasetIngest = createOperationScopedFacade(stores.datasetIngest, scope)
|
|
278
285
|
this.fileDownloadGrants = createOperationScopedFacade(stores.fileDownloadGrants, scope)
|
|
279
286
|
this.activity = createOperationScopedFacade(stores.activity, scope)
|
|
287
|
+
this.maintenanceHeartbeats = createOperationScopedFacade(stores.maintenanceHeartbeats, scope)
|
|
288
|
+
this.lakeRetentionRuns = createOperationScopedFacade(stores.lakeRetentionRuns, scope)
|
|
280
289
|
registerPostgresStorageTestingAdapter(this, (durationMs) =>
|
|
281
290
|
stores.connectorConnections.advanceTimeForTesting(durationMs)
|
|
282
291
|
)
|
|
@@ -450,8 +459,11 @@ function createPostgresStores(
|
|
|
450
459
|
shareGrants,
|
|
451
460
|
shareSessions,
|
|
452
461
|
fileUploadSessions: new PgFileUploadSessionStorage(sql),
|
|
462
|
+
datasetIngest: new PgDatasetIngestStorage(sql),
|
|
453
463
|
fileDownloadGrants: new PgFileDownloadGrantStorage(sql),
|
|
454
464
|
activity: new PgActivityStorage(sql),
|
|
465
|
+
maintenanceHeartbeats: new PgMaintenanceHeartbeatStorage(sql),
|
|
466
|
+
lakeRetentionRuns: new PgLakeRetentionRunStorage(sql),
|
|
455
467
|
}
|
|
456
468
|
}
|
|
457
469
|
|
|
@@ -478,8 +490,11 @@ interface PostgresStoreSet {
|
|
|
478
490
|
readonly shareGrants: PgShareGrantStorage
|
|
479
491
|
readonly shareSessions: PgShareSessionStorage
|
|
480
492
|
readonly fileUploadSessions: PgFileUploadSessionStorage
|
|
493
|
+
readonly datasetIngest: PgDatasetIngestStorage
|
|
481
494
|
readonly fileDownloadGrants: PgFileDownloadGrantStorage
|
|
482
495
|
readonly activity: PgActivityStorage
|
|
496
|
+
readonly maintenanceHeartbeats: PgMaintenanceHeartbeatStorage
|
|
497
|
+
readonly lakeRetentionRuns: PgLakeRetentionRunStorage
|
|
483
498
|
}
|
|
484
499
|
|
|
485
500
|
function resolveTimeoutMillis(value: number | undefined, label: string): number | undefined {
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
-- What each open sixb reports of its threads' background commands, which outlive the turn that
|
|
2
|
+
-- started them, and the ones someone asked to stop from the web: its renewals hand those back.
|
|
3
|
+
ALTER TABLE agent_computer_sessions ADD COLUMN background_commands JSONB NOT NULL DEFAULT '[]';
|
|
4
|
+
ALTER TABLE agent_computer_sessions ADD COLUMN stop_requests JSONB NOT NULL DEFAULT '[]';
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
ALTER TABLE ai_model_call_usage
|
|
2
|
+
DROP CONSTRAINT ai_model_call_usage_model_kind_check;
|
|
3
|
+
|
|
4
|
+
ALTER TABLE ai_model_call_usage
|
|
5
|
+
ADD CONSTRAINT ai_model_call_usage_model_kind_check
|
|
6
|
+
CHECK (
|
|
7
|
+
model_kind IS NULL
|
|
8
|
+
OR model_kind IN (
|
|
9
|
+
'language', 'image', 'video', 'embedding', 'decision', 'transcription', 'speech', 'reranking',
|
|
10
|
+
'web-search'
|
|
11
|
+
)
|
|
12
|
+
);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
-- One row per project and owner, rewritten by that owner's maintenance loop as it runs. Another
|
|
2
|
+
-- process reads it to tell a stopped loop from a quiet one.
|
|
3
|
+
CREATE TABLE maintenance_heartbeats (
|
|
4
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
5
|
+
owner TEXT NOT NULL CHECK (owner IN ('orchestrator', 'embedded')),
|
|
6
|
+
beat_at TIMESTAMPTZ NOT NULL,
|
|
7
|
+
every_ms INTEGER NOT NULL CHECK (every_ms > 0),
|
|
8
|
+
passes JSONB NOT NULL CHECK (jsonb_typeof(passes) = 'array'),
|
|
9
|
+
PRIMARY KEY (project_id, owner)
|
|
10
|
+
);
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
-- One row per project, rewritten by each automatic lake retention run. The API reads it to tell a
|
|
2
|
+
-- request nobody consumes from one a worker ran.
|
|
3
|
+
CREATE TABLE lake_retention_runs (
|
|
4
|
+
project_id TEXT PRIMARY KEY CHECK (length(trim(project_id)) > 0),
|
|
5
|
+
started_at TIMESTAMPTZ NOT NULL,
|
|
6
|
+
finished_at TIMESTAMPTZ,
|
|
7
|
+
failed BOOLEAN NOT NULL
|
|
8
|
+
);
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
-- Changes `sixb.datasets.ingest` stored and no flush has written to the lake yet. A flush tags the
|
|
2
|
+
-- deliveries it takes with its id, writes them as one dataset version, then deletes them. One
|
|
3
|
+
-- refused when written stays, marked, until the orchestrator purges it.
|
|
4
|
+
CREATE TABLE dataset_ingest_deliveries (
|
|
5
|
+
id BIGSERIAL,
|
|
6
|
+
project_id TEXT NOT NULL,
|
|
7
|
+
dataset_id TEXT NOT NULL,
|
|
8
|
+
execution_id TEXT NOT NULL,
|
|
9
|
+
correlation_id TEXT NOT NULL,
|
|
10
|
+
received_at TIMESTAMPTZ NOT NULL,
|
|
11
|
+
change_count INTEGER NOT NULL CHECK (change_count > 0),
|
|
12
|
+
definition_hash TEXT NOT NULL,
|
|
13
|
+
-- Text, not JSONB: nothing reads inside it, and JSONB refuses strings JSON allows.
|
|
14
|
+
changes TEXT NOT NULL,
|
|
15
|
+
flush_id TEXT,
|
|
16
|
+
rejected_at TIMESTAMPTZ,
|
|
17
|
+
rejection TEXT,
|
|
18
|
+
-- Led by the dataset, so no plan for one dataset walks the deliveries of another.
|
|
19
|
+
PRIMARY KEY (project_id, dataset_id, id)
|
|
20
|
+
);
|
|
21
|
+
CREATE INDEX dataset_ingest_deliveries_by_flush
|
|
22
|
+
ON dataset_ingest_deliveries (project_id, dataset_id, flush_id, id);
|
|
23
|
+
|
|
24
|
+
-- One row per dataset: its open flush request, the lease of the flush that holds it, the batch
|
|
25
|
+
-- that flush took, and when a batch last completed. Locking this row orders requests against batches.
|
|
26
|
+
CREATE TABLE dataset_ingest_state (
|
|
27
|
+
project_id TEXT NOT NULL,
|
|
28
|
+
dataset_id TEXT NOT NULL,
|
|
29
|
+
request BIGINT NOT NULL DEFAULT 0,
|
|
30
|
+
requested_at TIMESTAMPTZ,
|
|
31
|
+
lease_owner TEXT,
|
|
32
|
+
lease_expires_at TIMESTAMPTZ,
|
|
33
|
+
flush_id TEXT,
|
|
34
|
+
flush_started_at TIMESTAMPTZ,
|
|
35
|
+
failure_streak INTEGER NOT NULL DEFAULT 0,
|
|
36
|
+
last_flushed_at TIMESTAMPTZ,
|
|
37
|
+
PRIMARY KEY (project_id, dataset_id)
|
|
38
|
+
);
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
-- A sync or pipeline run now survives its worker: each execution holds the run under a token that
|
|
2
|
+
-- fences its writes, and projects its queue lease here so another delivery can tell a live holder
|
|
3
|
+
-- from a dead one. A run with neither was started by an older version: nothing backfills it, and the
|
|
4
|
+
-- recovery pass settles it once its old worker cannot still be alive.
|
|
5
|
+
--
|
|
6
|
+
-- Every column is added without touching a row. Constraints are `NOT VALID`: rows that exist cannot
|
|
7
|
+
-- break them, and validating would scan the table under this lock.
|
|
8
|
+
ALTER TABLE sync_runs
|
|
9
|
+
ADD COLUMN attempt BIGINT NOT NULL DEFAULT 0,
|
|
10
|
+
ADD COLUMN execution_token TEXT,
|
|
11
|
+
ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ,
|
|
12
|
+
ADD COLUMN commit_prepared_at TIMESTAMPTZ,
|
|
13
|
+
ADD CONSTRAINT sync_runs_attempt_check CHECK (attempt >= 0) NOT VALID,
|
|
14
|
+
ADD CONSTRAINT sync_runs_execution_lease_check
|
|
15
|
+
CHECK ((execution_token IS NULL) = (execution_queue_lease_expires_at IS NULL)) NOT VALID;
|
|
16
|
+
|
|
17
|
+
ALTER TABLE pipeline_runs
|
|
18
|
+
ADD COLUMN attempt BIGINT NOT NULL DEFAULT 0,
|
|
19
|
+
ADD COLUMN execution_token TEXT,
|
|
20
|
+
ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ,
|
|
21
|
+
ADD CONSTRAINT pipeline_runs_attempt_check CHECK (attempt >= 0) NOT VALID,
|
|
22
|
+
ADD CONSTRAINT pipeline_runs_execution_lease_check
|
|
23
|
+
CHECK ((execution_token IS NULL) = (execution_queue_lease_expires_at IS NULL)) NOT VALID;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
-- A webhook handler runs in the request, with no queue behind it, so the process running it keeps
|
|
2
|
+
-- its own lease. A run with none was started by an older API, and is judged from when it started.
|
|
3
|
+
-- A projection run holds a lease as other runs do, so a duplicate delivery leaves a live run alone.
|
|
4
|
+
--
|
|
5
|
+
-- These add nullable columns, which touches no row.
|
|
6
|
+
ALTER TABLE webhook_runs
|
|
7
|
+
ADD COLUMN execution_token TEXT,
|
|
8
|
+
ADD COLUMN execution_lease_expires_at TIMESTAMPTZ;
|
|
9
|
+
|
|
10
|
+
ALTER TABLE projection_runs ADD COLUMN execution_queue_lease_expires_at TIMESTAMPTZ;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
-- A picture of a canvas's board in one mode and size, kept under its render key: a hash of what the board
|
|
2
|
+
-- shows, which its version's outline names. Versions that keep a board as it was share its
|
|
3
|
+
-- picture, so no artifact owns one and deleting an artifact leaves them. The first picture kept
|
|
4
|
+
-- under a key stays.
|
|
5
|
+
CREATE TABLE artifact_board_previews (
|
|
6
|
+
project_id TEXT NOT NULL,
|
|
7
|
+
render_key TEXT NOT NULL CHECK (length(render_key) > 0),
|
|
8
|
+
theme TEXT NOT NULL CHECK (theme IN ('light', 'dark')),
|
|
9
|
+
-- `full`, a PNG of the board at its own size; `small`, a WebP at most 480 pixels wide.
|
|
10
|
+
size TEXT NOT NULL CHECK (size IN ('full', 'small')),
|
|
11
|
+
file JSONB NOT NULL,
|
|
12
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
13
|
+
PRIMARY KEY (project_id, render_key, theme, size)
|
|
14
|
+
);
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
-- An artifact's latest version is open until someone creates a version, which freezes it: a write
|
|
2
|
+
-- replaces an open version's content, and the first write after a freeze opens the next version.
|
|
3
|
+
-- `revision` counts every change to an artifact and only grows; a version's is the artifact's
|
|
4
|
+
-- when it last changed. Every existing version was published to stay as it is, so each is frozen
|
|
5
|
+
-- when it was published, and an address pinned to one keeps naming what it named.
|
|
6
|
+
ALTER TABLE artifacts ADD COLUMN revision INTEGER;
|
|
7
|
+
UPDATE artifacts SET revision = latest_version;
|
|
8
|
+
ALTER TABLE artifacts
|
|
9
|
+
ALTER COLUMN revision SET NOT NULL,
|
|
10
|
+
ADD CONSTRAINT artifacts_revision_check CHECK (revision > 0);
|
|
11
|
+
|
|
12
|
+
-- `created_at` is when a version opened, `updated_at` when its content was last written, and
|
|
13
|
+
-- `frozen_at` when someone created it, `frozen_by_*`: an existing version by whoever published it.
|
|
14
|
+
ALTER TABLE artifact_versions
|
|
15
|
+
ADD COLUMN revision INTEGER,
|
|
16
|
+
ADD COLUMN updated_at TIMESTAMPTZ,
|
|
17
|
+
ADD COLUMN frozen_at TIMESTAMPTZ,
|
|
18
|
+
ADD COLUMN frozen_by_principal_type TEXT
|
|
19
|
+
CHECK (frozen_by_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
20
|
+
ADD COLUMN frozen_by_principal_id TEXT CHECK (length(frozen_by_principal_id) > 0);
|
|
21
|
+
UPDATE artifact_versions
|
|
22
|
+
SET revision = version, updated_at = created_at, frozen_at = created_at,
|
|
23
|
+
frozen_by_principal_type = author_principal_type, frozen_by_principal_id = author_principal_id;
|
|
24
|
+
ALTER TABLE artifact_versions
|
|
25
|
+
ALTER COLUMN revision SET NOT NULL,
|
|
26
|
+
ALTER COLUMN updated_at SET NOT NULL,
|
|
27
|
+
ADD CONSTRAINT artifact_versions_revision_check CHECK (revision > 0),
|
|
28
|
+
ADD CONSTRAINT artifact_versions_frozen_by_check CHECK (
|
|
29
|
+
(frozen_at IS NULL) = (frozen_by_principal_type IS NULL)
|
|
30
|
+
AND (frozen_by_principal_type IS NULL) = (frozen_by_principal_id IS NULL)
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
-- At most one open version: the latest, which storage keeps so.
|
|
34
|
+
CREATE UNIQUE INDEX idx_artifact_versions_open
|
|
35
|
+
ON artifact_versions (project_id, artifact_id) WHERE frozen_at IS NULL;
|
|
36
|
+
|
|
37
|
+
-- A retry no longer finds what a turn wrote by its versions' origins, which the next write
|
|
38
|
+
-- replaces: `artifact_turn_writes` keeps it.
|
|
39
|
+
DROP INDEX idx_artifact_versions_origin_run;
|
|
40
|
+
DROP INDEX idx_artifact_versions_origin_thread;
|
|
41
|
+
|
|
42
|
+
-- A frame link serves one revision of its version, and of each file it lends. The links issued
|
|
43
|
+
-- before revisions name none, and expire within the hour: they go now, and the app asks again.
|
|
44
|
+
DELETE FROM artifact_frame_grants;
|
|
45
|
+
ALTER TABLE artifact_frame_grants
|
|
46
|
+
ADD COLUMN revision INTEGER NOT NULL CHECK (revision > 0);
|
|
47
|
+
|
|
48
|
+
-- The Agent turn whose write shared an artifact in a thread: a retry of the turn takes the link
|
|
49
|
+
-- back. Links made before turns were kept stay for as long as the artifact lasts.
|
|
50
|
+
ALTER TABLE artifact_threads ADD COLUMN linked_by_run_id TEXT;
|
|
51
|
+
CREATE INDEX idx_artifact_threads_linked_by_run
|
|
52
|
+
ON artifact_threads (project_id, linked_by_run_id) WHERE linked_by_run_id IS NOT NULL;
|
|
53
|
+
|
|
54
|
+
-- What an artifact's latest version held before a change that can be undone: an Agent turn's
|
|
55
|
+
-- first write to it, one per turn, or a restore. Kept by the revision whose content it holds; each
|
|
56
|
+
-- artifact keeps its newest 20. `refs`, as in `artifact_versions`.
|
|
57
|
+
CREATE TABLE artifact_restore_points (
|
|
58
|
+
project_id TEXT NOT NULL,
|
|
59
|
+
artifact_id TEXT NOT NULL,
|
|
60
|
+
revision INTEGER NOT NULL CHECK (revision > 0),
|
|
61
|
+
version INTEGER NOT NULL CHECK (version > 0),
|
|
62
|
+
frozen BOOLEAN NOT NULL,
|
|
63
|
+
title TEXT NOT NULL CHECK (length(trim(title)) > 0),
|
|
64
|
+
entry TEXT NOT NULL CHECK (length(entry) > 0),
|
|
65
|
+
files JSONB NOT NULL CHECK (jsonb_typeof(files) = 'array'),
|
|
66
|
+
outline JSONB NOT NULL CHECK (jsonb_typeof(outline) = 'object'),
|
|
67
|
+
refs JSONB NOT NULL CHECK (jsonb_typeof(refs) = 'array'),
|
|
68
|
+
author_principal_type TEXT NOT NULL
|
|
69
|
+
CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
70
|
+
author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
|
|
71
|
+
origin_thread_id TEXT,
|
|
72
|
+
origin_run_id TEXT,
|
|
73
|
+
updated_at TIMESTAMPTZ NOT NULL,
|
|
74
|
+
reason TEXT NOT NULL CHECK (reason IN ('turn', 'restore')),
|
|
75
|
+
turn_run_id TEXT,
|
|
76
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
77
|
+
PRIMARY KEY (project_id, artifact_id, revision),
|
|
78
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
|
|
79
|
+
CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL)),
|
|
80
|
+
CHECK ((reason = 'turn') = (turn_run_id IS NOT NULL))
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
CREATE UNIQUE INDEX idx_artifact_restore_points_turn
|
|
84
|
+
ON artifact_restore_points (project_id, artifact_id, turn_run_id) WHERE turn_run_id IS NOT NULL;
|
|
85
|
+
|
|
86
|
+
-- The artifacts each Agent turn wrote, its sub-agents' writes among them, the revision its last
|
|
87
|
+
-- write left and how many changes it made. Every change moves an artifact's revision by one, so a
|
|
88
|
+
-- retry undoes an artifact only while those changes are all its revision moved by since the turn
|
|
89
|
+
-- began: anyone's change in between keeps it. Each artifact keeps its newest 20 turns.
|
|
90
|
+
CREATE TABLE artifact_turn_writes (
|
|
91
|
+
project_id TEXT NOT NULL,
|
|
92
|
+
artifact_id TEXT NOT NULL,
|
|
93
|
+
turn_run_id TEXT NOT NULL CHECK (length(turn_run_id) > 0),
|
|
94
|
+
created BOOLEAN NOT NULL,
|
|
95
|
+
revision INTEGER NOT NULL CHECK (revision > 0),
|
|
96
|
+
writes INTEGER NOT NULL CHECK (writes > 0),
|
|
97
|
+
PRIMARY KEY (project_id, artifact_id, turn_run_id),
|
|
98
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
CREATE INDEX idx_artifact_turn_writes_turn ON artifact_turn_writes (project_id, turn_run_id);
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
-- A canvas's content as the Yjs document everyone editing it shares: its whole encoded state, in a
|
|
2
|
+
-- blob, replaced by each write that changes the content, and the revision whose content it holds.
|
|
3
|
+
-- It is not a journal of changes, only the document's own encoding, so a write merges into the
|
|
4
|
+
-- same history wherever it is made. One whose revision is not the artifact's was left behind by a
|
|
5
|
+
-- write that stored none: it is brought up to the content before it is used. Null until a write
|
|
6
|
+
-- stores one, and for kinds that are not edited live.
|
|
7
|
+
ALTER TABLE artifacts
|
|
8
|
+
ADD COLUMN document JSONB CHECK (document IS NULL OR jsonb_typeof(document) = 'object'),
|
|
9
|
+
ADD COLUMN document_revision INTEGER CHECK (document_revision > 0),
|
|
10
|
+
ADD CONSTRAINT artifacts_document_pair_check
|
|
11
|
+
CHECK ((document IS NULL) = (document_revision IS NULL));
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
-- A frame link names the files it serves, as a SHA-256 hex of their paths and digests, in place of
|
|
2
|
+
-- the revision it was issued at: a canvas edited live is written every few seconds, and a link
|
|
3
|
+
-- stays good while its boards' files are the same. Links last an hour, so those issued before are
|
|
4
|
+
-- dropped.
|
|
5
|
+
DELETE FROM artifact_frame_grants;
|
|
6
|
+
ALTER TABLE artifact_frame_grants
|
|
7
|
+
DROP COLUMN revision,
|
|
8
|
+
ADD COLUMN files_digest TEXT NOT NULL CHECK (length(files_digest) = 64);
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
-- Comments on an artifact, and replies to them (`parent_id`), each about a board or an element of
|
|
2
|
+
-- a canvas, or the artifact itself, at the revision its author saw (`anchor`). They go with the
|
|
3
|
+
-- artifact, and a reply with what it replies to; a copy of the artifact keeps none.
|
|
4
|
+
-- `seq` orders comments made in the same instant as they were made.
|
|
5
|
+
CREATE TABLE artifact_comments (
|
|
6
|
+
seq BIGINT GENERATED ALWAYS AS IDENTITY,
|
|
7
|
+
project_id TEXT NOT NULL,
|
|
8
|
+
artifact_id TEXT NOT NULL,
|
|
9
|
+
id TEXT NOT NULL CHECK (length(id) > 0),
|
|
10
|
+
parent_id TEXT,
|
|
11
|
+
anchor JSONB NOT NULL CHECK (jsonb_typeof(anchor) = 'object'),
|
|
12
|
+
body TEXT NOT NULL CHECK (length(body) > 0),
|
|
13
|
+
author_principal_type TEXT NOT NULL
|
|
14
|
+
CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
15
|
+
author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
|
|
16
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
17
|
+
resolved_at TIMESTAMPTZ,
|
|
18
|
+
resolved_by_principal_type TEXT
|
|
19
|
+
CHECK (resolved_by_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
20
|
+
resolved_by_principal_id TEXT CHECK (length(resolved_by_principal_id) > 0),
|
|
21
|
+
-- The Agent's run that wrote it for its author, and that run's conversation.
|
|
22
|
+
origin_thread_id TEXT,
|
|
23
|
+
origin_run_id TEXT,
|
|
24
|
+
PRIMARY KEY (project_id, artifact_id, id),
|
|
25
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
|
|
26
|
+
FOREIGN KEY (project_id, artifact_id, parent_id)
|
|
27
|
+
REFERENCES artifact_comments (project_id, artifact_id, id) ON DELETE CASCADE,
|
|
28
|
+
CHECK (
|
|
29
|
+
(resolved_at IS NULL) = (resolved_by_principal_type IS NULL)
|
|
30
|
+
AND (resolved_by_principal_type IS NULL) = (resolved_by_principal_id IS NULL)
|
|
31
|
+
),
|
|
32
|
+
CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL))
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
CREATE INDEX idx_artifact_comments_created
|
|
36
|
+
ON artifact_comments (project_id, artifact_id, created_at, seq);
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
-- The artifacts objects hold: one row per artifact an object property holds, wherever it sits in
|
|
2
|
+
-- the property's value. Every object write keeps them current in its own transaction, so whoever
|
|
3
|
+
-- reaches an artifact through the objects that hold it finds them here. Holders list by code
|
|
4
|
+
-- point, which the "C" collation keeps in the key's order.
|
|
5
|
+
CREATE TABLE object_artifact_refs (
|
|
6
|
+
project_id TEXT NOT NULL,
|
|
7
|
+
artifact_id TEXT NOT NULL,
|
|
8
|
+
object_type_id TEXT COLLATE "C" NOT NULL,
|
|
9
|
+
primary_id TEXT COLLATE "C" NOT NULL,
|
|
10
|
+
property_id TEXT COLLATE "C" NOT NULL,
|
|
11
|
+
PRIMARY KEY (project_id, artifact_id, object_type_id, primary_id, property_id)
|
|
12
|
+
);
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
-- A shared link's visitor has no account: they give a name once, which their session keeps.
|
|
2
|
+
ALTER TABLE share_sessions
|
|
3
|
+
ADD COLUMN guest_name TEXT CHECK (guest_name IS NULL OR length(guest_name) > 0);
|
|
4
|
+
|
|
5
|
+
-- They comment as a guest: the author's id is their session, and the comment keeps the name they
|
|
6
|
+
-- gave, whatever becomes of the session.
|
|
7
|
+
ALTER TABLE artifact_comments
|
|
8
|
+
DROP CONSTRAINT artifact_comments_author_principal_type_check;
|
|
9
|
+
|
|
10
|
+
ALTER TABLE artifact_comments
|
|
11
|
+
ADD COLUMN author_name TEXT,
|
|
12
|
+
ADD CONSTRAINT artifact_comments_author_principal_type_check
|
|
13
|
+
CHECK (author_principal_type IN ('user', 'serviceAccount', 'system', 'guest')),
|
|
14
|
+
ADD CONSTRAINT artifact_comments_author_name_check CHECK (
|
|
15
|
+
(author_principal_type = 'guest') = (author_name IS NOT NULL)
|
|
16
|
+
AND (author_name IS NULL OR length(author_name) > 0)
|
|
17
|
+
);
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
-- A connection belongs to the project or to one person. The project's owner_id is empty, so the
|
|
2
|
+
-- unique key over (owner_type, owner_id, slot) never meets a NULL. Existing rows become the
|
|
3
|
+
-- project's; the defaults are dropped afterwards so every write has to name its owner.
|
|
4
|
+
ALTER TABLE connector_connections
|
|
5
|
+
ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
|
|
6
|
+
ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
|
|
7
|
+
|
|
8
|
+
ALTER TABLE connector_connections
|
|
9
|
+
ALTER COLUMN owner_type DROP DEFAULT,
|
|
10
|
+
ALTER COLUMN owner_id DROP DEFAULT,
|
|
11
|
+
ADD CONSTRAINT connector_connections_owner_check CHECK (
|
|
12
|
+
(owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
|
|
13
|
+
),
|
|
14
|
+
DROP CONSTRAINT connector_connections_project_id_connector_id_slot_key,
|
|
15
|
+
ADD CONSTRAINT connector_connections_owner_slot_key
|
|
16
|
+
UNIQUE (project_id, connector_id, owner_type, owner_id, slot);
|
|
17
|
+
|
|
18
|
+
ALTER TABLE connector_connection_runs
|
|
19
|
+
ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
|
|
20
|
+
ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
|
|
21
|
+
|
|
22
|
+
ALTER TABLE connector_connection_runs
|
|
23
|
+
ALTER COLUMN owner_type DROP DEFAULT,
|
|
24
|
+
ALTER COLUMN owner_id DROP DEFAULT,
|
|
25
|
+
ADD CONSTRAINT connector_connection_runs_owner_check CHECK (
|
|
26
|
+
(owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
ALTER TABLE connector_authorization_attempts
|
|
30
|
+
ADD COLUMN owner_type TEXT NOT NULL DEFAULT 'project',
|
|
31
|
+
ADD COLUMN owner_id TEXT NOT NULL DEFAULT '';
|
|
32
|
+
|
|
33
|
+
ALTER TABLE connector_authorization_attempts
|
|
34
|
+
ALTER COLUMN owner_type DROP DEFAULT,
|
|
35
|
+
ALTER COLUMN owner_id DROP DEFAULT,
|
|
36
|
+
ADD CONSTRAINT connector_authorization_attempts_owner_check CHECK (
|
|
37
|
+
(owner_type = 'project' AND owner_id = '') OR (owner_type = 'user' AND owner_id <> '')
|
|
38
|
+
);
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
-- Who suspended a user: an administrator or server code ('manual'), or the identity provider's
|
|
2
|
+
-- directory ('directory'), which may lift only its own suspensions. Only a suspended user has one,
|
|
3
|
+
-- and every suspension made before this column existed was manual.
|
|
4
|
+
ALTER TABLE auth_users ADD COLUMN suspension_source TEXT;
|
|
5
|
+
|
|
6
|
+
UPDATE auth_users SET suspension_source = 'manual' WHERE status = 'suspended';
|
|
7
|
+
|
|
8
|
+
ALTER TABLE auth_users
|
|
9
|
+
ADD CONSTRAINT auth_users_suspension_source_check CHECK (
|
|
10
|
+
(status = 'suspended' AND suspension_source IN ('manual', 'directory'))
|
|
11
|
+
OR (status = 'active' AND suspension_source IS NULL)
|
|
12
|
+
);
|
package/src/migrations.ts
CHANGED
|
@@ -457,6 +457,22 @@ export const postgresStorageMigrations = defineMigrations<PostgresMigrationConte
|
|
|
457
457
|
pgSql("084-agent-token-rates", agentTokenRatesSql),
|
|
458
458
|
pgSql("085-agent-computer-sessions", agentComputerSessionsSql),
|
|
459
459
|
pgSql("086-partition-objects", partitionObjectsSql),
|
|
460
|
+
pgSql("087-agent-computer-background-commands", agentComputerBackgroundCommandsSql),
|
|
461
|
+
pgSql("088-web-search-model-kind", webSearchModelKindSql),
|
|
462
|
+
pgSql("089-maintenance-heartbeats", maintenanceHeartbeatsSql),
|
|
463
|
+
pgSql("090-lake-retention-runs", lakeRetentionRunsSql),
|
|
464
|
+
pgSql("091-dataset-ingest-inbox", datasetIngestInboxSql),
|
|
465
|
+
pgSql("092-run-recovery", runRecoverySql),
|
|
466
|
+
pgSql("093-webhook-and-projection-leases", webhookAndProjectionLeasesSql),
|
|
467
|
+
pgSql("094-artifact-board-previews", artifactBoardPreviewsSql),
|
|
468
|
+
pgSql("095-artifact-open-versions", artifactOpenVersionsSql),
|
|
469
|
+
pgSql("096-artifact-documents", artifactDocumentsSql),
|
|
470
|
+
pgSql("097-artifact-frame-files", artifactFrameFilesSql),
|
|
471
|
+
pgSql("098-artifact-comments", artifactCommentsSql),
|
|
472
|
+
pgSql("099-object-artifact-refs", objectArtifactRefsSql),
|
|
473
|
+
pgSql("100-shared-link-guests", sharedLinkGuestsSql),
|
|
474
|
+
pgSql("101-connector-connection-owners", connectorConnectionOwnersSql),
|
|
475
|
+
pgSql("102-user-suspension-source", userSuspensionSourceSql),
|
|
460
476
|
],
|
|
461
477
|
})
|
|
462
478
|
|
|
@@ -529,3 +545,33 @@ import agentComputerSessionsSql from "./migrations/085-agent-computer-sessions.s
|
|
|
529
545
|
type: "text",
|
|
530
546
|
}
|
|
531
547
|
import partitionObjectsSql from "./migrations/086-partition-objects.sql" with { type: "text" }
|
|
548
|
+
import agentComputerBackgroundCommandsSql from "./migrations/087-agent-computer-background-commands.sql" with {
|
|
549
|
+
type: "text",
|
|
550
|
+
}
|
|
551
|
+
import webSearchModelKindSql from "./migrations/088-web-search-model-kind.sql" with { type: "text" }
|
|
552
|
+
import maintenanceHeartbeatsSql from "./migrations/089-maintenance-heartbeats.sql" with {
|
|
553
|
+
type: "text",
|
|
554
|
+
}
|
|
555
|
+
import lakeRetentionRunsSql from "./migrations/090-lake-retention-runs.sql" with { type: "text" }
|
|
556
|
+
import datasetIngestInboxSql from "./migrations/091-dataset-ingest-inbox.sql" with { type: "text" }
|
|
557
|
+
import runRecoverySql from "./migrations/092-run-recovery.sql" with { type: "text" }
|
|
558
|
+
import webhookAndProjectionLeasesSql from "./migrations/093-webhook-and-projection-leases.sql" with {
|
|
559
|
+
type: "text",
|
|
560
|
+
}
|
|
561
|
+
import artifactBoardPreviewsSql from "./migrations/094-artifact-board-previews.sql" with {
|
|
562
|
+
type: "text",
|
|
563
|
+
}
|
|
564
|
+
import artifactOpenVersionsSql from "./migrations/095-artifact-open-versions.sql" with {
|
|
565
|
+
type: "text",
|
|
566
|
+
}
|
|
567
|
+
import artifactDocumentsSql from "./migrations/096-artifact-documents.sql" with { type: "text" }
|
|
568
|
+
import artifactFrameFilesSql from "./migrations/097-artifact-frame-files.sql" with { type: "text" }
|
|
569
|
+
import artifactCommentsSql from "./migrations/098-artifact-comments.sql" with { type: "text" }
|
|
570
|
+
import objectArtifactRefsSql from "./migrations/099-object-artifact-refs.sql" with { type: "text" }
|
|
571
|
+
import sharedLinkGuestsSql from "./migrations/100-shared-link-guests.sql" with { type: "text" }
|
|
572
|
+
import connectorConnectionOwnersSql from "./migrations/101-connector-connection-owners.sql" with {
|
|
573
|
+
type: "text",
|
|
574
|
+
}
|
|
575
|
+
import userSuspensionSourceSql from "./migrations/102-user-suspension-source.sql" with {
|
|
576
|
+
type: "text",
|
|
577
|
+
}
|
|
@@ -5,17 +5,21 @@ import type {
|
|
|
5
5
|
ObjectQueryCapabilities,
|
|
6
6
|
ObjectReadStorage,
|
|
7
7
|
} from "@sixb/core/storage"
|
|
8
|
+
import type { SQLClient } from "../pg-client"
|
|
8
9
|
import { type PgStoreClient, runPgRepeatableReadTransaction } from "../transactions"
|
|
9
10
|
import type { PgAggregateAdmission } from "./aggregate-admission"
|
|
11
|
+
import type { PgSortIndexCatalog } from "./query-indexes"
|
|
10
12
|
import { compilePgConditionalObjectReadSource } from "./read-scope"
|
|
11
13
|
import { PgObjectReader } from "./reader"
|
|
12
14
|
import { createPgScopedReader } from "./scoped-reader"
|
|
15
|
+
import { type PgVisibilityStatisticsCatalog, readsFewObjects } from "./visibility-statistics"
|
|
13
16
|
|
|
14
17
|
/**
|
|
15
18
|
* Read objects and links through relations that apply the scope's conditions as they are read.
|
|
16
19
|
*
|
|
17
|
-
* Reads run as unscoped ones do, under the same aggregate admission
|
|
18
|
-
* query on one repeatable-read snapshot for its candidate probe
|
|
20
|
+
* Reads run as unscoped ones do, under the same aggregate admission and over the same sort
|
|
21
|
+
* indexes: on the pool, and a vector query on one repeatable-read snapshot for its candidate probe
|
|
22
|
+
* and its rows.
|
|
19
23
|
*/
|
|
20
24
|
export function createPgConditionalReader(
|
|
21
25
|
sql: PgStoreClient,
|
|
@@ -25,18 +29,36 @@ export function createPgConditionalReader(
|
|
|
25
29
|
},
|
|
26
30
|
options: {
|
|
27
31
|
aggregateAdmission?: PgAggregateAdmission
|
|
32
|
+
sortIndexes: PgSortIndexCatalog
|
|
33
|
+
visibilityStatistics: PgVisibilityStatisticsCatalog
|
|
28
34
|
queryCapabilities: () => ObjectQueryCapabilities
|
|
29
35
|
}
|
|
30
36
|
): ObjectReadStorage {
|
|
31
37
|
const source = compilePgConditionalObjectReadSource(params.projectId, params.scope)
|
|
32
|
-
|
|
38
|
+
|
|
39
|
+
const fewObjects = async (client: SQLClient, objectTypeId: string) => {
|
|
40
|
+
const condition = params.scope.conditions.find((item) => item.objectTypeId === objectTypeId)
|
|
41
|
+
if (!condition) return false
|
|
42
|
+
|
|
43
|
+
return readsFewObjects(condition, await options.visibilityStatistics.commonKeys(client))
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const reader = new PgObjectReader(
|
|
47
|
+
sql,
|
|
48
|
+
source,
|
|
49
|
+
options.aggregateAdmission,
|
|
50
|
+
options.sortIndexes,
|
|
51
|
+
fewObjects
|
|
52
|
+
)
|
|
33
53
|
|
|
34
54
|
return createPgScopedReader(
|
|
35
55
|
{
|
|
36
56
|
projectId: params.projectId,
|
|
37
57
|
read: (run, query?: ObjectQuery) =>
|
|
38
58
|
query && hasVectorProfile(query)
|
|
39
|
-
? runPgRepeatableReadTransaction(sql, (tx) =>
|
|
59
|
+
? runPgRepeatableReadTransaction(sql, (tx) =>
|
|
60
|
+
run(new PgObjectReader(tx, source, undefined, options.sortIndexes, fewObjects))
|
|
61
|
+
)
|
|
40
62
|
: run(reader),
|
|
41
63
|
},
|
|
42
64
|
options.queryCapabilities
|
|
@@ -11,6 +11,12 @@ const LOCK_TIMEOUT = "1s"
|
|
|
11
11
|
const LOCK_ATTEMPTS = 120
|
|
12
12
|
const LOCK_RETRY_MS = 1_000
|
|
13
13
|
|
|
14
|
+
// Autovacuum analyzes a partition once 50 rows plus 2% of it changed, instead of 10%. A value the
|
|
15
|
+
// statistics have never seen is estimated at a row or two: once 5% of a 30,000-row type held one,
|
|
16
|
+
// a read planned on that took 6 s instead of 75 ms (at 4% it was still fine). ANALYZE samples at
|
|
17
|
+
// most 30,000 rows, so on any large partition it takes about half a second to a second.
|
|
18
|
+
const ANALYZE_SCALE_FACTOR = "autovacuum_analyze_scale_factor=0.02"
|
|
19
|
+
|
|
14
20
|
/** Migration 086 derives the same name in SQL: `'objects_' || left(md5(type), 24)`. */
|
|
15
21
|
export function objectPartitionName(objectTypeId: string): string {
|
|
16
22
|
return `objects_${createHash("md5").update(objectTypeId).digest("hex").slice(0, 24)}`
|
|
@@ -28,9 +34,16 @@ export async function ensureObjectPartition(
|
|
|
28
34
|
): Promise<string> {
|
|
29
35
|
const name = objectPartitionName(objectTypeId)
|
|
30
36
|
const partition = `${identifier(schema)}.${identifier(name)}`
|
|
31
|
-
const [existing] = await connection<{
|
|
32
|
-
SELECT
|
|
33
|
-
|
|
37
|
+
const [existing] = await connection<{ tuned: boolean }[]>`
|
|
38
|
+
SELECT coalesce(reloptions, '{}') @> ARRAY[${ANALYZE_SCALE_FACTOR}] AS tuned
|
|
39
|
+
FROM pg_class WHERE oid = to_regclass(${partition})`
|
|
40
|
+
if (existing) {
|
|
41
|
+
// Partitions created without it, such as migration 086's, take it on their next preparation.
|
|
42
|
+
if (!existing.tuned) {
|
|
43
|
+
await connection.unsafe(`ALTER TABLE ${partition} SET (${ANALYZE_SCALE_FACTOR})`)
|
|
44
|
+
}
|
|
45
|
+
return name
|
|
46
|
+
}
|
|
34
47
|
|
|
35
48
|
for (let attempt = 1; ; attempt++) {
|
|
36
49
|
try {
|
|
@@ -100,7 +113,7 @@ async function createObjectPartition(
|
|
|
100
113
|
}
|
|
101
114
|
|
|
102
115
|
await connection.unsafe(
|
|
103
|
-
`CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE)`
|
|
116
|
+
`CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE) WITH (${ANALYZE_SCALE_FACTOR})`
|
|
104
117
|
)
|
|
105
118
|
if (moving) await moveRowsFromDefault(connection, parent, fallback, partition, type, references)
|
|
106
119
|
await connection.unsafe(
|
|
@@ -8,6 +8,8 @@ import { ensureObjectPartition } from "./partitions"
|
|
|
8
8
|
import { lockPgPreparation } from "./preparation-lock"
|
|
9
9
|
import { ensurePgObjectQueryIndexes, type PgObjectQueryIndex } from "./query-indexes"
|
|
10
10
|
import { type PgObjectTextCountIndex, preparePgObjectTextCounts } from "./text-count-index"
|
|
11
|
+
import { compilePgVisibilityKeys } from "./visibility"
|
|
12
|
+
import { preparePgVisibilityIndexes } from "./visibility-indexes"
|
|
11
13
|
|
|
12
14
|
/** Reconcile ontology indexes and text columns once per plan, coordinating concurrent startups. */
|
|
13
15
|
export async function preparePgObjectQueries(
|
|
@@ -61,11 +63,13 @@ function preparationFingerprint(plan: ObjectQueryPreparationPlan): string {
|
|
|
61
63
|
.map(normalizeIndex)
|
|
62
64
|
.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
|
|
63
65
|
links: [...type.links].sort((a, b) => a.linkId.localeCompare(b.linkId)),
|
|
66
|
+
// The SQL each condition's index is built on, so a change to its compilation rebuilds it.
|
|
67
|
+
...(type.visibility ? { visibility: type.visibility.map(compilePgVisibilityKeys) } : {}),
|
|
64
68
|
}))
|
|
65
69
|
.sort((a, b) => a.objectTypeId.localeCompare(b.objectTypeId))
|
|
66
70
|
|
|
67
71
|
// Bump the version when physical generation changes, even if the ontology stays the same.
|
|
68
|
-
const definition = { version:
|
|
72
|
+
const definition = { version: 5, projectId: plan.projectId, objectTypes }
|
|
69
73
|
return createHash("sha256").update(JSON.stringify(definition)).digest("hex")
|
|
70
74
|
}
|
|
71
75
|
|
|
@@ -117,11 +121,12 @@ async function prepareStructures(
|
|
|
117
121
|
|
|
118
122
|
const names = await ensurePgObjectQueryIndexes(sql, schema, indexes)
|
|
119
123
|
const texts = await preparePgObjectTextCounts(sql, schema, textCounts)
|
|
124
|
+
const visibility = await preparePgVisibilityIndexes(sql, schema, plan)
|
|
120
125
|
return {
|
|
121
126
|
status: "prepared",
|
|
122
127
|
objectTypes: plan.objectTypes.length,
|
|
123
|
-
indexes: new Set([...names, ...texts]).size,
|
|
124
|
-
warnings,
|
|
128
|
+
indexes: new Set([...names, ...texts, ...visibility.names]).size,
|
|
129
|
+
warnings: [...warnings, ...visibility.warnings],
|
|
125
130
|
}
|
|
126
131
|
}
|
|
127
132
|
|