@sixb/pg 0.1.19 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/threads.d.ts +4 -0
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/magic-links.d.ts.map +1 -1
- package/dist/auth-storage/oidc-attempts.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +0 -3
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/sessions.d.ts +2 -11
- package/dist/auth-storage/sessions.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts +1 -2
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +984 -80
- package/dist/index.js.map +20 -18
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/replacement-plans.d.ts.map +1 -1
- package/dist/ontology-storage/sources.d.ts +2 -1
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/pg-action-run-storage.d.ts +4 -0
- package/dist/pg-action-run-storage.d.ts.map +1 -1
- package/dist/pg-activity-storage.d.ts +22 -0
- package/dist/pg-activity-storage.d.ts.map +1 -0
- package/dist/pg-artifact-storage.d.ts +19 -0
- package/dist/pg-artifact-storage.d.ts.map +1 -0
- package/dist/pg-execution-storage.d.ts +2 -1
- package/dist/pg-execution-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +4 -0
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/share-grant-storage.d.ts +2 -1
- package/dist/share-grant-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/threads.ts +23 -0
- package/src/auth-storage/auth-storage.ts +1 -22
- package/src/auth-storage/magic-links.ts +0 -4
- package/src/auth-storage/oidc-attempts.ts +0 -4
- package/src/auth-storage/rows.ts +0 -6
- package/src/auth-storage/sessions.ts +18 -30
- package/src/auth-storage/shared.ts +3 -35
- package/src/index.ts +10 -0
- package/src/migrations/062-drop-auth-audience.sql +19 -0
- package/src/migrations/063-artifacts.sql +67 -0
- package/src/migrations/064-drop-output-diagnostics.sql +14 -0
- package/src/migrations/065-plan-work-payload-statistics.sql +5 -0
- package/src/migrations/066-activity.sql +61 -0
- package/src/migrations/067-activity-timeline.sql +15 -0
- package/src/migrations/068-activity-interactions.sql +20 -0
- package/src/migrations.ts +20 -0
- package/src/ontology-storage/replacement-plans.ts +25 -7
- package/src/ontology-storage/sources.ts +37 -0
- package/src/pg-action-run-storage.ts +23 -0
- package/src/pg-activity-storage.ts +486 -0
- package/src/pg-artifact-storage.ts +325 -0
- package/src/pg-execution-storage.ts +52 -0
- package/src/pg-workflow-run-storage.ts +23 -0
- package/src/share-grant-storage.ts +21 -0
package/src/index.ts
CHANGED
|
@@ -49,9 +49,11 @@ export type { PgObjectTextCountIndex } from "./objects/text-count-index"
|
|
|
49
49
|
|
|
50
50
|
import { PgOntologyStorage, type PgOntologyTransactionContext } from "./ontology-storage"
|
|
51
51
|
import { PgActionRunStorage } from "./pg-action-run-storage"
|
|
52
|
+
import { PgActivityStorage } from "./pg-activity-storage"
|
|
52
53
|
import { PgAiCostStorage } from "./pg-ai-cost-storage"
|
|
53
54
|
import { PgAiLimitStorage } from "./pg-ai-limit-storage"
|
|
54
55
|
import { PgAiUsageStorage } from "./pg-ai-usage-storage"
|
|
56
|
+
import { PgArtifactStorage } from "./pg-artifact-storage"
|
|
55
57
|
import { createPgClient, type SQL, type SQLClient } from "./pg-client"
|
|
56
58
|
import { PgExecutionStorage } from "./pg-execution-storage"
|
|
57
59
|
import { PgFileDownloadGrantStorage } from "./pg-file-download-grant-storage"
|
|
@@ -166,6 +168,7 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
|
|
|
166
168
|
readonly auth: PgAuthStorage
|
|
167
169
|
readonly executions: PgExecutionStorage
|
|
168
170
|
readonly agents: PgAgentStorage
|
|
171
|
+
readonly artifacts: PgArtifactStorage
|
|
169
172
|
readonly aiUsage: AiUsageStorage
|
|
170
173
|
readonly aiCosts: AiCostStorage
|
|
171
174
|
readonly aiLimits: AiLimitStorage
|
|
@@ -183,6 +186,7 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
|
|
|
183
186
|
readonly shareSessions: PgShareSessionStorage
|
|
184
187
|
readonly fileUploadSessions: PgFileUploadSessionStorage
|
|
185
188
|
readonly fileDownloadGrants: PgFileDownloadGrantStorage
|
|
189
|
+
readonly activity: PgActivityStorage
|
|
186
190
|
readonly migrators: readonly StorageMigrator[]
|
|
187
191
|
|
|
188
192
|
private readonly sql: SQL
|
|
@@ -252,6 +256,7 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
|
|
|
252
256
|
this.auth = createAuthOperationScope(stores.auth, scope)
|
|
253
257
|
this.executions = createOperationScopedFacade(stores.executions, scope)
|
|
254
258
|
this.agents = createAgentOperationScope(stores.agents, scope)
|
|
259
|
+
this.artifacts = createOperationScopedFacade(stores.artifacts, scope)
|
|
255
260
|
this.aiUsage = createOperationScopedFacade(stores.aiUsage, scope)
|
|
256
261
|
this.aiCosts = createOperationScopedFacade(stores.aiCosts, scope)
|
|
257
262
|
this.aiLimits = createOperationScopedFacade(stores.aiLimits, scope)
|
|
@@ -269,6 +274,7 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
|
|
|
269
274
|
this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope)
|
|
270
275
|
this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope)
|
|
271
276
|
this.fileDownloadGrants = createOperationScopedFacade(stores.fileDownloadGrants, scope)
|
|
277
|
+
this.activity = createOperationScopedFacade(stores.activity, scope)
|
|
272
278
|
registerPostgresStorageTestingAdapter(this, (durationMs) =>
|
|
273
279
|
stores.connectorConnections.advanceTimeForTesting(durationMs)
|
|
274
280
|
)
|
|
@@ -414,6 +420,7 @@ function createPostgresStores(
|
|
|
414
420
|
auth,
|
|
415
421
|
executions,
|
|
416
422
|
agents: new PgAgentStorage({ sql, executions }),
|
|
423
|
+
artifacts: new PgArtifactStorage(sql),
|
|
417
424
|
aiUsage: new PgAiUsageStorage(sql),
|
|
418
425
|
aiCosts: new PgAiCostStorage(sql),
|
|
419
426
|
aiLimits: new PgAiLimitStorage(sql),
|
|
@@ -431,6 +438,7 @@ function createPostgresStores(
|
|
|
431
438
|
shareSessions,
|
|
432
439
|
fileUploadSessions: new PgFileUploadSessionStorage(sql),
|
|
433
440
|
fileDownloadGrants: new PgFileDownloadGrantStorage(sql),
|
|
441
|
+
activity: new PgActivityStorage(sql),
|
|
434
442
|
}
|
|
435
443
|
}
|
|
436
444
|
|
|
@@ -440,6 +448,7 @@ interface PostgresStoreSet {
|
|
|
440
448
|
readonly auth: PgAuthStorage
|
|
441
449
|
readonly executions: PgExecutionStorage
|
|
442
450
|
readonly agents: PgAgentStorage
|
|
451
|
+
readonly artifacts: PgArtifactStorage
|
|
443
452
|
readonly aiUsage: PgAiUsageStorage
|
|
444
453
|
readonly aiCosts: PgAiCostStorage
|
|
445
454
|
readonly aiLimits: PgAiLimitStorage
|
|
@@ -457,6 +466,7 @@ interface PostgresStoreSet {
|
|
|
457
466
|
readonly shareSessions: PgShareSessionStorage
|
|
458
467
|
readonly fileUploadSessions: PgFileUploadSessionStorage
|
|
459
468
|
readonly fileDownloadGrants: PgFileDownloadGrantStorage
|
|
469
|
+
readonly activity: PgActivityStorage
|
|
460
470
|
}
|
|
461
471
|
|
|
462
472
|
function resolveTimeoutMillis(value: number | undefined, label: string): number | undefined {
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
-- A project has one browser app, so sessions, magic links and sign-in attempts no longer name the
|
|
2
|
+
-- one they belong to. Atlas's browser sessions end here: their cookie is no longer read. Native
|
|
3
|
+
-- sessions carried Atlas's name only because the column needed a value, so they stay.
|
|
4
|
+
UPDATE auth_sessions
|
|
5
|
+
SET revoked_at = now()
|
|
6
|
+
WHERE audience = 'atlas'
|
|
7
|
+
AND refresh_token_hash IS NULL
|
|
8
|
+
AND revoked_at IS NULL;
|
|
9
|
+
|
|
10
|
+
-- Dropping the column drops the two indexes that name it.
|
|
11
|
+
ALTER TABLE auth_sessions DROP COLUMN audience;
|
|
12
|
+
CREATE INDEX idx_auth_sessions_user_active
|
|
13
|
+
ON auth_sessions (project_id, user_id, expires_at DESC, created_at DESC, id DESC)
|
|
14
|
+
WHERE revoked_at IS NULL;
|
|
15
|
+
CREATE INDEX idx_auth_sessions_user_created
|
|
16
|
+
ON auth_sessions (project_id, user_id, created_at DESC, id DESC);
|
|
17
|
+
|
|
18
|
+
ALTER TABLE auth_magic_links DROP COLUMN audience;
|
|
19
|
+
ALTER TABLE auth_oidc_authorization_attempts DROP COLUMN audience;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
-- Artifacts: versioned documents, sheets, decks, and files. An artifact's kind never
|
|
2
|
+
-- changes. Versions take `next_version`, which only grows, so a number is never reused after its
|
|
3
|
+
-- version is deleted.
|
|
4
|
+
CREATE TABLE artifacts (
|
|
5
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
6
|
+
id TEXT NOT NULL CHECK (length(trim(id)) > 0),
|
|
7
|
+
kind TEXT NOT NULL CHECK (kind IN ('document', 'sheet', 'slides', 'file')),
|
|
8
|
+
title TEXT NOT NULL CHECK (length(trim(title)) > 0),
|
|
9
|
+
owner_principal_type TEXT NOT NULL
|
|
10
|
+
CHECK (owner_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
11
|
+
owner_principal_id TEXT NOT NULL CHECK (length(owner_principal_id) > 0),
|
|
12
|
+
latest_version INTEGER NOT NULL CHECK (latest_version > 0),
|
|
13
|
+
next_version INTEGER NOT NULL CHECK (next_version > latest_version),
|
|
14
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
15
|
+
updated_at TIMESTAMPTZ NOT NULL,
|
|
16
|
+
PRIMARY KEY (project_id, id)
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
CREATE INDEX idx_artifacts_project_updated ON artifacts (project_id, updated_at DESC, id);
|
|
20
|
+
CREATE INDEX idx_artifacts_owner_updated
|
|
21
|
+
ON artifacts (project_id, owner_principal_type, owner_principal_id, updated_at DESC, id);
|
|
22
|
+
|
|
23
|
+
-- `refs`, not `references`: that word is reserved in SQL.
|
|
24
|
+
CREATE TABLE artifact_versions (
|
|
25
|
+
project_id TEXT NOT NULL,
|
|
26
|
+
artifact_id TEXT NOT NULL,
|
|
27
|
+
version INTEGER NOT NULL CHECK (version > 0),
|
|
28
|
+
title TEXT NOT NULL CHECK (length(trim(title)) > 0),
|
|
29
|
+
entry TEXT NOT NULL CHECK (length(entry) > 0),
|
|
30
|
+
files JSONB NOT NULL CHECK (jsonb_typeof(files) = 'array'),
|
|
31
|
+
outline JSONB NOT NULL CHECK (jsonb_typeof(outline) = 'object'),
|
|
32
|
+
refs JSONB NOT NULL CHECK (jsonb_typeof(refs) = 'array'),
|
|
33
|
+
author_principal_type TEXT NOT NULL
|
|
34
|
+
CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
35
|
+
author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
|
|
36
|
+
origin_thread_id TEXT,
|
|
37
|
+
origin_run_id TEXT,
|
|
38
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
39
|
+
PRIMARY KEY (project_id, artifact_id, version),
|
|
40
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
|
|
41
|
+
CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL))
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
CREATE INDEX idx_artifact_versions_origin_run
|
|
45
|
+
ON artifact_versions (project_id, origin_run_id) WHERE origin_run_id IS NOT NULL;
|
|
46
|
+
CREATE INDEX idx_artifact_versions_origin_thread
|
|
47
|
+
ON artifact_versions (project_id, origin_thread_id, artifact_id) WHERE origin_thread_id IS NOT NULL;
|
|
48
|
+
|
|
49
|
+
-- Short-lived links that a sandboxed frame loads a deck's files through. Only the SHA-256 of each
|
|
50
|
+
-- link's token is stored. `targets` is what the frame may show from other artifacts, decided for
|
|
51
|
+
-- its reader when the link was issued: a JSON array of { address, artifactId, version }.
|
|
52
|
+
CREATE TABLE artifact_frame_grants (
|
|
53
|
+
project_id TEXT NOT NULL,
|
|
54
|
+
token_hash TEXT NOT NULL CHECK (token_hash ~ '^[0-9a-f]{64}$'),
|
|
55
|
+
artifact_id TEXT NOT NULL,
|
|
56
|
+
version INTEGER NOT NULL CHECK (version > 0),
|
|
57
|
+
targets JSONB NOT NULL CHECK (jsonb_typeof(targets) = 'array'),
|
|
58
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
59
|
+
expires_at TIMESTAMPTZ NOT NULL,
|
|
60
|
+
PRIMARY KEY (project_id, token_hash),
|
|
61
|
+
FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
|
|
62
|
+
CHECK (expires_at > created_at)
|
|
63
|
+
);
|
|
64
|
+
|
|
65
|
+
CREATE INDEX idx_artifact_frame_grants_expiry ON artifact_frame_grants (project_id, expires_at);
|
|
66
|
+
-- Deleting an artifact cascades here, and Postgres does not index a foreign key by itself.
|
|
67
|
+
CREATE INDEX idx_artifact_frame_grants_artifact ON artifact_frame_grants (project_id, artifact_id);
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
-- The Agent delivers files by publishing artifacts, so a run no longer collects an output
|
|
2
|
+
-- directory or reports on it. Stored `output_*` diagnostics go; a run left with none stores none.
|
|
3
|
+
UPDATE agent_runs
|
|
4
|
+
SET diagnostics = (
|
|
5
|
+
SELECT jsonb_agg(item.value ORDER BY item.position)
|
|
6
|
+
FROM jsonb_array_elements(agent_runs.diagnostics) WITH ORDINALITY AS item(value, position)
|
|
7
|
+
WHERE item.value->>'code' NOT LIKE 'output\_%'
|
|
8
|
+
)
|
|
9
|
+
WHERE jsonb_typeof(diagnostics) = 'array'
|
|
10
|
+
AND EXISTS (
|
|
11
|
+
SELECT 1
|
|
12
|
+
FROM jsonb_array_elements(agent_runs.diagnostics) AS item(value)
|
|
13
|
+
WHERE item.value->>'code' LIKE 'output\_%'
|
|
14
|
+
);
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
-- A plan's work is analyzed whenever a stream has staged enough of it, and most of that ANALYZE
|
|
2
|
+
-- went into the statistics of `payload`, which no query reads: the plan queries filter on the
|
|
3
|
+
-- typed columns and only extract values from `payload`, and an extracted value has no statistics.
|
|
4
|
+
-- A query that filters `payload` itself (`@>`, `?`) would need them back.
|
|
5
|
+
ALTER TABLE ontology_replacement_plan_work ALTER COLUMN payload SET STATISTICS 0;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
-- App activity: one source per verified session, the visits it opened, and their page views.
|
|
2
|
+
-- Foreground activity moves a visit's `last_seen_at` and stores no row, unless it opens a visit.
|
|
3
|
+
-- A session id names an `auth_sessions` row for a user and a `share_sessions` row for a share; it
|
|
4
|
+
-- is not a secret, and no foreign key holds it, so history outlives its sessions. Ids compare
|
|
5
|
+
-- byte by byte, as the other providers sort them.
|
|
6
|
+
CREATE TABLE activity_sources (
|
|
7
|
+
project_id TEXT NOT NULL,
|
|
8
|
+
actor_type TEXT NOT NULL CHECK (actor_type IN ('user', 'share')),
|
|
9
|
+
session_id TEXT NOT NULL CHECK (length(session_id) > 0),
|
|
10
|
+
visit_id TEXT NOT NULL,
|
|
11
|
+
last_seen_at TIMESTAMPTZ NOT NULL,
|
|
12
|
+
sequence INTEGER NOT NULL CHECK (sequence >= 0),
|
|
13
|
+
window_start TIMESTAMPTZ NOT NULL,
|
|
14
|
+
window_count INTEGER NOT NULL CHECK (window_count >= 0),
|
|
15
|
+
PRIMARY KEY (project_id, actor_type, session_id)
|
|
16
|
+
);
|
|
17
|
+
|
|
18
|
+
CREATE TABLE activity_visits (
|
|
19
|
+
project_id TEXT NOT NULL,
|
|
20
|
+
id TEXT COLLATE "C" NOT NULL,
|
|
21
|
+
actor_type TEXT NOT NULL CHECK (actor_type IN ('user', 'share')),
|
|
22
|
+
user_id TEXT COLLATE "C",
|
|
23
|
+
grant_id TEXT,
|
|
24
|
+
session_id TEXT NOT NULL CHECK (length(session_id) > 0),
|
|
25
|
+
started_at TIMESTAMPTZ NOT NULL,
|
|
26
|
+
last_seen_at TIMESTAMPTZ NOT NULL CHECK (last_seen_at >= started_at),
|
|
27
|
+
page_views INTEGER NOT NULL CHECK (page_views >= 0),
|
|
28
|
+
attribution JSONB CHECK (attribution IS NULL OR jsonb_typeof(attribution) = 'object'),
|
|
29
|
+
PRIMARY KEY (project_id, id),
|
|
30
|
+
CHECK (
|
|
31
|
+
(actor_type = 'user' AND user_id IS NOT NULL AND grant_id IS NULL)
|
|
32
|
+
OR (actor_type = 'share' AND user_id IS NULL AND grant_id IS NOT NULL)
|
|
33
|
+
)
|
|
34
|
+
);
|
|
35
|
+
|
|
36
|
+
-- A period reads the visits it overlaps. A visit lasts less than a day, so they started in
|
|
37
|
+
-- `[from - 1 day, to)`: both period indexes bound `started_at` on both sides. `last_seen_at` is
|
|
38
|
+
-- left out of every index, so the update each batch makes touches none of them. The user index
|
|
39
|
+
-- also serves deleting one user's history.
|
|
40
|
+
CREATE INDEX idx_activity_visits_started
|
|
41
|
+
ON activity_visits (project_id, started_at, id);
|
|
42
|
+
CREATE INDEX idx_activity_visits_user
|
|
43
|
+
ON activity_visits (project_id, user_id, started_at, id);
|
|
44
|
+
-- A session's visits, to join them with the session and the executions it authorized.
|
|
45
|
+
CREATE INDEX idx_activity_visits_session
|
|
46
|
+
ON activity_visits (project_id, actor_type, session_id);
|
|
47
|
+
|
|
48
|
+
CREATE TABLE activity_page_views (
|
|
49
|
+
project_id TEXT NOT NULL,
|
|
50
|
+
visit_id TEXT COLLATE "C" NOT NULL,
|
|
51
|
+
sequence INTEGER NOT NULL CHECK (sequence > 0),
|
|
52
|
+
id TEXT NOT NULL,
|
|
53
|
+
route TEXT NOT NULL,
|
|
54
|
+
received_at TIMESTAMPTZ NOT NULL,
|
|
55
|
+
attribution JSONB CHECK (attribution IS NULL OR jsonb_typeof(attribution) = 'object'),
|
|
56
|
+
PRIMARY KEY (project_id, visit_id, sequence),
|
|
57
|
+
FOREIGN KEY (project_id, visit_id) REFERENCES activity_visits (project_id, id) ON DELETE CASCADE
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
-- Finds a retried page view by the id its browser chose.
|
|
61
|
+
CREATE INDEX idx_activity_page_views_id ON activity_page_views (project_id, id);
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
-- A visit's timeline reads what its session started: the roots it authorized in a window, then
|
|
2
|
+
-- everything they caused. A signed-in session is the credential of its roots, a shared link's
|
|
3
|
+
-- session their delegation; a child names its parent. Agent executions copy their parent's
|
|
4
|
+
-- session: the root indexes hold only executions without a parent, so they never see those, and
|
|
5
|
+
-- end with `id` so the root read's order needs no sort. A user's connection history reads
|
|
6
|
+
-- `idx_auth_sessions_user_created`, which already exists.
|
|
7
|
+
CREATE INDEX idx_executions_session_roots
|
|
8
|
+
ON executions (project_id, authority_session_id, created_at, id)
|
|
9
|
+
WHERE authority_session_id IS NOT NULL AND parent_execution_id IS NULL;
|
|
10
|
+
CREATE INDEX idx_executions_delegation_roots
|
|
11
|
+
ON executions (project_id, authority_delegation_session_id, created_at, id)
|
|
12
|
+
WHERE authority_delegation_session_id IS NOT NULL AND parent_execution_id IS NULL;
|
|
13
|
+
CREATE INDEX idx_executions_parent
|
|
14
|
+
ON executions (project_id, parent_execution_id)
|
|
15
|
+
WHERE parent_execution_id IS NOT NULL;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
-- Interactions an app marks, such as a button: their name and route, never a value. They share
|
|
2
|
+
-- the visit's sequence with its page views, so a visit reads both in the order they happened, and
|
|
3
|
+
-- cascade with the visit, which counts them as it counts its page views.
|
|
4
|
+
ALTER TABLE activity_visits
|
|
5
|
+
ADD COLUMN interactions INTEGER NOT NULL DEFAULT 0 CHECK (interactions >= 0);
|
|
6
|
+
|
|
7
|
+
CREATE TABLE activity_interactions (
|
|
8
|
+
project_id TEXT NOT NULL,
|
|
9
|
+
visit_id TEXT COLLATE "C" NOT NULL,
|
|
10
|
+
sequence INTEGER NOT NULL CHECK (sequence > 0),
|
|
11
|
+
id TEXT NOT NULL,
|
|
12
|
+
name TEXT NOT NULL CHECK (length(name) BETWEEN 1 AND 100),
|
|
13
|
+
route TEXT NOT NULL,
|
|
14
|
+
received_at TIMESTAMPTZ NOT NULL,
|
|
15
|
+
PRIMARY KEY (project_id, visit_id, sequence),
|
|
16
|
+
FOREIGN KEY (project_id, visit_id) REFERENCES activity_visits (project_id, id) ON DELETE CASCADE
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
-- Finds a retried interaction by the id its browser chose.
|
|
20
|
+
CREATE INDEX idx_activity_interactions_id ON activity_interactions (project_id, id);
|
package/src/migrations.ts
CHANGED
|
@@ -432,6 +432,13 @@ export const postgresStorageMigrations = defineMigrations<PostgresMigrationConte
|
|
|
432
432
|
pgSql("059-file-download-grants", fileDownloadGrantsSql),
|
|
433
433
|
pgSql("060-drained-source-versions", drainedSourceVersionsSql),
|
|
434
434
|
pgSql("061-terminal-action-runs", terminalActionRunsSql),
|
|
435
|
+
pgSql("062-drop-auth-audience", dropAuthAudienceSql),
|
|
436
|
+
pgSql("063-artifacts", artifactsSql),
|
|
437
|
+
pgSql("064-drop-output-diagnostics", dropOutputDiagnosticsSql),
|
|
438
|
+
pgSql("065-plan-work-payload-statistics", planWorkPayloadStatisticsSql),
|
|
439
|
+
pgSql("066-activity", activitySql),
|
|
440
|
+
pgSql("067-activity-timeline", activityTimelineSql),
|
|
441
|
+
pgSql("068-activity-interactions", activityInteractionsSql),
|
|
435
442
|
],
|
|
436
443
|
})
|
|
437
444
|
|
|
@@ -461,3 +468,16 @@ import drainedSourceVersionsSql from "./migrations/060-drained-source-versions.s
|
|
|
461
468
|
type: "text",
|
|
462
469
|
}
|
|
463
470
|
import terminalActionRunsSql from "./migrations/061-terminal-action-runs.sql" with { type: "text" }
|
|
471
|
+
import dropAuthAudienceSql from "./migrations/062-drop-auth-audience.sql" with { type: "text" }
|
|
472
|
+
import artifactsSql from "./migrations/063-artifacts.sql" with { type: "text" }
|
|
473
|
+
import dropOutputDiagnosticsSql from "./migrations/064-drop-output-diagnostics.sql" with {
|
|
474
|
+
type: "text",
|
|
475
|
+
}
|
|
476
|
+
import planWorkPayloadStatisticsSql from "./migrations/065-plan-work-payload-statistics.sql" with {
|
|
477
|
+
type: "text",
|
|
478
|
+
}
|
|
479
|
+
import activitySql from "./migrations/066-activity.sql" with { type: "text" }
|
|
480
|
+
import activityTimelineSql from "./migrations/067-activity-timeline.sql" with { type: "text" }
|
|
481
|
+
import activityInteractionsSql from "./migrations/068-activity-interactions.sql" with {
|
|
482
|
+
type: "text",
|
|
483
|
+
}
|
|
@@ -50,11 +50,13 @@ const IDENTITIES = "ontology_replacement_plan_identities"
|
|
|
50
50
|
const SORT_KEY = "encode(convert_to(identity_key, 'UTF8'), 'hex')"
|
|
51
51
|
|
|
52
52
|
/**
|
|
53
|
-
* Rows a plan writes
|
|
54
|
-
*
|
|
55
|
-
*
|
|
53
|
+
* Rows a plan writes before its tables are analyzed again: identities when it opens or expands,
|
|
54
|
+
* work once a stream has staged it. Every plan fills and empties the same tables, so their
|
|
55
|
+
* statistics describe the plans that were there when they were last analyzed. Until they describe
|
|
56
|
+
* this one, the planner expects none of its rows and reads its whole work for each key it looks
|
|
57
|
+
* up. Below this many rows, those reads cost less than an ANALYZE.
|
|
56
58
|
*/
|
|
57
|
-
const ANALYZE_ROWS =
|
|
59
|
+
const ANALYZE_ROWS = 1_000
|
|
58
60
|
|
|
59
61
|
/** The plan a session applies: its work rows carry the candidate's source version as `work_id`. */
|
|
60
62
|
export interface PgReplacementPlan {
|
|
@@ -240,7 +242,7 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
|
|
|
240
242
|
input: StreamReplacementPlanStateInput
|
|
241
243
|
): AsyncIterable<ReplacementPlanStatePage> {
|
|
242
244
|
assertPageRows(input.pageRows)
|
|
243
|
-
await runPgTransaction(this.sql, async (sql) => {
|
|
245
|
+
const versionId = await runPgTransaction(this.sql, async (sql) => {
|
|
244
246
|
const { candidate } = await requirePlan(sql, input)
|
|
245
247
|
if (input.entityKind === "object" && candidate.projection_kind !== "object") {
|
|
246
248
|
throw new MaterializationConflictError(
|
|
@@ -258,6 +260,7 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
|
|
|
258
260
|
const expanded = await expandLinks(sql, candidate.version_id, input.projectId, "pending")
|
|
259
261
|
if (expanded >= ANALYZE_ROWS && !isWithinPgTransaction(this.sql)) await analyzePlans(sql)
|
|
260
262
|
}
|
|
263
|
+
return candidate.version_id
|
|
261
264
|
})
|
|
262
265
|
let after = ""
|
|
263
266
|
let streamed = 0
|
|
@@ -296,8 +299,8 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
|
|
|
296
299
|
if (!page) {
|
|
297
300
|
// Every page is staged by now: later lookups and the commit join this work. Never in the
|
|
298
301
|
// commit transaction, where ANALYZE would hold a lock other commits queue behind.
|
|
299
|
-
if (streamed
|
|
300
|
-
await
|
|
302
|
+
if (streamed > 0 && !isWithinPgTransaction(this.sql)) {
|
|
303
|
+
await analyzeStagedWork(this.sql, versionId)
|
|
301
304
|
}
|
|
302
305
|
return
|
|
303
306
|
}
|
|
@@ -994,6 +997,21 @@ async function analyzePlans(sql: SQLClient): Promise<void> {
|
|
|
994
997
|
await sql`ANALYZE ${sql(IDENTITIES)}, ${sql(PG_PLAN_WORK_TABLE)}`
|
|
995
998
|
}
|
|
996
999
|
|
|
1000
|
+
/**
|
|
1001
|
+
* Analyzes the plan tables once the plan holds `ANALYZE_ROWS` rows of work. Counted in work rows,
|
|
1002
|
+
* not identities: an object stages its classification, its existence, its change and its event.
|
|
1003
|
+
*/
|
|
1004
|
+
async function analyzeStagedWork(sql: SQLClient, versionId: string): Promise<void> {
|
|
1005
|
+
const [staged] = await sql<{ readonly rows: number | string }[]>`
|
|
1006
|
+
SELECT COUNT(*) AS rows FROM (
|
|
1007
|
+
SELECT 1 FROM ${sql(PG_PLAN_WORK_TABLE)} WHERE work_id = ${versionId} LIMIT ${ANALYZE_ROWS}
|
|
1008
|
+
) AS work
|
|
1009
|
+
`
|
|
1010
|
+
|
|
1011
|
+
if (Number(staged?.rows ?? 0) < ANALYZE_ROWS) return
|
|
1012
|
+
await analyzePlans(sql)
|
|
1013
|
+
}
|
|
1014
|
+
|
|
997
1015
|
/**
|
|
998
1016
|
* Deletes the work of the identities of `kind` left to plan: an identity to plan holds none. The
|
|
999
1017
|
* keys come first, so the work is sought by key whatever the plan tables' statistics.
|
|
@@ -19,6 +19,7 @@ import type {
|
|
|
19
19
|
AbandonRunSourceMaterializationInput,
|
|
20
20
|
AbandonSourceMaterializationCandidateInput,
|
|
21
21
|
AbandonSourceMaterializationInput,
|
|
22
|
+
ActiveOntologySource,
|
|
22
23
|
AdoptedSourceMaterialization,
|
|
23
24
|
AdoptSourceMaterializationInput,
|
|
24
25
|
AssertSourceMaterializationExecutionInput,
|
|
@@ -26,6 +27,7 @@ import type {
|
|
|
26
27
|
CleanupTerminalSourceMaterializationsInput,
|
|
27
28
|
CleanupTerminalSourceMaterializationsResult,
|
|
28
29
|
GetActiveOntologySourceInput,
|
|
30
|
+
ListActiveOntologySourcesInput,
|
|
29
31
|
MarkSourceMaterializationReadyInput,
|
|
30
32
|
OntologySourceRecord,
|
|
31
33
|
OntologySourceStorage,
|
|
@@ -237,6 +239,32 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
237
239
|
})
|
|
238
240
|
}
|
|
239
241
|
|
|
242
|
+
async listActive(
|
|
243
|
+
input: ListActiveOntologySourcesInput
|
|
244
|
+
): Promise<readonly ActiveOntologySource[]> {
|
|
245
|
+
return this.runRootOperation(async (sql) => {
|
|
246
|
+
assertNonblank(input.projectId, "Source project id")
|
|
247
|
+
|
|
248
|
+
// Served by the partial index on active sources.
|
|
249
|
+
const rows = await sql<ActiveSourceRow[]>`
|
|
250
|
+
SELECT source_id, dataset_id, dataset_version_id, dataset_version_created_at,
|
|
251
|
+
projection_revision
|
|
252
|
+
FROM ontology_sources
|
|
253
|
+
WHERE project_id = ${input.projectId} AND status = 'active'
|
|
254
|
+
`
|
|
255
|
+
|
|
256
|
+
return rows.map((row) => ({
|
|
257
|
+
source: { projectionId: row.source_id },
|
|
258
|
+
datasetVersion: {
|
|
259
|
+
datasetId: row.dataset_id,
|
|
260
|
+
versionId: row.dataset_version_id,
|
|
261
|
+
createdAt: toIsoString(row.dataset_version_created_at),
|
|
262
|
+
},
|
|
263
|
+
projectionRevision: row.projection_revision,
|
|
264
|
+
}))
|
|
265
|
+
})
|
|
266
|
+
}
|
|
267
|
+
|
|
240
268
|
async adopt(
|
|
241
269
|
input: AdoptSourceMaterializationInput
|
|
242
270
|
): Promise<AdoptedSourceMaterialization | null> {
|
|
@@ -620,3 +648,12 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
|
|
|
620
648
|
function numberOrNull(value: number | string | null): number | null {
|
|
621
649
|
return value === null ? null : Number(value)
|
|
622
650
|
}
|
|
651
|
+
|
|
652
|
+
type ActiveSourceRow = Pick<
|
|
653
|
+
PgOntologySourceRow,
|
|
654
|
+
| "source_id"
|
|
655
|
+
| "dataset_id"
|
|
656
|
+
| "dataset_version_id"
|
|
657
|
+
| "dataset_version_created_at"
|
|
658
|
+
| "projection_revision"
|
|
659
|
+
>
|
|
@@ -136,6 +136,29 @@ export class PgActionRunStorage implements ActionRunStorage {
|
|
|
136
136
|
return row ? rowToActionRunRecord(row) : null
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
+
async getByExecutionIds(params: {
|
|
140
|
+
readonly projectId: string
|
|
141
|
+
readonly executionIds: readonly string[]
|
|
142
|
+
}): Promise<readonly ActionRunRecord[]> {
|
|
143
|
+
const executionIds = [...new Set(params.executionIds)]
|
|
144
|
+
if (executionIds.length === 0) {
|
|
145
|
+
return []
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const rows = await this.sql<DatabaseRow[]>`
|
|
149
|
+
SELECT *
|
|
150
|
+
FROM action_runs
|
|
151
|
+
WHERE project_id = ${params.projectId}
|
|
152
|
+
AND execution_id = ANY(${this.sql.array(executionIds)}::text[])
|
|
153
|
+
`
|
|
154
|
+
|
|
155
|
+
const byExecution = new Map(rows.map((row) => [row.execution_id, row]))
|
|
156
|
+
return executionIds.flatMap((executionId) => {
|
|
157
|
+
const row = byExecution.get(executionId)
|
|
158
|
+
return row ? [rowToActionRunRecord(row)] : []
|
|
159
|
+
})
|
|
160
|
+
}
|
|
161
|
+
|
|
139
162
|
async list(input: ListActionRunsInput): Promise<ListActionRunsResult> {
|
|
140
163
|
if ((input.statuses && input.statuses.length === 0) || input.actionIds?.length === 0) {
|
|
141
164
|
return {
|