@sixb/pg 0.1.19 → 0.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/dist/agents/threads.d.ts +4 -0
  2. package/dist/agents/threads.d.ts.map +1 -1
  3. package/dist/auth-storage/auth-storage.d.ts.map +1 -1
  4. package/dist/auth-storage/magic-links.d.ts.map +1 -1
  5. package/dist/auth-storage/oidc-attempts.d.ts.map +1 -1
  6. package/dist/auth-storage/rows.d.ts +0 -3
  7. package/dist/auth-storage/rows.d.ts.map +1 -1
  8. package/dist/auth-storage/sessions.d.ts +2 -11
  9. package/dist/auth-storage/sessions.d.ts.map +1 -1
  10. package/dist/auth-storage/shared.d.ts +1 -2
  11. package/dist/auth-storage/shared.d.ts.map +1 -1
  12. package/dist/index.d.ts +4 -0
  13. package/dist/index.d.ts.map +1 -1
  14. package/dist/index.js +984 -80
  15. package/dist/index.js.map +20 -18
  16. package/dist/migrations.d.ts.map +1 -1
  17. package/dist/ontology-storage/replacement-plans.d.ts.map +1 -1
  18. package/dist/ontology-storage/sources.d.ts +2 -1
  19. package/dist/ontology-storage/sources.d.ts.map +1 -1
  20. package/dist/pg-action-run-storage.d.ts +4 -0
  21. package/dist/pg-action-run-storage.d.ts.map +1 -1
  22. package/dist/pg-activity-storage.d.ts +22 -0
  23. package/dist/pg-activity-storage.d.ts.map +1 -0
  24. package/dist/pg-artifact-storage.d.ts +19 -0
  25. package/dist/pg-artifact-storage.d.ts.map +1 -0
  26. package/dist/pg-execution-storage.d.ts +2 -1
  27. package/dist/pg-execution-storage.d.ts.map +1 -1
  28. package/dist/pg-workflow-run-storage.d.ts +4 -0
  29. package/dist/pg-workflow-run-storage.d.ts.map +1 -1
  30. package/dist/share-grant-storage.d.ts +2 -1
  31. package/dist/share-grant-storage.d.ts.map +1 -1
  32. package/package.json +3 -3
  33. package/src/agents/threads.ts +23 -0
  34. package/src/auth-storage/auth-storage.ts +1 -22
  35. package/src/auth-storage/magic-links.ts +0 -4
  36. package/src/auth-storage/oidc-attempts.ts +0 -4
  37. package/src/auth-storage/rows.ts +0 -6
  38. package/src/auth-storage/sessions.ts +18 -30
  39. package/src/auth-storage/shared.ts +3 -35
  40. package/src/index.ts +10 -0
  41. package/src/migrations/062-drop-auth-audience.sql +19 -0
  42. package/src/migrations/063-artifacts.sql +67 -0
  43. package/src/migrations/064-drop-output-diagnostics.sql +14 -0
  44. package/src/migrations/065-plan-work-payload-statistics.sql +5 -0
  45. package/src/migrations/066-activity.sql +61 -0
  46. package/src/migrations/067-activity-timeline.sql +15 -0
  47. package/src/migrations/068-activity-interactions.sql +20 -0
  48. package/src/migrations.ts +20 -0
  49. package/src/ontology-storage/replacement-plans.ts +25 -7
  50. package/src/ontology-storage/sources.ts +37 -0
  51. package/src/pg-action-run-storage.ts +23 -0
  52. package/src/pg-activity-storage.ts +486 -0
  53. package/src/pg-artifact-storage.ts +325 -0
  54. package/src/pg-execution-storage.ts +52 -0
  55. package/src/pg-workflow-run-storage.ts +23 -0
  56. package/src/share-grant-storage.ts +21 -0
package/src/index.ts CHANGED
@@ -49,9 +49,11 @@ export type { PgObjectTextCountIndex } from "./objects/text-count-index"
49
49
 
50
50
  import { PgOntologyStorage, type PgOntologyTransactionContext } from "./ontology-storage"
51
51
  import { PgActionRunStorage } from "./pg-action-run-storage"
52
+ import { PgActivityStorage } from "./pg-activity-storage"
52
53
  import { PgAiCostStorage } from "./pg-ai-cost-storage"
53
54
  import { PgAiLimitStorage } from "./pg-ai-limit-storage"
54
55
  import { PgAiUsageStorage } from "./pg-ai-usage-storage"
56
+ import { PgArtifactStorage } from "./pg-artifact-storage"
55
57
  import { createPgClient, type SQL, type SQLClient } from "./pg-client"
56
58
  import { PgExecutionStorage } from "./pg-execution-storage"
57
59
  import { PgFileDownloadGrantStorage } from "./pg-file-download-grant-storage"
@@ -166,6 +168,7 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
166
168
  readonly auth: PgAuthStorage
167
169
  readonly executions: PgExecutionStorage
168
170
  readonly agents: PgAgentStorage
171
+ readonly artifacts: PgArtifactStorage
169
172
  readonly aiUsage: AiUsageStorage
170
173
  readonly aiCosts: AiCostStorage
171
174
  readonly aiLimits: AiLimitStorage
@@ -183,6 +186,7 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
183
186
  readonly shareSessions: PgShareSessionStorage
184
187
  readonly fileUploadSessions: PgFileUploadSessionStorage
185
188
  readonly fileDownloadGrants: PgFileDownloadGrantStorage
189
+ readonly activity: PgActivityStorage
186
190
  readonly migrators: readonly StorageMigrator[]
187
191
 
188
192
  private readonly sql: SQL
@@ -252,6 +256,7 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
252
256
  this.auth = createAuthOperationScope(stores.auth, scope)
253
257
  this.executions = createOperationScopedFacade(stores.executions, scope)
254
258
  this.agents = createAgentOperationScope(stores.agents, scope)
259
+ this.artifacts = createOperationScopedFacade(stores.artifacts, scope)
255
260
  this.aiUsage = createOperationScopedFacade(stores.aiUsage, scope)
256
261
  this.aiCosts = createOperationScopedFacade(stores.aiCosts, scope)
257
262
  this.aiLimits = createOperationScopedFacade(stores.aiLimits, scope)
@@ -269,6 +274,7 @@ export class PostgresStorage implements MigrationCapableStorage, QueryPreparatio
269
274
  this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope)
270
275
  this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope)
271
276
  this.fileDownloadGrants = createOperationScopedFacade(stores.fileDownloadGrants, scope)
277
+ this.activity = createOperationScopedFacade(stores.activity, scope)
272
278
  registerPostgresStorageTestingAdapter(this, (durationMs) =>
273
279
  stores.connectorConnections.advanceTimeForTesting(durationMs)
274
280
  )
@@ -414,6 +420,7 @@ function createPostgresStores(
414
420
  auth,
415
421
  executions,
416
422
  agents: new PgAgentStorage({ sql, executions }),
423
+ artifacts: new PgArtifactStorage(sql),
417
424
  aiUsage: new PgAiUsageStorage(sql),
418
425
  aiCosts: new PgAiCostStorage(sql),
419
426
  aiLimits: new PgAiLimitStorage(sql),
@@ -431,6 +438,7 @@ function createPostgresStores(
431
438
  shareSessions,
432
439
  fileUploadSessions: new PgFileUploadSessionStorage(sql),
433
440
  fileDownloadGrants: new PgFileDownloadGrantStorage(sql),
441
+ activity: new PgActivityStorage(sql),
434
442
  }
435
443
  }
436
444
 
@@ -440,6 +448,7 @@ interface PostgresStoreSet {
440
448
  readonly auth: PgAuthStorage
441
449
  readonly executions: PgExecutionStorage
442
450
  readonly agents: PgAgentStorage
451
+ readonly artifacts: PgArtifactStorage
443
452
  readonly aiUsage: PgAiUsageStorage
444
453
  readonly aiCosts: PgAiCostStorage
445
454
  readonly aiLimits: PgAiLimitStorage
@@ -457,6 +466,7 @@ interface PostgresStoreSet {
457
466
  readonly shareSessions: PgShareSessionStorage
458
467
  readonly fileUploadSessions: PgFileUploadSessionStorage
459
468
  readonly fileDownloadGrants: PgFileDownloadGrantStorage
469
+ readonly activity: PgActivityStorage
460
470
  }
461
471
 
462
472
  function resolveTimeoutMillis(value: number | undefined, label: string): number | undefined {
@@ -0,0 +1,19 @@
1
+ -- A project has one browser app, so sessions, magic links and sign-in attempts no longer name the
2
+ -- one they belong to. Atlas's browser sessions end here: their cookie is no longer read. Native
3
+ -- sessions carried Atlas's name only because the column needed a value, so they stay.
4
+ UPDATE auth_sessions
5
+ SET revoked_at = now()
6
+ WHERE audience = 'atlas'
7
+ AND refresh_token_hash IS NULL
8
+ AND revoked_at IS NULL;
9
+
10
+ -- Dropping the column drops the two indexes that name it.
11
+ ALTER TABLE auth_sessions DROP COLUMN audience;
12
+ CREATE INDEX idx_auth_sessions_user_active
13
+ ON auth_sessions (project_id, user_id, expires_at DESC, created_at DESC, id DESC)
14
+ WHERE revoked_at IS NULL;
15
+ CREATE INDEX idx_auth_sessions_user_created
16
+ ON auth_sessions (project_id, user_id, created_at DESC, id DESC);
17
+
18
+ ALTER TABLE auth_magic_links DROP COLUMN audience;
19
+ ALTER TABLE auth_oidc_authorization_attempts DROP COLUMN audience;
@@ -0,0 +1,67 @@
1
+ -- Artifacts: versioned documents, sheets, decks, and files. An artifact's kind never
2
+ -- changes. Versions take `next_version`, which only grows, so a number is never reused after its
3
+ -- version is deleted.
4
+ CREATE TABLE artifacts (
5
+ project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
6
+ id TEXT NOT NULL CHECK (length(trim(id)) > 0),
7
+ kind TEXT NOT NULL CHECK (kind IN ('document', 'sheet', 'slides', 'file')),
8
+ title TEXT NOT NULL CHECK (length(trim(title)) > 0),
9
+ owner_principal_type TEXT NOT NULL
10
+ CHECK (owner_principal_type IN ('user', 'serviceAccount', 'system')),
11
+ owner_principal_id TEXT NOT NULL CHECK (length(owner_principal_id) > 0),
12
+ latest_version INTEGER NOT NULL CHECK (latest_version > 0),
13
+ next_version INTEGER NOT NULL CHECK (next_version > latest_version),
14
+ created_at TIMESTAMPTZ NOT NULL,
15
+ updated_at TIMESTAMPTZ NOT NULL,
16
+ PRIMARY KEY (project_id, id)
17
+ );
18
+
19
+ CREATE INDEX idx_artifacts_project_updated ON artifacts (project_id, updated_at DESC, id);
20
+ CREATE INDEX idx_artifacts_owner_updated
21
+ ON artifacts (project_id, owner_principal_type, owner_principal_id, updated_at DESC, id);
22
+
23
+ -- `refs`, not `references`: that word is reserved in SQL.
24
+ CREATE TABLE artifact_versions (
25
+ project_id TEXT NOT NULL,
26
+ artifact_id TEXT NOT NULL,
27
+ version INTEGER NOT NULL CHECK (version > 0),
28
+ title TEXT NOT NULL CHECK (length(trim(title)) > 0),
29
+ entry TEXT NOT NULL CHECK (length(entry) > 0),
30
+ files JSONB NOT NULL CHECK (jsonb_typeof(files) = 'array'),
31
+ outline JSONB NOT NULL CHECK (jsonb_typeof(outline) = 'object'),
32
+ refs JSONB NOT NULL CHECK (jsonb_typeof(refs) = 'array'),
33
+ author_principal_type TEXT NOT NULL
34
+ CHECK (author_principal_type IN ('user', 'serviceAccount', 'system')),
35
+ author_principal_id TEXT NOT NULL CHECK (length(author_principal_id) > 0),
36
+ origin_thread_id TEXT,
37
+ origin_run_id TEXT,
38
+ created_at TIMESTAMPTZ NOT NULL,
39
+ PRIMARY KEY (project_id, artifact_id, version),
40
+ FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
41
+ CHECK ((origin_thread_id IS NULL) = (origin_run_id IS NULL))
42
+ );
43
+
44
+ CREATE INDEX idx_artifact_versions_origin_run
45
+ ON artifact_versions (project_id, origin_run_id) WHERE origin_run_id IS NOT NULL;
46
+ CREATE INDEX idx_artifact_versions_origin_thread
47
+ ON artifact_versions (project_id, origin_thread_id, artifact_id) WHERE origin_thread_id IS NOT NULL;
48
+
49
+ -- Short-lived links that a sandboxed frame loads a deck's files through. Only the SHA-256 of each
50
+ -- link's token is stored. `targets` is what the frame may show from other artifacts, decided for
51
+ -- its reader when the link was issued: a JSON array of { address, artifactId, version }.
52
+ CREATE TABLE artifact_frame_grants (
53
+ project_id TEXT NOT NULL,
54
+ token_hash TEXT NOT NULL CHECK (token_hash ~ '^[0-9a-f]{64}$'),
55
+ artifact_id TEXT NOT NULL,
56
+ version INTEGER NOT NULL CHECK (version > 0),
57
+ targets JSONB NOT NULL CHECK (jsonb_typeof(targets) = 'array'),
58
+ created_at TIMESTAMPTZ NOT NULL,
59
+ expires_at TIMESTAMPTZ NOT NULL,
60
+ PRIMARY KEY (project_id, token_hash),
61
+ FOREIGN KEY (project_id, artifact_id) REFERENCES artifacts (project_id, id) ON DELETE CASCADE,
62
+ CHECK (expires_at > created_at)
63
+ );
64
+
65
+ CREATE INDEX idx_artifact_frame_grants_expiry ON artifact_frame_grants (project_id, expires_at);
66
+ -- Deleting an artifact cascades here, and Postgres does not index a foreign key by itself.
67
+ CREATE INDEX idx_artifact_frame_grants_artifact ON artifact_frame_grants (project_id, artifact_id);
@@ -0,0 +1,14 @@
1
+ -- The Agent delivers files by publishing artifacts, so a run no longer collects an output
2
+ -- directory or reports on it. Stored `output_*` diagnostics go; a run left with none stores none.
3
+ UPDATE agent_runs
4
+ SET diagnostics = (
5
+ SELECT jsonb_agg(item.value ORDER BY item.position)
6
+ FROM jsonb_array_elements(agent_runs.diagnostics) WITH ORDINALITY AS item(value, position)
7
+ WHERE item.value->>'code' NOT LIKE 'output\_%'
8
+ )
9
+ WHERE jsonb_typeof(diagnostics) = 'array'
10
+ AND EXISTS (
11
+ SELECT 1
12
+ FROM jsonb_array_elements(agent_runs.diagnostics) AS item(value)
13
+ WHERE item.value->>'code' LIKE 'output\_%'
14
+ );
@@ -0,0 +1,5 @@
1
+ -- A plan's work is analyzed whenever a stream has staged enough of it, and most of that ANALYZE
2
+ -- went into the statistics of `payload`, which no query reads: the plan queries filter on the
3
+ -- typed columns and only extract values from `payload`, and an extracted value has no statistics.
4
+ -- A query that filters `payload` itself (`@>`, `?`) would need them back.
5
+ ALTER TABLE ontology_replacement_plan_work ALTER COLUMN payload SET STATISTICS 0;
@@ -0,0 +1,61 @@
1
+ -- App activity: one source per verified session, the visits it opened, and their page views.
2
+ -- Foreground activity moves a visit's `last_seen_at` and stores no row, unless it opens a visit.
3
+ -- A session id names an `auth_sessions` row for a user and a `share_sessions` row for a share; it
4
+ -- is not a secret, and no foreign key holds it, so history outlives its sessions. Ids compare
5
+ -- byte by byte, as the other providers sort them.
6
+ CREATE TABLE activity_sources (
7
+ project_id TEXT NOT NULL,
8
+ actor_type TEXT NOT NULL CHECK (actor_type IN ('user', 'share')),
9
+ session_id TEXT NOT NULL CHECK (length(session_id) > 0),
10
+ visit_id TEXT NOT NULL,
11
+ last_seen_at TIMESTAMPTZ NOT NULL,
12
+ sequence INTEGER NOT NULL CHECK (sequence >= 0),
13
+ window_start TIMESTAMPTZ NOT NULL,
14
+ window_count INTEGER NOT NULL CHECK (window_count >= 0),
15
+ PRIMARY KEY (project_id, actor_type, session_id)
16
+ );
17
+
18
+ CREATE TABLE activity_visits (
19
+ project_id TEXT NOT NULL,
20
+ id TEXT COLLATE "C" NOT NULL,
21
+ actor_type TEXT NOT NULL CHECK (actor_type IN ('user', 'share')),
22
+ user_id TEXT COLLATE "C",
23
+ grant_id TEXT,
24
+ session_id TEXT NOT NULL CHECK (length(session_id) > 0),
25
+ started_at TIMESTAMPTZ NOT NULL,
26
+ last_seen_at TIMESTAMPTZ NOT NULL CHECK (last_seen_at >= started_at),
27
+ page_views INTEGER NOT NULL CHECK (page_views >= 0),
28
+ attribution JSONB CHECK (attribution IS NULL OR jsonb_typeof(attribution) = 'object'),
29
+ PRIMARY KEY (project_id, id),
30
+ CHECK (
31
+ (actor_type = 'user' AND user_id IS NOT NULL AND grant_id IS NULL)
32
+ OR (actor_type = 'share' AND user_id IS NULL AND grant_id IS NOT NULL)
33
+ )
34
+ );
35
+
36
+ -- A period reads the visits it overlaps. A visit lasts less than a day, so they started in
37
+ -- `[from - 1 day, to)`: both period indexes bound `started_at` on both sides. `last_seen_at` is
38
+ -- left out of every index, so the update each batch makes touches none of them. The user index
39
+ -- also serves deleting one user's history.
40
+ CREATE INDEX idx_activity_visits_started
41
+ ON activity_visits (project_id, started_at, id);
42
+ CREATE INDEX idx_activity_visits_user
43
+ ON activity_visits (project_id, user_id, started_at, id);
44
+ -- A session's visits, to join them with the session and the executions it authorized.
45
+ CREATE INDEX idx_activity_visits_session
46
+ ON activity_visits (project_id, actor_type, session_id);
47
+
48
+ CREATE TABLE activity_page_views (
49
+ project_id TEXT NOT NULL,
50
+ visit_id TEXT COLLATE "C" NOT NULL,
51
+ sequence INTEGER NOT NULL CHECK (sequence > 0),
52
+ id TEXT NOT NULL,
53
+ route TEXT NOT NULL,
54
+ received_at TIMESTAMPTZ NOT NULL,
55
+ attribution JSONB CHECK (attribution IS NULL OR jsonb_typeof(attribution) = 'object'),
56
+ PRIMARY KEY (project_id, visit_id, sequence),
57
+ FOREIGN KEY (project_id, visit_id) REFERENCES activity_visits (project_id, id) ON DELETE CASCADE
58
+ );
59
+
60
+ -- Finds a retried page view by the id its browser chose.
61
+ CREATE INDEX idx_activity_page_views_id ON activity_page_views (project_id, id);
@@ -0,0 +1,15 @@
1
+ -- A visit's timeline reads what its session started: the roots it authorized in a window, then
2
+ -- everything they caused. A signed-in session is the credential of its roots, a shared link's
3
+ -- session their delegation; a child names its parent. Agent executions copy their parent's
4
+ -- session: the root indexes hold only executions without a parent, so they never see those, and
5
+ -- end with `id` so the root read's order needs no sort. A user's connection history reads
6
+ -- `idx_auth_sessions_user_created`, which already exists.
7
+ CREATE INDEX idx_executions_session_roots
8
+ ON executions (project_id, authority_session_id, created_at, id)
9
+ WHERE authority_session_id IS NOT NULL AND parent_execution_id IS NULL;
10
+ CREATE INDEX idx_executions_delegation_roots
11
+ ON executions (project_id, authority_delegation_session_id, created_at, id)
12
+ WHERE authority_delegation_session_id IS NOT NULL AND parent_execution_id IS NULL;
13
+ CREATE INDEX idx_executions_parent
14
+ ON executions (project_id, parent_execution_id)
15
+ WHERE parent_execution_id IS NOT NULL;
@@ -0,0 +1,20 @@
1
+ -- Interactions an app marks, such as a button: their name and route, never a value. They share
2
+ -- the visit's sequence with its page views, so a visit reads both in the order they happened, and
3
+ -- cascade with the visit, which counts them as it counts its page views.
4
+ ALTER TABLE activity_visits
5
+ ADD COLUMN interactions INTEGER NOT NULL DEFAULT 0 CHECK (interactions >= 0);
6
+
7
+ CREATE TABLE activity_interactions (
8
+ project_id TEXT NOT NULL,
9
+ visit_id TEXT COLLATE "C" NOT NULL,
10
+ sequence INTEGER NOT NULL CHECK (sequence > 0),
11
+ id TEXT NOT NULL,
12
+ name TEXT NOT NULL CHECK (length(name) BETWEEN 1 AND 100),
13
+ route TEXT NOT NULL,
14
+ received_at TIMESTAMPTZ NOT NULL,
15
+ PRIMARY KEY (project_id, visit_id, sequence),
16
+ FOREIGN KEY (project_id, visit_id) REFERENCES activity_visits (project_id, id) ON DELETE CASCADE
17
+ );
18
+
19
+ -- Finds a retried interaction by the id its browser chose.
20
+ CREATE INDEX idx_activity_interactions_id ON activity_interactions (project_id, id);
package/src/migrations.ts CHANGED
@@ -432,6 +432,13 @@ export const postgresStorageMigrations = defineMigrations<PostgresMigrationConte
432
432
  pgSql("059-file-download-grants", fileDownloadGrantsSql),
433
433
  pgSql("060-drained-source-versions", drainedSourceVersionsSql),
434
434
  pgSql("061-terminal-action-runs", terminalActionRunsSql),
435
+ pgSql("062-drop-auth-audience", dropAuthAudienceSql),
436
+ pgSql("063-artifacts", artifactsSql),
437
+ pgSql("064-drop-output-diagnostics", dropOutputDiagnosticsSql),
438
+ pgSql("065-plan-work-payload-statistics", planWorkPayloadStatisticsSql),
439
+ pgSql("066-activity", activitySql),
440
+ pgSql("067-activity-timeline", activityTimelineSql),
441
+ pgSql("068-activity-interactions", activityInteractionsSql),
435
442
  ],
436
443
  })
437
444
 
@@ -461,3 +468,16 @@ import drainedSourceVersionsSql from "./migrations/060-drained-source-versions.s
461
468
  type: "text",
462
469
  }
463
470
  import terminalActionRunsSql from "./migrations/061-terminal-action-runs.sql" with { type: "text" }
471
+ import dropAuthAudienceSql from "./migrations/062-drop-auth-audience.sql" with { type: "text" }
472
+ import artifactsSql from "./migrations/063-artifacts.sql" with { type: "text" }
473
+ import dropOutputDiagnosticsSql from "./migrations/064-drop-output-diagnostics.sql" with {
474
+ type: "text",
475
+ }
476
+ import planWorkPayloadStatisticsSql from "./migrations/065-plan-work-payload-statistics.sql" with {
477
+ type: "text",
478
+ }
479
+ import activitySql from "./migrations/066-activity.sql" with { type: "text" }
480
+ import activityTimelineSql from "./migrations/067-activity-timeline.sql" with { type: "text" }
481
+ import activityInteractionsSql from "./migrations/068-activity-interactions.sql" with {
482
+ type: "text",
483
+ }
@@ -50,11 +50,13 @@ const IDENTITIES = "ontology_replacement_plan_identities"
50
50
  const SORT_KEY = "encode(convert_to(identity_key, 'UTF8'), 'hex')"
51
51
 
52
52
  /**
53
- * Rows a plan writes at once before its tables are analyzed again. Autovacuum lags behind a bulk
54
- * write, and with the statistics of near-empty tables the planner scans a plan's whole work for
55
- * each keyed lookup instead of seeking it.
53
+ * Rows a plan writes before its tables are analyzed again: identities when it opens or expands,
54
+ * work once a stream has staged it. Every plan fills and empties the same tables, so their
55
+ * statistics describe the plans that were there when they were last analyzed. Until they describe
56
+ * this one, the planner expects none of its rows and reads its whole work for each key it looks
57
+ * up. Below this many rows, those reads cost less than an ANALYZE.
56
58
  */
57
- const ANALYZE_ROWS = 10_000
59
+ const ANALYZE_ROWS = 1_000
58
60
 
59
61
  /** The plan a session applies: its work rows carry the candidate's source version as `work_id`. */
60
62
  export interface PgReplacementPlan {
@@ -240,7 +242,7 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
240
242
  input: StreamReplacementPlanStateInput
241
243
  ): AsyncIterable<ReplacementPlanStatePage> {
242
244
  assertPageRows(input.pageRows)
243
- await runPgTransaction(this.sql, async (sql) => {
245
+ const versionId = await runPgTransaction(this.sql, async (sql) => {
244
246
  const { candidate } = await requirePlan(sql, input)
245
247
  if (input.entityKind === "object" && candidate.projection_kind !== "object") {
246
248
  throw new MaterializationConflictError(
@@ -258,6 +260,7 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
258
260
  const expanded = await expandLinks(sql, candidate.version_id, input.projectId, "pending")
259
261
  if (expanded >= ANALYZE_ROWS && !isWithinPgTransaction(this.sql)) await analyzePlans(sql)
260
262
  }
263
+ return candidate.version_id
261
264
  })
262
265
  let after = ""
263
266
  let streamed = 0
@@ -296,8 +299,8 @@ export class PgOntologyReplacementPlanStorage implements OntologyReplacementPlan
296
299
  if (!page) {
297
300
  // Every page is staged by now: later lookups and the commit join this work. Never in the
298
301
  // commit transaction, where ANALYZE would hold a lock other commits queue behind.
299
- if (streamed >= ANALYZE_ROWS && !isWithinPgTransaction(this.sql)) {
300
- await analyzePlans(this.sql)
302
+ if (streamed > 0 && !isWithinPgTransaction(this.sql)) {
303
+ await analyzeStagedWork(this.sql, versionId)
301
304
  }
302
305
  return
303
306
  }
@@ -994,6 +997,21 @@ async function analyzePlans(sql: SQLClient): Promise<void> {
994
997
  await sql`ANALYZE ${sql(IDENTITIES)}, ${sql(PG_PLAN_WORK_TABLE)}`
995
998
  }
996
999
 
1000
+ /**
1001
+ * Analyzes the plan tables once the plan holds `ANALYZE_ROWS` rows of work. Counted in work rows,
1002
+ * not identities: an object stages its classification, its existence, its change and its event.
1003
+ */
1004
+ async function analyzeStagedWork(sql: SQLClient, versionId: string): Promise<void> {
1005
+ const [staged] = await sql<{ readonly rows: number | string }[]>`
1006
+ SELECT COUNT(*) AS rows FROM (
1007
+ SELECT 1 FROM ${sql(PG_PLAN_WORK_TABLE)} WHERE work_id = ${versionId} LIMIT ${ANALYZE_ROWS}
1008
+ ) AS work
1009
+ `
1010
+
1011
+ if (Number(staged?.rows ?? 0) < ANALYZE_ROWS) return
1012
+ await analyzePlans(sql)
1013
+ }
1014
+
997
1015
  /**
998
1016
  * Deletes the work of the identities of `kind` left to plan: an identity to plan holds none. The
999
1017
  * keys come first, so the work is sought by key whatever the plan tables' statistics.
@@ -19,6 +19,7 @@ import type {
19
19
  AbandonRunSourceMaterializationInput,
20
20
  AbandonSourceMaterializationCandidateInput,
21
21
  AbandonSourceMaterializationInput,
22
+ ActiveOntologySource,
22
23
  AdoptedSourceMaterialization,
23
24
  AdoptSourceMaterializationInput,
24
25
  AssertSourceMaterializationExecutionInput,
@@ -26,6 +27,7 @@ import type {
26
27
  CleanupTerminalSourceMaterializationsInput,
27
28
  CleanupTerminalSourceMaterializationsResult,
28
29
  GetActiveOntologySourceInput,
30
+ ListActiveOntologySourcesInput,
29
31
  MarkSourceMaterializationReadyInput,
30
32
  OntologySourceRecord,
31
33
  OntologySourceStorage,
@@ -237,6 +239,32 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
237
239
  })
238
240
  }
239
241
 
242
+ async listActive(
243
+ input: ListActiveOntologySourcesInput
244
+ ): Promise<readonly ActiveOntologySource[]> {
245
+ return this.runRootOperation(async (sql) => {
246
+ assertNonblank(input.projectId, "Source project id")
247
+
248
+ // Served by the partial index on active sources.
249
+ const rows = await sql<ActiveSourceRow[]>`
250
+ SELECT source_id, dataset_id, dataset_version_id, dataset_version_created_at,
251
+ projection_revision
252
+ FROM ontology_sources
253
+ WHERE project_id = ${input.projectId} AND status = 'active'
254
+ `
255
+
256
+ return rows.map((row) => ({
257
+ source: { projectionId: row.source_id },
258
+ datasetVersion: {
259
+ datasetId: row.dataset_id,
260
+ versionId: row.dataset_version_id,
261
+ createdAt: toIsoString(row.dataset_version_created_at),
262
+ },
263
+ projectionRevision: row.projection_revision,
264
+ }))
265
+ })
266
+ }
267
+
240
268
  async adopt(
241
269
  input: AdoptSourceMaterializationInput
242
270
  ): Promise<AdoptedSourceMaterialization | null> {
@@ -620,3 +648,12 @@ export class PgOntologySourceStorage implements OntologySourceStorage {
620
648
  function numberOrNull(value: number | string | null): number | null {
621
649
  return value === null ? null : Number(value)
622
650
  }
651
+
652
+ type ActiveSourceRow = Pick<
653
+ PgOntologySourceRow,
654
+ | "source_id"
655
+ | "dataset_id"
656
+ | "dataset_version_id"
657
+ | "dataset_version_created_at"
658
+ | "projection_revision"
659
+ >
@@ -136,6 +136,29 @@ export class PgActionRunStorage implements ActionRunStorage {
136
136
  return row ? rowToActionRunRecord(row) : null
137
137
  }
138
138
 
139
+ async getByExecutionIds(params: {
140
+ readonly projectId: string
141
+ readonly executionIds: readonly string[]
142
+ }): Promise<readonly ActionRunRecord[]> {
143
+ const executionIds = [...new Set(params.executionIds)]
144
+ if (executionIds.length === 0) {
145
+ return []
146
+ }
147
+
148
+ const rows = await this.sql<DatabaseRow[]>`
149
+ SELECT *
150
+ FROM action_runs
151
+ WHERE project_id = ${params.projectId}
152
+ AND execution_id = ANY(${this.sql.array(executionIds)}::text[])
153
+ `
154
+
155
+ const byExecution = new Map(rows.map((row) => [row.execution_id, row]))
156
+ return executionIds.flatMap((executionId) => {
157
+ const row = byExecution.get(executionId)
158
+ return row ? [rowToActionRunRecord(row)] : []
159
+ })
160
+ }
161
+
139
162
  async list(input: ListActionRunsInput): Promise<ListActionRunsResult> {
140
163
  if ((input.statuses && input.statuses.length === 0) || input.actionIds?.length === 0) {
141
164
  return {