@sixb/pg 0.1.26 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +25 -1
  2. package/dist/agents/agent-storage.d.ts +8 -3
  3. package/dist/agents/agent-storage.d.ts.map +1 -1
  4. package/dist/agents/computers.d.ts +22 -0
  5. package/dist/agents/computers.d.ts.map +1 -0
  6. package/dist/agents/rows.d.ts +3 -0
  7. package/dist/agents/rows.d.ts.map +1 -1
  8. package/dist/agents/runs.d.ts.map +1 -1
  9. package/dist/agents/threads.d.ts +2 -1
  10. package/dist/agents/threads.d.ts.map +1 -1
  11. package/dist/agents/token-rates.d.ts +13 -0
  12. package/dist/agents/token-rates.d.ts.map +1 -0
  13. package/dist/ai-accounting-filters.d.ts +12 -0
  14. package/dist/ai-accounting-filters.d.ts.map +1 -0
  15. package/dist/ai-cost-groups.d.ts +1 -1
  16. package/dist/ai-cost-groups.d.ts.map +1 -1
  17. package/dist/index.d.ts +1 -1
  18. package/dist/index.js +886 -158
  19. package/dist/index.js.map +26 -21
  20. package/dist/migrations.d.ts.map +1 -1
  21. package/dist/objects/partitions.d.ts +10 -0
  22. package/dist/objects/partitions.d.ts.map +1 -0
  23. package/dist/objects/prepare-queries.d.ts.map +1 -1
  24. package/dist/objects/query-compiler.d.ts +5 -0
  25. package/dist/objects/query-compiler.d.ts.map +1 -1
  26. package/dist/objects/query-indexes.d.ts.map +1 -1
  27. package/dist/objects/sql-text.d.ts +3 -0
  28. package/dist/objects/sql-text.d.ts.map +1 -0
  29. package/dist/objects/text-count-index.d.ts.map +1 -1
  30. package/dist/ontology-storage/commits.d.ts.map +1 -1
  31. package/dist/ontology-storage/materialization-session.d.ts +7 -1
  32. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  33. package/dist/ontology-storage/materialization-writer.d.ts +2 -0
  34. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  35. package/dist/ontology-storage/materializations.d.ts +2 -1
  36. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  37. package/dist/pg-ai-cost-storage.d.ts.map +1 -1
  38. package/dist/pg-client.d.ts +1 -1
  39. package/dist/pg-client.d.ts.map +1 -1
  40. package/dist/storage-errors.d.ts +4 -1
  41. package/dist/storage-errors.d.ts.map +1 -1
  42. package/package.json +3 -3
  43. package/src/agents/agent-storage.ts +10 -3
  44. package/src/agents/computers.ts +95 -0
  45. package/src/agents/rows.ts +9 -1
  46. package/src/agents/runs.ts +25 -3
  47. package/src/agents/threads.ts +111 -20
  48. package/src/agents/token-rates.ts +76 -0
  49. package/src/ai-accounting-filters.ts +48 -0
  50. package/src/ai-cost-groups.ts +37 -12
  51. package/src/index.ts +1 -1
  52. package/src/migrations/084-agent-token-rates.sql +16 -0
  53. package/src/migrations/085-agent-computer-sessions.sql +27 -0
  54. package/src/migrations/086-partition-objects.sql +93 -0
  55. package/src/migrations.ts +8 -0
  56. package/src/objects/partitions.ts +163 -0
  57. package/src/objects/prepare-queries.ts +50 -2
  58. package/src/objects/query-compiler.ts +80 -66
  59. package/src/objects/query-indexes.ts +59 -46
  60. package/src/objects/sql-text.ts +8 -0
  61. package/src/objects/text-count-index.ts +17 -12
  62. package/src/ontology-storage/commits.ts +11 -2
  63. package/src/ontology-storage/materialization-session.ts +10 -1
  64. package/src/ontology-storage/materialization-writer.ts +14 -0
  65. package/src/ontology-storage/materializations.ts +81 -7
  66. package/src/pg-ai-cost-storage.ts +119 -15
  67. package/src/pg-client.ts +9 -2
  68. package/src/storage-errors.ts +28 -1
@@ -1,10 +1,13 @@
1
1
  import { createHash } from "node:crypto"
2
2
  import type { ObjectQuery, QueryScalarKind } from "@sixb/core"
3
+ import { linkEndpointObjectTypeIds } from "@sixb/core/internal/query"
3
4
  import type { ReservedSQL, SQL, SQLClient } from "../pg-client"
4
5
  import { isConnectionLost } from "../storage-errors"
5
6
  import { withReservedPgConnection } from "../transactions"
7
+ import { ensureObjectPartition, objectPartitionName } from "./partitions"
6
8
  import { lockPgPreparation } from "./preparation-lock"
7
- import { isTextScalarKind } from "./query-compiler"
9
+ import { hasJsonValueIndex, isTextScalarKind } from "./query-compiler"
10
+ import { identifier, literal } from "./sql-text"
8
11
 
9
12
  export interface PgObjectQueryIndexScope {
10
13
  readonly projectId: string
@@ -39,7 +42,7 @@ export async function ensurePgObjectQueryIndexes(
39
42
  indexes: readonly PgObjectQueryIndex[]
40
43
  ): Promise<readonly string[]> {
41
44
  const definitions = indexes.map((index) => ({
42
- table: index.kind === "incomingLinks" ? "links" : "objects",
45
+ table: index.kind === "incomingLinks" ? "links" : objectPartitionName(index.objectTypeId),
43
46
  expression: compileIndex(index),
44
47
  index,
45
48
  }))
@@ -62,6 +65,11 @@ export async function ensurePgObjectQueryIndexes(
62
65
  try {
63
66
  // One concurrent build per table, across definitions and across processes.
64
67
  await lockPgPreparation(connection, `${schema}.objects.query-indexes`)
68
+ for (const objectTypeId of new Set(
69
+ indexes.flatMap((index) => (index.kind === "incomingLinks" ? [] : [index.objectTypeId]))
70
+ )) {
71
+ await ensureObjectPartition(connection, schema, objectTypeId)
72
+ }
65
73
  for (const definition of definitions) {
66
74
  names.push(await createQueryIndex(connection, schema, definition))
67
75
  }
@@ -83,16 +91,14 @@ async function createQueryIndex(
83
91
  schema: string,
84
92
  definition: { table: string; expression: string; index: PgObjectQueryIndex }
85
93
  ): Promise<string> {
94
+ // A partition's name stands for its type, which an index's expression no longer names.
95
+ const { table } = definition
86
96
  const name = `sixb_query_${createHash("sha256")
87
- .update(
88
- definition.table === "objects"
89
- ? definition.expression
90
- : `${definition.table}:${definition.expression}`
91
- )
97
+ .update(`${table}:${definition.expression}`)
92
98
  .digest("hex")
93
99
  .slice(0, 32)}`
94
100
  await connection.unsafe(
95
- `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${identifier(schema)}.${identifier(definition.table)} ${definition.expression}`
101
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${identifier(schema)}.${identifier(table)} ${definition.expression}`
96
102
  )
97
103
  const [index] = await connection<{ valid: boolean }[]>`
98
104
  SELECT i.indisvalid AS valid FROM pg_index i
@@ -107,9 +113,8 @@ async function createQueryIndex(
107
113
  await connection.unsafe(
108
114
  `COMMENT ON INDEX ${identifier(schema)}.${identifier(name)} IS ${literal(JSON.stringify(definition.index))}`
109
115
  )
110
- const scope = `project_id = ${literal(definition.index.projectId)} AND object_type_id = ${literal(definition.index.objectTypeId)}`
111
116
  await connection.unsafe(
112
- `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}.objects (primary_id) WHERE ${scope} AND NOT (${sortIndexBound(definition.index)})`
117
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}.${identifier(table)} (primary_id) WHERE project_id = ${literal(definition.index.projectId)} AND NOT (${sortIndexBound(definition.index)})`
113
118
  )
114
119
 
115
120
  // The same definition under another name is an earlier generation of this index, built
@@ -136,27 +141,42 @@ async function ensureExpressionStatistics(
136
141
  schema: string,
137
142
  indexes: readonly PgObjectQueryIndex[]
138
143
  ): Promise<void> {
139
- let statisticsChanged = false
140
144
  // Partial-index statistics do not describe expressions for the whole table. Without
141
145
  // expression statistics JSON's default selectivity can turn a 4M-row view into an
142
- // estimated 125-row sort, even when the right ordered index exists.
143
- const expressions = new Set(indexes.flatMap(statisticsExpressions))
144
- for (const expression of expressions) {
145
- const name = `sixb_query_stats_${createHash("sha256").update(expression).digest("hex").slice(0, 24)}`
146
- const [existing] = await connection`
147
- SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
148
- WHERE n.nspname = ${schema} AND s.stxname = ${name}`
149
- if (existing) continue
150
- await connection.unsafe(
151
- `CREATE STATISTICS ${identifier(schema)}.${identifier(name)} ON project_id, object_type_id, ${expression} FROM ${identifier(schema)}.objects`
152
- )
153
- statisticsChanged = true
146
+ // estimated 125-row sort, even when the right ordered index exists. PostgreSQL plans a pruned
147
+ // query with its partition's statistics, so they live there, and ANALYZE stays per type.
148
+ const expressionsByType = new Map<string, Set<string>>()
149
+ for (const index of indexes) {
150
+ if (index.kind === "incomingLinks") continue
151
+ const expressions = expressionsByType.get(index.objectTypeId) ?? new Set<string>()
152
+ for (const expression of statisticsExpressions(index)) expressions.add(expression)
153
+ expressionsByType.set(index.objectTypeId, expressions)
154
+ }
155
+
156
+ for (const [objectTypeId, expressions] of expressionsByType) {
157
+ const partition = `${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`
158
+ let statisticsChanged = false
159
+ for (const expression of expressions) {
160
+ const name = `sixb_query_stats_${createHash("sha256")
161
+ .update(`${objectTypeId}:${expression}`)
162
+ .digest("hex")
163
+ .slice(0, 24)}`
164
+ const [existing] = await connection`
165
+ SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
166
+ WHERE n.nspname = ${schema} AND s.stxname = ${name}`
167
+ if (existing) continue
168
+ await connection.unsafe(
169
+ `CREATE STATISTICS ${identifier(schema)}.${identifier(name)} ON project_id, ${expression} FROM ${partition}`
170
+ )
171
+ statisticsChanged = true
172
+ }
173
+ if (statisticsChanged) await connection.unsafe(`ANALYZE ${partition}`)
154
174
  }
155
- if (statisticsChanged) await connection.unsafe(`ANALYZE ${identifier(schema)}.objects`)
156
175
  }
157
176
 
158
- function statisticsExpressions(index: PgObjectQueryIndex): string[] {
159
- if (index.kind === "incomingLinks") return []
177
+ function statisticsExpressions(
178
+ index: Exclude<PgObjectQueryIndex, { kind: "incomingLinks" }>
179
+ ): string[] {
160
180
  const equality = new Set(Object.keys(index.where ?? {}))
161
181
  if (index.kind === "filter") for (const property of index.properties) equality.add(property)
162
182
  if (index.kind === "sort") for (const property of index.equality ?? []) equality.add(property)
@@ -177,10 +197,7 @@ function compileIndex(index: PgObjectQueryIndex): string {
177
197
  throw new Error("[SixbPg] Incoming link indexes do not accept object property conditions")
178
198
  return `(target_id, link_id, source_type_id, source_id) INCLUDE (project_id, target_type_id) WHERE project_id = ${literal(index.projectId)} AND target_type_id = ${literal(index.objectTypeId)}`
179
199
  }
180
- const predicate = [
181
- `project_id = ${literal(index.projectId)}`,
182
- `object_type_id = ${literal(index.objectTypeId)}`,
183
- ]
200
+ const predicate = [`project_id = ${literal(index.projectId)}`]
184
201
  for (const [propertyId, value] of Object.entries(index.where ?? {}).sort(([a], [b]) =>
185
202
  a.localeCompare(b)
186
203
  )) {
@@ -246,16 +263,6 @@ function sortValue(field: Extract<PgObjectQueryIndex, { kind: "sort" }>["fields"
246
263
  return `(NULLIF(properties -> ${literal(field.propertyId)}, 'null'::jsonb))`
247
264
  }
248
265
 
249
- function literal(value: string): string {
250
- if (value.includes("\0"))
251
- throw new Error("[SixbPg] Query index names and values cannot contain NUL")
252
- return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`
253
- }
254
-
255
- function identifier(value: string): string {
256
- return `"${value.replace(/"/g, '""')}"`
257
- }
258
-
259
266
  /** Keep complete B-tree entries below PostgreSQL's page limit, without constraining valid data.
260
267
  * Ordered reads merge a small indexed page with the exact overflow page. */
261
268
  export function sortIndexBound(index: Extract<PgObjectQueryIndex, { kind: "sort" }>): string {
@@ -313,8 +320,8 @@ async function readPgSortIndexes(sql: SQLClient): Promise<readonly PgSortIndex[]
313
320
  JOIN pg_class c ON c.oid = i.indexrelid
314
321
  JOIN pg_description d
315
322
  ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
316
- WHERE i.indrelid = 'objects'::regclass AND i.indisvalid
317
- AND c.relname LIKE 'sixb_query_%'
323
+ WHERE i.indrelid IN (SELECT inhrelid FROM pg_inherits WHERE inhparent = 'objects'::regclass)
324
+ AND i.indisvalid AND c.relname LIKE 'sixb_query_%'
318
325
  `
319
326
  const indexes: PgSortIndex[] = []
320
327
  for (const row of rows) {
@@ -351,12 +358,17 @@ export async function findPgSortIndexBound(
351
358
  if (input.kind === "filter") {
352
359
  const inspect = (predicate: Extract<ObjectQuery, { kind: "filter" }>["predicate"]): void => {
353
360
  if (predicate.op === "and") predicate.items.forEach(inspect)
354
- else if (predicate.op === "eq" && typeof predicate.value === "string")
361
+ else if (
362
+ predicate.op === "eq" &&
363
+ typeof predicate.value === "string" &&
364
+ !hasJsonValueIndex(predicate.scalarKind)
365
+ )
355
366
  equality.add(predicate.propertyId)
356
367
  else if (
357
368
  predicate.op === "in" &&
358
369
  predicate.values.length === 1 &&
359
- typeof predicate.values[0] === "string"
370
+ typeof predicate.values[0] === "string" &&
371
+ !hasJsonValueIndex(predicate.scalarKind)
360
372
  )
361
373
  equality.add(predicate.propertyId)
362
374
  }
@@ -367,8 +379,9 @@ export async function findPgSortIndexBound(
367
379
  let objectTypeId: string | undefined
368
380
  if (input.kind === "start" && !input.includeSubtypes) {
369
381
  objectTypeId = input.objectTypeId
370
- } else if (input.kind === "traverse" && input.direction === "incoming") {
371
- objectTypeId = input.sourceObjectTypeId
382
+ } else if (input.kind === "traverse") {
383
+ const endpoints = linkEndpointObjectTypeIds(input)
384
+ objectTypeId = endpoints?.length === 1 ? endpoints[0] : undefined
372
385
  }
373
386
 
374
387
  if (!objectTypeId || !fields?.length || fields.some((field) => field.kind !== "property"))
@@ -0,0 +1,8 @@
1
+ export function identifier(value: string): string {
2
+ return `"${value.replace(/"/g, '""')}"`
3
+ }
4
+
5
+ export function literal(value: string): string {
6
+ if (value.includes("\0")) throw new Error("[SixbPg] SQL literals cannot contain NUL")
7
+ return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`
8
+ }
@@ -3,6 +3,7 @@ import type { ObjectQuery, ObjectQueryPredicate } from "@sixb/core"
3
3
  import type { ReservedSQL, SQL, SQLClient } from "../pg-client"
4
4
  import { isConnectionLost } from "../storage-errors"
5
5
  import { withReservedPgConnection } from "../transactions"
6
+ import { ensureObjectPartition, objectPartitionName } from "./partitions"
6
7
  import { lockPgPreparation } from "./preparation-lock"
7
8
  import {
8
9
  compilePgFacetSummary,
@@ -10,6 +11,7 @@ import {
10
11
  compilePgObjectQuery,
11
12
  DEFAULT_OBJECT_QUERY_SOURCE,
12
13
  } from "./query-compiler"
14
+ import { identifier, literal } from "./sql-text"
13
15
 
14
16
  export interface PgObjectTextCountIndex {
15
17
  readonly projectId: string
@@ -106,11 +108,19 @@ export async function preparePgObjectTextCounts(
106
108
  )
107
109
  }
108
110
 
111
+ const objectTypeIds = new Set(definitions.map((item) => item.definition.objectTypeId))
112
+ for (const objectTypeId of objectTypeIds) {
113
+ await ensureObjectPartition(connection, schema, objectTypeId)
114
+ }
115
+
109
116
  for (const item of definitions) {
110
117
  const statements = textCountIndexStatements(item)
118
+ const partition = `${identifier(schema)}.${identifier(
119
+ objectPartitionName(item.definition.objectTypeId)
120
+ )}`
111
121
  for (const [name, expression] of statements) {
112
122
  await connection.unsafe(
113
- `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${table} ${expression}`
123
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${partition} ${expression}`
114
124
  )
115
125
  const [valid] = await connection<{ indisvalid: boolean }[]>`
116
126
  SELECT indisvalid FROM pg_index
@@ -126,7 +136,11 @@ export async function preparePgObjectTextCounts(
126
136
  )
127
137
  names.push(item.name)
128
138
  }
129
- await connection.unsafe(`ANALYZE ${table}`)
139
+ for (const objectTypeId of objectTypeIds) {
140
+ await connection.unsafe(
141
+ `ANALYZE ${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`
142
+ )
143
+ }
130
144
  } catch (error) {
131
145
  connected = !isConnectionLost(error)
132
146
  throw error
@@ -204,7 +218,7 @@ async function compilePgIndexedTextAggregate(
204
218
  const rows = await sql<{ name: string; definition: string }[]>`
205
219
  SELECT c.relname AS name, obj_description(c.oid,'pg_class') AS definition
206
220
  FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid
207
- WHERE i.indrelid='objects'::regclass AND i.indisvalid
221
+ WHERE i.indrelid=to_regclass(${objectPartitionName(shape.objectTypeId)}) AND i.indisvalid
208
222
  AND c.relname LIKE 'sixb_text_count_%' AND obj_description(c.oid,'pg_class') IS NOT NULL`
209
223
  for (const row of rows) {
210
224
  const definition = parseDefinition(row.definition)
@@ -445,12 +459,3 @@ function parseDefinition(text: string): PgObjectTextCountIndex | undefined {
445
459
  function hash(value: string): string {
446
460
  return createHash("sha256").update(value).digest("hex").slice(0, 24)
447
461
  }
448
-
449
- function identifier(value: string): string {
450
- return `"${value.replace(/"/g, '""')}"`
451
- }
452
-
453
- function literal(value: string): string {
454
- if (value.includes("\0")) throw new Error("[SixbPg] Text count identifiers cannot contain NUL")
455
- return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`
456
- }
@@ -10,6 +10,7 @@ import type {
10
10
  import {
11
11
  assertNonblank,
12
12
  assertNonnegativeInteger,
13
+ assertTimestamp,
13
14
  commitRecord,
14
15
  originWhere,
15
16
  type PgOntologyCommitRow,
@@ -81,6 +82,9 @@ export class PgOntologyCommitStorage implements OntologyCommitStorage {
81
82
  const limit = input.limit
82
83
  assertNonnegativeInteger(offset, "Ontology commit list offset")
83
84
  if (limit !== undefined) assertNonnegativeInteger(limit, "Ontology commit list limit")
85
+ if (input.committedAt !== undefined) {
86
+ assertTimestamp(input.committedAt, "Ontology commit list committedAt")
87
+ }
84
88
 
85
89
  const runFilter =
86
90
  input.run?.kind === "action"
@@ -89,10 +93,15 @@ export class PgOntologyCommitStorage implements OntologyCommitStorage {
89
93
  ? sql`AND origin_run_id = ${input.run.id}
90
94
  AND origin_kind IN ('projection', 'telemetry')`
91
95
  : sql``
96
+ // The (project_id, committed_at, id) index serves this exact match.
97
+ const committedAtFilter =
98
+ input.committedAt === undefined
99
+ ? sql``
100
+ : sql`AND committed_at = ${input.committedAt}::timestamptz`
92
101
  const [{ total }] = await sql<{ readonly total: number | string }[]>`
93
102
  SELECT COUNT(*) AS total
94
103
  FROM ontology_commits
95
- WHERE project_id = ${input.projectId} ${runFilter}
104
+ WHERE project_id = ${input.projectId} ${runFilter} ${committedAtFilter}
96
105
  `
97
106
  const direction = input.order === "desc" ? sql`DESC` : sql`ASC`
98
107
  const order =
@@ -103,7 +112,7 @@ export class PgOntologyCommitStorage implements OntologyCommitStorage {
103
112
  const limitFragment = limit === undefined ? sql`` : sql`LIMIT ${limit}`
104
113
  const rows = await sql<PgOntologyCommitRow[]>`
105
114
  SELECT * FROM ontology_commits
106
- WHERE project_id = ${input.projectId} ${runFilter}
115
+ WHERE project_id = ${input.projectId} ${runFilter} ${committedAtFilter}
107
116
  ORDER BY ${order}
108
117
  ${limitFragment} OFFSET ${offset}
109
118
  `
@@ -33,7 +33,7 @@ export interface PgOntologyTransactionContext {
33
33
 
34
34
  export class PgMaterializationSessionState extends ProviderMaterializationSessionState {
35
35
  stagedWorkCount = 0
36
- changedObjects = 0
36
+ changedObjects: ReadonlyMap<string, number> = new Map()
37
37
  changedLinks = 0
38
38
  changedUniqueValues = 0
39
39
  /** What unique value items this session staged, known without reading its work back. */
@@ -106,6 +106,15 @@ export class PgMaterializationSessions {
106
106
  if (!session.active) return
107
107
  // A plan's rows outlive the session: maintenance deletes them once the candidate moved on.
108
108
  if (!session.plan) await this.deleteWork(session.id)
109
+ this.discard(session)
110
+ }
111
+
112
+ /**
113
+ * Ends a session without sending SQL, for a session that failed before staging any work. The
114
+ * failure may be a statement PostgreSQL already aborted the transaction for: anything sent after
115
+ * it fails with 25P02 and replaces the original error, a retryable 40001 included.
116
+ */
117
+ discard(session: PgMaterializationSessionState): void {
109
118
  session.active = false
110
119
  this.live.delete(session)
111
120
  this.context?.materializations.complete(session.providerToken)
@@ -76,6 +76,20 @@ export class PgMaterializationWriter {
76
76
  return { objectWrites, linkWrites, uniqueValueWrites, eventCount }
77
77
  }
78
78
 
79
+ /** How many staged object writes each type has, for types a publication changed substantially. */
80
+ async stagedObjectWritesByType(target: PgStagedPlanTarget): Promise<ReadonlyMap<string, number>> {
81
+ const rows = await this.sql<{ readonly object_type_id: string; readonly writes: string }[]>`
82
+ SELECT object_type_id, COUNT(*) AS writes FROM (
83
+ SELECT value->'row'->'ref'->>'objectTypeId' AS object_type_id
84
+ FROM (${this.staged(target, "object-upsert")}) AS staged
85
+ UNION ALL
86
+ SELECT value->'ref'->>'objectTypeId'
87
+ FROM (${this.staged(target, "object-delete")}) AS staged
88
+ ) AS writes
89
+ GROUP BY object_type_id`
90
+ return new Map(rows.map((row) => [row.object_type_id, Number(row.writes)]))
91
+ }
92
+
79
93
  /** Two probes of the work table's lane index: a commit with no unique values runs nothing else. */
80
94
  private async stagedUniqueWork(target: PgStagedPlanTarget): Promise<PgUniqueWork> {
81
95
  const probe = (kind: "unique-value-release" | "unique-value-claim") => this.sql`
@@ -22,6 +22,7 @@ import {
22
22
  beginMaterializationVectorChanges,
23
23
  cardinalityViolation,
24
24
  invalidCorrelation,
25
+ prepareMaterializationStateRead,
25
26
  sameNonnegativeCounts as sameCounts,
26
27
  uniqueSorted,
27
28
  } from "@sixb/core/internal/ontology-storage-provider"
@@ -32,18 +33,21 @@ import type {
32
33
  FinalizeMaterializationInput,
33
34
  MaterializationPlanHeader,
34
35
  MaterializationSession,
36
+ MaterializationState,
35
37
  MaterializationStatePage,
36
38
  MaterializationVectorChangePage,
37
39
  OntologyCommitRecord,
38
40
  OntologyMaterializationStorage,
41
+ ReadMaterializationStateInput,
39
42
  SourceActivationWrite,
40
43
  StageMaterializationWorkInput,
41
44
  StreamMaterializationStateInput,
42
45
  StreamMaterializationVectorChangesInput,
43
46
  } from "@sixb/core/storage"
47
+ import { objectPartitionName } from "../objects/partitions"
44
48
  import type { SQLClient } from "../pg-client"
45
49
  import { isUniqueViolation } from "../storage-errors"
46
- import { lockAdvisoryKeys } from "../transactions"
50
+ import { lockAdvisoryKeys, runPgRepeatableReadTransaction } from "../transactions"
47
51
  import {
48
52
  type PgMaterializationSessionState,
49
53
  PgMaterializationSessions,
@@ -147,7 +151,7 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
147
151
  }
148
152
  return session.publicSession()
149
153
  } catch (error) {
150
- await this.sessions.release(session)
154
+ this.sessions.discard(session)
151
155
  throw error
152
156
  }
153
157
  }
@@ -218,6 +222,20 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
218
222
  }
219
223
  }
220
224
 
225
+ async readState(input: ReadMaterializationStateInput): Promise<MaterializationState> {
226
+ const request = prepareMaterializationStateRead(input)
227
+
228
+ // One snapshot for every statement, so the read never straddles a commit.
229
+ return runPgRepeatableReadTransaction(this.sql, async (tx) => {
230
+ const reader = new PgMaterializationStateReader(tx, request.projectId)
231
+ return {
232
+ objects: await reader.objectStates(request.objects),
233
+ links: await reader.linkStates(request.links),
234
+ linkScopes: await reader.linkSlotStates(request.linkScopes),
235
+ }
236
+ })
237
+ }
238
+
221
239
  async stageWork(input: StageMaterializationWorkInput): Promise<void> {
222
240
  await this.sessions.stage(input)
223
241
  }
@@ -252,7 +270,16 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
252
270
  committedAt: commit.committedAt,
253
271
  ...(session.plan ? {} : { uniqueWork: session.uniqueWork }),
254
272
  })
255
- session.changedObjects = applied.objectWrites
273
+ // Below this no type can reach the statistics threshold, so the work table stays unread.
274
+ if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {
275
+ session.changedObjects = await this.writer.stagedObjectWritesByType({
276
+ workTable: session.workTable,
277
+ workId: session.workId,
278
+ projectId: commit.projectId,
279
+ commitId: commit.id,
280
+ committedAt: commit.committedAt,
281
+ })
282
+ }
256
283
  session.changedLinks = applied.linkWrites
257
284
  session.changedUniqueValues = applied.uniqueValueWrites
258
285
  session.appliedEventCount = applied.eventCount
@@ -268,7 +295,7 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
268
295
  }
269
296
  // The next sparse projection must see the distribution of a newly published bulk load.
270
297
  // With empty/stale statistics, a link lookup can scan every edge sharing its target.
271
- await refreshMaterializedStatistics(this.sql, "objects", session.changedObjects)
298
+ await refreshObjectStatistics(this.sql, session.changedObjects)
272
299
  await refreshMaterializedStatistics(this.sql, "links", session.changedLinks)
273
300
  await refreshMaterializedStatistics(
274
301
  this.sql,
@@ -756,18 +783,65 @@ function databaseCount(value: number | string | undefined): number {
756
783
  return Number(value ?? 0)
757
784
  }
758
785
 
786
+ const STATISTICS_MINIMUM_CHANGES = 1_000
787
+
759
788
  /** Keep the first bulk publication queryable without analyzing every small subsequent delta.
760
789
  * Expression indexes make full ANALYZE expensive. Only a substantial current publication
761
790
  * forces synchronous work. Autovacuum accumulates smaller writes separately; its modification
762
791
  * count can still include a previous transaction's manually analyzed writes at this point. */
763
792
  async function refreshMaterializedStatistics(
764
793
  sql: SQLClient,
765
- table: "objects" | "links" | "object_unique_values",
794
+ table: string,
766
795
  changed: number
767
796
  ): Promise<void> {
768
- if (changed < 1_000) return
797
+ if (changed < STATISTICS_MINIMUM_CHANGES) return
769
798
  const [state] = await sql<{ rows: number }[]>`
770
799
  SELECT reltuples AS rows FROM pg_class WHERE oid=${table}::regclass`
771
- if (!state || state.rows <= 0 || changed >= Math.max(10_000, state.rows * 0.1))
800
+ // At least doubling a small table changes its distribution, however few rows that took.
801
+ if (
802
+ !state ||
803
+ state.rows <= 0 ||
804
+ changed >= state.rows ||
805
+ changed >= Math.max(10_000, state.rows * 0.1)
806
+ )
772
807
  await sql.unsafe(`ANALYZE ${table}`)
773
808
  }
809
+
810
+ /**
811
+ * A substantial publication refreshes the parent too, as it always has: autovacuum never analyzes a
812
+ * partitioned parent, and its statistics plan queries that read across types. A smaller one only
813
+ * refreshes partitions it changed substantially, which autovacuum may not reach before the next
814
+ * projection. A routine update must not: ANALYZE over a partition's expression statistics takes
815
+ * seconds, and a thousand changed rows in ten thousand took five times as long with it.
816
+ */
817
+ async function refreshObjectStatistics(
818
+ sql: SQLClient,
819
+ changes: ReadonlyMap<string, number>
820
+ ): Promise<void> {
821
+ const total = [...changes.values()].reduce((sum, changed) => sum + changed, 0)
822
+ if (total < STATISTICS_MINIMUM_CHANGES) return
823
+
824
+ const [parent] = await sql<{ rows: number }[]>`
825
+ SELECT reltuples AS rows FROM pg_class WHERE oid = 'objects'::regclass`
826
+ if (!parent || parent.rows <= 0 || total >= Math.max(10_000, parent.rows * 0.1)) {
827
+ await sql.unsafe("ANALYZE objects")
828
+ return
829
+ }
830
+ for (const [objectTypeId, changed] of changes) {
831
+ if (changed < STATISTICS_MINIMUM_CHANGES) continue
832
+ await refreshMaterializedStatistics(
833
+ sql,
834
+ await objectStatisticsTable(sql, objectTypeId),
835
+ changed
836
+ )
837
+ }
838
+ }
839
+
840
+ /** A type's rows are in its partition, or in the default until its first preparation. */
841
+ async function objectStatisticsTable(sql: SQLClient, objectTypeId: string): Promise<string> {
842
+ const partition = objectPartitionName(objectTypeId)
843
+ const [row] = await sql<
844
+ { found: string | null }[]
845
+ >`SELECT to_regclass(${partition})::text AS found`
846
+ return row?.found ? partition : "objects_default"
847
+ }