@sixb/pg 0.1.26 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -1
- package/dist/agents/agent-storage.d.ts +8 -3
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/computers.d.ts +22 -0
- package/dist/agents/computers.d.ts.map +1 -0
- package/dist/agents/rows.d.ts +3 -0
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/agents/threads.d.ts +2 -1
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/agents/token-rates.d.ts +13 -0
- package/dist/agents/token-rates.d.ts.map +1 -0
- package/dist/ai-accounting-filters.d.ts +12 -0
- package/dist/ai-accounting-filters.d.ts.map +1 -0
- package/dist/ai-cost-groups.d.ts +1 -1
- package/dist/ai-cost-groups.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +886 -158
- package/dist/index.js.map +26 -21
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/partitions.d.ts +10 -0
- package/dist/objects/partitions.d.ts.map +1 -0
- package/dist/objects/prepare-queries.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +5 -0
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/sql-text.d.ts +3 -0
- package/dist/objects/sql-text.d.ts.map +1 -0
- package/dist/objects/text-count-index.d.ts.map +1 -1
- package/dist/ontology-storage/commits.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +7 -1
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +2 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +2 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-ai-cost-storage.d.ts.map +1 -1
- package/dist/pg-client.d.ts +1 -1
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/storage-errors.d.ts +4 -1
- package/dist/storage-errors.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/agent-storage.ts +10 -3
- package/src/agents/computers.ts +95 -0
- package/src/agents/rows.ts +9 -1
- package/src/agents/runs.ts +25 -3
- package/src/agents/threads.ts +111 -20
- package/src/agents/token-rates.ts +76 -0
- package/src/ai-accounting-filters.ts +48 -0
- package/src/ai-cost-groups.ts +37 -12
- package/src/index.ts +1 -1
- package/src/migrations/084-agent-token-rates.sql +16 -0
- package/src/migrations/085-agent-computer-sessions.sql +27 -0
- package/src/migrations/086-partition-objects.sql +93 -0
- package/src/migrations.ts +8 -0
- package/src/objects/partitions.ts +163 -0
- package/src/objects/prepare-queries.ts +50 -2
- package/src/objects/query-compiler.ts +80 -66
- package/src/objects/query-indexes.ts +59 -46
- package/src/objects/sql-text.ts +8 -0
- package/src/objects/text-count-index.ts +17 -12
- package/src/ontology-storage/commits.ts +11 -2
- package/src/ontology-storage/materialization-session.ts +10 -1
- package/src/ontology-storage/materialization-writer.ts +14 -0
- package/src/ontology-storage/materializations.ts +81 -7
- package/src/pg-ai-cost-storage.ts +119 -15
- package/src/pg-client.ts +9 -2
- package/src/storage-errors.ts +28 -1
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { createHash } from "node:crypto"
|
|
2
2
|
import type { ObjectQuery, QueryScalarKind } from "@sixb/core"
|
|
3
|
+
import { linkEndpointObjectTypeIds } from "@sixb/core/internal/query"
|
|
3
4
|
import type { ReservedSQL, SQL, SQLClient } from "../pg-client"
|
|
4
5
|
import { isConnectionLost } from "../storage-errors"
|
|
5
6
|
import { withReservedPgConnection } from "../transactions"
|
|
7
|
+
import { ensureObjectPartition, objectPartitionName } from "./partitions"
|
|
6
8
|
import { lockPgPreparation } from "./preparation-lock"
|
|
7
|
-
import { isTextScalarKind } from "./query-compiler"
|
|
9
|
+
import { hasJsonValueIndex, isTextScalarKind } from "./query-compiler"
|
|
10
|
+
import { identifier, literal } from "./sql-text"
|
|
8
11
|
|
|
9
12
|
export interface PgObjectQueryIndexScope {
|
|
10
13
|
readonly projectId: string
|
|
@@ -39,7 +42,7 @@ export async function ensurePgObjectQueryIndexes(
|
|
|
39
42
|
indexes: readonly PgObjectQueryIndex[]
|
|
40
43
|
): Promise<readonly string[]> {
|
|
41
44
|
const definitions = indexes.map((index) => ({
|
|
42
|
-
table: index.kind === "incomingLinks" ? "links" :
|
|
45
|
+
table: index.kind === "incomingLinks" ? "links" : objectPartitionName(index.objectTypeId),
|
|
43
46
|
expression: compileIndex(index),
|
|
44
47
|
index,
|
|
45
48
|
}))
|
|
@@ -62,6 +65,11 @@ export async function ensurePgObjectQueryIndexes(
|
|
|
62
65
|
try {
|
|
63
66
|
// One concurrent build per table, across definitions and across processes.
|
|
64
67
|
await lockPgPreparation(connection, `${schema}.objects.query-indexes`)
|
|
68
|
+
for (const objectTypeId of new Set(
|
|
69
|
+
indexes.flatMap((index) => (index.kind === "incomingLinks" ? [] : [index.objectTypeId]))
|
|
70
|
+
)) {
|
|
71
|
+
await ensureObjectPartition(connection, schema, objectTypeId)
|
|
72
|
+
}
|
|
65
73
|
for (const definition of definitions) {
|
|
66
74
|
names.push(await createQueryIndex(connection, schema, definition))
|
|
67
75
|
}
|
|
@@ -83,16 +91,14 @@ async function createQueryIndex(
|
|
|
83
91
|
schema: string,
|
|
84
92
|
definition: { table: string; expression: string; index: PgObjectQueryIndex }
|
|
85
93
|
): Promise<string> {
|
|
94
|
+
// A partition's name stands for its type, which an index's expression no longer names.
|
|
95
|
+
const { table } = definition
|
|
86
96
|
const name = `sixb_query_${createHash("sha256")
|
|
87
|
-
.update(
|
|
88
|
-
definition.table === "objects"
|
|
89
|
-
? definition.expression
|
|
90
|
-
: `${definition.table}:${definition.expression}`
|
|
91
|
-
)
|
|
97
|
+
.update(`${table}:${definition.expression}`)
|
|
92
98
|
.digest("hex")
|
|
93
99
|
.slice(0, 32)}`
|
|
94
100
|
await connection.unsafe(
|
|
95
|
-
`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${identifier(schema)}.${identifier(
|
|
101
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${identifier(schema)}.${identifier(table)} ${definition.expression}`
|
|
96
102
|
)
|
|
97
103
|
const [index] = await connection<{ valid: boolean }[]>`
|
|
98
104
|
SELECT i.indisvalid AS valid FROM pg_index i
|
|
@@ -107,9 +113,8 @@ async function createQueryIndex(
|
|
|
107
113
|
await connection.unsafe(
|
|
108
114
|
`COMMENT ON INDEX ${identifier(schema)}.${identifier(name)} IS ${literal(JSON.stringify(definition.index))}`
|
|
109
115
|
)
|
|
110
|
-
const scope = `project_id = ${literal(definition.index.projectId)} AND object_type_id = ${literal(definition.index.objectTypeId)}`
|
|
111
116
|
await connection.unsafe(
|
|
112
|
-
`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}
|
|
117
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}.${identifier(table)} (primary_id) WHERE project_id = ${literal(definition.index.projectId)} AND NOT (${sortIndexBound(definition.index)})`
|
|
113
118
|
)
|
|
114
119
|
|
|
115
120
|
// The same definition under another name is an earlier generation of this index, built
|
|
@@ -136,27 +141,42 @@ async function ensureExpressionStatistics(
|
|
|
136
141
|
schema: string,
|
|
137
142
|
indexes: readonly PgObjectQueryIndex[]
|
|
138
143
|
): Promise<void> {
|
|
139
|
-
let statisticsChanged = false
|
|
140
144
|
// Partial-index statistics do not describe expressions for the whole table. Without
|
|
141
145
|
// expression statistics JSON's default selectivity can turn a 4M-row view into an
|
|
142
|
-
// estimated 125-row sort, even when the right ordered index exists.
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
146
|
+
// estimated 125-row sort, even when the right ordered index exists. PostgreSQL plans a pruned
|
|
147
|
+
// query with its partition's statistics, so they live there, and ANALYZE stays per type.
|
|
148
|
+
const expressionsByType = new Map<string, Set<string>>()
|
|
149
|
+
for (const index of indexes) {
|
|
150
|
+
if (index.kind === "incomingLinks") continue
|
|
151
|
+
const expressions = expressionsByType.get(index.objectTypeId) ?? new Set<string>()
|
|
152
|
+
for (const expression of statisticsExpressions(index)) expressions.add(expression)
|
|
153
|
+
expressionsByType.set(index.objectTypeId, expressions)
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
for (const [objectTypeId, expressions] of expressionsByType) {
|
|
157
|
+
const partition = `${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`
|
|
158
|
+
let statisticsChanged = false
|
|
159
|
+
for (const expression of expressions) {
|
|
160
|
+
const name = `sixb_query_stats_${createHash("sha256")
|
|
161
|
+
.update(`${objectTypeId}:${expression}`)
|
|
162
|
+
.digest("hex")
|
|
163
|
+
.slice(0, 24)}`
|
|
164
|
+
const [existing] = await connection`
|
|
165
|
+
SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
|
|
166
|
+
WHERE n.nspname = ${schema} AND s.stxname = ${name}`
|
|
167
|
+
if (existing) continue
|
|
168
|
+
await connection.unsafe(
|
|
169
|
+
`CREATE STATISTICS ${identifier(schema)}.${identifier(name)} ON project_id, ${expression} FROM ${partition}`
|
|
170
|
+
)
|
|
171
|
+
statisticsChanged = true
|
|
172
|
+
}
|
|
173
|
+
if (statisticsChanged) await connection.unsafe(`ANALYZE ${partition}`)
|
|
154
174
|
}
|
|
155
|
-
if (statisticsChanged) await connection.unsafe(`ANALYZE ${identifier(schema)}.objects`)
|
|
156
175
|
}
|
|
157
176
|
|
|
158
|
-
function statisticsExpressions(
|
|
159
|
-
|
|
177
|
+
function statisticsExpressions(
|
|
178
|
+
index: Exclude<PgObjectQueryIndex, { kind: "incomingLinks" }>
|
|
179
|
+
): string[] {
|
|
160
180
|
const equality = new Set(Object.keys(index.where ?? {}))
|
|
161
181
|
if (index.kind === "filter") for (const property of index.properties) equality.add(property)
|
|
162
182
|
if (index.kind === "sort") for (const property of index.equality ?? []) equality.add(property)
|
|
@@ -177,10 +197,7 @@ function compileIndex(index: PgObjectQueryIndex): string {
|
|
|
177
197
|
throw new Error("[SixbPg] Incoming link indexes do not accept object property conditions")
|
|
178
198
|
return `(target_id, link_id, source_type_id, source_id) INCLUDE (project_id, target_type_id) WHERE project_id = ${literal(index.projectId)} AND target_type_id = ${literal(index.objectTypeId)}`
|
|
179
199
|
}
|
|
180
|
-
const predicate = [
|
|
181
|
-
`project_id = ${literal(index.projectId)}`,
|
|
182
|
-
`object_type_id = ${literal(index.objectTypeId)}`,
|
|
183
|
-
]
|
|
200
|
+
const predicate = [`project_id = ${literal(index.projectId)}`]
|
|
184
201
|
for (const [propertyId, value] of Object.entries(index.where ?? {}).sort(([a], [b]) =>
|
|
185
202
|
a.localeCompare(b)
|
|
186
203
|
)) {
|
|
@@ -246,16 +263,6 @@ function sortValue(field: Extract<PgObjectQueryIndex, { kind: "sort" }>["fields"
|
|
|
246
263
|
return `(NULLIF(properties -> ${literal(field.propertyId)}, 'null'::jsonb))`
|
|
247
264
|
}
|
|
248
265
|
|
|
249
|
-
function literal(value: string): string {
|
|
250
|
-
if (value.includes("\0"))
|
|
251
|
-
throw new Error("[SixbPg] Query index names and values cannot contain NUL")
|
|
252
|
-
return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
function identifier(value: string): string {
|
|
256
|
-
return `"${value.replace(/"/g, '""')}"`
|
|
257
|
-
}
|
|
258
|
-
|
|
259
266
|
/** Keep complete B-tree entries below PostgreSQL's page limit, without constraining valid data.
|
|
260
267
|
* Ordered reads merge a small indexed page with the exact overflow page. */
|
|
261
268
|
export function sortIndexBound(index: Extract<PgObjectQueryIndex, { kind: "sort" }>): string {
|
|
@@ -313,8 +320,8 @@ async function readPgSortIndexes(sql: SQLClient): Promise<readonly PgSortIndex[]
|
|
|
313
320
|
JOIN pg_class c ON c.oid = i.indexrelid
|
|
314
321
|
JOIN pg_description d
|
|
315
322
|
ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
|
|
316
|
-
WHERE i.indrelid = 'objects'::regclass
|
|
317
|
-
AND c.relname LIKE 'sixb_query_%'
|
|
323
|
+
WHERE i.indrelid IN (SELECT inhrelid FROM pg_inherits WHERE inhparent = 'objects'::regclass)
|
|
324
|
+
AND i.indisvalid AND c.relname LIKE 'sixb_query_%'
|
|
318
325
|
`
|
|
319
326
|
const indexes: PgSortIndex[] = []
|
|
320
327
|
for (const row of rows) {
|
|
@@ -351,12 +358,17 @@ export async function findPgSortIndexBound(
|
|
|
351
358
|
if (input.kind === "filter") {
|
|
352
359
|
const inspect = (predicate: Extract<ObjectQuery, { kind: "filter" }>["predicate"]): void => {
|
|
353
360
|
if (predicate.op === "and") predicate.items.forEach(inspect)
|
|
354
|
-
else if (
|
|
361
|
+
else if (
|
|
362
|
+
predicate.op === "eq" &&
|
|
363
|
+
typeof predicate.value === "string" &&
|
|
364
|
+
!hasJsonValueIndex(predicate.scalarKind)
|
|
365
|
+
)
|
|
355
366
|
equality.add(predicate.propertyId)
|
|
356
367
|
else if (
|
|
357
368
|
predicate.op === "in" &&
|
|
358
369
|
predicate.values.length === 1 &&
|
|
359
|
-
typeof predicate.values[0] === "string"
|
|
370
|
+
typeof predicate.values[0] === "string" &&
|
|
371
|
+
!hasJsonValueIndex(predicate.scalarKind)
|
|
360
372
|
)
|
|
361
373
|
equality.add(predicate.propertyId)
|
|
362
374
|
}
|
|
@@ -367,8 +379,9 @@ export async function findPgSortIndexBound(
|
|
|
367
379
|
let objectTypeId: string | undefined
|
|
368
380
|
if (input.kind === "start" && !input.includeSubtypes) {
|
|
369
381
|
objectTypeId = input.objectTypeId
|
|
370
|
-
} else if (input.kind === "traverse"
|
|
371
|
-
|
|
382
|
+
} else if (input.kind === "traverse") {
|
|
383
|
+
const endpoints = linkEndpointObjectTypeIds(input)
|
|
384
|
+
objectTypeId = endpoints?.length === 1 ? endpoints[0] : undefined
|
|
372
385
|
}
|
|
373
386
|
|
|
374
387
|
if (!objectTypeId || !fields?.length || fields.some((field) => field.kind !== "property"))
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export function identifier(value: string): string {
|
|
2
|
+
return `"${value.replace(/"/g, '""')}"`
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
export function literal(value: string): string {
|
|
6
|
+
if (value.includes("\0")) throw new Error("[SixbPg] SQL literals cannot contain NUL")
|
|
7
|
+
return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`
|
|
8
|
+
}
|
|
@@ -3,6 +3,7 @@ import type { ObjectQuery, ObjectQueryPredicate } from "@sixb/core"
|
|
|
3
3
|
import type { ReservedSQL, SQL, SQLClient } from "../pg-client"
|
|
4
4
|
import { isConnectionLost } from "../storage-errors"
|
|
5
5
|
import { withReservedPgConnection } from "../transactions"
|
|
6
|
+
import { ensureObjectPartition, objectPartitionName } from "./partitions"
|
|
6
7
|
import { lockPgPreparation } from "./preparation-lock"
|
|
7
8
|
import {
|
|
8
9
|
compilePgFacetSummary,
|
|
@@ -10,6 +11,7 @@ import {
|
|
|
10
11
|
compilePgObjectQuery,
|
|
11
12
|
DEFAULT_OBJECT_QUERY_SOURCE,
|
|
12
13
|
} from "./query-compiler"
|
|
14
|
+
import { identifier, literal } from "./sql-text"
|
|
13
15
|
|
|
14
16
|
export interface PgObjectTextCountIndex {
|
|
15
17
|
readonly projectId: string
|
|
@@ -106,11 +108,19 @@ export async function preparePgObjectTextCounts(
|
|
|
106
108
|
)
|
|
107
109
|
}
|
|
108
110
|
|
|
111
|
+
const objectTypeIds = new Set(definitions.map((item) => item.definition.objectTypeId))
|
|
112
|
+
for (const objectTypeId of objectTypeIds) {
|
|
113
|
+
await ensureObjectPartition(connection, schema, objectTypeId)
|
|
114
|
+
}
|
|
115
|
+
|
|
109
116
|
for (const item of definitions) {
|
|
110
117
|
const statements = textCountIndexStatements(item)
|
|
118
|
+
const partition = `${identifier(schema)}.${identifier(
|
|
119
|
+
objectPartitionName(item.definition.objectTypeId)
|
|
120
|
+
)}`
|
|
111
121
|
for (const [name, expression] of statements) {
|
|
112
122
|
await connection.unsafe(
|
|
113
|
-
`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${
|
|
123
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(name)} ON ${partition} ${expression}`
|
|
114
124
|
)
|
|
115
125
|
const [valid] = await connection<{ indisvalid: boolean }[]>`
|
|
116
126
|
SELECT indisvalid FROM pg_index
|
|
@@ -126,7 +136,11 @@ export async function preparePgObjectTextCounts(
|
|
|
126
136
|
)
|
|
127
137
|
names.push(item.name)
|
|
128
138
|
}
|
|
129
|
-
|
|
139
|
+
for (const objectTypeId of objectTypeIds) {
|
|
140
|
+
await connection.unsafe(
|
|
141
|
+
`ANALYZE ${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`
|
|
142
|
+
)
|
|
143
|
+
}
|
|
130
144
|
} catch (error) {
|
|
131
145
|
connected = !isConnectionLost(error)
|
|
132
146
|
throw error
|
|
@@ -204,7 +218,7 @@ async function compilePgIndexedTextAggregate(
|
|
|
204
218
|
const rows = await sql<{ name: string; definition: string }[]>`
|
|
205
219
|
SELECT c.relname AS name, obj_description(c.oid,'pg_class') AS definition
|
|
206
220
|
FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid
|
|
207
|
-
WHERE i.indrelid=
|
|
221
|
+
WHERE i.indrelid=to_regclass(${objectPartitionName(shape.objectTypeId)}) AND i.indisvalid
|
|
208
222
|
AND c.relname LIKE 'sixb_text_count_%' AND obj_description(c.oid,'pg_class') IS NOT NULL`
|
|
209
223
|
for (const row of rows) {
|
|
210
224
|
const definition = parseDefinition(row.definition)
|
|
@@ -445,12 +459,3 @@ function parseDefinition(text: string): PgObjectTextCountIndex | undefined {
|
|
|
445
459
|
function hash(value: string): string {
|
|
446
460
|
return createHash("sha256").update(value).digest("hex").slice(0, 24)
|
|
447
461
|
}
|
|
448
|
-
|
|
449
|
-
function identifier(value: string): string {
|
|
450
|
-
return `"${value.replace(/"/g, '""')}"`
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
function literal(value: string): string {
|
|
454
|
-
if (value.includes("\0")) throw new Error("[SixbPg] Text count identifiers cannot contain NUL")
|
|
455
|
-
return `E'${value.replace(/\\/g, "\\\\").replace(/'/g, "''")}'`
|
|
456
|
-
}
|
|
@@ -10,6 +10,7 @@ import type {
|
|
|
10
10
|
import {
|
|
11
11
|
assertNonblank,
|
|
12
12
|
assertNonnegativeInteger,
|
|
13
|
+
assertTimestamp,
|
|
13
14
|
commitRecord,
|
|
14
15
|
originWhere,
|
|
15
16
|
type PgOntologyCommitRow,
|
|
@@ -81,6 +82,9 @@ export class PgOntologyCommitStorage implements OntologyCommitStorage {
|
|
|
81
82
|
const limit = input.limit
|
|
82
83
|
assertNonnegativeInteger(offset, "Ontology commit list offset")
|
|
83
84
|
if (limit !== undefined) assertNonnegativeInteger(limit, "Ontology commit list limit")
|
|
85
|
+
if (input.committedAt !== undefined) {
|
|
86
|
+
assertTimestamp(input.committedAt, "Ontology commit list committedAt")
|
|
87
|
+
}
|
|
84
88
|
|
|
85
89
|
const runFilter =
|
|
86
90
|
input.run?.kind === "action"
|
|
@@ -89,10 +93,15 @@ export class PgOntologyCommitStorage implements OntologyCommitStorage {
|
|
|
89
93
|
? sql`AND origin_run_id = ${input.run.id}
|
|
90
94
|
AND origin_kind IN ('projection', 'telemetry')`
|
|
91
95
|
: sql``
|
|
96
|
+
// The (project_id, committed_at, id) index serves this exact match.
|
|
97
|
+
const committedAtFilter =
|
|
98
|
+
input.committedAt === undefined
|
|
99
|
+
? sql``
|
|
100
|
+
: sql`AND committed_at = ${input.committedAt}::timestamptz`
|
|
92
101
|
const [{ total }] = await sql<{ readonly total: number | string }[]>`
|
|
93
102
|
SELECT COUNT(*) AS total
|
|
94
103
|
FROM ontology_commits
|
|
95
|
-
WHERE project_id = ${input.projectId} ${runFilter}
|
|
104
|
+
WHERE project_id = ${input.projectId} ${runFilter} ${committedAtFilter}
|
|
96
105
|
`
|
|
97
106
|
const direction = input.order === "desc" ? sql`DESC` : sql`ASC`
|
|
98
107
|
const order =
|
|
@@ -103,7 +112,7 @@ export class PgOntologyCommitStorage implements OntologyCommitStorage {
|
|
|
103
112
|
const limitFragment = limit === undefined ? sql`` : sql`LIMIT ${limit}`
|
|
104
113
|
const rows = await sql<PgOntologyCommitRow[]>`
|
|
105
114
|
SELECT * FROM ontology_commits
|
|
106
|
-
WHERE project_id = ${input.projectId} ${runFilter}
|
|
115
|
+
WHERE project_id = ${input.projectId} ${runFilter} ${committedAtFilter}
|
|
107
116
|
ORDER BY ${order}
|
|
108
117
|
${limitFragment} OFFSET ${offset}
|
|
109
118
|
`
|
|
@@ -33,7 +33,7 @@ export interface PgOntologyTransactionContext {
|
|
|
33
33
|
|
|
34
34
|
export class PgMaterializationSessionState extends ProviderMaterializationSessionState {
|
|
35
35
|
stagedWorkCount = 0
|
|
36
|
-
changedObjects =
|
|
36
|
+
changedObjects: ReadonlyMap<string, number> = new Map()
|
|
37
37
|
changedLinks = 0
|
|
38
38
|
changedUniqueValues = 0
|
|
39
39
|
/** What unique value items this session staged, known without reading its work back. */
|
|
@@ -106,6 +106,15 @@ export class PgMaterializationSessions {
|
|
|
106
106
|
if (!session.active) return
|
|
107
107
|
// A plan's rows outlive the session: maintenance deletes them once the candidate moved on.
|
|
108
108
|
if (!session.plan) await this.deleteWork(session.id)
|
|
109
|
+
this.discard(session)
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Ends a session without sending SQL, for a session that failed before staging any work. The
|
|
114
|
+
* failure may be a statement PostgreSQL already aborted the transaction for: anything sent after
|
|
115
|
+
* it fails with 25P02 and replaces the original error, a retryable 40001 included.
|
|
116
|
+
*/
|
|
117
|
+
discard(session: PgMaterializationSessionState): void {
|
|
109
118
|
session.active = false
|
|
110
119
|
this.live.delete(session)
|
|
111
120
|
this.context?.materializations.complete(session.providerToken)
|
|
@@ -76,6 +76,20 @@ export class PgMaterializationWriter {
|
|
|
76
76
|
return { objectWrites, linkWrites, uniqueValueWrites, eventCount }
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
+
/** How many staged object writes each type has, for types a publication changed substantially. */
|
|
80
|
+
async stagedObjectWritesByType(target: PgStagedPlanTarget): Promise<ReadonlyMap<string, number>> {
|
|
81
|
+
const rows = await this.sql<{ readonly object_type_id: string; readonly writes: string }[]>`
|
|
82
|
+
SELECT object_type_id, COUNT(*) AS writes FROM (
|
|
83
|
+
SELECT value->'row'->'ref'->>'objectTypeId' AS object_type_id
|
|
84
|
+
FROM (${this.staged(target, "object-upsert")}) AS staged
|
|
85
|
+
UNION ALL
|
|
86
|
+
SELECT value->'ref'->>'objectTypeId'
|
|
87
|
+
FROM (${this.staged(target, "object-delete")}) AS staged
|
|
88
|
+
) AS writes
|
|
89
|
+
GROUP BY object_type_id`
|
|
90
|
+
return new Map(rows.map((row) => [row.object_type_id, Number(row.writes)]))
|
|
91
|
+
}
|
|
92
|
+
|
|
79
93
|
/** Two probes of the work table's lane index: a commit with no unique values runs nothing else. */
|
|
80
94
|
private async stagedUniqueWork(target: PgStagedPlanTarget): Promise<PgUniqueWork> {
|
|
81
95
|
const probe = (kind: "unique-value-release" | "unique-value-claim") => this.sql`
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
beginMaterializationVectorChanges,
|
|
23
23
|
cardinalityViolation,
|
|
24
24
|
invalidCorrelation,
|
|
25
|
+
prepareMaterializationStateRead,
|
|
25
26
|
sameNonnegativeCounts as sameCounts,
|
|
26
27
|
uniqueSorted,
|
|
27
28
|
} from "@sixb/core/internal/ontology-storage-provider"
|
|
@@ -32,18 +33,21 @@ import type {
|
|
|
32
33
|
FinalizeMaterializationInput,
|
|
33
34
|
MaterializationPlanHeader,
|
|
34
35
|
MaterializationSession,
|
|
36
|
+
MaterializationState,
|
|
35
37
|
MaterializationStatePage,
|
|
36
38
|
MaterializationVectorChangePage,
|
|
37
39
|
OntologyCommitRecord,
|
|
38
40
|
OntologyMaterializationStorage,
|
|
41
|
+
ReadMaterializationStateInput,
|
|
39
42
|
SourceActivationWrite,
|
|
40
43
|
StageMaterializationWorkInput,
|
|
41
44
|
StreamMaterializationStateInput,
|
|
42
45
|
StreamMaterializationVectorChangesInput,
|
|
43
46
|
} from "@sixb/core/storage"
|
|
47
|
+
import { objectPartitionName } from "../objects/partitions"
|
|
44
48
|
import type { SQLClient } from "../pg-client"
|
|
45
49
|
import { isUniqueViolation } from "../storage-errors"
|
|
46
|
-
import { lockAdvisoryKeys } from "../transactions"
|
|
50
|
+
import { lockAdvisoryKeys, runPgRepeatableReadTransaction } from "../transactions"
|
|
47
51
|
import {
|
|
48
52
|
type PgMaterializationSessionState,
|
|
49
53
|
PgMaterializationSessions,
|
|
@@ -147,7 +151,7 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
147
151
|
}
|
|
148
152
|
return session.publicSession()
|
|
149
153
|
} catch (error) {
|
|
150
|
-
|
|
154
|
+
this.sessions.discard(session)
|
|
151
155
|
throw error
|
|
152
156
|
}
|
|
153
157
|
}
|
|
@@ -218,6 +222,20 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
218
222
|
}
|
|
219
223
|
}
|
|
220
224
|
|
|
225
|
+
async readState(input: ReadMaterializationStateInput): Promise<MaterializationState> {
|
|
226
|
+
const request = prepareMaterializationStateRead(input)
|
|
227
|
+
|
|
228
|
+
// One snapshot for every statement, so the read never straddles a commit.
|
|
229
|
+
return runPgRepeatableReadTransaction(this.sql, async (tx) => {
|
|
230
|
+
const reader = new PgMaterializationStateReader(tx, request.projectId)
|
|
231
|
+
return {
|
|
232
|
+
objects: await reader.objectStates(request.objects),
|
|
233
|
+
links: await reader.linkStates(request.links),
|
|
234
|
+
linkScopes: await reader.linkSlotStates(request.linkScopes),
|
|
235
|
+
}
|
|
236
|
+
})
|
|
237
|
+
}
|
|
238
|
+
|
|
221
239
|
async stageWork(input: StageMaterializationWorkInput): Promise<void> {
|
|
222
240
|
await this.sessions.stage(input)
|
|
223
241
|
}
|
|
@@ -252,7 +270,16 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
252
270
|
committedAt: commit.committedAt,
|
|
253
271
|
...(session.plan ? {} : { uniqueWork: session.uniqueWork }),
|
|
254
272
|
})
|
|
255
|
-
|
|
273
|
+
// Below this no type can reach the statistics threshold, so the work table stays unread.
|
|
274
|
+
if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {
|
|
275
|
+
session.changedObjects = await this.writer.stagedObjectWritesByType({
|
|
276
|
+
workTable: session.workTable,
|
|
277
|
+
workId: session.workId,
|
|
278
|
+
projectId: commit.projectId,
|
|
279
|
+
commitId: commit.id,
|
|
280
|
+
committedAt: commit.committedAt,
|
|
281
|
+
})
|
|
282
|
+
}
|
|
256
283
|
session.changedLinks = applied.linkWrites
|
|
257
284
|
session.changedUniqueValues = applied.uniqueValueWrites
|
|
258
285
|
session.appliedEventCount = applied.eventCount
|
|
@@ -268,7 +295,7 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
268
295
|
}
|
|
269
296
|
// The next sparse projection must see the distribution of a newly published bulk load.
|
|
270
297
|
// With empty/stale statistics, a link lookup can scan every edge sharing its target.
|
|
271
|
-
await
|
|
298
|
+
await refreshObjectStatistics(this.sql, session.changedObjects)
|
|
272
299
|
await refreshMaterializedStatistics(this.sql, "links", session.changedLinks)
|
|
273
300
|
await refreshMaterializedStatistics(
|
|
274
301
|
this.sql,
|
|
@@ -756,18 +783,65 @@ function databaseCount(value: number | string | undefined): number {
|
|
|
756
783
|
return Number(value ?? 0)
|
|
757
784
|
}
|
|
758
785
|
|
|
786
|
+
const STATISTICS_MINIMUM_CHANGES = 1_000
|
|
787
|
+
|
|
759
788
|
/** Keep the first bulk publication queryable without analyzing every small subsequent delta.
|
|
760
789
|
* Expression indexes make full ANALYZE expensive. Only a substantial current publication
|
|
761
790
|
* forces synchronous work. Autovacuum accumulates smaller writes separately; its modification
|
|
762
791
|
* count can still include a previous transaction's manually analyzed writes at this point. */
|
|
763
792
|
async function refreshMaterializedStatistics(
|
|
764
793
|
sql: SQLClient,
|
|
765
|
-
table:
|
|
794
|
+
table: string,
|
|
766
795
|
changed: number
|
|
767
796
|
): Promise<void> {
|
|
768
|
-
if (changed <
|
|
797
|
+
if (changed < STATISTICS_MINIMUM_CHANGES) return
|
|
769
798
|
const [state] = await sql<{ rows: number }[]>`
|
|
770
799
|
SELECT reltuples AS rows FROM pg_class WHERE oid=${table}::regclass`
|
|
771
|
-
|
|
800
|
+
// At least doubling a small table changes its distribution, however few rows that took.
|
|
801
|
+
if (
|
|
802
|
+
!state ||
|
|
803
|
+
state.rows <= 0 ||
|
|
804
|
+
changed >= state.rows ||
|
|
805
|
+
changed >= Math.max(10_000, state.rows * 0.1)
|
|
806
|
+
)
|
|
772
807
|
await sql.unsafe(`ANALYZE ${table}`)
|
|
773
808
|
}
|
|
809
|
+
|
|
810
|
+
/**
|
|
811
|
+
* A substantial publication refreshes the parent too, as it always has: autovacuum never analyzes a
|
|
812
|
+
* partitioned parent, and its statistics plan queries that read across types. A smaller one only
|
|
813
|
+
* refreshes partitions it changed substantially, which autovacuum may not reach before the next
|
|
814
|
+
* projection. A routine update must not: ANALYZE over a partition's expression statistics takes
|
|
815
|
+
* seconds, and a thousand changed rows in ten thousand took five times as long with it.
|
|
816
|
+
*/
|
|
817
|
+
async function refreshObjectStatistics(
|
|
818
|
+
sql: SQLClient,
|
|
819
|
+
changes: ReadonlyMap<string, number>
|
|
820
|
+
): Promise<void> {
|
|
821
|
+
const total = [...changes.values()].reduce((sum, changed) => sum + changed, 0)
|
|
822
|
+
if (total < STATISTICS_MINIMUM_CHANGES) return
|
|
823
|
+
|
|
824
|
+
const [parent] = await sql<{ rows: number }[]>`
|
|
825
|
+
SELECT reltuples AS rows FROM pg_class WHERE oid = 'objects'::regclass`
|
|
826
|
+
if (!parent || parent.rows <= 0 || total >= Math.max(10_000, parent.rows * 0.1)) {
|
|
827
|
+
await sql.unsafe("ANALYZE objects")
|
|
828
|
+
return
|
|
829
|
+
}
|
|
830
|
+
for (const [objectTypeId, changed] of changes) {
|
|
831
|
+
if (changed < STATISTICS_MINIMUM_CHANGES) continue
|
|
832
|
+
await refreshMaterializedStatistics(
|
|
833
|
+
sql,
|
|
834
|
+
await objectStatisticsTable(sql, objectTypeId),
|
|
835
|
+
changed
|
|
836
|
+
)
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
/** A type's rows are in its partition, or in the default until its first preparation. */
|
|
841
|
+
async function objectStatisticsTable(sql: SQLClient, objectTypeId: string): Promise<string> {
|
|
842
|
+
const partition = objectPartitionName(objectTypeId)
|
|
843
|
+
const [row] = await sql<
|
|
844
|
+
{ found: string | null }[]
|
|
845
|
+
>`SELECT to_regclass(${partition})::text AS found`
|
|
846
|
+
return row?.found ? partition : "objects_default"
|
|
847
|
+
}
|