@sixb/pg 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/agents/computers.d.ts +8 -2
- package/dist/agents/computers.d.ts.map +1 -1
- package/dist/index.js +566 -144
- package/dist/index.js.map +20 -17
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/conditional-reader.d.ts +7 -2
- package/dist/objects/conditional-reader.d.ts.map +1 -1
- package/dist/objects/partitions.d.ts.map +1 -1
- package/dist/objects/prepare-queries.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +14 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts +13 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +14 -1
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +1 -0
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/visibility-indexes.d.ts +22 -0
- package/dist/objects/visibility-indexes.d.ts.map +1 -0
- package/dist/objects/visibility-statistics.d.ts +26 -0
- package/dist/objects/visibility-statistics.d.ts.map +1 -0
- package/dist/objects/visibility.d.ts +34 -0
- package/dist/objects/visibility.d.ts.map +1 -0
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/computers.ts +54 -5
- package/src/migrations/087-agent-computer-background-commands.sql +4 -0
- package/src/migrations/088-web-search-model-kind.sql +12 -0
- package/src/migrations.ts +6 -0
- package/src/objects/conditional-reader.ts +26 -4
- package/src/objects/partitions.ts +17 -4
- package/src/objects/prepare-queries.ts +8 -3
- package/src/objects/query-compiler.ts +64 -23
- package/src/objects/query-indexes.ts +93 -27
- package/src/objects/read-scope.ts +147 -77
- package/src/objects/reader.ts +46 -18
- package/src/objects/storage.ts +4 -0
- package/src/objects/visibility-indexes.ts +222 -0
- package/src/objects/visibility-statistics.ts +73 -0
- package/src/objects/visibility.ts +135 -0
- package/src/ontology-storage/materializations.ts +3 -1
- package/src/pg-client.ts +4 -0
- package/src/pg-pipeline-run-storage.ts +5 -7
- package/src/pg-sync-run-storage.ts +2 -7
|
@@ -5,9 +5,9 @@ import type {
|
|
|
5
5
|
import {
|
|
6
6
|
type CompiledPgObjectQuery,
|
|
7
7
|
compilePgObjectStatement,
|
|
8
|
-
compilePredicate,
|
|
9
8
|
type PgObjectQuerySource,
|
|
10
9
|
} from "./query-compiler"
|
|
10
|
+
import { compilePgConditionFilter } from "./visibility"
|
|
11
11
|
|
|
12
12
|
export interface PgSelectedObjectReadSource extends PgObjectQuerySource {
|
|
13
13
|
readonly objectPropertyPermissionsTable: string
|
|
@@ -34,8 +34,9 @@ export function compilePgSelectedObjectReadSource(
|
|
|
34
34
|
objects: scope.objects,
|
|
35
35
|
steps: scope.steps,
|
|
36
36
|
})
|
|
37
|
+
const relations = compileSelectedReadScopeCte(scopeJson, projectId)
|
|
37
38
|
const source: PgSelectedObjectReadSource = {
|
|
38
|
-
...scopedQuerySource(
|
|
39
|
+
...scopedQuerySource(() => relations),
|
|
39
40
|
objectPropertyPermissionsTable: "_sixb_scope_object_permissions",
|
|
40
41
|
traversalProbe: { sql: "", args: [] },
|
|
41
42
|
}
|
|
@@ -77,12 +78,18 @@ export function compilePgConditionalObjectReadSource(
|
|
|
77
78
|
const filters = new Map(
|
|
78
79
|
scope.conditions.map((condition) => [
|
|
79
80
|
condition.objectTypeId,
|
|
80
|
-
|
|
81
|
+
compilePgConditionFilter(condition),
|
|
81
82
|
])
|
|
82
83
|
)
|
|
83
84
|
|
|
85
|
+
const relations = compileConditionalReadScopeRelations(projectId, filters)
|
|
86
|
+
|
|
84
87
|
return Object.freeze({
|
|
85
|
-
...scopedQuerySource(
|
|
88
|
+
...scopedQuerySource((sql) => withReadRelations(relations, sql)),
|
|
89
|
+
// A link is visible while both endpoints are, so a statement that reads both through
|
|
90
|
+
// `_sixb_scope_objects` reads stored links. Checking each endpoint again plans every partition
|
|
91
|
+
// of objects: the endpoint's type comes from the link row, so PostgreSQL cannot prune them.
|
|
92
|
+
linksBetweenVisibleObjectsTable: "links",
|
|
86
93
|
// One type's visible objects are the stored ones its own condition matches.
|
|
87
94
|
objectsOfType: (objectTypeId: string) => {
|
|
88
95
|
const filter = filters.get(objectTypeId)
|
|
@@ -97,33 +104,68 @@ interface CompiledReadScopeCte {
|
|
|
97
104
|
}
|
|
98
105
|
|
|
99
106
|
/** Prefix every statement of a query with the scope's relations, ahead of its own arguments. */
|
|
100
|
-
function scopedQuerySource(
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
107
|
+
function scopedQuerySource(
|
|
108
|
+
withClause: (statementSql: string) => CompiledReadScopeCte
|
|
109
|
+
): PgObjectQuerySource {
|
|
110
|
+
const wrapStatement = (sql: string, args: readonly unknown[] = []) => {
|
|
111
|
+
const compiled = withClause(sql)
|
|
112
|
+
return {
|
|
113
|
+
sql: `${compiled.sql}\n${sql}`,
|
|
114
|
+
args: [...compiled.args, ...args],
|
|
115
|
+
}
|
|
116
|
+
}
|
|
105
117
|
|
|
106
118
|
return {
|
|
107
119
|
objectsTable: "_sixb_scope_objects",
|
|
108
120
|
linksTable: "_sixb_scope_links",
|
|
109
121
|
wrapStatement,
|
|
110
|
-
wrapQuery: (query: CompiledPgObjectQuery): CompiledPgObjectQuery =>
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
122
|
+
wrapQuery: (query: CompiledPgObjectQuery): CompiledPgObjectQuery => {
|
|
123
|
+
const total = wrapStatement(query.totalSql, query.totalArgs)
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
...query,
|
|
127
|
+
...wrapStatement(query.sql, query.args),
|
|
128
|
+
vectorProbe: query.vectorProbe
|
|
129
|
+
? {
|
|
130
|
+
...query.vectorProbe,
|
|
131
|
+
...wrapStatement(query.vectorProbe.sql, query.vectorProbe.args),
|
|
132
|
+
}
|
|
133
|
+
: undefined,
|
|
134
|
+
totalSql: total.sql,
|
|
135
|
+
totalArgs: total.args,
|
|
136
|
+
...(query.hasMoreProbe
|
|
137
|
+
? {
|
|
138
|
+
hasMoreProbe: {
|
|
139
|
+
...query.hasMoreProbe,
|
|
140
|
+
...wrapStatement(query.hasMoreProbe.sql, query.hasMoreProbe.args),
|
|
141
|
+
},
|
|
142
|
+
}
|
|
143
|
+
: {}),
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
interface ReadRelation extends CompiledReadScopeCte {
|
|
150
|
+
readonly name: string
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* The WITH clause naming the relations a statement reads. PostgreSQL parses and analyzes every
|
|
155
|
+
* relation a WITH clause names, read or not, and each one repeats every condition.
|
|
156
|
+
*/
|
|
157
|
+
function withReadRelations(
|
|
158
|
+
relations: readonly ReadRelation[],
|
|
159
|
+
statementSql: string
|
|
160
|
+
): CompiledReadScopeCte {
|
|
161
|
+
const read = relations.filter((relation) =>
|
|
162
|
+
new RegExp(`\\b${relation.name}\\b`).test(statementSql)
|
|
163
|
+
)
|
|
164
|
+
if (read.length === 0) return { sql: "", args: [] }
|
|
165
|
+
|
|
166
|
+
return {
|
|
167
|
+
sql: `WITH ${read.map((relation) => relation.sql).join(",\n")}`,
|
|
168
|
+
args: read.flatMap((relation) => relation.args),
|
|
127
169
|
}
|
|
128
170
|
}
|
|
129
171
|
|
|
@@ -133,23 +175,31 @@ type ConditionFilters = ReadonlyMap<
|
|
|
133
175
|
{ readonly sql: string; readonly args: readonly unknown[] }
|
|
134
176
|
>
|
|
135
177
|
|
|
136
|
-
function
|
|
178
|
+
function compileConditionalReadScopeRelations(
|
|
137
179
|
projectId: string,
|
|
138
180
|
filters: ConditionFilters
|
|
139
|
-
):
|
|
181
|
+
): readonly ReadRelation[] {
|
|
140
182
|
if (filters.size === 0) {
|
|
141
|
-
return
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
183
|
+
return [
|
|
184
|
+
{
|
|
185
|
+
name: "_sixb_scope_objects",
|
|
186
|
+
sql: `
|
|
187
|
+
_sixb_scope_objects AS NOT MATERIALIZED (
|
|
188
|
+
SELECT * FROM objects WHERE project_id = ?
|
|
189
|
+
)
|
|
190
|
+
`,
|
|
191
|
+
args: [projectId],
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
name: "_sixb_scope_links",
|
|
195
|
+
sql: `
|
|
196
|
+
_sixb_scope_links AS NOT MATERIALIZED (
|
|
197
|
+
SELECT * FROM links WHERE project_id = ?
|
|
198
|
+
)
|
|
199
|
+
`,
|
|
200
|
+
args: [projectId],
|
|
201
|
+
},
|
|
202
|
+
]
|
|
153
203
|
}
|
|
154
204
|
|
|
155
205
|
const conditionedTypeArgs = [...filters.keys()]
|
|
@@ -162,45 +212,65 @@ function compileConditionalReadScopeCte(
|
|
|
162
212
|
matchArgs.push(objectTypeId, ...filter.args)
|
|
163
213
|
}
|
|
164
214
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
WHERE endpoint.project_id = stored.project_id
|
|
172
|
-
AND endpoint.object_type_id = stored.${end}_type_id
|
|
173
|
-
AND endpoint.primary_id = stored.${end}_id
|
|
174
|
-
)
|
|
175
|
-
)
|
|
176
|
-
`
|
|
215
|
+
// An endpoint of a conditioned type is a visible object of that type. Each check names its type,
|
|
216
|
+
// so PostgreSQL plans that type's partition alone: matched on the link row's type, one check
|
|
217
|
+
// would plan every partition of objects.
|
|
218
|
+
const visibleEndpoint = (end: "source" | "target"): CompiledReadScopeCte => {
|
|
219
|
+
const checks: string[] = []
|
|
220
|
+
const args: unknown[] = [...conditionedTypeArgs]
|
|
177
221
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
projectId,
|
|
200
|
-
...conditionedTypeArgs,
|
|
201
|
-
...conditionedTypeArgs,
|
|
202
|
-
],
|
|
222
|
+
for (const [objectTypeId, filter] of filters) {
|
|
223
|
+
checks.push(`
|
|
224
|
+
(
|
|
225
|
+
stored.${end}_type_id = ?::text
|
|
226
|
+
AND EXISTS (
|
|
227
|
+
SELECT 1
|
|
228
|
+
FROM objects AS endpoint
|
|
229
|
+
WHERE endpoint.project_id = stored.project_id
|
|
230
|
+
AND endpoint.object_type_id = ?::text
|
|
231
|
+
AND endpoint.primary_id = stored.${end}_id
|
|
232
|
+
AND (${filter.sql})
|
|
233
|
+
)
|
|
234
|
+
)
|
|
235
|
+
`)
|
|
236
|
+
args.push(objectTypeId, objectTypeId, ...filter.args)
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
return {
|
|
240
|
+
sql: `(stored.${end}_type_id NOT IN (${conditionedTypes}) OR ${checks.join(" OR ")})`,
|
|
241
|
+
args,
|
|
242
|
+
}
|
|
203
243
|
}
|
|
244
|
+
const visibleSource = visibleEndpoint("source")
|
|
245
|
+
const visibleTarget = visibleEndpoint("target")
|
|
246
|
+
|
|
247
|
+
return [
|
|
248
|
+
{
|
|
249
|
+
name: "_sixb_scope_objects",
|
|
250
|
+
sql: `
|
|
251
|
+
_sixb_scope_objects AS NOT MATERIALIZED (
|
|
252
|
+
SELECT *
|
|
253
|
+
FROM objects
|
|
254
|
+
WHERE project_id = ?
|
|
255
|
+
AND (object_type_id NOT IN (${conditionedTypes}) OR ${matches.join(" OR ")})
|
|
256
|
+
)
|
|
257
|
+
`,
|
|
258
|
+
args: [projectId, ...conditionedTypeArgs, ...matchArgs],
|
|
259
|
+
},
|
|
260
|
+
{
|
|
261
|
+
name: "_sixb_scope_links",
|
|
262
|
+
sql: `
|
|
263
|
+
_sixb_scope_links AS NOT MATERIALIZED (
|
|
264
|
+
SELECT stored.*
|
|
265
|
+
FROM links AS stored
|
|
266
|
+
WHERE stored.project_id = ?
|
|
267
|
+
AND ${visibleSource.sql}
|
|
268
|
+
AND ${visibleTarget.sql}
|
|
269
|
+
)
|
|
270
|
+
`,
|
|
271
|
+
args: [projectId, ...visibleSource.args, ...visibleTarget.args],
|
|
272
|
+
},
|
|
273
|
+
]
|
|
204
274
|
}
|
|
205
275
|
|
|
206
276
|
function compileSelectedReadScopeCte(scopeJson: string, projectId: string): CompiledReadScopeCte {
|
package/src/objects/reader.ts
CHANGED
|
@@ -32,7 +32,7 @@ import {
|
|
|
32
32
|
type PgObjectQuerySource,
|
|
33
33
|
pgObjectsOfType,
|
|
34
34
|
} from "./query-compiler"
|
|
35
|
-
import {
|
|
35
|
+
import { findPgSortIndexRead, type PgSortIndexCatalog } from "./query-indexes"
|
|
36
36
|
import type { PgSelectedObjectReadSource } from "./read-scope"
|
|
37
37
|
import {
|
|
38
38
|
type FacetDatabaseRow,
|
|
@@ -55,7 +55,13 @@ export class PgObjectReader {
|
|
|
55
55
|
private readonly sql: SQLClient,
|
|
56
56
|
private readonly source: PgObjectQuerySource | PgSelectedObjectReadSource,
|
|
57
57
|
private readonly aggregateAdmission?: PgAggregateAdmission,
|
|
58
|
-
|
|
58
|
+
/**
|
|
59
|
+
* Ordered pages walk a prepared sort index when one fits. A selected source takes none: its
|
|
60
|
+
* objects are the ones its paths reach, which no index over a type's rows orders.
|
|
61
|
+
*/
|
|
62
|
+
private readonly sortIndexes?: PgSortIndexCatalog,
|
|
63
|
+
/** Whether the reader sees so few objects of a type that its pages read them before ordering. */
|
|
64
|
+
private readonly readsFewObjects?: (sql: SQLClient, objectTypeId: string) => Promise<boolean>
|
|
59
65
|
) {}
|
|
60
66
|
|
|
61
67
|
private async assertVectorSearch(
|
|
@@ -115,12 +121,19 @@ export class PgObjectReader {
|
|
|
115
121
|
private async compilePageQuery(params: QueryObjectsInput): Promise<CompiledPgObjectQuery> {
|
|
116
122
|
const sql = this.sql
|
|
117
123
|
const source = this.source
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
124
|
+
let sortIndex = this.sortIndexes
|
|
125
|
+
? await findPgSortIndexRead(sql, this.sortIndexes, params.projectId, params.query)
|
|
126
|
+
: undefined
|
|
127
|
+
|
|
128
|
+
// Walking the index would check nearly every object to find the few this reader sees.
|
|
129
|
+
const unorderedStart =
|
|
130
|
+
sortIndex !== undefined &&
|
|
131
|
+
(await this.readsFewObjects?.(sql, sortIndex.objectTypeId)) === true
|
|
132
|
+
if (unorderedStart) sortIndex = undefined
|
|
133
|
+
|
|
122
134
|
let compiled = compilePgObjectQuery(params.projectId, params.query, {
|
|
123
|
-
|
|
135
|
+
sortIndex,
|
|
136
|
+
unorderedStart,
|
|
124
137
|
includeTotal: params.includeTotal,
|
|
125
138
|
source,
|
|
126
139
|
})
|
|
@@ -132,7 +145,8 @@ export class PgObjectReader {
|
|
|
132
145
|
includeTotal: params.includeTotal,
|
|
133
146
|
source,
|
|
134
147
|
correlatedTraversal: true,
|
|
135
|
-
|
|
148
|
+
sortIndex,
|
|
149
|
+
unorderedStart,
|
|
136
150
|
})
|
|
137
151
|
}
|
|
138
152
|
}
|
|
@@ -402,20 +416,34 @@ export class PgObjectReader {
|
|
|
402
416
|
const source = this.source
|
|
403
417
|
const result = new Map<ObjectBatchKey, ObjectRow>()
|
|
404
418
|
if (params.items.length === 0) return result
|
|
405
|
-
|
|
406
|
-
const
|
|
407
|
-
|
|
419
|
+
|
|
420
|
+
const requestedByType = new Map<string, { primaryId: string; batchIndex: number }[]>()
|
|
421
|
+
for (const [batchIndex, item] of params.items.entries()) {
|
|
422
|
+
const requested = requestedByType.get(item.objectTypeId) ?? []
|
|
423
|
+
requested.push({ primaryId: item.primaryId, batchIndex })
|
|
424
|
+
requestedByType.set(item.objectTypeId, requested)
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// One branch per type, each naming it, so PostgreSQL plans only the requested types'
|
|
428
|
+
// partitions. Matched on a requested row's type, it would plan every partition of objects.
|
|
429
|
+
const branches: string[] = []
|
|
430
|
+
const args: unknown[] = []
|
|
431
|
+
for (const [objectTypeId, requested] of requestedByType) {
|
|
432
|
+
const objects = pgObjectsOfType(source, objectTypeId)
|
|
433
|
+
branches.push(`
|
|
408
434
|
SELECT object.*, requested."batchIndex" AS _batch_index
|
|
409
435
|
FROM jsonb_to_recordset(?::text::jsonb)
|
|
410
|
-
AS requested("
|
|
411
|
-
JOIN ${
|
|
436
|
+
AS requested("primaryId" text, "batchIndex" integer)
|
|
437
|
+
JOIN ${objects.table} AS object
|
|
412
438
|
ON object.project_id = ?
|
|
413
|
-
AND object.object_type_id =
|
|
439
|
+
AND object.object_type_id = ?::text
|
|
414
440
|
AND object.primary_id = requested."primaryId"
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
441
|
+
${objects.filter}
|
|
442
|
+
`)
|
|
443
|
+
args.push(JSON.stringify(requested), params.projectId, objectTypeId, ...objects.args)
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
const statement = compilePgObjectStatement(branches.join(" UNION ALL "), args, source)
|
|
419
447
|
const rows = await sql.unsafe<ObjectBatchDatabaseRow[]>(
|
|
420
448
|
statement.sql,
|
|
421
449
|
statement.args as SqlParameter[]
|
package/src/objects/storage.ts
CHANGED
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
rowToObject,
|
|
42
42
|
} from "./rows"
|
|
43
43
|
import { createPgSelectedReader } from "./selected-reader"
|
|
44
|
+
import { PgVisibilityStatisticsCatalog } from "./visibility-statistics"
|
|
44
45
|
|
|
45
46
|
const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
|
|
46
47
|
queryObjects: true,
|
|
@@ -116,6 +117,7 @@ const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
|
|
|
116
117
|
export class PgObjectStorage implements ObjectStorage {
|
|
117
118
|
private readonly aggregateAdmission?: PgAggregateAdmission
|
|
118
119
|
private readonly reader: PgObjectReader
|
|
120
|
+
private readonly visibilityStatistics = new PgVisibilityStatisticsCatalog()
|
|
119
121
|
|
|
120
122
|
constructor(
|
|
121
123
|
private readonly sql: PgStoreClient,
|
|
@@ -151,6 +153,8 @@ export class PgObjectStorage implements ObjectStorage {
|
|
|
151
153
|
}): ObjectReadStorage {
|
|
152
154
|
return createPgConditionalReader(this.sql, params, {
|
|
153
155
|
aggregateAdmission: this.aggregateAdmission,
|
|
156
|
+
sortIndexes: this.sortIndexes,
|
|
157
|
+
visibilityStatistics: this.visibilityStatistics,
|
|
154
158
|
queryCapabilities: () => this.queryCapabilities(),
|
|
155
159
|
})
|
|
156
160
|
}
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { createHash } from "node:crypto"
|
|
2
|
+
import type { ObjectQueryPreparationPlan, ObjectVisibilityKeys } from "@sixb/core/storage"
|
|
3
|
+
import type { ReservedSQL } from "../pg-client"
|
|
4
|
+
import { runPgTransactionOn } from "../transactions"
|
|
5
|
+
import { objectPartitionName } from "./partitions"
|
|
6
|
+
import { identifier, literal } from "./sql-text"
|
|
7
|
+
import { compilePgVisibilityKeys } from "./visibility"
|
|
8
|
+
|
|
9
|
+
interface VisibilityIndex {
|
|
10
|
+
readonly objectTypeId: string
|
|
11
|
+
readonly partition: string
|
|
12
|
+
readonly expression: string
|
|
13
|
+
readonly name: string
|
|
14
|
+
readonly statistics: string
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** What an index comment records, so a later preparation finds the indexes it no longer needs. */
|
|
18
|
+
interface VisibilityIndexComment {
|
|
19
|
+
readonly kind: "visibility"
|
|
20
|
+
readonly projectId: string
|
|
21
|
+
readonly statistics: string
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Index the keys of every view condition, so a member's read finds the rows it sees instead of
|
|
26
|
+
* checking each one.
|
|
27
|
+
*
|
|
28
|
+
* One GIN index per condition and type, partial on the project and built concurrently, with
|
|
29
|
+
* statistics on its expression so the planner knows how many rows a reader's keys show. A read
|
|
30
|
+
* repeats the indexed expression, so it is served by an index only once it exists, and is right
|
|
31
|
+
* either way. Indexes no condition of the project needs any more are dropped. Runs under the
|
|
32
|
+
* preparation lock, on the connection that holds it.
|
|
33
|
+
*/
|
|
34
|
+
export async function preparePgVisibilityIndexes(
|
|
35
|
+
connection: ReservedSQL,
|
|
36
|
+
schema: string,
|
|
37
|
+
plan: ObjectQueryPreparationPlan
|
|
38
|
+
): Promise<{ readonly names: readonly string[]; readonly warnings: readonly string[] }> {
|
|
39
|
+
// Prepared without the view conditions: keep the indexes a preparation that had them built.
|
|
40
|
+
if (plan.objectTypes.every((type) => type.visibility === undefined)) {
|
|
41
|
+
return { names: [], warnings: [] }
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const warnings: string[] = []
|
|
45
|
+
const built: VisibilityIndex[] = []
|
|
46
|
+
for (const index of visibilityIndexes(plan)) {
|
|
47
|
+
const warning = await createVisibilityIndex(connection, schema, plan.projectId, index)
|
|
48
|
+
if (warning) warnings.push(warning)
|
|
49
|
+
else built.push(index)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
await ensureVisibilityStatistics(connection, schema, built)
|
|
53
|
+
await dropUnusedVisibilityIndexes(connection, schema, plan.projectId, built)
|
|
54
|
+
|
|
55
|
+
return { names: built.map((index) => index.name), warnings }
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function visibilityIndexes(plan: ObjectQueryPreparationPlan): VisibilityIndex[] {
|
|
59
|
+
return plan.objectTypes.flatMap((type) => {
|
|
60
|
+
const partition = objectPartitionName(type.objectTypeId)
|
|
61
|
+
|
|
62
|
+
return (type.visibility ?? []).map((keys) => {
|
|
63
|
+
const expression = compilePgVisibilityKeys(keys)
|
|
64
|
+
const statistics = visibilityStatisticsName(type.objectTypeId, keys)
|
|
65
|
+
const name = `sixb_visibility_${digest(`${partition}:${plan.projectId}:${expression}`, 32)}`
|
|
66
|
+
return { objectTypeId: type.objectTypeId, partition, expression, name, statistics }
|
|
67
|
+
})
|
|
68
|
+
})
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The statistics on a condition's keys. Statistics cannot be partial, so projects sharing a schema
|
|
73
|
+
* share those of one expression.
|
|
74
|
+
*/
|
|
75
|
+
export function visibilityStatisticsName(objectTypeId: string, keys: ObjectVisibilityKeys): string {
|
|
76
|
+
const expression = compilePgVisibilityKeys(keys)
|
|
77
|
+
return `sixb_visibility_stats_${digest(`${objectPartitionName(objectTypeId)}:${expression}`, 24)}`
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Build one index; a condition PostgreSQL cannot index keeps its reads right, and is reported. */
|
|
81
|
+
async function createVisibilityIndex(
|
|
82
|
+
connection: ReservedSQL,
|
|
83
|
+
schema: string,
|
|
84
|
+
projectId: string,
|
|
85
|
+
index: VisibilityIndex
|
|
86
|
+
): Promise<string | undefined> {
|
|
87
|
+
const table = `${identifier(schema)}.${identifier(index.partition)}`
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
await connection.unsafe(
|
|
91
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(index.name)} ON ${table} USING gin ((${index.expression})) WHERE project_id = ${literal(projectId)}`
|
|
92
|
+
)
|
|
93
|
+
} catch (error) {
|
|
94
|
+
// An expression with a function that is not IMMUTABLE fails before anything is built.
|
|
95
|
+
if ((error as { code?: unknown }).code !== FUNCTION_NOT_IMMUTABLE) throw error
|
|
96
|
+
return conditionNotIndexed(index.objectTypeId, error)
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const [built] = await connection<{ valid: boolean }[]>`
|
|
100
|
+
SELECT i.indisvalid AS valid FROM pg_index i
|
|
101
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
102
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
103
|
+
WHERE n.nspname = ${schema} AND c.relname = ${index.name}`
|
|
104
|
+
if (!built?.valid) {
|
|
105
|
+
throw new Error(
|
|
106
|
+
`[SixbPg] Visibility index ${schema}.${index.name} is invalid after an interrupted build. Drop that index concurrently and prepare again.`
|
|
107
|
+
)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const comment: VisibilityIndexComment = {
|
|
111
|
+
kind: "visibility",
|
|
112
|
+
projectId,
|
|
113
|
+
statistics: index.statistics,
|
|
114
|
+
}
|
|
115
|
+
await connection.unsafe(
|
|
116
|
+
`COMMENT ON INDEX ${identifier(schema)}.${identifier(index.name)} IS ${literal(JSON.stringify(comment))}`
|
|
117
|
+
)
|
|
118
|
+
return undefined
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const FUNCTION_NOT_IMMUTABLE = "42P17"
|
|
122
|
+
|
|
123
|
+
/** One transaction per type, so its queries plan with the old statistics until the new exist. */
|
|
124
|
+
async function ensureVisibilityStatistics(
|
|
125
|
+
connection: ReservedSQL,
|
|
126
|
+
schema: string,
|
|
127
|
+
indexes: readonly VisibilityIndex[]
|
|
128
|
+
): Promise<void> {
|
|
129
|
+
const partitions = new Map<string, VisibilityIndex[]>()
|
|
130
|
+
for (const index of indexes) {
|
|
131
|
+
partitions.set(index.partition, [...(partitions.get(index.partition) ?? []), index])
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
for (const [partition, partitionIndexes] of partitions) {
|
|
135
|
+
const table = `${identifier(schema)}.${identifier(partition)}`
|
|
136
|
+
|
|
137
|
+
await runPgTransactionOn(connection, async () => {
|
|
138
|
+
let created = false
|
|
139
|
+
for (const index of partitionIndexes) {
|
|
140
|
+
const [existing] = await connection`
|
|
141
|
+
SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
|
|
142
|
+
WHERE n.nspname = ${schema} AND s.stxname = ${index.statistics}`
|
|
143
|
+
if (existing) continue
|
|
144
|
+
|
|
145
|
+
await connection.unsafe(
|
|
146
|
+
`CREATE STATISTICS ${identifier(schema)}.${identifier(index.statistics)} ON (${index.expression}) FROM ${table}`
|
|
147
|
+
)
|
|
148
|
+
created = true
|
|
149
|
+
}
|
|
150
|
+
if (created) await connection.unsafe(`ANALYZE ${table}`)
|
|
151
|
+
})
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Drop this project's visibility indexes that no current condition needs: writes would keep
|
|
157
|
+
* maintaining them. Statistics go with the last index of any project that uses them.
|
|
158
|
+
*/
|
|
159
|
+
async function dropUnusedVisibilityIndexes(
|
|
160
|
+
connection: ReservedSQL,
|
|
161
|
+
schema: string,
|
|
162
|
+
projectId: string,
|
|
163
|
+
current: readonly VisibilityIndex[]
|
|
164
|
+
): Promise<void> {
|
|
165
|
+
const currentNames = new Set(current.map((index) => index.name))
|
|
166
|
+
const existing = await readVisibilityIndexes(connection, schema)
|
|
167
|
+
|
|
168
|
+
const unused = existing.filter(
|
|
169
|
+
(index) => index.comment.projectId === projectId && !currentNames.has(index.name)
|
|
170
|
+
)
|
|
171
|
+
for (const index of unused) {
|
|
172
|
+
await connection.unsafe(
|
|
173
|
+
`DROP INDEX CONCURRENTLY IF EXISTS ${identifier(schema)}.${identifier(index.name)}`
|
|
174
|
+
)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const kept = existing.filter((index) => !unused.includes(index))
|
|
178
|
+
for (const statistics of new Set(unused.map((index) => index.comment.statistics))) {
|
|
179
|
+
if (kept.some((index) => index.comment.statistics === statistics)) continue
|
|
180
|
+
await connection.unsafe(
|
|
181
|
+
`DROP STATISTICS IF EXISTS ${identifier(schema)}.${identifier(statistics)}`
|
|
182
|
+
)
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async function readVisibilityIndexes(connection: ReservedSQL, schema: string) {
|
|
187
|
+
const rows = await connection<{ name: string; definition: string }[]>`
|
|
188
|
+
SELECT c.relname AS name, d.description AS definition
|
|
189
|
+
FROM pg_index i
|
|
190
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
191
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
192
|
+
JOIN pg_description d
|
|
193
|
+
ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
|
|
194
|
+
WHERE n.nspname = ${schema} AND c.relname LIKE 'sixb_visibility_%'`
|
|
195
|
+
|
|
196
|
+
return rows.flatMap((row) => {
|
|
197
|
+
const comment = parseComment(row.definition)
|
|
198
|
+
return comment ? [{ name: row.name, comment }] : []
|
|
199
|
+
})
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function parseComment(definition: string): VisibilityIndexComment | undefined {
|
|
203
|
+
try {
|
|
204
|
+
const comment = JSON.parse(definition) as Partial<VisibilityIndexComment> | null
|
|
205
|
+
return comment?.kind === "visibility" &&
|
|
206
|
+
typeof comment.projectId === "string" &&
|
|
207
|
+
typeof comment.statistics === "string"
|
|
208
|
+
? (comment as VisibilityIndexComment)
|
|
209
|
+
: undefined
|
|
210
|
+
} catch {
|
|
211
|
+
return undefined
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function conditionNotIndexed(objectTypeId: string, error: unknown): string {
|
|
216
|
+
const reason = error instanceof Error ? error.message : String(error)
|
|
217
|
+
return `${objectTypeId}: a view condition cannot be indexed (${reason}); members read the type by checking each object.`
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function digest(value: string, length: number): string {
|
|
221
|
+
return createHash("sha256").update(value).digest("hex").slice(0, length)
|
|
222
|
+
}
|