@sixb/pg 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +9 -8
  2. package/dist/agents/computers.d.ts +8 -2
  3. package/dist/agents/computers.d.ts.map +1 -1
  4. package/dist/index.js +566 -144
  5. package/dist/index.js.map +20 -17
  6. package/dist/migrations.d.ts.map +1 -1
  7. package/dist/objects/conditional-reader.d.ts +7 -2
  8. package/dist/objects/conditional-reader.d.ts.map +1 -1
  9. package/dist/objects/partitions.d.ts.map +1 -1
  10. package/dist/objects/prepare-queries.d.ts.map +1 -1
  11. package/dist/objects/query-compiler.d.ts +14 -1
  12. package/dist/objects/query-compiler.d.ts.map +1 -1
  13. package/dist/objects/query-indexes.d.ts +13 -1
  14. package/dist/objects/query-indexes.d.ts.map +1 -1
  15. package/dist/objects/read-scope.d.ts.map +1 -1
  16. package/dist/objects/reader.d.ts +14 -1
  17. package/dist/objects/reader.d.ts.map +1 -1
  18. package/dist/objects/storage.d.ts +1 -0
  19. package/dist/objects/storage.d.ts.map +1 -1
  20. package/dist/objects/visibility-indexes.d.ts +22 -0
  21. package/dist/objects/visibility-indexes.d.ts.map +1 -0
  22. package/dist/objects/visibility-statistics.d.ts +26 -0
  23. package/dist/objects/visibility-statistics.d.ts.map +1 -0
  24. package/dist/objects/visibility.d.ts +34 -0
  25. package/dist/objects/visibility.d.ts.map +1 -0
  26. package/dist/pg-client.d.ts.map +1 -1
  27. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  28. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  29. package/package.json +3 -3
  30. package/src/agents/computers.ts +54 -5
  31. package/src/migrations/087-agent-computer-background-commands.sql +4 -0
  32. package/src/migrations/088-web-search-model-kind.sql +12 -0
  33. package/src/migrations.ts +6 -0
  34. package/src/objects/conditional-reader.ts +26 -4
  35. package/src/objects/partitions.ts +17 -4
  36. package/src/objects/prepare-queries.ts +8 -3
  37. package/src/objects/query-compiler.ts +64 -23
  38. package/src/objects/query-indexes.ts +93 -27
  39. package/src/objects/read-scope.ts +147 -77
  40. package/src/objects/reader.ts +46 -18
  41. package/src/objects/storage.ts +4 -0
  42. package/src/objects/visibility-indexes.ts +222 -0
  43. package/src/objects/visibility-statistics.ts +73 -0
  44. package/src/objects/visibility.ts +135 -0
  45. package/src/ontology-storage/materializations.ts +3 -1
  46. package/src/pg-client.ts +4 -0
  47. package/src/pg-pipeline-run-storage.ts +5 -7
  48. package/src/pg-sync-run-storage.ts +2 -7
@@ -5,9 +5,9 @@ import type {
5
5
  import {
6
6
  type CompiledPgObjectQuery,
7
7
  compilePgObjectStatement,
8
- compilePredicate,
9
8
  type PgObjectQuerySource,
10
9
  } from "./query-compiler"
10
+ import { compilePgConditionFilter } from "./visibility"
11
11
 
12
12
  export interface PgSelectedObjectReadSource extends PgObjectQuerySource {
13
13
  readonly objectPropertyPermissionsTable: string
@@ -34,8 +34,9 @@ export function compilePgSelectedObjectReadSource(
34
34
  objects: scope.objects,
35
35
  steps: scope.steps,
36
36
  })
37
+ const relations = compileSelectedReadScopeCte(scopeJson, projectId)
37
38
  const source: PgSelectedObjectReadSource = {
38
- ...scopedQuerySource(compileSelectedReadScopeCte(scopeJson, projectId)),
39
+ ...scopedQuerySource(() => relations),
39
40
  objectPropertyPermissionsTable: "_sixb_scope_object_permissions",
40
41
  traversalProbe: { sql: "", args: [] },
41
42
  }
@@ -77,12 +78,18 @@ export function compilePgConditionalObjectReadSource(
77
78
  const filters = new Map(
78
79
  scope.conditions.map((condition) => [
79
80
  condition.objectTypeId,
80
- compilePredicate(condition.predicate),
81
+ compilePgConditionFilter(condition),
81
82
  ])
82
83
  )
83
84
 
85
+ const relations = compileConditionalReadScopeRelations(projectId, filters)
86
+
84
87
  return Object.freeze({
85
- ...scopedQuerySource(compileConditionalReadScopeCte(projectId, filters)),
88
+ ...scopedQuerySource((sql) => withReadRelations(relations, sql)),
89
+ // A link is visible while both endpoints are, so a statement that reads both through
90
+ // `_sixb_scope_objects` reads stored links. Checking each endpoint again plans every partition
91
+ // of objects: the endpoint's type comes from the link row, so PostgreSQL cannot prune them.
92
+ linksBetweenVisibleObjectsTable: "links",
86
93
  // One type's visible objects are the stored ones its own condition matches.
87
94
  objectsOfType: (objectTypeId: string) => {
88
95
  const filter = filters.get(objectTypeId)
@@ -97,33 +104,68 @@ interface CompiledReadScopeCte {
97
104
  }
98
105
 
99
106
  /** Prefix every statement of a query with the scope's relations, ahead of its own arguments. */
100
- function scopedQuerySource(compiled: CompiledReadScopeCte): PgObjectQuerySource {
101
- const wrapStatement = (sql: string, args: readonly unknown[] = []) => ({
102
- sql: `${compiled.sql}\n${sql}`,
103
- args: [...compiled.args, ...args],
104
- })
107
+ function scopedQuerySource(
108
+ withClause: (statementSql: string) => CompiledReadScopeCte
109
+ ): PgObjectQuerySource {
110
+ const wrapStatement = (sql: string, args: readonly unknown[] = []) => {
111
+ const compiled = withClause(sql)
112
+ return {
113
+ sql: `${compiled.sql}\n${sql}`,
114
+ args: [...compiled.args, ...args],
115
+ }
116
+ }
105
117
 
106
118
  return {
107
119
  objectsTable: "_sixb_scope_objects",
108
120
  linksTable: "_sixb_scope_links",
109
121
  wrapStatement,
110
- wrapQuery: (query: CompiledPgObjectQuery): CompiledPgObjectQuery => ({
111
- ...query,
112
- ...wrapStatement(query.sql, query.args),
113
- vectorProbe: query.vectorProbe
114
- ? { ...query.vectorProbe, ...wrapStatement(query.vectorProbe.sql, query.vectorProbe.args) }
115
- : undefined,
116
- totalSql: `${compiled.sql}\n${query.totalSql}`,
117
- totalArgs: [...compiled.args, ...query.totalArgs],
118
- ...(query.hasMoreProbe
119
- ? {
120
- hasMoreProbe: {
121
- ...query.hasMoreProbe,
122
- ...wrapStatement(query.hasMoreProbe.sql, query.hasMoreProbe.args),
123
- },
124
- }
125
- : {}),
126
- }),
122
+ wrapQuery: (query: CompiledPgObjectQuery): CompiledPgObjectQuery => {
123
+ const total = wrapStatement(query.totalSql, query.totalArgs)
124
+
125
+ return {
126
+ ...query,
127
+ ...wrapStatement(query.sql, query.args),
128
+ vectorProbe: query.vectorProbe
129
+ ? {
130
+ ...query.vectorProbe,
131
+ ...wrapStatement(query.vectorProbe.sql, query.vectorProbe.args),
132
+ }
133
+ : undefined,
134
+ totalSql: total.sql,
135
+ totalArgs: total.args,
136
+ ...(query.hasMoreProbe
137
+ ? {
138
+ hasMoreProbe: {
139
+ ...query.hasMoreProbe,
140
+ ...wrapStatement(query.hasMoreProbe.sql, query.hasMoreProbe.args),
141
+ },
142
+ }
143
+ : {}),
144
+ }
145
+ },
146
+ }
147
+ }
148
+
149
+ interface ReadRelation extends CompiledReadScopeCte {
150
+ readonly name: string
151
+ }
152
+
153
+ /**
154
+ * The WITH clause naming the relations a statement reads. PostgreSQL parses and analyzes every
155
+ * relation a WITH clause names, read or not, and each one repeats every condition.
156
+ */
157
+ function withReadRelations(
158
+ relations: readonly ReadRelation[],
159
+ statementSql: string
160
+ ): CompiledReadScopeCte {
161
+ const read = relations.filter((relation) =>
162
+ new RegExp(`\\b${relation.name}\\b`).test(statementSql)
163
+ )
164
+ if (read.length === 0) return { sql: "", args: [] }
165
+
166
+ return {
167
+ sql: `WITH ${read.map((relation) => relation.sql).join(",\n")}`,
168
+ args: read.flatMap((relation) => relation.args),
127
169
  }
128
170
  }
129
171
 
@@ -133,23 +175,31 @@ type ConditionFilters = ReadonlyMap<
133
175
  { readonly sql: string; readonly args: readonly unknown[] }
134
176
  >
135
177
 
136
- function compileConditionalReadScopeCte(
178
+ function compileConditionalReadScopeRelations(
137
179
  projectId: string,
138
180
  filters: ConditionFilters
139
- ): CompiledReadScopeCte {
181
+ ): readonly ReadRelation[] {
140
182
  if (filters.size === 0) {
141
- return {
142
- sql: `
143
- WITH
144
- _sixb_scope_objects AS NOT MATERIALIZED (
145
- SELECT * FROM objects WHERE project_id = ?
146
- ),
147
- _sixb_scope_links AS NOT MATERIALIZED (
148
- SELECT * FROM links WHERE project_id = ?
149
- )
150
- `,
151
- args: [projectId, projectId],
152
- }
183
+ return [
184
+ {
185
+ name: "_sixb_scope_objects",
186
+ sql: `
187
+ _sixb_scope_objects AS NOT MATERIALIZED (
188
+ SELECT * FROM objects WHERE project_id = ?
189
+ )
190
+ `,
191
+ args: [projectId],
192
+ },
193
+ {
194
+ name: "_sixb_scope_links",
195
+ sql: `
196
+ _sixb_scope_links AS NOT MATERIALIZED (
197
+ SELECT * FROM links WHERE project_id = ?
198
+ )
199
+ `,
200
+ args: [projectId],
201
+ },
202
+ ]
153
203
  }
154
204
 
155
205
  const conditionedTypeArgs = [...filters.keys()]
@@ -162,45 +212,65 @@ function compileConditionalReadScopeCte(
162
212
  matchArgs.push(objectTypeId, ...filter.args)
163
213
  }
164
214
 
165
- const visibleEndpoint = (end: "source" | "target") => `
166
- (
167
- stored.${end}_type_id NOT IN (${conditionedTypes})
168
- OR EXISTS (
169
- SELECT 1
170
- FROM _sixb_scope_objects AS endpoint
171
- WHERE endpoint.project_id = stored.project_id
172
- AND endpoint.object_type_id = stored.${end}_type_id
173
- AND endpoint.primary_id = stored.${end}_id
174
- )
175
- )
176
- `
215
+ // An endpoint of a conditioned type is a visible object of that type. Each check names its type,
216
+ // so PostgreSQL plans that type's partition alone: matched on the link row's type, one check
217
+ // would plan every partition of objects.
218
+ const visibleEndpoint = (end: "source" | "target"): CompiledReadScopeCte => {
219
+ const checks: string[] = []
220
+ const args: unknown[] = [...conditionedTypeArgs]
177
221
 
178
- return {
179
- sql: `
180
- WITH
181
- _sixb_scope_objects AS NOT MATERIALIZED (
182
- SELECT *
183
- FROM objects
184
- WHERE project_id = ?
185
- AND (object_type_id NOT IN (${conditionedTypes}) OR ${matches.join(" OR ")})
186
- ),
187
- _sixb_scope_links AS NOT MATERIALIZED (
188
- SELECT stored.*
189
- FROM links AS stored
190
- WHERE stored.project_id = ?
191
- AND ${visibleEndpoint("source")}
192
- AND ${visibleEndpoint("target")}
193
- )
194
- `,
195
- args: [
196
- projectId,
197
- ...conditionedTypeArgs,
198
- ...matchArgs,
199
- projectId,
200
- ...conditionedTypeArgs,
201
- ...conditionedTypeArgs,
202
- ],
222
+ for (const [objectTypeId, filter] of filters) {
223
+ checks.push(`
224
+ (
225
+ stored.${end}_type_id = ?::text
226
+ AND EXISTS (
227
+ SELECT 1
228
+ FROM objects AS endpoint
229
+ WHERE endpoint.project_id = stored.project_id
230
+ AND endpoint.object_type_id = ?::text
231
+ AND endpoint.primary_id = stored.${end}_id
232
+ AND (${filter.sql})
233
+ )
234
+ )
235
+ `)
236
+ args.push(objectTypeId, objectTypeId, ...filter.args)
237
+ }
238
+
239
+ return {
240
+ sql: `(stored.${end}_type_id NOT IN (${conditionedTypes}) OR ${checks.join(" OR ")})`,
241
+ args,
242
+ }
203
243
  }
244
+ const visibleSource = visibleEndpoint("source")
245
+ const visibleTarget = visibleEndpoint("target")
246
+
247
+ return [
248
+ {
249
+ name: "_sixb_scope_objects",
250
+ sql: `
251
+ _sixb_scope_objects AS NOT MATERIALIZED (
252
+ SELECT *
253
+ FROM objects
254
+ WHERE project_id = ?
255
+ AND (object_type_id NOT IN (${conditionedTypes}) OR ${matches.join(" OR ")})
256
+ )
257
+ `,
258
+ args: [projectId, ...conditionedTypeArgs, ...matchArgs],
259
+ },
260
+ {
261
+ name: "_sixb_scope_links",
262
+ sql: `
263
+ _sixb_scope_links AS NOT MATERIALIZED (
264
+ SELECT stored.*
265
+ FROM links AS stored
266
+ WHERE stored.project_id = ?
267
+ AND ${visibleSource.sql}
268
+ AND ${visibleTarget.sql}
269
+ )
270
+ `,
271
+ args: [projectId, ...visibleSource.args, ...visibleTarget.args],
272
+ },
273
+ ]
204
274
  }
205
275
 
206
276
  function compileSelectedReadScopeCte(scopeJson: string, projectId: string): CompiledReadScopeCte {
@@ -32,7 +32,7 @@ import {
32
32
  type PgObjectQuerySource,
33
33
  pgObjectsOfType,
34
34
  } from "./query-compiler"
35
- import { findPgSortIndexBound, type PgSortIndexCatalog } from "./query-indexes"
35
+ import { findPgSortIndexRead, type PgSortIndexCatalog } from "./query-indexes"
36
36
  import type { PgSelectedObjectReadSource } from "./read-scope"
37
37
  import {
38
38
  type FacetDatabaseRow,
@@ -55,7 +55,13 @@ export class PgObjectReader {
55
55
  private readonly sql: SQLClient,
56
56
  private readonly source: PgObjectQuerySource | PgSelectedObjectReadSource,
57
57
  private readonly aggregateAdmission?: PgAggregateAdmission,
58
- private readonly sortIndexes?: PgSortIndexCatalog
58
+ /**
59
+ * Ordered pages walk a prepared sort index when one fits. A selected source takes none: its
60
+ * objects are the ones its paths reach, which no index over a type's rows orders.
61
+ */
62
+ private readonly sortIndexes?: PgSortIndexCatalog,
63
+ /** Whether the reader sees so few objects of a type that its pages read them before ordering. */
64
+ private readonly readsFewObjects?: (sql: SQLClient, objectTypeId: string) => Promise<boolean>
59
65
  ) {}
60
66
 
61
67
  private async assertVectorSearch(
@@ -115,12 +121,19 @@ export class PgObjectReader {
115
121
  private async compilePageQuery(params: QueryObjectsInput): Promise<CompiledPgObjectQuery> {
116
122
  const sql = this.sql
117
123
  const source = this.source
118
- const sortIndexBound =
119
- source === DEFAULT_OBJECT_QUERY_SOURCE && this.sortIndexes
120
- ? await findPgSortIndexBound(sql, this.sortIndexes, params.projectId, params.query)
121
- : undefined
124
+ let sortIndex = this.sortIndexes
125
+ ? await findPgSortIndexRead(sql, this.sortIndexes, params.projectId, params.query)
126
+ : undefined
127
+
128
+ // Walking the index would check nearly every object to find the few this reader sees.
129
+ const unorderedStart =
130
+ sortIndex !== undefined &&
131
+ (await this.readsFewObjects?.(sql, sortIndex.objectTypeId)) === true
132
+ if (unorderedStart) sortIndex = undefined
133
+
122
134
  let compiled = compilePgObjectQuery(params.projectId, params.query, {
123
- sortIndexBound,
135
+ sortIndex,
136
+ unorderedStart,
124
137
  includeTotal: params.includeTotal,
125
138
  source,
126
139
  })
@@ -132,7 +145,8 @@ export class PgObjectReader {
132
145
  includeTotal: params.includeTotal,
133
146
  source,
134
147
  correlatedTraversal: true,
135
- sortIndexBound,
148
+ sortIndex,
149
+ unorderedStart,
136
150
  })
137
151
  }
138
152
  }
@@ -402,20 +416,34 @@ export class PgObjectReader {
402
416
  const source = this.source
403
417
  const result = new Map<ObjectBatchKey, ObjectRow>()
404
418
  if (params.items.length === 0) return result
405
- const items = params.items.map((item, batchIndex) => ({ ...item, batchIndex }))
406
- const statement = compilePgObjectStatement(
407
- `
419
+
420
+ const requestedByType = new Map<string, { primaryId: string; batchIndex: number }[]>()
421
+ for (const [batchIndex, item] of params.items.entries()) {
422
+ const requested = requestedByType.get(item.objectTypeId) ?? []
423
+ requested.push({ primaryId: item.primaryId, batchIndex })
424
+ requestedByType.set(item.objectTypeId, requested)
425
+ }
426
+
427
+ // One branch per type, each naming it, so PostgreSQL plans only the requested types'
428
+ // partitions. Matched on a requested row's type, it would plan every partition of objects.
429
+ const branches: string[] = []
430
+ const args: unknown[] = []
431
+ for (const [objectTypeId, requested] of requestedByType) {
432
+ const objects = pgObjectsOfType(source, objectTypeId)
433
+ branches.push(`
408
434
  SELECT object.*, requested."batchIndex" AS _batch_index
409
435
  FROM jsonb_to_recordset(?::text::jsonb)
410
- AS requested("objectTypeId" text, "primaryId" text, "batchIndex" integer)
411
- JOIN ${source.objectsTable} AS object
436
+ AS requested("primaryId" text, "batchIndex" integer)
437
+ JOIN ${objects.table} AS object
412
438
  ON object.project_id = ?
413
- AND object.object_type_id = requested."objectTypeId"
439
+ AND object.object_type_id = ?::text
414
440
  AND object.primary_id = requested."primaryId"
415
- `,
416
- [JSON.stringify(items), params.projectId],
417
- source
418
- )
441
+ ${objects.filter}
442
+ `)
443
+ args.push(JSON.stringify(requested), params.projectId, objectTypeId, ...objects.args)
444
+ }
445
+
446
+ const statement = compilePgObjectStatement(branches.join(" UNION ALL "), args, source)
419
447
  const rows = await sql.unsafe<ObjectBatchDatabaseRow[]>(
420
448
  statement.sql,
421
449
  statement.args as SqlParameter[]
@@ -41,6 +41,7 @@ import {
41
41
  rowToObject,
42
42
  } from "./rows"
43
43
  import { createPgSelectedReader } from "./selected-reader"
44
+ import { PgVisibilityStatisticsCatalog } from "./visibility-statistics"
44
45
 
45
46
  const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
46
47
  queryObjects: true,
@@ -116,6 +117,7 @@ const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
116
117
  export class PgObjectStorage implements ObjectStorage {
117
118
  private readonly aggregateAdmission?: PgAggregateAdmission
118
119
  private readonly reader: PgObjectReader
120
+ private readonly visibilityStatistics = new PgVisibilityStatisticsCatalog()
119
121
 
120
122
  constructor(
121
123
  private readonly sql: PgStoreClient,
@@ -151,6 +153,8 @@ export class PgObjectStorage implements ObjectStorage {
151
153
  }): ObjectReadStorage {
152
154
  return createPgConditionalReader(this.sql, params, {
153
155
  aggregateAdmission: this.aggregateAdmission,
156
+ sortIndexes: this.sortIndexes,
157
+ visibilityStatistics: this.visibilityStatistics,
154
158
  queryCapabilities: () => this.queryCapabilities(),
155
159
  })
156
160
  }
@@ -0,0 +1,222 @@
1
+ import { createHash } from "node:crypto"
2
+ import type { ObjectQueryPreparationPlan, ObjectVisibilityKeys } from "@sixb/core/storage"
3
+ import type { ReservedSQL } from "../pg-client"
4
+ import { runPgTransactionOn } from "../transactions"
5
+ import { objectPartitionName } from "./partitions"
6
+ import { identifier, literal } from "./sql-text"
7
+ import { compilePgVisibilityKeys } from "./visibility"
8
+
9
+ interface VisibilityIndex {
10
+ readonly objectTypeId: string
11
+ readonly partition: string
12
+ readonly expression: string
13
+ readonly name: string
14
+ readonly statistics: string
15
+ }
16
+
17
+ /** What an index comment records, so a later preparation finds the indexes it no longer needs. */
18
+ interface VisibilityIndexComment {
19
+ readonly kind: "visibility"
20
+ readonly projectId: string
21
+ readonly statistics: string
22
+ }
23
+
24
+ /**
25
+ * Index the keys of every view condition, so a member's read finds the rows it sees instead of
26
+ * checking each one.
27
+ *
28
+ * One GIN index per condition and type, partial on the project and built concurrently, with
29
+ * statistics on its expression so the planner knows how many rows a reader's keys show. A read
30
+ * repeats the indexed expression, so it is served by an index only once it exists, and is right
31
+ * either way. Indexes no condition of the project needs any more are dropped. Runs under the
32
+ * preparation lock, on the connection that holds it.
33
+ */
34
+ export async function preparePgVisibilityIndexes(
35
+ connection: ReservedSQL,
36
+ schema: string,
37
+ plan: ObjectQueryPreparationPlan
38
+ ): Promise<{ readonly names: readonly string[]; readonly warnings: readonly string[] }> {
39
+ // Prepared without the view conditions: keep the indexes a preparation that had them built.
40
+ if (plan.objectTypes.every((type) => type.visibility === undefined)) {
41
+ return { names: [], warnings: [] }
42
+ }
43
+
44
+ const warnings: string[] = []
45
+ const built: VisibilityIndex[] = []
46
+ for (const index of visibilityIndexes(plan)) {
47
+ const warning = await createVisibilityIndex(connection, schema, plan.projectId, index)
48
+ if (warning) warnings.push(warning)
49
+ else built.push(index)
50
+ }
51
+
52
+ await ensureVisibilityStatistics(connection, schema, built)
53
+ await dropUnusedVisibilityIndexes(connection, schema, plan.projectId, built)
54
+
55
+ return { names: built.map((index) => index.name), warnings }
56
+ }
57
+
58
+ function visibilityIndexes(plan: ObjectQueryPreparationPlan): VisibilityIndex[] {
59
+ return plan.objectTypes.flatMap((type) => {
60
+ const partition = objectPartitionName(type.objectTypeId)
61
+
62
+ return (type.visibility ?? []).map((keys) => {
63
+ const expression = compilePgVisibilityKeys(keys)
64
+ const statistics = visibilityStatisticsName(type.objectTypeId, keys)
65
+ const name = `sixb_visibility_${digest(`${partition}:${plan.projectId}:${expression}`, 32)}`
66
+ return { objectTypeId: type.objectTypeId, partition, expression, name, statistics }
67
+ })
68
+ })
69
+ }
70
+
71
+ /**
72
+ * The statistics on a condition's keys. Statistics cannot be partial, so projects sharing a schema
73
+ * share those of one expression.
74
+ */
75
+ export function visibilityStatisticsName(objectTypeId: string, keys: ObjectVisibilityKeys): string {
76
+ const expression = compilePgVisibilityKeys(keys)
77
+ return `sixb_visibility_stats_${digest(`${objectPartitionName(objectTypeId)}:${expression}`, 24)}`
78
+ }
79
+
80
+ /** Build one index; a condition PostgreSQL cannot index keeps its reads right, and is reported. */
81
+ async function createVisibilityIndex(
82
+ connection: ReservedSQL,
83
+ schema: string,
84
+ projectId: string,
85
+ index: VisibilityIndex
86
+ ): Promise<string | undefined> {
87
+ const table = `${identifier(schema)}.${identifier(index.partition)}`
88
+
89
+ try {
90
+ await connection.unsafe(
91
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(index.name)} ON ${table} USING gin ((${index.expression})) WHERE project_id = ${literal(projectId)}`
92
+ )
93
+ } catch (error) {
94
+ // An expression with a function that is not IMMUTABLE fails before anything is built.
95
+ if ((error as { code?: unknown }).code !== FUNCTION_NOT_IMMUTABLE) throw error
96
+ return conditionNotIndexed(index.objectTypeId, error)
97
+ }
98
+
99
+ const [built] = await connection<{ valid: boolean }[]>`
100
+ SELECT i.indisvalid AS valid FROM pg_index i
101
+ JOIN pg_class c ON c.oid = i.indexrelid
102
+ JOIN pg_namespace n ON n.oid = c.relnamespace
103
+ WHERE n.nspname = ${schema} AND c.relname = ${index.name}`
104
+ if (!built?.valid) {
105
+ throw new Error(
106
+ `[SixbPg] Visibility index ${schema}.${index.name} is invalid after an interrupted build. Drop that index concurrently and prepare again.`
107
+ )
108
+ }
109
+
110
+ const comment: VisibilityIndexComment = {
111
+ kind: "visibility",
112
+ projectId,
113
+ statistics: index.statistics,
114
+ }
115
+ await connection.unsafe(
116
+ `COMMENT ON INDEX ${identifier(schema)}.${identifier(index.name)} IS ${literal(JSON.stringify(comment))}`
117
+ )
118
+ return undefined
119
+ }
120
+
121
+ const FUNCTION_NOT_IMMUTABLE = "42P17"
122
+
123
+ /** One transaction per type, so its queries plan with the old statistics until the new exist. */
124
+ async function ensureVisibilityStatistics(
125
+ connection: ReservedSQL,
126
+ schema: string,
127
+ indexes: readonly VisibilityIndex[]
128
+ ): Promise<void> {
129
+ const partitions = new Map<string, VisibilityIndex[]>()
130
+ for (const index of indexes) {
131
+ partitions.set(index.partition, [...(partitions.get(index.partition) ?? []), index])
132
+ }
133
+
134
+ for (const [partition, partitionIndexes] of partitions) {
135
+ const table = `${identifier(schema)}.${identifier(partition)}`
136
+
137
+ await runPgTransactionOn(connection, async () => {
138
+ let created = false
139
+ for (const index of partitionIndexes) {
140
+ const [existing] = await connection`
141
+ SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
142
+ WHERE n.nspname = ${schema} AND s.stxname = ${index.statistics}`
143
+ if (existing) continue
144
+
145
+ await connection.unsafe(
146
+ `CREATE STATISTICS ${identifier(schema)}.${identifier(index.statistics)} ON (${index.expression}) FROM ${table}`
147
+ )
148
+ created = true
149
+ }
150
+ if (created) await connection.unsafe(`ANALYZE ${table}`)
151
+ })
152
+ }
153
+ }
154
+
155
+ /**
156
+ * Drop this project's visibility indexes that no current condition needs: writes would keep
157
+ * maintaining them. Statistics go with the last index of any project that uses them.
158
+ */
159
+ async function dropUnusedVisibilityIndexes(
160
+ connection: ReservedSQL,
161
+ schema: string,
162
+ projectId: string,
163
+ current: readonly VisibilityIndex[]
164
+ ): Promise<void> {
165
+ const currentNames = new Set(current.map((index) => index.name))
166
+ const existing = await readVisibilityIndexes(connection, schema)
167
+
168
+ const unused = existing.filter(
169
+ (index) => index.comment.projectId === projectId && !currentNames.has(index.name)
170
+ )
171
+ for (const index of unused) {
172
+ await connection.unsafe(
173
+ `DROP INDEX CONCURRENTLY IF EXISTS ${identifier(schema)}.${identifier(index.name)}`
174
+ )
175
+ }
176
+
177
+ const kept = existing.filter((index) => !unused.includes(index))
178
+ for (const statistics of new Set(unused.map((index) => index.comment.statistics))) {
179
+ if (kept.some((index) => index.comment.statistics === statistics)) continue
180
+ await connection.unsafe(
181
+ `DROP STATISTICS IF EXISTS ${identifier(schema)}.${identifier(statistics)}`
182
+ )
183
+ }
184
+ }
185
+
186
+ async function readVisibilityIndexes(connection: ReservedSQL, schema: string) {
187
+ const rows = await connection<{ name: string; definition: string }[]>`
188
+ SELECT c.relname AS name, d.description AS definition
189
+ FROM pg_index i
190
+ JOIN pg_class c ON c.oid = i.indexrelid
191
+ JOIN pg_namespace n ON n.oid = c.relnamespace
192
+ JOIN pg_description d
193
+ ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
194
+ WHERE n.nspname = ${schema} AND c.relname LIKE 'sixb_visibility_%'`
195
+
196
+ return rows.flatMap((row) => {
197
+ const comment = parseComment(row.definition)
198
+ return comment ? [{ name: row.name, comment }] : []
199
+ })
200
+ }
201
+
202
+ function parseComment(definition: string): VisibilityIndexComment | undefined {
203
+ try {
204
+ const comment = JSON.parse(definition) as Partial<VisibilityIndexComment> | null
205
+ return comment?.kind === "visibility" &&
206
+ typeof comment.projectId === "string" &&
207
+ typeof comment.statistics === "string"
208
+ ? (comment as VisibilityIndexComment)
209
+ : undefined
210
+ } catch {
211
+ return undefined
212
+ }
213
+ }
214
+
215
+ function conditionNotIndexed(objectTypeId: string, error: unknown): string {
216
+ const reason = error instanceof Error ? error.message : String(error)
217
+ return `${objectTypeId}: a view condition cannot be indexed (${reason}); members read the type by checking each object.`
218
+ }
219
+
220
+ function digest(value: string, length: number): string {
221
+ return createHash("sha256").update(value).digest("hex").slice(0, length)
222
+ }