@sixb/pg 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +9 -8
  2. package/dist/agents/computers.d.ts +8 -2
  3. package/dist/agents/computers.d.ts.map +1 -1
  4. package/dist/auth-storage/auth-storage.d.ts +2 -1
  5. package/dist/auth-storage/auth-storage.d.ts.map +1 -1
  6. package/dist/auth-storage/identities.d.ts +4 -0
  7. package/dist/auth-storage/identities.d.ts.map +1 -1
  8. package/dist/auth-storage/rows.d.ts +2 -1
  9. package/dist/auth-storage/rows.d.ts.map +1 -1
  10. package/dist/auth-storage/users.d.ts.map +1 -1
  11. package/dist/connector-connection-storage.d.ts.map +1 -1
  12. package/dist/index.d.ts +6 -0
  13. package/dist/index.d.ts.map +1 -1
  14. package/dist/index.js +2377 -428
  15. package/dist/index.js.map +37 -30
  16. package/dist/migrations.d.ts.map +1 -1
  17. package/dist/objects/conditional-reader.d.ts +7 -2
  18. package/dist/objects/conditional-reader.d.ts.map +1 -1
  19. package/dist/objects/partitions.d.ts.map +1 -1
  20. package/dist/objects/prepare-queries.d.ts.map +1 -1
  21. package/dist/objects/query-compiler.d.ts +14 -1
  22. package/dist/objects/query-compiler.d.ts.map +1 -1
  23. package/dist/objects/query-indexes.d.ts +13 -1
  24. package/dist/objects/query-indexes.d.ts.map +1 -1
  25. package/dist/objects/read-scope.d.ts.map +1 -1
  26. package/dist/objects/reader.d.ts +14 -1
  27. package/dist/objects/reader.d.ts.map +1 -1
  28. package/dist/objects/storage.d.ts +11 -1
  29. package/dist/objects/storage.d.ts.map +1 -1
  30. package/dist/objects/visibility-indexes.d.ts +22 -0
  31. package/dist/objects/visibility-indexes.d.ts.map +1 -0
  32. package/dist/objects/visibility-statistics.d.ts +26 -0
  33. package/dist/objects/visibility-statistics.d.ts.map +1 -0
  34. package/dist/objects/visibility.d.ts +34 -0
  35. package/dist/objects/visibility.d.ts.map +1 -0
  36. package/dist/ontology-storage/materialization-session.d.ts +5 -0
  37. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  38. package/dist/ontology-storage/materialization-writer.d.ts +14 -0
  39. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  40. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  41. package/dist/pg-artifact-storage.d.ts +14 -2
  42. package/dist/pg-artifact-storage.d.ts.map +1 -1
  43. package/dist/pg-client.d.ts.map +1 -1
  44. package/dist/pg-dataset-ingest-storage.d.ts +6 -0
  45. package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
  46. package/dist/pg-lake-retention-run-storage.d.ts +11 -0
  47. package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
  48. package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
  49. package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
  50. package/dist/pg-pipeline-run-storage.d.ts +5 -2
  51. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  52. package/dist/pg-projection-run-storage.d.ts +5 -1
  53. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  54. package/dist/pg-sync-run-storage.d.ts +6 -2
  55. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  56. package/dist/pg-webhook-run-storage.d.ts +2 -1
  57. package/dist/pg-webhook-run-storage.d.ts.map +1 -1
  58. package/dist/pg-workflow-run-storage.d.ts +4 -3
  59. package/dist/pg-workflow-run-storage.d.ts.map +1 -1
  60. package/dist/run-claim-sql.d.ts +37 -0
  61. package/dist/run-claim-sql.d.ts.map +1 -0
  62. package/dist/share-session-storage.d.ts +2 -1
  63. package/dist/share-session-storage.d.ts.map +1 -1
  64. package/package.json +3 -3
  65. package/src/agents/computers.ts +54 -5
  66. package/src/auth-storage/auth-storage.ts +66 -0
  67. package/src/auth-storage/identities.ts +15 -0
  68. package/src/auth-storage/rows.ts +3 -0
  69. package/src/auth-storage/users.ts +3 -0
  70. package/src/connector-connection-storage.ts +33 -16
  71. package/src/index.ts +15 -0
  72. package/src/migrations/087-agent-computer-background-commands.sql +4 -0
  73. package/src/migrations/088-web-search-model-kind.sql +12 -0
  74. package/src/migrations/089-maintenance-heartbeats.sql +10 -0
  75. package/src/migrations/090-lake-retention-runs.sql +8 -0
  76. package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
  77. package/src/migrations/092-run-recovery.sql +23 -0
  78. package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
  79. package/src/migrations/094-artifact-board-previews.sql +14 -0
  80. package/src/migrations/095-artifact-open-versions.sql +101 -0
  81. package/src/migrations/096-artifact-documents.sql +11 -0
  82. package/src/migrations/097-artifact-frame-files.sql +8 -0
  83. package/src/migrations/098-artifact-comments.sql +36 -0
  84. package/src/migrations/099-object-artifact-refs.sql +12 -0
  85. package/src/migrations/100-shared-link-guests.sql +17 -0
  86. package/src/migrations/101-connector-connection-owners.sql +38 -0
  87. package/src/migrations/102-user-suspension-source.sql +12 -0
  88. package/src/migrations.ts +46 -0
  89. package/src/objects/conditional-reader.ts +26 -4
  90. package/src/objects/partitions.ts +17 -4
  91. package/src/objects/prepare-queries.ts +8 -3
  92. package/src/objects/query-compiler.ts +64 -23
  93. package/src/objects/query-indexes.ts +93 -27
  94. package/src/objects/read-scope.ts +147 -77
  95. package/src/objects/reader.ts +46 -18
  96. package/src/objects/storage.ts +53 -0
  97. package/src/objects/visibility-indexes.ts +222 -0
  98. package/src/objects/visibility-statistics.ts +73 -0
  99. package/src/objects/visibility.ts +135 -0
  100. package/src/ontology-storage/materialization-session.ts +4 -0
  101. package/src/ontology-storage/materialization-writer.ts +68 -1
  102. package/src/ontology-storage/materializations.ts +6 -2
  103. package/src/pg-artifact-storage.ts +798 -137
  104. package/src/pg-client.ts +4 -0
  105. package/src/pg-dataset-ingest-storage.ts +305 -0
  106. package/src/pg-lake-retention-run-storage.ts +40 -0
  107. package/src/pg-maintenance-heartbeat-storage.ts +53 -0
  108. package/src/pg-pipeline-run-storage.ts +85 -42
  109. package/src/pg-projection-run-storage.ts +61 -2
  110. package/src/pg-sync-run-storage.ts +100 -37
  111. package/src/pg-webhook-run-storage.ts +37 -11
  112. package/src/pg-workflow-run-storage.ts +39 -100
  113. package/src/run-claim-sql.ts +173 -0
  114. package/src/share-session-storage.ts +20 -0
@@ -0,0 +1,222 @@
1
+ import { createHash } from "node:crypto"
2
+ import type { ObjectQueryPreparationPlan, ObjectVisibilityKeys } from "@sixb/core/storage"
3
+ import type { ReservedSQL } from "../pg-client"
4
+ import { runPgTransactionOn } from "../transactions"
5
+ import { objectPartitionName } from "./partitions"
6
+ import { identifier, literal } from "./sql-text"
7
+ import { compilePgVisibilityKeys } from "./visibility"
8
+
9
+ interface VisibilityIndex {
10
+ readonly objectTypeId: string
11
+ readonly partition: string
12
+ readonly expression: string
13
+ readonly name: string
14
+ readonly statistics: string
15
+ }
16
+
17
+ /** What an index comment records, so a later preparation finds the indexes it no longer needs. */
18
+ interface VisibilityIndexComment {
19
+ readonly kind: "visibility"
20
+ readonly projectId: string
21
+ readonly statistics: string
22
+ }
23
+
24
+ /**
25
+ * Index the keys of every view condition, so a member's read finds the rows it sees instead of
26
+ * checking each one.
27
+ *
28
+ * One GIN index per condition and type, partial on the project and built concurrently, with
29
+ * statistics on its expression so the planner knows how many rows a reader's keys show. A read
30
+ * repeats the indexed expression, so it is served by an index only once it exists, and is right
31
+ * either way. Indexes no condition of the project needs any more are dropped. Runs under the
32
+ * preparation lock, on the connection that holds it.
33
+ */
34
+ export async function preparePgVisibilityIndexes(
35
+ connection: ReservedSQL,
36
+ schema: string,
37
+ plan: ObjectQueryPreparationPlan
38
+ ): Promise<{ readonly names: readonly string[]; readonly warnings: readonly string[] }> {
39
+ // Prepared without the view conditions: keep the indexes a preparation that had them built.
40
+ if (plan.objectTypes.every((type) => type.visibility === undefined)) {
41
+ return { names: [], warnings: [] }
42
+ }
43
+
44
+ const warnings: string[] = []
45
+ const built: VisibilityIndex[] = []
46
+ for (const index of visibilityIndexes(plan)) {
47
+ const warning = await createVisibilityIndex(connection, schema, plan.projectId, index)
48
+ if (warning) warnings.push(warning)
49
+ else built.push(index)
50
+ }
51
+
52
+ await ensureVisibilityStatistics(connection, schema, built)
53
+ await dropUnusedVisibilityIndexes(connection, schema, plan.projectId, built)
54
+
55
+ return { names: built.map((index) => index.name), warnings }
56
+ }
57
+
58
+ function visibilityIndexes(plan: ObjectQueryPreparationPlan): VisibilityIndex[] {
59
+ return plan.objectTypes.flatMap((type) => {
60
+ const partition = objectPartitionName(type.objectTypeId)
61
+
62
+ return (type.visibility ?? []).map((keys) => {
63
+ const expression = compilePgVisibilityKeys(keys)
64
+ const statistics = visibilityStatisticsName(type.objectTypeId, keys)
65
+ const name = `sixb_visibility_${digest(`${partition}:${plan.projectId}:${expression}`, 32)}`
66
+ return { objectTypeId: type.objectTypeId, partition, expression, name, statistics }
67
+ })
68
+ })
69
+ }
70
+
71
+ /**
72
+ * The statistics on a condition's keys. Statistics cannot be partial, so projects sharing a schema
73
+ * share those of one expression.
74
+ */
75
+ export function visibilityStatisticsName(objectTypeId: string, keys: ObjectVisibilityKeys): string {
76
+ const expression = compilePgVisibilityKeys(keys)
77
+ return `sixb_visibility_stats_${digest(`${objectPartitionName(objectTypeId)}:${expression}`, 24)}`
78
+ }
79
+
80
+ /** Build one index; a condition PostgreSQL cannot index keeps its reads right, and is reported. */
81
+ async function createVisibilityIndex(
82
+ connection: ReservedSQL,
83
+ schema: string,
84
+ projectId: string,
85
+ index: VisibilityIndex
86
+ ): Promise<string | undefined> {
87
+ const table = `${identifier(schema)}.${identifier(index.partition)}`
88
+
89
+ try {
90
+ await connection.unsafe(
91
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(index.name)} ON ${table} USING gin ((${index.expression})) WHERE project_id = ${literal(projectId)}`
92
+ )
93
+ } catch (error) {
94
+ // An expression with a function that is not IMMUTABLE fails before anything is built.
95
+ if ((error as { code?: unknown }).code !== FUNCTION_NOT_IMMUTABLE) throw error
96
+ return conditionNotIndexed(index.objectTypeId, error)
97
+ }
98
+
99
+ const [built] = await connection<{ valid: boolean }[]>`
100
+ SELECT i.indisvalid AS valid FROM pg_index i
101
+ JOIN pg_class c ON c.oid = i.indexrelid
102
+ JOIN pg_namespace n ON n.oid = c.relnamespace
103
+ WHERE n.nspname = ${schema} AND c.relname = ${index.name}`
104
+ if (!built?.valid) {
105
+ throw new Error(
106
+ `[SixbPg] Visibility index ${schema}.${index.name} is invalid after an interrupted build. Drop that index concurrently and prepare again.`
107
+ )
108
+ }
109
+
110
+ const comment: VisibilityIndexComment = {
111
+ kind: "visibility",
112
+ projectId,
113
+ statistics: index.statistics,
114
+ }
115
+ await connection.unsafe(
116
+ `COMMENT ON INDEX ${identifier(schema)}.${identifier(index.name)} IS ${literal(JSON.stringify(comment))}`
117
+ )
118
+ return undefined
119
+ }
120
+
121
+ const FUNCTION_NOT_IMMUTABLE = "42P17"
122
+
123
+ /** One transaction per type, so its queries plan with the old statistics until the new exist. */
124
+ async function ensureVisibilityStatistics(
125
+ connection: ReservedSQL,
126
+ schema: string,
127
+ indexes: readonly VisibilityIndex[]
128
+ ): Promise<void> {
129
+ const partitions = new Map<string, VisibilityIndex[]>()
130
+ for (const index of indexes) {
131
+ partitions.set(index.partition, [...(partitions.get(index.partition) ?? []), index])
132
+ }
133
+
134
+ for (const [partition, partitionIndexes] of partitions) {
135
+ const table = `${identifier(schema)}.${identifier(partition)}`
136
+
137
+ await runPgTransactionOn(connection, async () => {
138
+ let created = false
139
+ for (const index of partitionIndexes) {
140
+ const [existing] = await connection`
141
+ SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
142
+ WHERE n.nspname = ${schema} AND s.stxname = ${index.statistics}`
143
+ if (existing) continue
144
+
145
+ await connection.unsafe(
146
+ `CREATE STATISTICS ${identifier(schema)}.${identifier(index.statistics)} ON (${index.expression}) FROM ${table}`
147
+ )
148
+ created = true
149
+ }
150
+ if (created) await connection.unsafe(`ANALYZE ${table}`)
151
+ })
152
+ }
153
+ }
154
+
155
+ /**
156
+ * Drop this project's visibility indexes that no current condition needs: writes would keep
157
+ * maintaining them. Statistics go with the last index of any project that uses them.
158
+ */
159
+ async function dropUnusedVisibilityIndexes(
160
+ connection: ReservedSQL,
161
+ schema: string,
162
+ projectId: string,
163
+ current: readonly VisibilityIndex[]
164
+ ): Promise<void> {
165
+ const currentNames = new Set(current.map((index) => index.name))
166
+ const existing = await readVisibilityIndexes(connection, schema)
167
+
168
+ const unused = existing.filter(
169
+ (index) => index.comment.projectId === projectId && !currentNames.has(index.name)
170
+ )
171
+ for (const index of unused) {
172
+ await connection.unsafe(
173
+ `DROP INDEX CONCURRENTLY IF EXISTS ${identifier(schema)}.${identifier(index.name)}`
174
+ )
175
+ }
176
+
177
+ const kept = existing.filter((index) => !unused.includes(index))
178
+ for (const statistics of new Set(unused.map((index) => index.comment.statistics))) {
179
+ if (kept.some((index) => index.comment.statistics === statistics)) continue
180
+ await connection.unsafe(
181
+ `DROP STATISTICS IF EXISTS ${identifier(schema)}.${identifier(statistics)}`
182
+ )
183
+ }
184
+ }
185
+
186
+ async function readVisibilityIndexes(connection: ReservedSQL, schema: string) {
187
+ const rows = await connection<{ name: string; definition: string }[]>`
188
+ SELECT c.relname AS name, d.description AS definition
189
+ FROM pg_index i
190
+ JOIN pg_class c ON c.oid = i.indexrelid
191
+ JOIN pg_namespace n ON n.oid = c.relnamespace
192
+ JOIN pg_description d
193
+ ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
194
+ WHERE n.nspname = ${schema} AND c.relname LIKE 'sixb_visibility_%'`
195
+
196
+ return rows.flatMap((row) => {
197
+ const comment = parseComment(row.definition)
198
+ return comment ? [{ name: row.name, comment }] : []
199
+ })
200
+ }
201
+
202
+ function parseComment(definition: string): VisibilityIndexComment | undefined {
203
+ try {
204
+ const comment = JSON.parse(definition) as Partial<VisibilityIndexComment> | null
205
+ return comment?.kind === "visibility" &&
206
+ typeof comment.projectId === "string" &&
207
+ typeof comment.statistics === "string"
208
+ ? (comment as VisibilityIndexComment)
209
+ : undefined
210
+ } catch {
211
+ return undefined
212
+ }
213
+ }
214
+
215
+ function conditionNotIndexed(objectTypeId: string, error: unknown): string {
216
+ const reason = error instanceof Error ? error.message : String(error)
217
+ return `${objectTypeId}: a view condition cannot be indexed (${reason}); members read the type by checking each object.`
218
+ }
219
+
220
+ function digest(value: string, length: number): string {
221
+ return createHash("sha256").update(value).digest("hex").slice(0, length)
222
+ }
@@ -0,0 +1,73 @@
1
+ import type { ObjectReadCondition } from "@sixb/core/storage"
2
+ import type { SQLClient } from "../pg-client"
3
+ import { MAX_KEYED_ID_LENGTH, pgVisibilityReaderKeys } from "./visibility"
4
+ import { visibilityStatisticsName } from "./visibility-indexes"
5
+
6
+ // How long a read of the statistics serves queries before they are read again.
7
+ const VISIBILITY_STATISTICS_TTL_MS = 60_000
8
+
9
+ /**
10
+ * The keys the statistics of each view condition list as most common, read at most once a minute.
11
+ *
12
+ * ANALYZE lists every key above a small share of the rows. PostgreSQL estimates one it does not
13
+ * list at about half a percent of them, whatever its real share: a reader who sees nothing looks
14
+ * like one who sees thousands of objects.
15
+ */
16
+ export class PgVisibilityStatisticsCatalog {
17
+ #read:
18
+ | {
19
+ readonly at: number
20
+ readonly keys: Promise<ReadonlyMap<string, ReadonlySet<string>>>
21
+ }
22
+ | undefined
23
+
24
+ /** Read through the caller's client, so a transaction never waits for a second connection. */
25
+ commonKeys(sql: SQLClient): Promise<ReadonlyMap<string, ReadonlySet<string>>> {
26
+ const now = Date.now()
27
+ if (this.#read && now - this.#read.at < VISIBILITY_STATISTICS_TTL_MS) return this.#read.keys
28
+
29
+ const read = { at: now, keys: readCommonKeys(sql) }
30
+ this.#read = read
31
+ // A failed read is not kept: the next query reads again.
32
+ read.keys.catch(() => {
33
+ if (this.#read === read) this.#read = undefined
34
+ })
35
+ return read.keys
36
+ }
37
+ }
38
+
39
+ async function readCommonKeys(sql: SQLClient): Promise<ReadonlyMap<string, ReadonlySet<string>>> {
40
+ const rows = await sql<{ name: string; keys: string[] | null }[]>`
41
+ SELECT statistics_name AS name, most_common_elems::text::text[] AS keys
42
+ FROM pg_stats_ext_exprs
43
+ WHERE schemaname = current_schema() AND statistics_name LIKE 'sixb\\_visibility\\_stats\\_%'`
44
+
45
+ return new Map(rows.map((row) => [row.name, new Set(row.keys ?? [])]))
46
+ }
47
+
48
+ /**
49
+ * Whether a reader sees so few objects of a type that its ordered pages read the objects it sees
50
+ * first, then sort them.
51
+ *
52
+ * Walking a sort index stops at a page's last object, so it is the right plan for a reader who sees
53
+ * many. For one who sees few, it checks nearly every object, and PostgreSQL cannot tell them apart
54
+ * when the statistics list none of the reader's keys. Such a reader sees few objects: those its
55
+ * keys meet are under the share ANALYZE lists. Without statistics, or with a remainder read object
56
+ * by object, the reader's share is unknown and PostgreSQL chooses.
57
+ */
58
+ export function readsFewObjects(
59
+ condition: ObjectReadCondition,
60
+ common: ReadonlyMap<string, ReadonlySet<string>>
61
+ ): boolean {
62
+ const visibility = condition.visibility
63
+ if (!visibility || visibility.remainder) return false
64
+ if ((visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH) return false
65
+
66
+ const readerKeys = pgVisibilityReaderKeys(visibility.user)
67
+
68
+ for (const keys of visibility.keys) {
69
+ const listed = common.get(visibilityStatisticsName(condition.objectTypeId, keys))
70
+ if (!listed || readerKeys.some((key) => listed.has(key))) return false
71
+ }
72
+ return true
73
+ }
@@ -0,0 +1,135 @@
1
+ import type { UserRef } from "@sixb/core"
2
+ import type { ObjectReadCondition, ObjectVisibilityKeys } from "@sixb/core/storage"
3
+ import { compilePredicate } from "./query-compiler"
4
+ import { literal } from "./sql-text"
5
+
6
+ /**
7
+ * Visibility keys as SQL: one expression over `properties` returning who the condition shows the
8
+ * row to, `*` for everyone holding it and `u:<id>` for a user.
9
+ *
10
+ * The expression is IMMUTABLE and holds its values inline, so PostgreSQL can index it and match a
11
+ * read that repeats it, whatever the reader. A user's id is written as jsonb prints it between its
12
+ * quotes: that is how a list's ids come out of `jsonb_path_query_array` with no subquery, which an
13
+ * index expression cannot hold. An escaped quote holds a `"`, so `\\` and `\"` are first written as
14
+ * control characters jsonb never prints raw: `", "` then only separates two ids, and `\x1f` never
15
+ * appears in one. A reader's key is written the same way.
16
+ *
17
+ * A GIN entry holds at most 2,712 bytes, and jsonb prints a character in up to 6 bytes, so ids
18
+ * longer than `MAX_KEYED_ID_LENGTH` characters get no key: a reader with such an id reads the
19
+ * predicate instead. A short list keeps its long ids, which no reader looks up, rather than run a
20
+ * regex on each of its ids.
21
+ *
22
+ * Reads number every `?` of a statement as a parameter, so none is written: text goes in `E''`
23
+ * strings, where `?` is written `\x3f`.
24
+ */
25
+ export function compilePgVisibilityKeys(keys: ObjectVisibilityKeys): string {
26
+ switch (keys.kind) {
27
+ case "everyone":
28
+ return `CASE WHEN ${predicateSql(keys.when)} THEN ARRAY[${text(EVERYONE)}]::text[] ELSE ${NO_KEYS} END`
29
+
30
+ case "user": {
31
+ // `eq` compares the whole reference, so the property holds exactly a type and a string id.
32
+ const ref = propertySql(keys.propertyId)
33
+ return `CASE WHEN jsonb_typeof(${ref}) = 'object' AND (${ref} - 'type' - 'id') = '{}'::jsonb AND ${ref} -> 'type' = '"user"'::jsonb AND jsonb_typeof(${ref} -> 'id') = 'string' AND char_length(${ref} ->> 'id') <= ${MAX_KEYED_ID_LENGTH} THEN ARRAY[${text(USER_PREFIX)} || ${withoutEscapes(`left(substr((${ref} -> 'id')::text, 2), -1)`)}] ELSE ${NO_KEYS} END`
34
+ }
35
+
36
+ case "users": {
37
+ // Strict mode keeps the elements `@>` matches: objects whose type is "user" and id a string.
38
+ const users = `@.type == "user" && @.id.type() == "string"`
39
+ const keysOf = (filter: string) => {
40
+ const path = text(`strict $[*] ? (${filter}).id`)
41
+ const ids = withoutEscapes(
42
+ `jsonb_path_query_array(${propertySql(keys.propertyId)}, ${path}, '{}', true)::text`
43
+ )
44
+ const separator = `E'\\x1f'`
45
+ return `coalesce(string_to_array(${text(USER_PREFIX)} || replace(NULLIF(left(substr(${ids}, 3), -2), ''), '", "', ${separator} || ${text(USER_PREFIX)}), ${separator}), ${NO_KEYS})`
46
+ }
47
+
48
+ // No id of a short list is too long for an entry: only a longer one pays for the regex.
49
+ const bounded = `${users} && @.id like_regex "^.{0,${MAX_KEYED_ID_LENGTH}}$" flag "s"`
50
+ return `CASE WHEN octet_length(${propertySql(keys.propertyId)}::text) <= ${SHORT_LIST_BYTES} THEN ${keysOf(users)} ELSE ${keysOf(bounded)} END`
51
+ }
52
+
53
+ case "union":
54
+ return `(${keys.items.map((item) => `(${compilePgVisibilityKeys(item)})`).join(" || ")})`
55
+
56
+ case "when":
57
+ return `CASE WHEN ${predicateSql(keys.when)} THEN ${compilePgVisibilityKeys(keys.keys)} ELSE ${NO_KEYS} END`
58
+ }
59
+ }
60
+
61
+ /** The keys a reader holds: everyone's, and its user's. */
62
+ export function pgVisibilityReaderKeys(user: UserRef | undefined): string[] {
63
+ if (!user) return [EVERYONE]
64
+
65
+ // As `withoutEscapes` writes the ids the keys hold.
66
+ const id = JSON.stringify(user.id)
67
+ .slice(1, -1)
68
+ .replaceAll("\\\\", ESCAPED_BACKSLASH)
69
+ .replaceAll('\\"', ESCAPED_QUOTE)
70
+ return [EVERYONE, `${USER_PREFIX}${id}`]
71
+ }
72
+
73
+ /**
74
+ * A conditioned type's filter: each condition's keys meet the reader's, or the remainder matches.
75
+ * Without keys, or for a user whose id is too long to have one, the condition's predicate.
76
+ */
77
+ export function compilePgConditionFilter(condition: ObjectReadCondition): {
78
+ readonly sql: string
79
+ readonly args: readonly unknown[]
80
+ } {
81
+ const visibility = condition.visibility
82
+ // UTF-16 code units, never fewer than the characters PostgreSQL counts.
83
+ if (!visibility || (visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH) {
84
+ return compilePredicate(condition.predicate)
85
+ }
86
+
87
+ const readerKeys = pgVisibilityReaderKeys(visibility.user)
88
+ const clauses = visibility.keys.map((keys) => `((${compilePgVisibilityKeys(keys)}) && ?::text[])`)
89
+ const args: unknown[] = visibility.keys.map(() => readerKeys)
90
+
91
+ if (visibility.remainder) {
92
+ const remainder = compilePredicate(visibility.remainder)
93
+ clauses.push(remainder.sql)
94
+ args.push(...remainder.args)
95
+ }
96
+
97
+ return { sql: `(${clauses.join(" OR ")})`, args }
98
+ }
99
+
100
+ export const MAX_KEYED_ID_LENGTH = 200
101
+ const ESCAPED_BACKSLASH = "\x1e"
102
+ const ESCAPED_QUOTE = "\x1d"
103
+ // A list whose JSON is this short holds no id whose key would overflow an entry.
104
+ const SHORT_LIST_BYTES = 2000
105
+ const EVERYONE = "*"
106
+ const USER_PREFIX = "u:"
107
+ const NO_KEYS = "'{}'::text[]"
108
+
109
+ function propertySql(propertyId: string): string {
110
+ return `(properties -> ${text(propertyId)}::text)`
111
+ }
112
+
113
+ /** jsonb's `\\` and `\"` in printed ids, as control characters it never prints raw. */
114
+ function withoutEscapes(printed: string): string {
115
+ return `replace(replace(${printed}, ${text("\\\\")}, E'\\x1e'), ${text('\\"')}, E'\\x1d')`
116
+ }
117
+
118
+ function text(value: string): string {
119
+ return literal(value).replace(/\?/g, "\\x3f")
120
+ }
121
+
122
+ /** The reader's own predicate compiler, so a part means exactly what the condition does. */
123
+ function predicateSql(predicate: Parameters<typeof compilePredicate>[0]): string {
124
+ const compiled = compilePredicate(predicate)
125
+ let index = 0
126
+ return `(${compiled.sql.replace(/\?/g, () => inlineValue(compiled.args[index++]))})`
127
+ }
128
+
129
+ /** The values the predicate compiler binds: property ids and compared values, text or numbers. */
130
+ function inlineValue(value: unknown): string {
131
+ if (typeof value === "string") return text(value)
132
+ if (typeof value === "number" && Number.isFinite(value)) return `(${String(value)})`
133
+
134
+ throw new Error(`[SixbPg] A view condition compares a value it cannot index: ${String(value)}.`)
135
+ }
@@ -38,6 +38,8 @@ export class PgMaterializationSessionState extends ProviderMaterializationSessio
38
38
  changedUniqueValues = 0
39
39
  /** What unique value items this session staged, known without reading its work back. */
40
40
  readonly uniqueWork = { releases: false, claims: false }
41
+ /** What artifact holder items this session staged, likewise. */
42
+ readonly artifactHolderWork = { deletes: false, inserts: false }
41
43
  workAnalyzed = false
42
44
  /** Work staged on the session lives in a temp table; a plan-bound session reads its plan's. */
43
45
  readonly workTable: string
@@ -172,6 +174,8 @@ export class PgMaterializationSessions {
172
174
  if (record.kind !== "plan") continue
173
175
  if (record.item.kind === "unique-value-release") session.uniqueWork.releases = true
174
176
  if (record.item.kind === "unique-value-claim") session.uniqueWork.claims = true
177
+ if (record.item.kind === "artifact-holder-delete") session.artifactHolderWork.deletes = true
178
+ if (record.item.kind === "artifact-holder-insert") session.artifactHolderWork.inserts = true
175
179
  }
176
180
  }
177
181
 
@@ -27,6 +27,8 @@ export interface PgStagedPlanTarget {
27
27
  readonly committedAt: string
28
28
  /** Whether unique value items were staged; probed in the work table when unknown. */
29
29
  readonly uniqueWork?: PgUniqueWork
30
+ /** Whether artifact holder items were staged; probed in the work table when unknown. */
31
+ readonly artifactHolderWork?: PgArtifactHolderWork
30
32
  }
31
33
 
32
34
  export interface PgUniqueWork {
@@ -34,6 +36,11 @@ export interface PgUniqueWork {
34
36
  readonly claims: boolean
35
37
  }
36
38
 
39
+ export interface PgArtifactHolderWork {
40
+ readonly deletes: boolean
41
+ readonly inserts: boolean
42
+ }
43
+
37
44
  export interface PgAppliedPlan {
38
45
  readonly objectWrites: number
39
46
  readonly linkWrites: number
@@ -65,12 +72,15 @@ export class PgMaterializationWriter {
65
72
  await this.applyLinkOverrides(target)
66
73
  await this.applyPoints(target)
67
74
  const uniqueWork = target.uniqueWork ?? (await this.stagedUniqueWork(target))
75
+ const holderWork = target.artifactHolderWork ?? (await this.stagedArtifactHolderWork(target))
68
76
  if (uniqueWork.releases) await this.releaseUniqueValues(target)
77
+ if (holderWork.deletes) await this.deleteArtifactHolders(target)
69
78
  let linkWrites = await this.deleteLinks(target)
70
79
  let objectWrites = await this.deleteObjects(target)
71
80
  objectWrites += await this.upsertObjects(target)
72
81
  linkWrites += await this.upsertLinks(target)
73
82
  const uniqueValueWrites = uniqueWork.claims ? await this.claimUniqueValues(target) : 0
83
+ if (holderWork.inserts) await this.insertArtifactHolders(target)
74
84
  const eventCount = await this.writeOutbox(target)
75
85
  await this.writeTouches(target)
76
86
  return { objectWrites, linkWrites, uniqueValueWrites, eventCount }
@@ -106,6 +116,25 @@ export class PgMaterializationWriter {
106
116
  return { releases: row?.releases ?? false, claims: row?.claims ?? false }
107
117
  }
108
118
 
119
+ /** The same probes for the artifact holder index: most commits hold no artifact. */
120
+ private async stagedArtifactHolderWork(
121
+ target: PgStagedPlanTarget
122
+ ): Promise<PgArtifactHolderWork> {
123
+ const probe = (kind: "artifact-holder-delete" | "artifact-holder-insert") => this.sql`
124
+ EXISTS (
125
+ SELECT 1 FROM ${this.sql(target.workTable)}
126
+ WHERE work_id = ${target.workId} AND lane = 'apply'
127
+ AND major_order = ${materializationApplyPhase(kind)}
128
+ AND minor_order = ${materializationPlanKindRank(kind)}
129
+ )
130
+ `
131
+ const [row] = await this.sql<{ readonly deletes: boolean; readonly inserts: boolean }[]>`
132
+ SELECT ${probe("artifact-holder-delete")} AS deletes,
133
+ ${probe("artifact-holder-insert")} AS inserts
134
+ `
135
+ return { deletes: row?.deletes ?? false, inserts: row?.inserts ?? false }
136
+ }
137
+
109
138
  /** The staged values of one plan item kind, as `value`. */
110
139
  private staged(target: PgStagedPlanTarget, kind: MaterializationPlanWorkItem["kind"]) {
111
140
  return this.sql`
@@ -417,6 +446,43 @@ export class PgMaterializationWriter {
417
446
  )
418
447
  }
419
448
 
449
+ /** The staged rows of the artifact holder index of one item kind, as its columns. */
450
+ private stagedArtifactHolders(
451
+ target: PgStagedPlanTarget,
452
+ kind: "artifact-holder-delete" | "artifact-holder-insert"
453
+ ) {
454
+ return this.sql`
455
+ SELECT value->>'artifactId' AS artifact_id, value->>'objectTypeId' AS object_type_id,
456
+ value->>'primaryId' AS primary_id, value->>'propertyId' AS property_id
457
+ FROM (${this.staged(target, kind)}) AS staged
458
+ `
459
+ }
460
+
461
+ /** A row already gone is no conflict: the index only follows what objects hold. */
462
+ private async deleteArtifactHolders(target: PgStagedPlanTarget): Promise<void> {
463
+ await this.sql`
464
+ WITH candidates AS (${this.stagedArtifactHolders(target, "artifact-holder-delete")})
465
+ DELETE FROM object_artifact_refs AS stored USING candidates
466
+ WHERE stored.project_id = ${target.projectId}
467
+ AND stored.artifact_id = candidates.artifact_id
468
+ AND stored.object_type_id = candidates.object_type_id
469
+ AND stored.primary_id = candidates.primary_id
470
+ AND stored.property_id = candidates.property_id
471
+ `
472
+ }
473
+
474
+ /** A row already there is no conflict either. */
475
+ private async insertArtifactHolders(target: PgStagedPlanTarget): Promise<void> {
476
+ await this.sql`
477
+ INSERT INTO object_artifact_refs (
478
+ project_id, artifact_id, object_type_id, primary_id, property_id
479
+ )
480
+ SELECT ${target.projectId}, artifact_id, object_type_id, primary_id, property_id
481
+ FROM (${this.stagedArtifactHolders(target, "artifact-holder-insert")}) AS candidates
482
+ ON CONFLICT DO NOTHING
483
+ `
484
+ }
485
+
420
486
  private async deleteLinks(target: PgStagedPlanTarget): Promise<number> {
421
487
  const candidates = this.staged(target, "link-delete")
422
488
  return this.write(
@@ -673,7 +739,8 @@ export class PgMaterializationWriter {
673
739
  END AS ref
674
740
  ) AS entity
675
741
  WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
676
- -- Unique values are no plan input: a stale claim fails when it applies.
742
+ -- Unique values are no plan input: a stale claim fails when it applies. Nor are the
743
+ -- artifacts objects hold, which follow the objects.
677
744
  AND entity.kind IS NOT NULL
678
745
  `
679
746
  }
@@ -268,7 +268,9 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
268
268
  projectId: commit.projectId,
269
269
  commitId: commit.id,
270
270
  committedAt: commit.committedAt,
271
- ...(session.plan ? {} : { uniqueWork: session.uniqueWork }),
271
+ ...(session.plan
272
+ ? {}
273
+ : { uniqueWork: session.uniqueWork, artifactHolderWork: session.artifactHolderWork }),
272
274
  })
273
275
  // Below this no type can reach the statistics threshold, so the work table stays unread.
274
276
  if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {
@@ -812,7 +814,9 @@ async function refreshMaterializedStatistics(
812
814
  * partitioned parent, and its statistics plan queries that read across types. A smaller one only
813
815
  * refreshes partitions it changed substantially, which autovacuum may not reach before the next
814
816
  * projection. A routine update must not: ANALYZE over a partition's expression statistics takes
815
- * seconds, and a thousand changed rows in ten thousand took five times as long with it.
817
+ * seconds, and a thousand changed rows in ten thousand took five times as long with it. Autovacuum
818
+ * analyzes a partition once 2% of it changed (see `ensureObjectPartition`), and an ANALYZE here
819
+ * resets that count, so the same change is never analyzed twice.
816
820
  */
817
821
  async function refreshObjectStatistics(
818
822
  sql: SQLClient,