@sixb/pg 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/agents/computers.d.ts +8 -2
- package/dist/agents/computers.d.ts.map +1 -1
- package/dist/auth-storage/auth-storage.d.ts +2 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/identities.d.ts +4 -0
- package/dist/auth-storage/identities.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +2 -1
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/users.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2377 -428
- package/dist/index.js.map +37 -30
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/conditional-reader.d.ts +7 -2
- package/dist/objects/conditional-reader.d.ts.map +1 -1
- package/dist/objects/partitions.d.ts.map +1 -1
- package/dist/objects/prepare-queries.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +14 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts +13 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +14 -1
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +11 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/visibility-indexes.d.ts +22 -0
- package/dist/objects/visibility-indexes.d.ts.map +1 -0
- package/dist/objects/visibility-statistics.d.ts +26 -0
- package/dist/objects/visibility-statistics.d.ts.map +1 -0
- package/dist/objects/visibility.d.ts +34 -0
- package/dist/objects/visibility.d.ts.map +1 -0
- package/dist/ontology-storage/materialization-session.d.ts +5 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +14 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-artifact-storage.d.ts +14 -2
- package/dist/pg-artifact-storage.d.ts.map +1 -1
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/pg-dataset-ingest-storage.d.ts +6 -0
- package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
- package/dist/pg-lake-retention-run-storage.d.ts +11 -0
- package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +6 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +2 -1
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +4 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-claim-sql.d.ts +37 -0
- package/dist/run-claim-sql.d.ts.map +1 -0
- package/dist/share-session-storage.d.ts +2 -1
- package/dist/share-session-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/computers.ts +54 -5
- package/src/auth-storage/auth-storage.ts +66 -0
- package/src/auth-storage/identities.ts +15 -0
- package/src/auth-storage/rows.ts +3 -0
- package/src/auth-storage/users.ts +3 -0
- package/src/connector-connection-storage.ts +33 -16
- package/src/index.ts +15 -0
- package/src/migrations/087-agent-computer-background-commands.sql +4 -0
- package/src/migrations/088-web-search-model-kind.sql +12 -0
- package/src/migrations/089-maintenance-heartbeats.sql +10 -0
- package/src/migrations/090-lake-retention-runs.sql +8 -0
- package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
- package/src/migrations/092-run-recovery.sql +23 -0
- package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
- package/src/migrations/094-artifact-board-previews.sql +14 -0
- package/src/migrations/095-artifact-open-versions.sql +101 -0
- package/src/migrations/096-artifact-documents.sql +11 -0
- package/src/migrations/097-artifact-frame-files.sql +8 -0
- package/src/migrations/098-artifact-comments.sql +36 -0
- package/src/migrations/099-object-artifact-refs.sql +12 -0
- package/src/migrations/100-shared-link-guests.sql +17 -0
- package/src/migrations/101-connector-connection-owners.sql +38 -0
- package/src/migrations/102-user-suspension-source.sql +12 -0
- package/src/migrations.ts +46 -0
- package/src/objects/conditional-reader.ts +26 -4
- package/src/objects/partitions.ts +17 -4
- package/src/objects/prepare-queries.ts +8 -3
- package/src/objects/query-compiler.ts +64 -23
- package/src/objects/query-indexes.ts +93 -27
- package/src/objects/read-scope.ts +147 -77
- package/src/objects/reader.ts +46 -18
- package/src/objects/storage.ts +53 -0
- package/src/objects/visibility-indexes.ts +222 -0
- package/src/objects/visibility-statistics.ts +73 -0
- package/src/objects/visibility.ts +135 -0
- package/src/ontology-storage/materialization-session.ts +4 -0
- package/src/ontology-storage/materialization-writer.ts +68 -1
- package/src/ontology-storage/materializations.ts +6 -2
- package/src/pg-artifact-storage.ts +798 -137
- package/src/pg-client.ts +4 -0
- package/src/pg-dataset-ingest-storage.ts +305 -0
- package/src/pg-lake-retention-run-storage.ts +40 -0
- package/src/pg-maintenance-heartbeat-storage.ts +53 -0
- package/src/pg-pipeline-run-storage.ts +85 -42
- package/src/pg-projection-run-storage.ts +61 -2
- package/src/pg-sync-run-storage.ts +100 -37
- package/src/pg-webhook-run-storage.ts +37 -11
- package/src/pg-workflow-run-storage.ts +39 -100
- package/src/run-claim-sql.ts +173 -0
- package/src/share-session-storage.ts +20 -0
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { createHash } from "node:crypto"
|
|
2
|
+
import type { ObjectQueryPreparationPlan, ObjectVisibilityKeys } from "@sixb/core/storage"
|
|
3
|
+
import type { ReservedSQL } from "../pg-client"
|
|
4
|
+
import { runPgTransactionOn } from "../transactions"
|
|
5
|
+
import { objectPartitionName } from "./partitions"
|
|
6
|
+
import { identifier, literal } from "./sql-text"
|
|
7
|
+
import { compilePgVisibilityKeys } from "./visibility"
|
|
8
|
+
|
|
9
|
+
interface VisibilityIndex {
|
|
10
|
+
readonly objectTypeId: string
|
|
11
|
+
readonly partition: string
|
|
12
|
+
readonly expression: string
|
|
13
|
+
readonly name: string
|
|
14
|
+
readonly statistics: string
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** What an index comment records, so a later preparation finds the indexes it no longer needs. */
|
|
18
|
+
interface VisibilityIndexComment {
|
|
19
|
+
readonly kind: "visibility"
|
|
20
|
+
readonly projectId: string
|
|
21
|
+
readonly statistics: string
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Index the keys of every view condition, so a member's read finds the rows it sees instead of
|
|
26
|
+
* checking each one.
|
|
27
|
+
*
|
|
28
|
+
* One GIN index per condition and type, partial on the project and built concurrently, with
|
|
29
|
+
* statistics on its expression so the planner knows how many rows a reader's keys show. A read
|
|
30
|
+
* repeats the indexed expression, so it is served by an index only once it exists, and is right
|
|
31
|
+
* either way. Indexes no condition of the project needs any more are dropped. Runs under the
|
|
32
|
+
* preparation lock, on the connection that holds it.
|
|
33
|
+
*/
|
|
34
|
+
export async function preparePgVisibilityIndexes(
|
|
35
|
+
connection: ReservedSQL,
|
|
36
|
+
schema: string,
|
|
37
|
+
plan: ObjectQueryPreparationPlan
|
|
38
|
+
): Promise<{ readonly names: readonly string[]; readonly warnings: readonly string[] }> {
|
|
39
|
+
// Prepared without the view conditions: keep the indexes a preparation that had them built.
|
|
40
|
+
if (plan.objectTypes.every((type) => type.visibility === undefined)) {
|
|
41
|
+
return { names: [], warnings: [] }
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const warnings: string[] = []
|
|
45
|
+
const built: VisibilityIndex[] = []
|
|
46
|
+
for (const index of visibilityIndexes(plan)) {
|
|
47
|
+
const warning = await createVisibilityIndex(connection, schema, plan.projectId, index)
|
|
48
|
+
if (warning) warnings.push(warning)
|
|
49
|
+
else built.push(index)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
await ensureVisibilityStatistics(connection, schema, built)
|
|
53
|
+
await dropUnusedVisibilityIndexes(connection, schema, plan.projectId, built)
|
|
54
|
+
|
|
55
|
+
return { names: built.map((index) => index.name), warnings }
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function visibilityIndexes(plan: ObjectQueryPreparationPlan): VisibilityIndex[] {
|
|
59
|
+
return plan.objectTypes.flatMap((type) => {
|
|
60
|
+
const partition = objectPartitionName(type.objectTypeId)
|
|
61
|
+
|
|
62
|
+
return (type.visibility ?? []).map((keys) => {
|
|
63
|
+
const expression = compilePgVisibilityKeys(keys)
|
|
64
|
+
const statistics = visibilityStatisticsName(type.objectTypeId, keys)
|
|
65
|
+
const name = `sixb_visibility_${digest(`${partition}:${plan.projectId}:${expression}`, 32)}`
|
|
66
|
+
return { objectTypeId: type.objectTypeId, partition, expression, name, statistics }
|
|
67
|
+
})
|
|
68
|
+
})
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The statistics on a condition's keys. Statistics cannot be partial, so projects sharing a schema
|
|
73
|
+
* share those of one expression.
|
|
74
|
+
*/
|
|
75
|
+
export function visibilityStatisticsName(objectTypeId: string, keys: ObjectVisibilityKeys): string {
|
|
76
|
+
const expression = compilePgVisibilityKeys(keys)
|
|
77
|
+
return `sixb_visibility_stats_${digest(`${objectPartitionName(objectTypeId)}:${expression}`, 24)}`
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Build one index; a condition PostgreSQL cannot index keeps its reads right, and is reported. */
|
|
81
|
+
async function createVisibilityIndex(
|
|
82
|
+
connection: ReservedSQL,
|
|
83
|
+
schema: string,
|
|
84
|
+
projectId: string,
|
|
85
|
+
index: VisibilityIndex
|
|
86
|
+
): Promise<string | undefined> {
|
|
87
|
+
const table = `${identifier(schema)}.${identifier(index.partition)}`
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
await connection.unsafe(
|
|
91
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(index.name)} ON ${table} USING gin ((${index.expression})) WHERE project_id = ${literal(projectId)}`
|
|
92
|
+
)
|
|
93
|
+
} catch (error) {
|
|
94
|
+
// An expression with a function that is not IMMUTABLE fails before anything is built.
|
|
95
|
+
if ((error as { code?: unknown }).code !== FUNCTION_NOT_IMMUTABLE) throw error
|
|
96
|
+
return conditionNotIndexed(index.objectTypeId, error)
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const [built] = await connection<{ valid: boolean }[]>`
|
|
100
|
+
SELECT i.indisvalid AS valid FROM pg_index i
|
|
101
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
102
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
103
|
+
WHERE n.nspname = ${schema} AND c.relname = ${index.name}`
|
|
104
|
+
if (!built?.valid) {
|
|
105
|
+
throw new Error(
|
|
106
|
+
`[SixbPg] Visibility index ${schema}.${index.name} is invalid after an interrupted build. Drop that index concurrently and prepare again.`
|
|
107
|
+
)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const comment: VisibilityIndexComment = {
|
|
111
|
+
kind: "visibility",
|
|
112
|
+
projectId,
|
|
113
|
+
statistics: index.statistics,
|
|
114
|
+
}
|
|
115
|
+
await connection.unsafe(
|
|
116
|
+
`COMMENT ON INDEX ${identifier(schema)}.${identifier(index.name)} IS ${literal(JSON.stringify(comment))}`
|
|
117
|
+
)
|
|
118
|
+
return undefined
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const FUNCTION_NOT_IMMUTABLE = "42P17"
|
|
122
|
+
|
|
123
|
+
/** One transaction per type, so its queries plan with the old statistics until the new exist. */
|
|
124
|
+
async function ensureVisibilityStatistics(
|
|
125
|
+
connection: ReservedSQL,
|
|
126
|
+
schema: string,
|
|
127
|
+
indexes: readonly VisibilityIndex[]
|
|
128
|
+
): Promise<void> {
|
|
129
|
+
const partitions = new Map<string, VisibilityIndex[]>()
|
|
130
|
+
for (const index of indexes) {
|
|
131
|
+
partitions.set(index.partition, [...(partitions.get(index.partition) ?? []), index])
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
for (const [partition, partitionIndexes] of partitions) {
|
|
135
|
+
const table = `${identifier(schema)}.${identifier(partition)}`
|
|
136
|
+
|
|
137
|
+
await runPgTransactionOn(connection, async () => {
|
|
138
|
+
let created = false
|
|
139
|
+
for (const index of partitionIndexes) {
|
|
140
|
+
const [existing] = await connection`
|
|
141
|
+
SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
|
|
142
|
+
WHERE n.nspname = ${schema} AND s.stxname = ${index.statistics}`
|
|
143
|
+
if (existing) continue
|
|
144
|
+
|
|
145
|
+
await connection.unsafe(
|
|
146
|
+
`CREATE STATISTICS ${identifier(schema)}.${identifier(index.statistics)} ON (${index.expression}) FROM ${table}`
|
|
147
|
+
)
|
|
148
|
+
created = true
|
|
149
|
+
}
|
|
150
|
+
if (created) await connection.unsafe(`ANALYZE ${table}`)
|
|
151
|
+
})
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Drop this project's visibility indexes that no current condition needs: writes would keep
|
|
157
|
+
* maintaining them. Statistics go with the last index of any project that uses them.
|
|
158
|
+
*/
|
|
159
|
+
async function dropUnusedVisibilityIndexes(
|
|
160
|
+
connection: ReservedSQL,
|
|
161
|
+
schema: string,
|
|
162
|
+
projectId: string,
|
|
163
|
+
current: readonly VisibilityIndex[]
|
|
164
|
+
): Promise<void> {
|
|
165
|
+
const currentNames = new Set(current.map((index) => index.name))
|
|
166
|
+
const existing = await readVisibilityIndexes(connection, schema)
|
|
167
|
+
|
|
168
|
+
const unused = existing.filter(
|
|
169
|
+
(index) => index.comment.projectId === projectId && !currentNames.has(index.name)
|
|
170
|
+
)
|
|
171
|
+
for (const index of unused) {
|
|
172
|
+
await connection.unsafe(
|
|
173
|
+
`DROP INDEX CONCURRENTLY IF EXISTS ${identifier(schema)}.${identifier(index.name)}`
|
|
174
|
+
)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const kept = existing.filter((index) => !unused.includes(index))
|
|
178
|
+
for (const statistics of new Set(unused.map((index) => index.comment.statistics))) {
|
|
179
|
+
if (kept.some((index) => index.comment.statistics === statistics)) continue
|
|
180
|
+
await connection.unsafe(
|
|
181
|
+
`DROP STATISTICS IF EXISTS ${identifier(schema)}.${identifier(statistics)}`
|
|
182
|
+
)
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async function readVisibilityIndexes(connection: ReservedSQL, schema: string) {
|
|
187
|
+
const rows = await connection<{ name: string; definition: string }[]>`
|
|
188
|
+
SELECT c.relname AS name, d.description AS definition
|
|
189
|
+
FROM pg_index i
|
|
190
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
191
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
192
|
+
JOIN pg_description d
|
|
193
|
+
ON d.objoid = c.oid AND d.classoid = 'pg_class'::regclass AND d.objsubid = 0
|
|
194
|
+
WHERE n.nspname = ${schema} AND c.relname LIKE 'sixb_visibility_%'`
|
|
195
|
+
|
|
196
|
+
return rows.flatMap((row) => {
|
|
197
|
+
const comment = parseComment(row.definition)
|
|
198
|
+
return comment ? [{ name: row.name, comment }] : []
|
|
199
|
+
})
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function parseComment(definition: string): VisibilityIndexComment | undefined {
|
|
203
|
+
try {
|
|
204
|
+
const comment = JSON.parse(definition) as Partial<VisibilityIndexComment> | null
|
|
205
|
+
return comment?.kind === "visibility" &&
|
|
206
|
+
typeof comment.projectId === "string" &&
|
|
207
|
+
typeof comment.statistics === "string"
|
|
208
|
+
? (comment as VisibilityIndexComment)
|
|
209
|
+
: undefined
|
|
210
|
+
} catch {
|
|
211
|
+
return undefined
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function conditionNotIndexed(objectTypeId: string, error: unknown): string {
|
|
216
|
+
const reason = error instanceof Error ? error.message : String(error)
|
|
217
|
+
return `${objectTypeId}: a view condition cannot be indexed (${reason}); members read the type by checking each object.`
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function digest(value: string, length: number): string {
|
|
221
|
+
return createHash("sha256").update(value).digest("hex").slice(0, length)
|
|
222
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { ObjectReadCondition } from "@sixb/core/storage"
|
|
2
|
+
import type { SQLClient } from "../pg-client"
|
|
3
|
+
import { MAX_KEYED_ID_LENGTH, pgVisibilityReaderKeys } from "./visibility"
|
|
4
|
+
import { visibilityStatisticsName } from "./visibility-indexes"
|
|
5
|
+
|
|
6
|
+
// How long a read of the statistics serves queries before they are read again.
|
|
7
|
+
const VISIBILITY_STATISTICS_TTL_MS = 60_000
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The keys the statistics of each view condition list as most common, read at most once a minute.
|
|
11
|
+
*
|
|
12
|
+
* ANALYZE lists every key above a small share of the rows. PostgreSQL estimates one it does not
|
|
13
|
+
* list at about half a percent of them, whatever its real share: a reader who sees nothing looks
|
|
14
|
+
* like one who sees thousands of objects.
|
|
15
|
+
*/
|
|
16
|
+
export class PgVisibilityStatisticsCatalog {
|
|
17
|
+
#read:
|
|
18
|
+
| {
|
|
19
|
+
readonly at: number
|
|
20
|
+
readonly keys: Promise<ReadonlyMap<string, ReadonlySet<string>>>
|
|
21
|
+
}
|
|
22
|
+
| undefined
|
|
23
|
+
|
|
24
|
+
/** Read through the caller's client, so a transaction never waits for a second connection. */
|
|
25
|
+
commonKeys(sql: SQLClient): Promise<ReadonlyMap<string, ReadonlySet<string>>> {
|
|
26
|
+
const now = Date.now()
|
|
27
|
+
if (this.#read && now - this.#read.at < VISIBILITY_STATISTICS_TTL_MS) return this.#read.keys
|
|
28
|
+
|
|
29
|
+
const read = { at: now, keys: readCommonKeys(sql) }
|
|
30
|
+
this.#read = read
|
|
31
|
+
// A failed read is not kept: the next query reads again.
|
|
32
|
+
read.keys.catch(() => {
|
|
33
|
+
if (this.#read === read) this.#read = undefined
|
|
34
|
+
})
|
|
35
|
+
return read.keys
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function readCommonKeys(sql: SQLClient): Promise<ReadonlyMap<string, ReadonlySet<string>>> {
|
|
40
|
+
const rows = await sql<{ name: string; keys: string[] | null }[]>`
|
|
41
|
+
SELECT statistics_name AS name, most_common_elems::text::text[] AS keys
|
|
42
|
+
FROM pg_stats_ext_exprs
|
|
43
|
+
WHERE schemaname = current_schema() AND statistics_name LIKE 'sixb\\_visibility\\_stats\\_%'`
|
|
44
|
+
|
|
45
|
+
return new Map(rows.map((row) => [row.name, new Set(row.keys ?? [])]))
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Whether a reader sees so few objects of a type that its ordered pages read the objects it sees
|
|
50
|
+
* first, then sort them.
|
|
51
|
+
*
|
|
52
|
+
* Walking a sort index stops at a page's last object, so it is the right plan for a reader who sees
|
|
53
|
+
* many. For one who sees few, it checks nearly every object, and PostgreSQL cannot tell them apart
|
|
54
|
+
* when the statistics list none of the reader's keys. Such a reader sees few objects: those its
|
|
55
|
+
* keys meet are under the share ANALYZE lists. Without statistics, or with a remainder read object
|
|
56
|
+
* by object, the reader's share is unknown and PostgreSQL chooses.
|
|
57
|
+
*/
|
|
58
|
+
export function readsFewObjects(
|
|
59
|
+
condition: ObjectReadCondition,
|
|
60
|
+
common: ReadonlyMap<string, ReadonlySet<string>>
|
|
61
|
+
): boolean {
|
|
62
|
+
const visibility = condition.visibility
|
|
63
|
+
if (!visibility || visibility.remainder) return false
|
|
64
|
+
if ((visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH) return false
|
|
65
|
+
|
|
66
|
+
const readerKeys = pgVisibilityReaderKeys(visibility.user)
|
|
67
|
+
|
|
68
|
+
for (const keys of visibility.keys) {
|
|
69
|
+
const listed = common.get(visibilityStatisticsName(condition.objectTypeId, keys))
|
|
70
|
+
if (!listed || readerKeys.some((key) => listed.has(key))) return false
|
|
71
|
+
}
|
|
72
|
+
return true
|
|
73
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import type { UserRef } from "@sixb/core"
|
|
2
|
+
import type { ObjectReadCondition, ObjectVisibilityKeys } from "@sixb/core/storage"
|
|
3
|
+
import { compilePredicate } from "./query-compiler"
|
|
4
|
+
import { literal } from "./sql-text"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Visibility keys as SQL: one expression over `properties` returning who the condition shows the
|
|
8
|
+
* row to, `*` for everyone holding it and `u:<id>` for a user.
|
|
9
|
+
*
|
|
10
|
+
* The expression is IMMUTABLE and holds its values inline, so PostgreSQL can index it and match a
|
|
11
|
+
* read that repeats it, whatever the reader. A user's id is written as jsonb prints it between its
|
|
12
|
+
* quotes: that is how a list's ids come out of `jsonb_path_query_array` with no subquery, which an
|
|
13
|
+
* index expression cannot hold. An escaped quote holds a `"`, so `\\` and `\"` are first written as
|
|
14
|
+
* control characters jsonb never prints raw: `", "` then only separates two ids, and `\x1f` never
|
|
15
|
+
* appears in one. A reader's key is written the same way.
|
|
16
|
+
*
|
|
17
|
+
* A GIN entry holds at most 2,712 bytes, and jsonb prints a character in up to 6 bytes, so ids
|
|
18
|
+
* longer than `MAX_KEYED_ID_LENGTH` characters get no key: a reader with such an id reads the
|
|
19
|
+
* predicate instead. A short list keeps its long ids, which no reader looks up, rather than run a
|
|
20
|
+
* regex on each of its ids.
|
|
21
|
+
*
|
|
22
|
+
* Reads number every `?` of a statement as a parameter, so none is written: text goes in `E''`
|
|
23
|
+
* strings, where `?` is written `\x3f`.
|
|
24
|
+
*/
|
|
25
|
+
export function compilePgVisibilityKeys(keys: ObjectVisibilityKeys): string {
|
|
26
|
+
switch (keys.kind) {
|
|
27
|
+
case "everyone":
|
|
28
|
+
return `CASE WHEN ${predicateSql(keys.when)} THEN ARRAY[${text(EVERYONE)}]::text[] ELSE ${NO_KEYS} END`
|
|
29
|
+
|
|
30
|
+
case "user": {
|
|
31
|
+
// `eq` compares the whole reference, so the property holds exactly a type and a string id.
|
|
32
|
+
const ref = propertySql(keys.propertyId)
|
|
33
|
+
return `CASE WHEN jsonb_typeof(${ref}) = 'object' AND (${ref} - 'type' - 'id') = '{}'::jsonb AND ${ref} -> 'type' = '"user"'::jsonb AND jsonb_typeof(${ref} -> 'id') = 'string' AND char_length(${ref} ->> 'id') <= ${MAX_KEYED_ID_LENGTH} THEN ARRAY[${text(USER_PREFIX)} || ${withoutEscapes(`left(substr((${ref} -> 'id')::text, 2), -1)`)}] ELSE ${NO_KEYS} END`
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
case "users": {
|
|
37
|
+
// Strict mode keeps the elements `@>` matches: objects whose type is "user" and id a string.
|
|
38
|
+
const users = `@.type == "user" && @.id.type() == "string"`
|
|
39
|
+
const keysOf = (filter: string) => {
|
|
40
|
+
const path = text(`strict $[*] ? (${filter}).id`)
|
|
41
|
+
const ids = withoutEscapes(
|
|
42
|
+
`jsonb_path_query_array(${propertySql(keys.propertyId)}, ${path}, '{}', true)::text`
|
|
43
|
+
)
|
|
44
|
+
const separator = `E'\\x1f'`
|
|
45
|
+
return `coalesce(string_to_array(${text(USER_PREFIX)} || replace(NULLIF(left(substr(${ids}, 3), -2), ''), '", "', ${separator} || ${text(USER_PREFIX)}), ${separator}), ${NO_KEYS})`
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// No id of a short list is too long for an entry: only a longer one pays for the regex.
|
|
49
|
+
const bounded = `${users} && @.id like_regex "^.{0,${MAX_KEYED_ID_LENGTH}}$" flag "s"`
|
|
50
|
+
return `CASE WHEN octet_length(${propertySql(keys.propertyId)}::text) <= ${SHORT_LIST_BYTES} THEN ${keysOf(users)} ELSE ${keysOf(bounded)} END`
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
case "union":
|
|
54
|
+
return `(${keys.items.map((item) => `(${compilePgVisibilityKeys(item)})`).join(" || ")})`
|
|
55
|
+
|
|
56
|
+
case "when":
|
|
57
|
+
return `CASE WHEN ${predicateSql(keys.when)} THEN ${compilePgVisibilityKeys(keys.keys)} ELSE ${NO_KEYS} END`
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The keys a reader holds: everyone's, and its user's. */
|
|
62
|
+
export function pgVisibilityReaderKeys(user: UserRef | undefined): string[] {
|
|
63
|
+
if (!user) return [EVERYONE]
|
|
64
|
+
|
|
65
|
+
// As `withoutEscapes` writes the ids the keys hold.
|
|
66
|
+
const id = JSON.stringify(user.id)
|
|
67
|
+
.slice(1, -1)
|
|
68
|
+
.replaceAll("\\\\", ESCAPED_BACKSLASH)
|
|
69
|
+
.replaceAll('\\"', ESCAPED_QUOTE)
|
|
70
|
+
return [EVERYONE, `${USER_PREFIX}${id}`]
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* A conditioned type's filter: each condition's keys meet the reader's, or the remainder matches.
|
|
75
|
+
* Without keys, or for a user whose id is too long to have one, the condition's predicate.
|
|
76
|
+
*/
|
|
77
|
+
export function compilePgConditionFilter(condition: ObjectReadCondition): {
|
|
78
|
+
readonly sql: string
|
|
79
|
+
readonly args: readonly unknown[]
|
|
80
|
+
} {
|
|
81
|
+
const visibility = condition.visibility
|
|
82
|
+
// UTF-16 code units, never fewer than the characters PostgreSQL counts.
|
|
83
|
+
if (!visibility || (visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH) {
|
|
84
|
+
return compilePredicate(condition.predicate)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const readerKeys = pgVisibilityReaderKeys(visibility.user)
|
|
88
|
+
const clauses = visibility.keys.map((keys) => `((${compilePgVisibilityKeys(keys)}) && ?::text[])`)
|
|
89
|
+
const args: unknown[] = visibility.keys.map(() => readerKeys)
|
|
90
|
+
|
|
91
|
+
if (visibility.remainder) {
|
|
92
|
+
const remainder = compilePredicate(visibility.remainder)
|
|
93
|
+
clauses.push(remainder.sql)
|
|
94
|
+
args.push(...remainder.args)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return { sql: `(${clauses.join(" OR ")})`, args }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export const MAX_KEYED_ID_LENGTH = 200
|
|
101
|
+
const ESCAPED_BACKSLASH = "\x1e"
|
|
102
|
+
const ESCAPED_QUOTE = "\x1d"
|
|
103
|
+
// A list whose JSON is this short holds no id whose key would overflow an entry.
|
|
104
|
+
const SHORT_LIST_BYTES = 2000
|
|
105
|
+
const EVERYONE = "*"
|
|
106
|
+
const USER_PREFIX = "u:"
|
|
107
|
+
const NO_KEYS = "'{}'::text[]"
|
|
108
|
+
|
|
109
|
+
function propertySql(propertyId: string): string {
|
|
110
|
+
return `(properties -> ${text(propertyId)}::text)`
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** jsonb's `\\` and `\"` in printed ids, as control characters it never prints raw. */
|
|
114
|
+
function withoutEscapes(printed: string): string {
|
|
115
|
+
return `replace(replace(${printed}, ${text("\\\\")}, E'\\x1e'), ${text('\\"')}, E'\\x1d')`
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function text(value: string): string {
|
|
119
|
+
return literal(value).replace(/\?/g, "\\x3f")
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The reader's own predicate compiler, so a part means exactly what the condition does. */
|
|
123
|
+
function predicateSql(predicate: Parameters<typeof compilePredicate>[0]): string {
|
|
124
|
+
const compiled = compilePredicate(predicate)
|
|
125
|
+
let index = 0
|
|
126
|
+
return `(${compiled.sql.replace(/\?/g, () => inlineValue(compiled.args[index++]))})`
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** The values the predicate compiler binds: property ids and compared values, text or numbers. */
|
|
130
|
+
function inlineValue(value: unknown): string {
|
|
131
|
+
if (typeof value === "string") return text(value)
|
|
132
|
+
if (typeof value === "number" && Number.isFinite(value)) return `(${String(value)})`
|
|
133
|
+
|
|
134
|
+
throw new Error(`[SixbPg] A view condition compares a value it cannot index: ${String(value)}.`)
|
|
135
|
+
}
|
|
@@ -38,6 +38,8 @@ export class PgMaterializationSessionState extends ProviderMaterializationSessio
|
|
|
38
38
|
changedUniqueValues = 0
|
|
39
39
|
/** What unique value items this session staged, known without reading its work back. */
|
|
40
40
|
readonly uniqueWork = { releases: false, claims: false }
|
|
41
|
+
/** What artifact holder items this session staged, likewise. */
|
|
42
|
+
readonly artifactHolderWork = { deletes: false, inserts: false }
|
|
41
43
|
workAnalyzed = false
|
|
42
44
|
/** Work staged on the session lives in a temp table; a plan-bound session reads its plan's. */
|
|
43
45
|
readonly workTable: string
|
|
@@ -172,6 +174,8 @@ export class PgMaterializationSessions {
|
|
|
172
174
|
if (record.kind !== "plan") continue
|
|
173
175
|
if (record.item.kind === "unique-value-release") session.uniqueWork.releases = true
|
|
174
176
|
if (record.item.kind === "unique-value-claim") session.uniqueWork.claims = true
|
|
177
|
+
if (record.item.kind === "artifact-holder-delete") session.artifactHolderWork.deletes = true
|
|
178
|
+
if (record.item.kind === "artifact-holder-insert") session.artifactHolderWork.inserts = true
|
|
175
179
|
}
|
|
176
180
|
}
|
|
177
181
|
|
|
@@ -27,6 +27,8 @@ export interface PgStagedPlanTarget {
|
|
|
27
27
|
readonly committedAt: string
|
|
28
28
|
/** Whether unique value items were staged; probed in the work table when unknown. */
|
|
29
29
|
readonly uniqueWork?: PgUniqueWork
|
|
30
|
+
/** Whether artifact holder items were staged; probed in the work table when unknown. */
|
|
31
|
+
readonly artifactHolderWork?: PgArtifactHolderWork
|
|
30
32
|
}
|
|
31
33
|
|
|
32
34
|
export interface PgUniqueWork {
|
|
@@ -34,6 +36,11 @@ export interface PgUniqueWork {
|
|
|
34
36
|
readonly claims: boolean
|
|
35
37
|
}
|
|
36
38
|
|
|
39
|
+
export interface PgArtifactHolderWork {
|
|
40
|
+
readonly deletes: boolean
|
|
41
|
+
readonly inserts: boolean
|
|
42
|
+
}
|
|
43
|
+
|
|
37
44
|
export interface PgAppliedPlan {
|
|
38
45
|
readonly objectWrites: number
|
|
39
46
|
readonly linkWrites: number
|
|
@@ -65,12 +72,15 @@ export class PgMaterializationWriter {
|
|
|
65
72
|
await this.applyLinkOverrides(target)
|
|
66
73
|
await this.applyPoints(target)
|
|
67
74
|
const uniqueWork = target.uniqueWork ?? (await this.stagedUniqueWork(target))
|
|
75
|
+
const holderWork = target.artifactHolderWork ?? (await this.stagedArtifactHolderWork(target))
|
|
68
76
|
if (uniqueWork.releases) await this.releaseUniqueValues(target)
|
|
77
|
+
if (holderWork.deletes) await this.deleteArtifactHolders(target)
|
|
69
78
|
let linkWrites = await this.deleteLinks(target)
|
|
70
79
|
let objectWrites = await this.deleteObjects(target)
|
|
71
80
|
objectWrites += await this.upsertObjects(target)
|
|
72
81
|
linkWrites += await this.upsertLinks(target)
|
|
73
82
|
const uniqueValueWrites = uniqueWork.claims ? await this.claimUniqueValues(target) : 0
|
|
83
|
+
if (holderWork.inserts) await this.insertArtifactHolders(target)
|
|
74
84
|
const eventCount = await this.writeOutbox(target)
|
|
75
85
|
await this.writeTouches(target)
|
|
76
86
|
return { objectWrites, linkWrites, uniqueValueWrites, eventCount }
|
|
@@ -106,6 +116,25 @@ export class PgMaterializationWriter {
|
|
|
106
116
|
return { releases: row?.releases ?? false, claims: row?.claims ?? false }
|
|
107
117
|
}
|
|
108
118
|
|
|
119
|
+
/** The same probes for the artifact holder index: most commits hold no artifact. */
|
|
120
|
+
private async stagedArtifactHolderWork(
|
|
121
|
+
target: PgStagedPlanTarget
|
|
122
|
+
): Promise<PgArtifactHolderWork> {
|
|
123
|
+
const probe = (kind: "artifact-holder-delete" | "artifact-holder-insert") => this.sql`
|
|
124
|
+
EXISTS (
|
|
125
|
+
SELECT 1 FROM ${this.sql(target.workTable)}
|
|
126
|
+
WHERE work_id = ${target.workId} AND lane = 'apply'
|
|
127
|
+
AND major_order = ${materializationApplyPhase(kind)}
|
|
128
|
+
AND minor_order = ${materializationPlanKindRank(kind)}
|
|
129
|
+
)
|
|
130
|
+
`
|
|
131
|
+
const [row] = await this.sql<{ readonly deletes: boolean; readonly inserts: boolean }[]>`
|
|
132
|
+
SELECT ${probe("artifact-holder-delete")} AS deletes,
|
|
133
|
+
${probe("artifact-holder-insert")} AS inserts
|
|
134
|
+
`
|
|
135
|
+
return { deletes: row?.deletes ?? false, inserts: row?.inserts ?? false }
|
|
136
|
+
}
|
|
137
|
+
|
|
109
138
|
/** The staged values of one plan item kind, as `value`. */
|
|
110
139
|
private staged(target: PgStagedPlanTarget, kind: MaterializationPlanWorkItem["kind"]) {
|
|
111
140
|
return this.sql`
|
|
@@ -417,6 +446,43 @@ export class PgMaterializationWriter {
|
|
|
417
446
|
)
|
|
418
447
|
}
|
|
419
448
|
|
|
449
|
+
/** The staged rows of the artifact holder index of one item kind, as its columns. */
|
|
450
|
+
private stagedArtifactHolders(
|
|
451
|
+
target: PgStagedPlanTarget,
|
|
452
|
+
kind: "artifact-holder-delete" | "artifact-holder-insert"
|
|
453
|
+
) {
|
|
454
|
+
return this.sql`
|
|
455
|
+
SELECT value->>'artifactId' AS artifact_id, value->>'objectTypeId' AS object_type_id,
|
|
456
|
+
value->>'primaryId' AS primary_id, value->>'propertyId' AS property_id
|
|
457
|
+
FROM (${this.staged(target, kind)}) AS staged
|
|
458
|
+
`
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/** A row already gone is no conflict: the index only follows what objects hold. */
|
|
462
|
+
private async deleteArtifactHolders(target: PgStagedPlanTarget): Promise<void> {
|
|
463
|
+
await this.sql`
|
|
464
|
+
WITH candidates AS (${this.stagedArtifactHolders(target, "artifact-holder-delete")})
|
|
465
|
+
DELETE FROM object_artifact_refs AS stored USING candidates
|
|
466
|
+
WHERE stored.project_id = ${target.projectId}
|
|
467
|
+
AND stored.artifact_id = candidates.artifact_id
|
|
468
|
+
AND stored.object_type_id = candidates.object_type_id
|
|
469
|
+
AND stored.primary_id = candidates.primary_id
|
|
470
|
+
AND stored.property_id = candidates.property_id
|
|
471
|
+
`
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/** A row already there is no conflict either. */
|
|
475
|
+
private async insertArtifactHolders(target: PgStagedPlanTarget): Promise<void> {
|
|
476
|
+
await this.sql`
|
|
477
|
+
INSERT INTO object_artifact_refs (
|
|
478
|
+
project_id, artifact_id, object_type_id, primary_id, property_id
|
|
479
|
+
)
|
|
480
|
+
SELECT ${target.projectId}, artifact_id, object_type_id, primary_id, property_id
|
|
481
|
+
FROM (${this.stagedArtifactHolders(target, "artifact-holder-insert")}) AS candidates
|
|
482
|
+
ON CONFLICT DO NOTHING
|
|
483
|
+
`
|
|
484
|
+
}
|
|
485
|
+
|
|
420
486
|
private async deleteLinks(target: PgStagedPlanTarget): Promise<number> {
|
|
421
487
|
const candidates = this.staged(target, "link-delete")
|
|
422
488
|
return this.write(
|
|
@@ -673,7 +739,8 @@ export class PgMaterializationWriter {
|
|
|
673
739
|
END AS ref
|
|
674
740
|
) AS entity
|
|
675
741
|
WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
|
|
676
|
-
-- Unique values are no plan input: a stale claim fails when it applies.
|
|
742
|
+
-- Unique values are no plan input: a stale claim fails when it applies. Nor are the
|
|
743
|
+
-- artifacts objects hold, which follow the objects.
|
|
677
744
|
AND entity.kind IS NOT NULL
|
|
678
745
|
`
|
|
679
746
|
}
|
|
@@ -268,7 +268,9 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
268
268
|
projectId: commit.projectId,
|
|
269
269
|
commitId: commit.id,
|
|
270
270
|
committedAt: commit.committedAt,
|
|
271
|
-
...(session.plan
|
|
271
|
+
...(session.plan
|
|
272
|
+
? {}
|
|
273
|
+
: { uniqueWork: session.uniqueWork, artifactHolderWork: session.artifactHolderWork }),
|
|
272
274
|
})
|
|
273
275
|
// Below this no type can reach the statistics threshold, so the work table stays unread.
|
|
274
276
|
if (applied.objectWrites >= STATISTICS_MINIMUM_CHANGES) {
|
|
@@ -812,7 +814,9 @@ async function refreshMaterializedStatistics(
|
|
|
812
814
|
* partitioned parent, and its statistics plan queries that read across types. A smaller one only
|
|
813
815
|
* refreshes partitions it changed substantially, which autovacuum may not reach before the next
|
|
814
816
|
* projection. A routine update must not: ANALYZE over a partition's expression statistics takes
|
|
815
|
-
* seconds, and a thousand changed rows in ten thousand took five times as long with it.
|
|
817
|
+
* seconds, and a thousand changed rows in ten thousand took five times as long with it. Autovacuum
|
|
818
|
+
* analyzes a partition once 2% of it changed (see `ensureObjectPartition`), and an ANALYZE here
|
|
819
|
+
* resets that count, so the same change is never analyzed twice.
|
|
816
820
|
*/
|
|
817
821
|
async function refreshObjectStatistics(
|
|
818
822
|
sql: SQLClient,
|