@sixb/pg 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/agents/computers.d.ts +8 -2
- package/dist/agents/computers.d.ts.map +1 -1
- package/dist/index.js +566 -144
- package/dist/index.js.map +20 -17
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/conditional-reader.d.ts +7 -2
- package/dist/objects/conditional-reader.d.ts.map +1 -1
- package/dist/objects/partitions.d.ts.map +1 -1
- package/dist/objects/prepare-queries.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +14 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts +13 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +14 -1
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +1 -0
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/visibility-indexes.d.ts +22 -0
- package/dist/objects/visibility-indexes.d.ts.map +1 -0
- package/dist/objects/visibility-statistics.d.ts +26 -0
- package/dist/objects/visibility-statistics.d.ts.map +1 -0
- package/dist/objects/visibility.d.ts +34 -0
- package/dist/objects/visibility.d.ts.map +1 -0
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/computers.ts +54 -5
- package/src/migrations/087-agent-computer-background-commands.sql +4 -0
- package/src/migrations/088-web-search-model-kind.sql +12 -0
- package/src/migrations.ts +6 -0
- package/src/objects/conditional-reader.ts +26 -4
- package/src/objects/partitions.ts +17 -4
- package/src/objects/prepare-queries.ts +8 -3
- package/src/objects/query-compiler.ts +64 -23
- package/src/objects/query-indexes.ts +93 -27
- package/src/objects/read-scope.ts +147 -77
- package/src/objects/reader.ts +46 -18
- package/src/objects/storage.ts +4 -0
- package/src/objects/visibility-indexes.ts +222 -0
- package/src/objects/visibility-statistics.ts +73 -0
- package/src/objects/visibility.ts +135 -0
- package/src/ontology-storage/materializations.ts +3 -1
- package/src/pg-client.ts +4 -0
- package/src/pg-pipeline-run-storage.ts +5 -7
- package/src/pg-sync-run-storage.ts +2 -7
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { ObjectReadCondition } from "@sixb/core/storage"
|
|
2
|
+
import type { SQLClient } from "../pg-client"
|
|
3
|
+
import { MAX_KEYED_ID_LENGTH, pgVisibilityReaderKeys } from "./visibility"
|
|
4
|
+
import { visibilityStatisticsName } from "./visibility-indexes"
|
|
5
|
+
|
|
6
|
+
// How long a read of the statistics serves queries before they are read again.
|
|
7
|
+
const VISIBILITY_STATISTICS_TTL_MS = 60_000
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The keys the statistics of each view condition list as most common, read at most once a minute.
|
|
11
|
+
*
|
|
12
|
+
* ANALYZE lists every key above a small share of the rows. PostgreSQL estimates one it does not
|
|
13
|
+
* list at about half a percent of them, whatever its real share: a reader who sees nothing looks
|
|
14
|
+
* like one who sees thousands of objects.
|
|
15
|
+
*/
|
|
16
|
+
export class PgVisibilityStatisticsCatalog {
|
|
17
|
+
#read:
|
|
18
|
+
| {
|
|
19
|
+
readonly at: number
|
|
20
|
+
readonly keys: Promise<ReadonlyMap<string, ReadonlySet<string>>>
|
|
21
|
+
}
|
|
22
|
+
| undefined
|
|
23
|
+
|
|
24
|
+
/** Read through the caller's client, so a transaction never waits for a second connection. */
|
|
25
|
+
commonKeys(sql: SQLClient): Promise<ReadonlyMap<string, ReadonlySet<string>>> {
|
|
26
|
+
const now = Date.now()
|
|
27
|
+
if (this.#read && now - this.#read.at < VISIBILITY_STATISTICS_TTL_MS) return this.#read.keys
|
|
28
|
+
|
|
29
|
+
const read = { at: now, keys: readCommonKeys(sql) }
|
|
30
|
+
this.#read = read
|
|
31
|
+
// A failed read is not kept: the next query reads again.
|
|
32
|
+
read.keys.catch(() => {
|
|
33
|
+
if (this.#read === read) this.#read = undefined
|
|
34
|
+
})
|
|
35
|
+
return read.keys
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function readCommonKeys(sql: SQLClient): Promise<ReadonlyMap<string, ReadonlySet<string>>> {
|
|
40
|
+
const rows = await sql<{ name: string; keys: string[] | null }[]>`
|
|
41
|
+
SELECT statistics_name AS name, most_common_elems::text::text[] AS keys
|
|
42
|
+
FROM pg_stats_ext_exprs
|
|
43
|
+
WHERE schemaname = current_schema() AND statistics_name LIKE 'sixb\\_visibility\\_stats\\_%'`
|
|
44
|
+
|
|
45
|
+
return new Map(rows.map((row) => [row.name, new Set(row.keys ?? [])]))
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Whether a reader sees so few objects of a type that its ordered pages read the objects it sees
|
|
50
|
+
* first, then sort them.
|
|
51
|
+
*
|
|
52
|
+
* Walking a sort index stops at a page's last object, so it is the right plan for a reader who sees
|
|
53
|
+
* many. For one who sees few, it checks nearly every object, and PostgreSQL cannot tell them apart
|
|
54
|
+
* when the statistics list none of the reader's keys. Such a reader sees few objects: those its
|
|
55
|
+
* keys meet are under the share ANALYZE lists. Without statistics, or with a remainder read object
|
|
56
|
+
* by object, the reader's share is unknown and PostgreSQL chooses.
|
|
57
|
+
*/
|
|
58
|
+
export function readsFewObjects(
|
|
59
|
+
condition: ObjectReadCondition,
|
|
60
|
+
common: ReadonlyMap<string, ReadonlySet<string>>
|
|
61
|
+
): boolean {
|
|
62
|
+
const visibility = condition.visibility
|
|
63
|
+
if (!visibility || visibility.remainder) return false
|
|
64
|
+
if ((visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH) return false
|
|
65
|
+
|
|
66
|
+
const readerKeys = pgVisibilityReaderKeys(visibility.user)
|
|
67
|
+
|
|
68
|
+
for (const keys of visibility.keys) {
|
|
69
|
+
const listed = common.get(visibilityStatisticsName(condition.objectTypeId, keys))
|
|
70
|
+
if (!listed || readerKeys.some((key) => listed.has(key))) return false
|
|
71
|
+
}
|
|
72
|
+
return true
|
|
73
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import type { UserRef } from "@sixb/core"
|
|
2
|
+
import type { ObjectReadCondition, ObjectVisibilityKeys } from "@sixb/core/storage"
|
|
3
|
+
import { compilePredicate } from "./query-compiler"
|
|
4
|
+
import { literal } from "./sql-text"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Visibility keys as SQL: one expression over `properties` returning who the condition shows the
|
|
8
|
+
* row to, `*` for everyone holding it and `u:<id>` for a user.
|
|
9
|
+
*
|
|
10
|
+
* The expression is IMMUTABLE and holds its values inline, so PostgreSQL can index it and match a
|
|
11
|
+
* read that repeats it, whatever the reader. A user's id is written as jsonb prints it between its
|
|
12
|
+
* quotes: that is how a list's ids come out of `jsonb_path_query_array` with no subquery, which an
|
|
13
|
+
* index expression cannot hold. An escaped quote holds a `"`, so `\\` and `\"` are first written as
|
|
14
|
+
* control characters jsonb never prints raw: `", "` then only separates two ids, and `\x1f` never
|
|
15
|
+
* appears in one. A reader's key is written the same way.
|
|
16
|
+
*
|
|
17
|
+
* A GIN entry holds at most 2,712 bytes, and jsonb prints a character in up to 6 bytes, so ids
|
|
18
|
+
* longer than `MAX_KEYED_ID_LENGTH` characters get no key: a reader with such an id reads the
|
|
19
|
+
* predicate instead. A short list keeps its long ids, which no reader looks up, rather than run a
|
|
20
|
+
* regex on each of its ids.
|
|
21
|
+
*
|
|
22
|
+
* Reads number every `?` of a statement as a parameter, so none is written: text goes in `E''`
|
|
23
|
+
* strings, where `?` is written `\x3f`.
|
|
24
|
+
*/
|
|
25
|
+
export function compilePgVisibilityKeys(keys: ObjectVisibilityKeys): string {
|
|
26
|
+
switch (keys.kind) {
|
|
27
|
+
case "everyone":
|
|
28
|
+
return `CASE WHEN ${predicateSql(keys.when)} THEN ARRAY[${text(EVERYONE)}]::text[] ELSE ${NO_KEYS} END`
|
|
29
|
+
|
|
30
|
+
case "user": {
|
|
31
|
+
// `eq` compares the whole reference, so the property holds exactly a type and a string id.
|
|
32
|
+
const ref = propertySql(keys.propertyId)
|
|
33
|
+
return `CASE WHEN jsonb_typeof(${ref}) = 'object' AND (${ref} - 'type' - 'id') = '{}'::jsonb AND ${ref} -> 'type' = '"user"'::jsonb AND jsonb_typeof(${ref} -> 'id') = 'string' AND char_length(${ref} ->> 'id') <= ${MAX_KEYED_ID_LENGTH} THEN ARRAY[${text(USER_PREFIX)} || ${withoutEscapes(`left(substr((${ref} -> 'id')::text, 2), -1)`)}] ELSE ${NO_KEYS} END`
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
case "users": {
|
|
37
|
+
// Strict mode keeps the elements `@>` matches: objects whose type is "user" and id a string.
|
|
38
|
+
const users = `@.type == "user" && @.id.type() == "string"`
|
|
39
|
+
const keysOf = (filter: string) => {
|
|
40
|
+
const path = text(`strict $[*] ? (${filter}).id`)
|
|
41
|
+
const ids = withoutEscapes(
|
|
42
|
+
`jsonb_path_query_array(${propertySql(keys.propertyId)}, ${path}, '{}', true)::text`
|
|
43
|
+
)
|
|
44
|
+
const separator = `E'\\x1f'`
|
|
45
|
+
return `coalesce(string_to_array(${text(USER_PREFIX)} || replace(NULLIF(left(substr(${ids}, 3), -2), ''), '", "', ${separator} || ${text(USER_PREFIX)}), ${separator}), ${NO_KEYS})`
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// No id of a short list is too long for an entry: only a longer one pays for the regex.
|
|
49
|
+
const bounded = `${users} && @.id like_regex "^.{0,${MAX_KEYED_ID_LENGTH}}$" flag "s"`
|
|
50
|
+
return `CASE WHEN octet_length(${propertySql(keys.propertyId)}::text) <= ${SHORT_LIST_BYTES} THEN ${keysOf(users)} ELSE ${keysOf(bounded)} END`
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
case "union":
|
|
54
|
+
return `(${keys.items.map((item) => `(${compilePgVisibilityKeys(item)})`).join(" || ")})`
|
|
55
|
+
|
|
56
|
+
case "when":
|
|
57
|
+
return `CASE WHEN ${predicateSql(keys.when)} THEN ${compilePgVisibilityKeys(keys.keys)} ELSE ${NO_KEYS} END`
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The keys a reader holds: everyone's, and its user's. */
|
|
62
|
+
export function pgVisibilityReaderKeys(user: UserRef | undefined): string[] {
|
|
63
|
+
if (!user) return [EVERYONE]
|
|
64
|
+
|
|
65
|
+
// As `withoutEscapes` writes the ids the keys hold.
|
|
66
|
+
const id = JSON.stringify(user.id)
|
|
67
|
+
.slice(1, -1)
|
|
68
|
+
.replaceAll("\\\\", ESCAPED_BACKSLASH)
|
|
69
|
+
.replaceAll('\\"', ESCAPED_QUOTE)
|
|
70
|
+
return [EVERYONE, `${USER_PREFIX}${id}`]
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* A conditioned type's filter: each condition's keys meet the reader's, or the remainder matches.
|
|
75
|
+
* Without keys, or for a user whose id is too long to have one, the condition's predicate.
|
|
76
|
+
*/
|
|
77
|
+
export function compilePgConditionFilter(condition: ObjectReadCondition): {
|
|
78
|
+
readonly sql: string
|
|
79
|
+
readonly args: readonly unknown[]
|
|
80
|
+
} {
|
|
81
|
+
const visibility = condition.visibility
|
|
82
|
+
// UTF-16 code units, never fewer than the characters PostgreSQL counts.
|
|
83
|
+
if (!visibility || (visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH) {
|
|
84
|
+
return compilePredicate(condition.predicate)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const readerKeys = pgVisibilityReaderKeys(visibility.user)
|
|
88
|
+
const clauses = visibility.keys.map((keys) => `((${compilePgVisibilityKeys(keys)}) && ?::text[])`)
|
|
89
|
+
const args: unknown[] = visibility.keys.map(() => readerKeys)
|
|
90
|
+
|
|
91
|
+
if (visibility.remainder) {
|
|
92
|
+
const remainder = compilePredicate(visibility.remainder)
|
|
93
|
+
clauses.push(remainder.sql)
|
|
94
|
+
args.push(...remainder.args)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return { sql: `(${clauses.join(" OR ")})`, args }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export const MAX_KEYED_ID_LENGTH = 200
|
|
101
|
+
const ESCAPED_BACKSLASH = "\x1e"
|
|
102
|
+
const ESCAPED_QUOTE = "\x1d"
|
|
103
|
+
// A list whose JSON is this short holds no id whose key would overflow an entry.
|
|
104
|
+
const SHORT_LIST_BYTES = 2000
|
|
105
|
+
const EVERYONE = "*"
|
|
106
|
+
const USER_PREFIX = "u:"
|
|
107
|
+
const NO_KEYS = "'{}'::text[]"
|
|
108
|
+
|
|
109
|
+
function propertySql(propertyId: string): string {
|
|
110
|
+
return `(properties -> ${text(propertyId)}::text)`
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** jsonb's `\\` and `\"` in printed ids, as control characters it never prints raw. */
|
|
114
|
+
function withoutEscapes(printed: string): string {
|
|
115
|
+
return `replace(replace(${printed}, ${text("\\\\")}, E'\\x1e'), ${text('\\"')}, E'\\x1d')`
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function text(value: string): string {
|
|
119
|
+
return literal(value).replace(/\?/g, "\\x3f")
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The reader's own predicate compiler, so a part means exactly what the condition does. */
|
|
123
|
+
function predicateSql(predicate: Parameters<typeof compilePredicate>[0]): string {
|
|
124
|
+
const compiled = compilePredicate(predicate)
|
|
125
|
+
let index = 0
|
|
126
|
+
return `(${compiled.sql.replace(/\?/g, () => inlineValue(compiled.args[index++]))})`
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** The values the predicate compiler binds: property ids and compared values, text or numbers. */
|
|
130
|
+
function inlineValue(value: unknown): string {
|
|
131
|
+
if (typeof value === "string") return text(value)
|
|
132
|
+
if (typeof value === "number" && Number.isFinite(value)) return `(${String(value)})`
|
|
133
|
+
|
|
134
|
+
throw new Error(`[SixbPg] A view condition compares a value it cannot index: ${String(value)}.`)
|
|
135
|
+
}
|
|
@@ -812,7 +812,9 @@ async function refreshMaterializedStatistics(
|
|
|
812
812
|
* partitioned parent, and its statistics plan queries that read across types. A smaller one only
|
|
813
813
|
* refreshes partitions it changed substantially, which autovacuum may not reach before the next
|
|
814
814
|
* projection. A routine update must not: ANALYZE over a partition's expression statistics takes
|
|
815
|
-
* seconds, and a thousand changed rows in ten thousand took five times as long with it.
|
|
815
|
+
* seconds, and a thousand changed rows in ten thousand took five times as long with it. Autovacuum
|
|
816
|
+
* analyzes a partition once 2% of it changed (see `ensureObjectPartition`), and an ANALYZE here
|
|
817
|
+
* resets that count, so the same change is never analyzed twice.
|
|
816
818
|
*/
|
|
817
819
|
async function refreshObjectStatistics(
|
|
818
820
|
sql: SQLClient,
|
package/src/pg-client.ts
CHANGED
|
@@ -88,6 +88,10 @@ export function createPgClient(options: CreatePgClientOptions): SQL {
|
|
|
88
88
|
// index the first key is a null rank with one value, so a query with a sparse filter reads
|
|
89
89
|
// every row of the type where the planner expected a few.
|
|
90
90
|
enable_incremental_sort: "off",
|
|
91
|
+
// Reads are page-sized, so compiling a plan never pays for itself. A plan over many partitions
|
|
92
|
+
// crosses the JIT cost threshold anyway, and compiling it took seconds and enough memory for
|
|
93
|
+
// the server to kill the backend, which drops every connection.
|
|
94
|
+
jit: "off",
|
|
91
95
|
}
|
|
92
96
|
if (options.statementTimeoutMillis !== undefined) {
|
|
93
97
|
connection.statement_timeout = options.statementTimeoutMillis
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import type { JsonValue } from "@sixb/core"
|
|
2
2
|
import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
|
|
3
3
|
import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider"
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
canFinishPipelineRun,
|
|
6
|
+
canRequeuePipelineRunAfterEnqueueFailure,
|
|
7
|
+
} from "@sixb/core/internal/storage"
|
|
5
8
|
import type { DatasetVersionRef } from "@sixb/core/lake-storage"
|
|
6
9
|
import type {
|
|
7
10
|
ExecutionStorage,
|
|
@@ -137,12 +140,7 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
137
140
|
)
|
|
138
141
|
}
|
|
139
142
|
|
|
140
|
-
|
|
141
|
-
input.status === "failed" && input.error.code === "queue.enqueue_failed"
|
|
142
|
-
if (
|
|
143
|
-
(enqueueFailure && existing.status !== "queued") ||
|
|
144
|
-
(!enqueueFailure && existing.status !== "running")
|
|
145
|
-
) {
|
|
143
|
+
if (!canFinishPipelineRun(existing.status, input)) {
|
|
146
144
|
throw new PipelineRunError(
|
|
147
145
|
`[SixbPg] Pipeline run '${input.id}' cannot finish from status '${existing.status}'.`
|
|
148
146
|
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { JsonValue } from "@sixb/core"
|
|
2
2
|
import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
|
|
3
|
-
import { canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
|
|
3
|
+
import { canFinishSyncRun, canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
|
|
4
4
|
import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider"
|
|
5
5
|
import type {
|
|
6
6
|
ExecutionStorage,
|
|
@@ -134,12 +134,7 @@ export class PgSyncRunStorage implements SyncRunStorage {
|
|
|
134
134
|
)
|
|
135
135
|
}
|
|
136
136
|
|
|
137
|
-
|
|
138
|
-
input.status === "failed" && input.error.code === "queue.enqueue_failed"
|
|
139
|
-
if (
|
|
140
|
-
(enqueueFailure && existing.status !== "queued") ||
|
|
141
|
-
(!enqueueFailure && existing.status !== "running")
|
|
142
|
-
) {
|
|
137
|
+
if (!canFinishSyncRun(existing.status, input)) {
|
|
143
138
|
throw new SyncRunError(
|
|
144
139
|
`[SixbPg] Sync run '${input.id}' cannot finish from status '${existing.status}'.`
|
|
145
140
|
)
|