@sixb/pg 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +9 -8
  2. package/dist/agents/computers.d.ts +8 -2
  3. package/dist/agents/computers.d.ts.map +1 -1
  4. package/dist/index.js +566 -144
  5. package/dist/index.js.map +20 -17
  6. package/dist/migrations.d.ts.map +1 -1
  7. package/dist/objects/conditional-reader.d.ts +7 -2
  8. package/dist/objects/conditional-reader.d.ts.map +1 -1
  9. package/dist/objects/partitions.d.ts.map +1 -1
  10. package/dist/objects/prepare-queries.d.ts.map +1 -1
  11. package/dist/objects/query-compiler.d.ts +14 -1
  12. package/dist/objects/query-compiler.d.ts.map +1 -1
  13. package/dist/objects/query-indexes.d.ts +13 -1
  14. package/dist/objects/query-indexes.d.ts.map +1 -1
  15. package/dist/objects/read-scope.d.ts.map +1 -1
  16. package/dist/objects/reader.d.ts +14 -1
  17. package/dist/objects/reader.d.ts.map +1 -1
  18. package/dist/objects/storage.d.ts +1 -0
  19. package/dist/objects/storage.d.ts.map +1 -1
  20. package/dist/objects/visibility-indexes.d.ts +22 -0
  21. package/dist/objects/visibility-indexes.d.ts.map +1 -0
  22. package/dist/objects/visibility-statistics.d.ts +26 -0
  23. package/dist/objects/visibility-statistics.d.ts.map +1 -0
  24. package/dist/objects/visibility.d.ts +34 -0
  25. package/dist/objects/visibility.d.ts.map +1 -0
  26. package/dist/pg-client.d.ts.map +1 -1
  27. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  28. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  29. package/package.json +3 -3
  30. package/src/agents/computers.ts +54 -5
  31. package/src/migrations/087-agent-computer-background-commands.sql +4 -0
  32. package/src/migrations/088-web-search-model-kind.sql +12 -0
  33. package/src/migrations.ts +6 -0
  34. package/src/objects/conditional-reader.ts +26 -4
  35. package/src/objects/partitions.ts +17 -4
  36. package/src/objects/prepare-queries.ts +8 -3
  37. package/src/objects/query-compiler.ts +64 -23
  38. package/src/objects/query-indexes.ts +93 -27
  39. package/src/objects/read-scope.ts +147 -77
  40. package/src/objects/reader.ts +46 -18
  41. package/src/objects/storage.ts +4 -0
  42. package/src/objects/visibility-indexes.ts +222 -0
  43. package/src/objects/visibility-statistics.ts +73 -0
  44. package/src/objects/visibility.ts +135 -0
  45. package/src/ontology-storage/materializations.ts +3 -1
  46. package/src/pg-client.ts +4 -0
  47. package/src/pg-pipeline-run-storage.ts +5 -7
  48. package/src/pg-sync-run-storage.ts +2 -7
@@ -0,0 +1,73 @@
1
+ import type { ObjectReadCondition } from "@sixb/core/storage"
2
+ import type { SQLClient } from "../pg-client"
3
+ import { MAX_KEYED_ID_LENGTH, pgVisibilityReaderKeys } from "./visibility"
4
+ import { visibilityStatisticsName } from "./visibility-indexes"
5
+
6
+ // How long a read of the statistics serves queries before they are read again.
7
+ const VISIBILITY_STATISTICS_TTL_MS = 60_000
8
+
9
+ /**
10
+ * The keys the statistics of each view condition list as most common, read at most once a minute.
11
+ *
12
+ * ANALYZE lists every key above a small share of the rows. PostgreSQL estimates one it does not
13
+ * list at about half a percent of them, whatever its real share: a reader who sees nothing looks
14
+ * like one who sees thousands of objects.
15
+ */
16
+ export class PgVisibilityStatisticsCatalog {
17
+ #read:
18
+ | {
19
+ readonly at: number
20
+ readonly keys: Promise<ReadonlyMap<string, ReadonlySet<string>>>
21
+ }
22
+ | undefined
23
+
24
+ /** Read through the caller's client, so a transaction never waits for a second connection. */
25
+ commonKeys(sql: SQLClient): Promise<ReadonlyMap<string, ReadonlySet<string>>> {
26
+ const now = Date.now()
27
+ if (this.#read && now - this.#read.at < VISIBILITY_STATISTICS_TTL_MS) return this.#read.keys
28
+
29
+ const read = { at: now, keys: readCommonKeys(sql) }
30
+ this.#read = read
31
+ // A failed read is not kept: the next query reads again.
32
+ read.keys.catch(() => {
33
+ if (this.#read === read) this.#read = undefined
34
+ })
35
+ return read.keys
36
+ }
37
+ }
38
+
39
+ async function readCommonKeys(sql: SQLClient): Promise<ReadonlyMap<string, ReadonlySet<string>>> {
40
+ const rows = await sql<{ name: string; keys: string[] | null }[]>`
41
+ SELECT statistics_name AS name, most_common_elems::text::text[] AS keys
42
+ FROM pg_stats_ext_exprs
43
+ WHERE schemaname = current_schema() AND statistics_name LIKE 'sixb\\_visibility\\_stats\\_%'`
44
+
45
+ return new Map(rows.map((row) => [row.name, new Set(row.keys ?? [])]))
46
+ }
47
+
48
+ /**
49
+ * Whether a reader sees so few objects of a type that its ordered pages read the objects it sees
50
+ * first, then sort them.
51
+ *
52
+ * Walking a sort index stops at a page's last object, so it is the right plan for a reader who sees
53
+ * many. For one who sees few, it checks nearly every object, and PostgreSQL cannot tell them apart
54
+ * when the statistics list none of the reader's keys. Such a reader sees few objects: those its
55
+ * keys meet are under the share ANALYZE lists. Without statistics, or with a remainder read object
56
+ * by object, the reader's share is unknown and PostgreSQL chooses.
57
+ */
58
+ export function readsFewObjects(
59
+ condition: ObjectReadCondition,
60
+ common: ReadonlyMap<string, ReadonlySet<string>>
61
+ ): boolean {
62
+ const visibility = condition.visibility
63
+ if (!visibility || visibility.remainder) return false
64
+ if ((visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH) return false
65
+
66
+ const readerKeys = pgVisibilityReaderKeys(visibility.user)
67
+
68
+ for (const keys of visibility.keys) {
69
+ const listed = common.get(visibilityStatisticsName(condition.objectTypeId, keys))
70
+ if (!listed || readerKeys.some((key) => listed.has(key))) return false
71
+ }
72
+ return true
73
+ }
@@ -0,0 +1,135 @@
1
+ import type { UserRef } from "@sixb/core"
2
+ import type { ObjectReadCondition, ObjectVisibilityKeys } from "@sixb/core/storage"
3
+ import { compilePredicate } from "./query-compiler"
4
+ import { literal } from "./sql-text"
5
+
6
+ /**
7
+ * Visibility keys as SQL: one expression over `properties` returning who the condition shows the
8
+ * row to, `*` for everyone holding it and `u:<id>` for a user.
9
+ *
10
+ * The expression is IMMUTABLE and holds its values inline, so PostgreSQL can index it and match a
11
+ * read that repeats it, whatever the reader. A user's id is written as jsonb prints it between its
12
+ * quotes: that is how a list's ids come out of `jsonb_path_query_array` with no subquery, which an
13
+ * index expression cannot hold. An escaped quote holds a `"`, so `\\` and `\"` are first written as
14
+ * control characters jsonb never prints raw: `", "` then only separates two ids, and `\x1f` never
15
+ * appears in one. A reader's key is written the same way.
16
+ *
17
+ * A GIN entry holds at most 2,712 bytes, and jsonb prints a character in up to 6 bytes, so ids
18
+ * longer than `MAX_KEYED_ID_LENGTH` characters get no key: a reader with such an id reads the
19
+ * predicate instead. A short list keeps its long ids, which no reader looks up, rather than run a
20
+ * regex on each of its ids.
21
+ *
22
+ * Reads number every `?` of a statement as a parameter, so none is written: text goes in `E''`
23
+ * strings, where `?` is written `\x3f`.
24
+ */
25
+ export function compilePgVisibilityKeys(keys: ObjectVisibilityKeys): string {
26
+ switch (keys.kind) {
27
+ case "everyone":
28
+ return `CASE WHEN ${predicateSql(keys.when)} THEN ARRAY[${text(EVERYONE)}]::text[] ELSE ${NO_KEYS} END`
29
+
30
+ case "user": {
31
+ // `eq` compares the whole reference, so the property holds exactly a type and a string id.
32
+ const ref = propertySql(keys.propertyId)
33
+ return `CASE WHEN jsonb_typeof(${ref}) = 'object' AND (${ref} - 'type' - 'id') = '{}'::jsonb AND ${ref} -> 'type' = '"user"'::jsonb AND jsonb_typeof(${ref} -> 'id') = 'string' AND char_length(${ref} ->> 'id') <= ${MAX_KEYED_ID_LENGTH} THEN ARRAY[${text(USER_PREFIX)} || ${withoutEscapes(`left(substr((${ref} -> 'id')::text, 2), -1)`)}] ELSE ${NO_KEYS} END`
34
+ }
35
+
36
+ case "users": {
37
+ // Strict mode keeps the elements `@>` matches: objects whose type is "user" and id a string.
38
+ const users = `@.type == "user" && @.id.type() == "string"`
39
+ const keysOf = (filter: string) => {
40
+ const path = text(`strict $[*] ? (${filter}).id`)
41
+ const ids = withoutEscapes(
42
+ `jsonb_path_query_array(${propertySql(keys.propertyId)}, ${path}, '{}', true)::text`
43
+ )
44
+ const separator = `E'\\x1f'`
45
+ return `coalesce(string_to_array(${text(USER_PREFIX)} || replace(NULLIF(left(substr(${ids}, 3), -2), ''), '", "', ${separator} || ${text(USER_PREFIX)}), ${separator}), ${NO_KEYS})`
46
+ }
47
+
48
+ // No id of a short list is too long for an entry: only a longer one pays for the regex.
49
+ const bounded = `${users} && @.id like_regex "^.{0,${MAX_KEYED_ID_LENGTH}}$" flag "s"`
50
+ return `CASE WHEN octet_length(${propertySql(keys.propertyId)}::text) <= ${SHORT_LIST_BYTES} THEN ${keysOf(users)} ELSE ${keysOf(bounded)} END`
51
+ }
52
+
53
+ case "union":
54
+ return `(${keys.items.map((item) => `(${compilePgVisibilityKeys(item)})`).join(" || ")})`
55
+
56
+ case "when":
57
+ return `CASE WHEN ${predicateSql(keys.when)} THEN ${compilePgVisibilityKeys(keys.keys)} ELSE ${NO_KEYS} END`
58
+ }
59
+ }
60
+
61
+ /** The keys a reader holds: everyone's, and its user's. */
62
+ export function pgVisibilityReaderKeys(user: UserRef | undefined): string[] {
63
+ if (!user) return [EVERYONE]
64
+
65
+ // As `withoutEscapes` writes the ids the keys hold.
66
+ const id = JSON.stringify(user.id)
67
+ .slice(1, -1)
68
+ .replaceAll("\\\\", ESCAPED_BACKSLASH)
69
+ .replaceAll('\\"', ESCAPED_QUOTE)
70
+ return [EVERYONE, `${USER_PREFIX}${id}`]
71
+ }
72
+
73
+ /**
74
+ * A conditioned type's filter: each condition's keys meet the reader's, or the remainder matches.
75
+ * Without keys, or for a user whose id is too long to have one, the condition's predicate.
76
+ */
77
+ export function compilePgConditionFilter(condition: ObjectReadCondition): {
78
+ readonly sql: string
79
+ readonly args: readonly unknown[]
80
+ } {
81
+ const visibility = condition.visibility
82
+ // UTF-16 code units, never fewer than the characters PostgreSQL counts.
83
+ if (!visibility || (visibility.user?.id.length ?? 0) > MAX_KEYED_ID_LENGTH) {
84
+ return compilePredicate(condition.predicate)
85
+ }
86
+
87
+ const readerKeys = pgVisibilityReaderKeys(visibility.user)
88
+ const clauses = visibility.keys.map((keys) => `((${compilePgVisibilityKeys(keys)}) && ?::text[])`)
89
+ const args: unknown[] = visibility.keys.map(() => readerKeys)
90
+
91
+ if (visibility.remainder) {
92
+ const remainder = compilePredicate(visibility.remainder)
93
+ clauses.push(remainder.sql)
94
+ args.push(...remainder.args)
95
+ }
96
+
97
+ return { sql: `(${clauses.join(" OR ")})`, args }
98
+ }
99
+
100
+ export const MAX_KEYED_ID_LENGTH = 200
101
+ const ESCAPED_BACKSLASH = "\x1e"
102
+ const ESCAPED_QUOTE = "\x1d"
103
+ // A list whose JSON is this short holds no id whose key would overflow an entry.
104
+ const SHORT_LIST_BYTES = 2000
105
+ const EVERYONE = "*"
106
+ const USER_PREFIX = "u:"
107
+ const NO_KEYS = "'{}'::text[]"
108
+
109
+ function propertySql(propertyId: string): string {
110
+ return `(properties -> ${text(propertyId)}::text)`
111
+ }
112
+
113
+ /** jsonb's `\\` and `\"` in printed ids, as control characters it never prints raw. */
114
+ function withoutEscapes(printed: string): string {
115
+ return `replace(replace(${printed}, ${text("\\\\")}, E'\\x1e'), ${text('\\"')}, E'\\x1d')`
116
+ }
117
+
118
+ function text(value: string): string {
119
+ return literal(value).replace(/\?/g, "\\x3f")
120
+ }
121
+
122
+ /** The reader's own predicate compiler, so a part means exactly what the condition does. */
123
+ function predicateSql(predicate: Parameters<typeof compilePredicate>[0]): string {
124
+ const compiled = compilePredicate(predicate)
125
+ let index = 0
126
+ return `(${compiled.sql.replace(/\?/g, () => inlineValue(compiled.args[index++]))})`
127
+ }
128
+
129
+ /** The values the predicate compiler binds: property ids and compared values, text or numbers. */
130
+ function inlineValue(value: unknown): string {
131
+ if (typeof value === "string") return text(value)
132
+ if (typeof value === "number" && Number.isFinite(value)) return `(${String(value)})`
133
+
134
+ throw new Error(`[SixbPg] A view condition compares a value it cannot index: ${String(value)}.`)
135
+ }
@@ -812,7 +812,9 @@ async function refreshMaterializedStatistics(
812
812
  * partitioned parent, and its statistics plan queries that read across types. A smaller one only
813
813
  * refreshes partitions it changed substantially, which autovacuum may not reach before the next
814
814
  * projection. A routine update must not: ANALYZE over a partition's expression statistics takes
815
- * seconds, and a thousand changed rows in ten thousand took five times as long with it.
815
+ * seconds, and a thousand changed rows in ten thousand took five times as long with it. Autovacuum
816
+ * analyzes a partition once 2% of it changed (see `ensureObjectPartition`), and an ANALYZE here
817
+ * resets that count, so the same change is never analyzed twice.
816
818
  */
817
819
  async function refreshObjectStatistics(
818
820
  sql: SQLClient,
package/src/pg-client.ts CHANGED
@@ -88,6 +88,10 @@ export function createPgClient(options: CreatePgClientOptions): SQL {
88
88
  // index the first key is a null rank with one value, so a query with a sparse filter reads
89
89
  // every row of the type where the planner expected a few.
90
90
  enable_incremental_sort: "off",
91
+ // Reads are page-sized, so compiling a plan never pays for itself. A plan over many partitions
92
+ // crosses the JIT cost threshold anyway, and compiling it took seconds and enough memory for
93
+ // the server to kill the backend, which drops every connection.
94
+ jit: "off",
91
95
  }
92
96
  if (options.statementTimeoutMillis !== undefined) {
93
97
  connection.statement_timeout = options.statementTimeoutMillis
@@ -1,7 +1,10 @@
1
1
  import type { JsonValue } from "@sixb/core"
2
2
  import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
3
3
  import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider"
4
- import { canRequeuePipelineRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
4
+ import {
5
+ canFinishPipelineRun,
6
+ canRequeuePipelineRunAfterEnqueueFailure,
7
+ } from "@sixb/core/internal/storage"
5
8
  import type { DatasetVersionRef } from "@sixb/core/lake-storage"
6
9
  import type {
7
10
  ExecutionStorage,
@@ -137,12 +140,7 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
137
140
  )
138
141
  }
139
142
 
140
- const enqueueFailure =
141
- input.status === "failed" && input.error.code === "queue.enqueue_failed"
142
- if (
143
- (enqueueFailure && existing.status !== "queued") ||
144
- (!enqueueFailure && existing.status !== "running")
145
- ) {
143
+ if (!canFinishPipelineRun(existing.status, input)) {
146
144
  throw new PipelineRunError(
147
145
  `[SixbPg] Pipeline run '${input.id}' cannot finish from status '${existing.status}'.`
148
146
  )
@@ -1,6 +1,6 @@
1
1
  import type { JsonValue } from "@sixb/core"
2
2
  import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
3
- import { canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
3
+ import { canFinishSyncRun, canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
4
4
  import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider"
5
5
  import type {
6
6
  ExecutionStorage,
@@ -134,12 +134,7 @@ export class PgSyncRunStorage implements SyncRunStorage {
134
134
  )
135
135
  }
136
136
 
137
- const enqueueFailure =
138
- input.status === "failed" && input.error.code === "queue.enqueue_failed"
139
- if (
140
- (enqueueFailure && existing.status !== "queued") ||
141
- (!enqueueFailure && existing.status !== "running")
142
- ) {
137
+ if (!canFinishSyncRun(existing.status, input)) {
143
138
  throw new SyncRunError(
144
139
  `[SixbPg] Sync run '${input.id}' cannot finish from status '${existing.status}'.`
145
140
  )