@sixb/pg 0.1.26 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -1
- package/dist/agents/agent-storage.d.ts +8 -3
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/computers.d.ts +22 -0
- package/dist/agents/computers.d.ts.map +1 -0
- package/dist/agents/rows.d.ts +3 -0
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/agents/threads.d.ts +2 -1
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/agents/token-rates.d.ts +13 -0
- package/dist/agents/token-rates.d.ts.map +1 -0
- package/dist/ai-accounting-filters.d.ts +12 -0
- package/dist/ai-accounting-filters.d.ts.map +1 -0
- package/dist/ai-cost-groups.d.ts +1 -1
- package/dist/ai-cost-groups.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +886 -158
- package/dist/index.js.map +26 -21
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/partitions.d.ts +10 -0
- package/dist/objects/partitions.d.ts.map +1 -0
- package/dist/objects/prepare-queries.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +5 -0
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/sql-text.d.ts +3 -0
- package/dist/objects/sql-text.d.ts.map +1 -0
- package/dist/objects/text-count-index.d.ts.map +1 -1
- package/dist/ontology-storage/commits.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +7 -1
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +2 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +2 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-ai-cost-storage.d.ts.map +1 -1
- package/dist/pg-client.d.ts +1 -1
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/storage-errors.d.ts +4 -1
- package/dist/storage-errors.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/agent-storage.ts +10 -3
- package/src/agents/computers.ts +95 -0
- package/src/agents/rows.ts +9 -1
- package/src/agents/runs.ts +25 -3
- package/src/agents/threads.ts +111 -20
- package/src/agents/token-rates.ts +76 -0
- package/src/ai-accounting-filters.ts +48 -0
- package/src/ai-cost-groups.ts +37 -12
- package/src/index.ts +1 -1
- package/src/migrations/084-agent-token-rates.sql +16 -0
- package/src/migrations/085-agent-computer-sessions.sql +27 -0
- package/src/migrations/086-partition-objects.sql +93 -0
- package/src/migrations.ts +8 -0
- package/src/objects/partitions.ts +163 -0
- package/src/objects/prepare-queries.ts +50 -2
- package/src/objects/query-compiler.ts +80 -66
- package/src/objects/query-indexes.ts +59 -46
- package/src/objects/sql-text.ts +8 -0
- package/src/objects/text-count-index.ts +17 -12
- package/src/ontology-storage/commits.ts +11 -2
- package/src/ontology-storage/materialization-session.ts +10 -1
- package/src/ontology-storage/materialization-writer.ts +14 -0
- package/src/ontology-storage/materializations.ts +81 -7
- package/src/pg-ai-cost-storage.ts +119 -15
- package/src/pg-client.ts +9 -2
- package/src/storage-errors.ts +28 -1
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { createHash } from "node:crypto"
|
|
2
|
+
import type { ReservedSQL } from "../pg-client"
|
|
3
|
+
import { isLockConflict } from "../storage-errors"
|
|
4
|
+
import { runPgTransactionOn } from "../transactions"
|
|
5
|
+
import { identifier, literal } from "./sql-text"
|
|
6
|
+
|
|
7
|
+
// A query reading every partition, and any publication, holds the default partition until it
|
|
8
|
+
// commits, which can take minutes. While this waits such queries queue behind it, so it waits a
|
|
9
|
+
// moment at a time and steps aside between attempts: about four minutes in all.
|
|
10
|
+
const LOCK_TIMEOUT = "1s"
|
|
11
|
+
const LOCK_ATTEMPTS = 120
|
|
12
|
+
const LOCK_RETRY_MS = 1_000
|
|
13
|
+
|
|
14
|
+
/** Migration 086 derives the same name in SQL: `'objects_' || left(md5(type), 24)`. */
|
|
15
|
+
export function objectPartitionName(objectTypeId: string): string {
|
|
16
|
+
return `objects_${createHash("md5").update(objectTypeId).digest("hex").slice(0, 24)}`
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* The object type's partition, created on first use. Reads that name another type never wait. A
|
|
21
|
+
* short lock timeout and retries mean this gives way to long queries and publications instead of
|
|
22
|
+
* queuing other work behind itself, and preparation fails cleanly if they outlast every attempt.
|
|
23
|
+
*/
|
|
24
|
+
export async function ensureObjectPartition(
|
|
25
|
+
connection: ReservedSQL,
|
|
26
|
+
schema: string,
|
|
27
|
+
objectTypeId: string
|
|
28
|
+
): Promise<string> {
|
|
29
|
+
const name = objectPartitionName(objectTypeId)
|
|
30
|
+
const partition = `${identifier(schema)}.${identifier(name)}`
|
|
31
|
+
const [existing] = await connection<{ found: string | null }[]>`
|
|
32
|
+
SELECT to_regclass(${partition})::text AS found`
|
|
33
|
+
if (existing?.found) return name
|
|
34
|
+
|
|
35
|
+
for (let attempt = 1; ; attempt++) {
|
|
36
|
+
try {
|
|
37
|
+
await createObjectPartition(connection, schema, partition, objectTypeId)
|
|
38
|
+
return name
|
|
39
|
+
} catch (error) {
|
|
40
|
+
if (!isLockConflict(error)) throw error
|
|
41
|
+
if (attempt === LOCK_ATTEMPTS) {
|
|
42
|
+
throw new Error(
|
|
43
|
+
`[SixbPg] Could not add a partition for object type '${objectTypeId}': other queries held their locks through ${LOCK_ATTEMPTS} attempts. Retry when they finish.`,
|
|
44
|
+
{ cause: error }
|
|
45
|
+
)
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
await Bun.sleep(LOCK_RETRY_MS)
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async function createObjectPartition(
|
|
53
|
+
connection: ReservedSQL,
|
|
54
|
+
schema: string,
|
|
55
|
+
partition: string,
|
|
56
|
+
objectTypeId: string
|
|
57
|
+
): Promise<void> {
|
|
58
|
+
const parent = `${identifier(schema)}.objects`
|
|
59
|
+
const fallback = `${identifier(schema)}.objects_default`
|
|
60
|
+
const type = literal(objectTypeId)
|
|
61
|
+
const hasRows = async () => {
|
|
62
|
+
const [row] = await connection.unsafe(
|
|
63
|
+
`SELECT 1 FROM ${fallback} WHERE object_type_id = ${type} LIMIT 1`
|
|
64
|
+
)
|
|
65
|
+
return row !== undefined
|
|
66
|
+
}
|
|
67
|
+
const guarded = (run: () => Promise<void>) =>
|
|
68
|
+
runPgTransactionOn(connection, async () => {
|
|
69
|
+
await connection.unsafe(`SET LOCAL lock_timeout = '${LOCK_TIMEOUT}'`)
|
|
70
|
+
// Copying rows takes as long as it takes, whatever timeout this role's queries run under.
|
|
71
|
+
await connection.unsafe("SET LOCAL statement_timeout = 0")
|
|
72
|
+
await run()
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
let moving = false
|
|
76
|
+
await guarded(async () => {
|
|
77
|
+
moving = await hasRows()
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
// Moving rows must drop foreign keys, which takes the parent exclusively, so it asks for that from
|
|
81
|
+
// the start rather than upgrading a lock while queries wait behind it. Otherwise attaching only
|
|
82
|
+
// needs SHARE UPDATE EXCLUSIVE on the parent, so reads and writes of other types carry on.
|
|
83
|
+
await guarded(async () => {
|
|
84
|
+
const references = await referencesToObjects(connection, parent)
|
|
85
|
+
await connection.unsafe(
|
|
86
|
+
`LOCK TABLE ONLY ${parent} IN ${moving ? "ACCESS EXCLUSIVE" : "SHARE UPDATE EXCLUSIVE"} MODE`
|
|
87
|
+
)
|
|
88
|
+
// Before the default partition: a writer on these tables goes on to check objects, so taking
|
|
89
|
+
// them in this order leaves no cycle for it to be killed in.
|
|
90
|
+
for (const { table_name: table } of references) {
|
|
91
|
+
await connection.unsafe(`LOCK TABLE ${table} IN SHARE ROW EXCLUSIVE MODE`)
|
|
92
|
+
}
|
|
93
|
+
await connection.unsafe(`LOCK TABLE ${fallback} IN ACCESS EXCLUSIVE MODE`)
|
|
94
|
+
|
|
95
|
+
// A row arrived since the check above, so its type needs the moving path. The next attempt takes it.
|
|
96
|
+
if (!moving && (await hasRows())) {
|
|
97
|
+
throw Object.assign(new Error("[SixbPg] A row arrived while adding its partition."), {
|
|
98
|
+
code: "55P03",
|
|
99
|
+
})
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
await connection.unsafe(
|
|
103
|
+
`CREATE TABLE ${partition} (LIKE ${parent} INCLUDING DEFAULTS INCLUDING GENERATED INCLUDING CONSTRAINTS INCLUDING STORAGE)`
|
|
104
|
+
)
|
|
105
|
+
if (moving) await moveRowsFromDefault(connection, parent, fallback, partition, type, references)
|
|
106
|
+
await connection.unsafe(
|
|
107
|
+
`ALTER TABLE ${parent} ATTACH PARTITION ${partition} FOR VALUES IN (${type})`
|
|
108
|
+
)
|
|
109
|
+
if (moving) {
|
|
110
|
+
for (const reference of references) {
|
|
111
|
+
await connection.unsafe(
|
|
112
|
+
`ALTER TABLE ${reference.table_name} ADD CONSTRAINT ${identifier(reference.name)} ${reference.definition}`
|
|
113
|
+
)
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
// Not under the locks above: every query on every type is waiting behind them.
|
|
119
|
+
if (moving) await guarded(() => connection.unsafe(`ANALYZE ${partition}`).then(() => undefined))
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
interface ObjectReference {
|
|
123
|
+
readonly table_name: string
|
|
124
|
+
readonly name: string
|
|
125
|
+
readonly definition: string
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function referencesToObjects(connection: ReservedSQL, parent: string) {
|
|
129
|
+
return connection<ObjectReference[]>`
|
|
130
|
+
SELECT conrelid::regclass::text AS table_name, conname AS name,
|
|
131
|
+
pg_get_constraintdef(oid) AS definition
|
|
132
|
+
FROM pg_constraint
|
|
133
|
+
WHERE confrelid = ${parent}::regclass AND contype = 'f' AND conparentid = 0`
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Nothing about the rows changes, so foreign keys that reference objects must not see a delete:
|
|
138
|
+
* ON DELETE CASCADE would remove what they protect. The caller adds them back.
|
|
139
|
+
*/
|
|
140
|
+
async function moveRowsFromDefault(
|
|
141
|
+
connection: ReservedSQL,
|
|
142
|
+
parent: string,
|
|
143
|
+
fallback: string,
|
|
144
|
+
partition: string,
|
|
145
|
+
type: string,
|
|
146
|
+
references: readonly ObjectReference[]
|
|
147
|
+
): Promise<void> {
|
|
148
|
+
for (const reference of references) {
|
|
149
|
+
await connection.unsafe(
|
|
150
|
+
`ALTER TABLE ${reference.table_name} DROP CONSTRAINT ${identifier(reference.name)}`
|
|
151
|
+
)
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const columns = await connection<{ name: string }[]>`
|
|
155
|
+
SELECT attname AS name FROM pg_attribute
|
|
156
|
+
WHERE attrelid = ${parent}::regclass AND attnum > 0 AND NOT attisdropped AND attgenerated = ''
|
|
157
|
+
ORDER BY attnum`
|
|
158
|
+
const list = columns.map((column) => identifier(column.name)).join(", ")
|
|
159
|
+
await connection.unsafe(
|
|
160
|
+
`INSERT INTO ${partition} (${list}) SELECT ${list} FROM ${fallback} WHERE object_type_id = ${type}`
|
|
161
|
+
)
|
|
162
|
+
await connection.unsafe(`DELETE FROM ${fallback} WHERE object_type_id = ${type}`)
|
|
163
|
+
}
|
|
@@ -2,7 +2,9 @@ import { createHash } from "node:crypto"
|
|
|
2
2
|
import type { ObjectQueryIndexDefinition } from "@sixb/core"
|
|
3
3
|
import type { ObjectQueryPreparationPlan, ObjectQueryPreparationResult } from "@sixb/core/storage"
|
|
4
4
|
import type { ReservedSQL, SQL, SQLClient } from "../pg-client"
|
|
5
|
+
import { isConnectionLost } from "../storage-errors"
|
|
5
6
|
import { undoOnFailure, withReservedPgConnection } from "../transactions"
|
|
7
|
+
import { ensureObjectPartition } from "./partitions"
|
|
6
8
|
import { lockPgPreparation } from "./preparation-lock"
|
|
7
9
|
import { ensurePgObjectQueryIndexes, type PgObjectQueryIndex } from "./query-indexes"
|
|
8
10
|
import { type PgObjectTextCountIndex, preparePgObjectTextCounts } from "./text-count-index"
|
|
@@ -43,7 +45,10 @@ export async function preparePgObjectQueries(
|
|
|
43
45
|
}, unlock)
|
|
44
46
|
await unlock()
|
|
45
47
|
|
|
46
|
-
|
|
48
|
+
// Advice, so it runs after the lock: other instances starting up need not wait for this scan.
|
|
49
|
+
return result.status === "current"
|
|
50
|
+
? result
|
|
51
|
+
: { ...result, warnings: [...result.warnings, ...(await linkWarnings(connection, plan))] }
|
|
47
52
|
})
|
|
48
53
|
}
|
|
49
54
|
|
|
@@ -55,11 +60,12 @@ function preparationFingerprint(plan: ObjectQueryPreparationPlan): string {
|
|
|
55
60
|
indexes: type.indexes
|
|
56
61
|
.map(normalizeIndex)
|
|
57
62
|
.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
|
|
63
|
+
links: [...type.links].sort((a, b) => a.linkId.localeCompare(b.linkId)),
|
|
58
64
|
}))
|
|
59
65
|
.sort((a, b) => a.objectTypeId.localeCompare(b.objectTypeId))
|
|
60
66
|
|
|
61
67
|
// Bump the version when physical generation changes, even if the ontology stays the same.
|
|
62
|
-
const definition = { version:
|
|
68
|
+
const definition = { version: 3, projectId: plan.projectId, objectTypes }
|
|
63
69
|
return createHash("sha256").update(JSON.stringify(definition)).digest("hex")
|
|
64
70
|
}
|
|
65
71
|
|
|
@@ -106,6 +112,9 @@ async function prepareStructures(
|
|
|
106
112
|
await ensureTrigrams(sql)
|
|
107
113
|
}
|
|
108
114
|
|
|
115
|
+
// Every ontology type gets its partition, whether or not it declares an index.
|
|
116
|
+
for (const type of plan.objectTypes) await ensureObjectPartition(sql, schema, type.objectTypeId)
|
|
117
|
+
|
|
109
118
|
const names = await ensurePgObjectQueryIndexes(sql, schema, indexes)
|
|
110
119
|
const texts = await preparePgObjectTextCounts(sql, schema, textCounts)
|
|
111
120
|
return {
|
|
@@ -116,6 +125,45 @@ async function prepareStructures(
|
|
|
116
125
|
}
|
|
117
126
|
}
|
|
118
127
|
|
|
128
|
+
/**
|
|
129
|
+
* Queries read a link as the ontology declares it, so narrowing or removing one hides its older
|
|
130
|
+
* edges. Told once, when the ontology changes: not stored with the plan, so it cannot outlive the
|
|
131
|
+
* edges. This is advice, so failing to work it out does not fail the preparation.
|
|
132
|
+
*/
|
|
133
|
+
async function linkWarnings(
|
|
134
|
+
sql: ReservedSQL,
|
|
135
|
+
plan: ObjectQueryPreparationPlan
|
|
136
|
+
): Promise<readonly string[]> {
|
|
137
|
+
const declared = new Map(
|
|
138
|
+
plan.objectTypes.flatMap((type) =>
|
|
139
|
+
type.links.map((link) => [`${type.objectTypeId}\0${link.linkId}`, link] as const)
|
|
140
|
+
)
|
|
141
|
+
)
|
|
142
|
+
try {
|
|
143
|
+
const edges = await sql<
|
|
144
|
+
{ source_type_id: string; link_id: string; target_type_id: string; edges: string }[]
|
|
145
|
+
>`
|
|
146
|
+
SELECT source_type_id, link_id, target_type_id, COUNT(*) AS edges
|
|
147
|
+
FROM links WHERE project_id = ${plan.projectId}
|
|
148
|
+
GROUP BY source_type_id, link_id, target_type_id`
|
|
149
|
+
|
|
150
|
+
return edges.flatMap((edge) => {
|
|
151
|
+
const name = `${edge.source_type_id}.${edge.link_id}`
|
|
152
|
+
const link = declared.get(`${edge.source_type_id}\0${edge.link_id}`)
|
|
153
|
+
if (!link) {
|
|
154
|
+
return `${name}: ${edge.edges} stored edges to ${edge.target_type_id} belong to a link the ontology no longer declares. Queries may not return them.`
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return !link.targetObjectTypeIds || link.targetObjectTypeIds.includes(edge.target_type_id)
|
|
158
|
+
? []
|
|
159
|
+
: `${name}: ${edge.edges} stored edges point to ${edge.target_type_id}, which this link no longer allows. Queries may not return them.`
|
|
160
|
+
})
|
|
161
|
+
} catch (error) {
|
|
162
|
+
if (isConnectionLost(error)) throw error
|
|
163
|
+
return [`Could not check stored links against the ontology: ${String(error)}`]
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
119
167
|
function compilePreparationPlan(plan: ObjectQueryPreparationPlan) {
|
|
120
168
|
const indexes: PgObjectQueryIndex[] = []
|
|
121
169
|
const textCounts: PgObjectTextCountIndex[] = []
|
|
@@ -5,11 +5,13 @@ import {
|
|
|
5
5
|
type ObjectQueryPredicate,
|
|
6
6
|
type ObjectQuerySetOperation,
|
|
7
7
|
type ObjectQuerySortField,
|
|
8
|
+
type ObjectQueryTraverse,
|
|
8
9
|
type QueryScalarKind,
|
|
9
10
|
} from "@sixb/core"
|
|
10
11
|
import {
|
|
11
12
|
hasVectorProfile,
|
|
12
13
|
isVectorProfileQuery,
|
|
14
|
+
linkEndpointObjectTypeIds,
|
|
13
15
|
normalizeVector,
|
|
14
16
|
vectorSearchCandidateLimit,
|
|
15
17
|
} from "@sixb/core/internal/query"
|
|
@@ -395,14 +397,7 @@ function compileObjectQueryInternal(
|
|
|
395
397
|
case "page":
|
|
396
398
|
return compilePage(projectId, query.input, query.pageSize, query.pageToken, ctx)
|
|
397
399
|
case "traverse":
|
|
398
|
-
return compileTraversal(
|
|
399
|
-
projectId,
|
|
400
|
-
query.input,
|
|
401
|
-
query.linkId,
|
|
402
|
-
query.direction,
|
|
403
|
-
query.sourceObjectTypeId,
|
|
404
|
-
ctx
|
|
405
|
-
)
|
|
400
|
+
return compileTraversal(projectId, query, ctx)
|
|
406
401
|
case "set":
|
|
407
402
|
return compileSet(projectId, query.op, query.inputs, ctx)
|
|
408
403
|
case "project":
|
|
@@ -502,7 +497,7 @@ function compileStart(
|
|
|
502
497
|
const order = compileOrder(identityOrderFields())
|
|
503
498
|
const objects = pgObjectsOfType(ctx.source, query.objectTypeId)
|
|
504
499
|
const sql = `
|
|
505
|
-
SELECT
|
|
500
|
+
SELECT *
|
|
506
501
|
FROM ${objects.table}
|
|
507
502
|
WHERE project_id = ? AND object_type_id = ? ${objects.filter}
|
|
508
503
|
${ctx.startPredicate ? `AND (${ctx.startPredicate})` : ""}
|
|
@@ -548,7 +543,7 @@ function compileRefs(
|
|
|
548
543
|
FROM jsonb_array_elements(?::text::jsonb) AS ref(value)
|
|
549
544
|
`
|
|
550
545
|
const sql = `
|
|
551
|
-
SELECT selected
|
|
546
|
+
SELECT selected.*
|
|
552
547
|
FROM (${requested}) AS requested
|
|
553
548
|
JOIN ${ctx.source.objectsTable} AS selected
|
|
554
549
|
ON selected.project_id = ?
|
|
@@ -660,7 +655,6 @@ function compileSort(
|
|
|
660
655
|
input.object_type_id,
|
|
661
656
|
input.primary_id,
|
|
662
657
|
input.properties,
|
|
663
|
-
input.properties AS _cursor_properties,
|
|
664
658
|
input.created_at,
|
|
665
659
|
input.updated_at,
|
|
666
660
|
input.version,
|
|
@@ -794,13 +788,10 @@ function compilePage(
|
|
|
794
788
|
|
|
795
789
|
function compileTraversal(
|
|
796
790
|
projectId: string,
|
|
797
|
-
|
|
798
|
-
linkId: string,
|
|
799
|
-
direction: "outgoing" | "incoming",
|
|
800
|
-
sourceObjectTypeId: string | undefined,
|
|
791
|
+
traversal: ObjectQueryTraverse,
|
|
801
792
|
ctx: CompileContext
|
|
802
793
|
): CompiledPgObjectQuery {
|
|
803
|
-
const input = compileObjectQueryInternal(projectId,
|
|
794
|
+
const input = compileObjectQueryInternal(projectId, traversal.input, {
|
|
804
795
|
...exactContext(ctx),
|
|
805
796
|
unordered: true,
|
|
806
797
|
startPredicate: undefined,
|
|
@@ -809,16 +800,14 @@ function compileTraversal(
|
|
|
809
800
|
const relation = compileTraversalRelation(
|
|
810
801
|
projectId,
|
|
811
802
|
input,
|
|
812
|
-
|
|
813
|
-
direction,
|
|
814
|
-
sourceObjectTypeId,
|
|
803
|
+
traversal,
|
|
815
804
|
ctx.source,
|
|
816
805
|
ctx.correlatedTraversal,
|
|
817
806
|
ctx.startPredicate
|
|
818
807
|
)
|
|
819
808
|
const order = compileOrder(identityOrderFields())
|
|
820
809
|
return {
|
|
821
|
-
sql: `SELECT selected
|
|
810
|
+
sql: `SELECT selected.* ${relation.sql} ${ctx.unordered ? "" : `ORDER BY ${order.sql}`} ${ctx.isolateSortOverflow ? "OFFSET 0" : ""}`,
|
|
822
811
|
args: [...relation.args, ...(ctx.unordered ? [] : order.args)],
|
|
823
812
|
totalSql: `SELECT COUNT(*)::bigint AS total ${relation.sql}`,
|
|
824
813
|
totalArgs: relation.args,
|
|
@@ -833,21 +822,22 @@ function compileTraversal(
|
|
|
833
822
|
function compileTraversalRelation(
|
|
834
823
|
projectId: string,
|
|
835
824
|
input: CompiledAggregateSource,
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
825
|
+
traversal: Pick<
|
|
826
|
+
ObjectQueryTraverse,
|
|
827
|
+
"linkId" | "direction" | "sourceObjectTypeId" | "endpointObjectTypeIds"
|
|
828
|
+
>,
|
|
839
829
|
source: PgObjectQuerySource,
|
|
840
830
|
correlated = false,
|
|
841
831
|
endpointPredicate?: string
|
|
842
832
|
): CompiledAggregateSource {
|
|
843
|
-
const incoming = direction === "incoming"
|
|
833
|
+
const incoming = traversal.direction === "incoming"
|
|
844
834
|
const parent = incoming ? "target" : "source"
|
|
845
835
|
const endpoint = incoming ? "source" : "target"
|
|
846
|
-
const
|
|
836
|
+
const endpointType = compileEndpointTypes("selected", linkEndpointObjectTypeIds(traversal))
|
|
847
837
|
return {
|
|
848
838
|
sql: `FROM ${source.objectsTable} AS selected
|
|
849
839
|
WHERE selected.project_id = ?
|
|
850
|
-
${
|
|
840
|
+
${endpointType.sql}
|
|
851
841
|
${endpointPredicate ? `AND (${endpointPredicate})` : ""}
|
|
852
842
|
AND EXISTS (
|
|
853
843
|
SELECT 1 FROM (${input.sql}) AS input
|
|
@@ -861,15 +851,25 @@ function compileTraversalRelation(
|
|
|
861
851
|
AND edge.link_id = ?
|
|
862
852
|
${correlated ? "OFFSET 0" : ""}
|
|
863
853
|
)`,
|
|
864
|
-
args: [
|
|
865
|
-
projectId,
|
|
866
|
-
...(constrainedType === undefined ? [] : [constrainedType]),
|
|
867
|
-
...input.args,
|
|
868
|
-
linkId,
|
|
869
|
-
],
|
|
854
|
+
args: [projectId, ...endpointType.args, ...input.args, traversal.linkId],
|
|
870
855
|
}
|
|
871
856
|
}
|
|
872
857
|
|
|
858
|
+
/** Names the types a link can reach, so PostgreSQL looks only at their rows and their indexes. */
|
|
859
|
+
function compileEndpointTypes(
|
|
860
|
+
alias: string,
|
|
861
|
+
objectTypeIds: readonly string[] | undefined
|
|
862
|
+
): CompiledPredicate {
|
|
863
|
+
if (!objectTypeIds?.length) return { sql: "", args: [] }
|
|
864
|
+
|
|
865
|
+
return objectTypeIds.length === 1
|
|
866
|
+
? { sql: `AND ${alias}.object_type_id = ?`, args: [objectTypeIds[0]] }
|
|
867
|
+
: {
|
|
868
|
+
sql: `AND ${alias}.object_type_id IN (${objectTypeIds.map(() => "?").join(", ")})`,
|
|
869
|
+
args: [...objectTypeIds],
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
|
|
873
873
|
/** Only a bounded page benefits from choosing an ordered, correlated endpoint scan. */
|
|
874
874
|
function boundedTraversal(
|
|
875
875
|
query: ObjectQuery,
|
|
@@ -963,7 +963,11 @@ function expandRows(
|
|
|
963
963
|
"",
|
|
964
964
|
ctx
|
|
965
965
|
)
|
|
966
|
-
const inputOrder = compileOrder(
|
|
966
|
+
const inputOrder = compileOrder(
|
|
967
|
+
input.order.fields,
|
|
968
|
+
"input",
|
|
969
|
+
`input.${input.order.propertyColumn}`
|
|
970
|
+
)
|
|
967
971
|
return {
|
|
968
972
|
sql: `
|
|
969
973
|
SELECT input.*, ${expand.sql} AS _expand
|
|
@@ -1026,7 +1030,8 @@ function compileExpansionValue(
|
|
|
1026
1030
|
`${parentId} = ${parent.id}`,
|
|
1027
1031
|
`${edge}.link_id = ?::text`,
|
|
1028
1032
|
]
|
|
1029
|
-
const
|
|
1033
|
+
const endpointType = compileEndpointTypes(target, expansion.endpointObjectTypeIds)
|
|
1034
|
+
const whereArgs: unknown[] = [...endpointType.args, expansion.linkId]
|
|
1030
1035
|
if (incoming && expansion.sourceObjectTypeId !== undefined) {
|
|
1031
1036
|
whereParts.push(`${edge}.source_type_id = ?::text`)
|
|
1032
1037
|
whereArgs.push(expansion.sourceObjectTypeId)
|
|
@@ -1038,6 +1043,7 @@ function compileExpansionValue(
|
|
|
1038
1043
|
ON ${target}.project_id = ${edge}.project_id
|
|
1039
1044
|
AND ${target}.object_type_id = ${neighborType}
|
|
1040
1045
|
AND ${target}.primary_id = ${neighborId}
|
|
1046
|
+
${endpointType.sql}
|
|
1041
1047
|
WHERE ${whereParts.join(" AND ")}
|
|
1042
1048
|
`
|
|
1043
1049
|
|
|
@@ -1142,7 +1148,7 @@ function compileSet(
|
|
|
1142
1148
|
const order = compileOrder(identityOrderFields())
|
|
1143
1149
|
return {
|
|
1144
1150
|
sql: `
|
|
1145
|
-
SELECT
|
|
1151
|
+
SELECT *
|
|
1146
1152
|
FROM ${ctx.source.objectsTable}
|
|
1147
1153
|
WHERE 1 = 0
|
|
1148
1154
|
ORDER BY ${order.sql}
|
|
@@ -1164,7 +1170,7 @@ function compileSet(
|
|
|
1164
1170
|
const order = compileOrder(identityOrderFields())
|
|
1165
1171
|
const selectedOrder = compileOrder(identityOrderFields(), "selected")
|
|
1166
1172
|
const sql = `
|
|
1167
|
-
SELECT selected
|
|
1173
|
+
SELECT selected.*
|
|
1168
1174
|
FROM (${identities.sql}) AS ids
|
|
1169
1175
|
JOIN ${ctx.source.objectsTable} AS selected
|
|
1170
1176
|
ON selected.project_id = ?
|
|
@@ -1195,7 +1201,9 @@ function compileProject(
|
|
|
1195
1201
|
if (!properties) return input
|
|
1196
1202
|
|
|
1197
1203
|
const projection = compileProjectionExpression(properties)
|
|
1198
|
-
|
|
1204
|
+
// Only a projection replaces the document, so only it keeps the original for ordering and cursors.
|
|
1205
|
+
const original = `input.${input.order.propertyColumn}`
|
|
1206
|
+
const inputOrder = compileOrder(input.order.fields, "input", original)
|
|
1199
1207
|
const outputOrder = compileOrder(input.order.fields, undefined, "_cursor_properties")
|
|
1200
1208
|
|
|
1201
1209
|
return {
|
|
@@ -1205,7 +1213,7 @@ function compileProject(
|
|
|
1205
1213
|
input.object_type_id,
|
|
1206
1214
|
input.primary_id,
|
|
1207
1215
|
${projection.sql} AS properties,
|
|
1208
|
-
|
|
1216
|
+
${original} AS _cursor_properties,
|
|
1209
1217
|
input.created_at,
|
|
1210
1218
|
input.updated_at,
|
|
1211
1219
|
input.version,
|
|
@@ -1256,14 +1264,7 @@ function compileAggregateSource(
|
|
|
1256
1264
|
)
|
|
1257
1265
|
}
|
|
1258
1266
|
case "traverse":
|
|
1259
|
-
return compileAggregateTraversal(
|
|
1260
|
-
projectId,
|
|
1261
|
-
query.input,
|
|
1262
|
-
query.linkId,
|
|
1263
|
-
query.direction,
|
|
1264
|
-
query.sourceObjectTypeId,
|
|
1265
|
-
source
|
|
1266
|
-
)
|
|
1267
|
+
return compileAggregateTraversal(projectId, query, source)
|
|
1267
1268
|
case "set":
|
|
1268
1269
|
return compileAggregateSet(projectId, query.op, query.inputs, source)
|
|
1269
1270
|
case "sort":
|
|
@@ -1353,13 +1354,13 @@ function compileAggregateWhere(
|
|
|
1353
1354
|
|
|
1354
1355
|
function compileAggregateTraversal(
|
|
1355
1356
|
projectId: string,
|
|
1356
|
-
|
|
1357
|
-
linkId: string,
|
|
1358
|
-
direction: "outgoing" | "incoming",
|
|
1359
|
-
sourceObjectTypeId: string | undefined,
|
|
1357
|
+
traversal: ObjectQueryTraverse,
|
|
1360
1358
|
source: PgObjectQuerySource
|
|
1361
1359
|
): CompiledAggregateSource {
|
|
1362
|
-
const
|
|
1360
|
+
const { linkId, direction } = traversal
|
|
1361
|
+
const input = compileAggregateSource(projectId, traversal.input, source)
|
|
1362
|
+
const endpoints = linkEndpointObjectTypeIds(traversal)
|
|
1363
|
+
const sourceObjectTypeId = endpoints?.length === 1 ? endpoints[0] : undefined
|
|
1363
1364
|
if (direction === "incoming" && sourceObjectTypeId !== undefined) {
|
|
1364
1365
|
// Deduplicate endpoint identities before fetching objects. Bounded arrays let PostgreSQL
|
|
1365
1366
|
// visit primary keys in groups instead of doing one random B-tree descent per edge.
|
|
@@ -1383,14 +1384,7 @@ function compileAggregateTraversal(
|
|
|
1383
1384
|
args: [...input.args, linkId, sourceObjectTypeId, projectId, sourceObjectTypeId],
|
|
1384
1385
|
}
|
|
1385
1386
|
}
|
|
1386
|
-
const relation = compileTraversalRelation(
|
|
1387
|
-
projectId,
|
|
1388
|
-
input,
|
|
1389
|
-
linkId,
|
|
1390
|
-
direction,
|
|
1391
|
-
sourceObjectTypeId,
|
|
1392
|
-
source
|
|
1393
|
-
)
|
|
1387
|
+
const relation = compileTraversalRelation(projectId, input, traversal, source)
|
|
1394
1388
|
return {
|
|
1395
1389
|
sql: `SELECT selected.project_id, selected.object_type_id, selected.primary_id, selected.properties ${relation.sql}`,
|
|
1396
1390
|
args: relation.args,
|
|
@@ -1523,7 +1517,12 @@ export function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredi
|
|
|
1523
1517
|
case "eq":
|
|
1524
1518
|
return predicate.scalarKind === "decimal"
|
|
1525
1519
|
? compileDecimalEqualityPredicate(predicate.propertyId, "eq", predicate.value)
|
|
1526
|
-
: compileEqualityPredicate(
|
|
1520
|
+
: compileEqualityPredicate(
|
|
1521
|
+
predicate.propertyId,
|
|
1522
|
+
"eq",
|
|
1523
|
+
predicate.value,
|
|
1524
|
+
predicate.scalarKind
|
|
1525
|
+
)
|
|
1527
1526
|
case "neq":
|
|
1528
1527
|
return predicate.scalarKind === "decimal"
|
|
1529
1528
|
? compileDecimalEqualityPredicate(predicate.propertyId, "neq", predicate.value)
|
|
@@ -1555,7 +1554,7 @@ export function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredi
|
|
|
1555
1554
|
case "in":
|
|
1556
1555
|
return predicate.scalarKind === "decimal"
|
|
1557
1556
|
? compileDecimalInPredicate(predicate.propertyId, predicate.values)
|
|
1558
|
-
: compileInPredicate(predicate.propertyId, predicate.values)
|
|
1557
|
+
: compileInPredicate(predicate.propertyId, predicate.values, predicate.scalarKind)
|
|
1559
1558
|
case "exists": {
|
|
1560
1559
|
const sql = `jsonb_exists(properties, ?::text)`
|
|
1561
1560
|
const item: CompiledPredicate = { sql, args: [predicate.propertyId] }
|
|
@@ -1629,7 +1628,8 @@ function negatePredicate(predicate: CompiledPredicate): CompiledPredicate {
|
|
|
1629
1628
|
function compileEqualityPredicate(
|
|
1630
1629
|
propertyId: string,
|
|
1631
1630
|
op: "eq" | "neq",
|
|
1632
|
-
value: unknown
|
|
1631
|
+
value: unknown,
|
|
1632
|
+
scalarKind?: QueryScalarKind
|
|
1633
1633
|
): CompiledPredicate {
|
|
1634
1634
|
if (value === null) {
|
|
1635
1635
|
const sql = `${jsonTypeExpression()} = 'null'`
|
|
@@ -1637,7 +1637,7 @@ function compileEqualityPredicate(
|
|
|
1637
1637
|
return op === "eq" ? item : negatePredicate(item)
|
|
1638
1638
|
}
|
|
1639
1639
|
|
|
1640
|
-
if (typeof value === "string") {
|
|
1640
|
+
if (typeof value === "string" && !(op === "eq" && hasJsonValueIndex(scalarKind))) {
|
|
1641
1641
|
if (op === "eq") {
|
|
1642
1642
|
return {
|
|
1643
1643
|
sql: `(${jsonTypeExpression()} = 'string' AND ${jsonTextExpression()} = ?::text AND md5(${jsonTextExpression()}) = md5(?::text))`,
|
|
@@ -1664,12 +1664,18 @@ function compileEqualityPredicate(
|
|
|
1664
1664
|
}
|
|
1665
1665
|
}
|
|
1666
1666
|
|
|
1667
|
-
function compileInPredicate(
|
|
1667
|
+
function compileInPredicate(
|
|
1668
|
+
propertyId: string,
|
|
1669
|
+
values: readonly unknown[],
|
|
1670
|
+
scalarKind?: QueryScalarKind
|
|
1671
|
+
): CompiledPredicate {
|
|
1668
1672
|
if (values.length === 0) return { sql: "0 = 1", args: [] }
|
|
1669
1673
|
|
|
1670
1674
|
const nonNullValues = values.filter((value) => value !== null)
|
|
1671
|
-
const
|
|
1672
|
-
|
|
1675
|
+
const comparedAsText = (value: unknown): value is string =>
|
|
1676
|
+
typeof value === "string" && !hasJsonValueIndex(scalarKind)
|
|
1677
|
+
const stringValues = nonNullValues.filter(comparedAsText)
|
|
1678
|
+
const jsonbValues = nonNullValues.filter((value) => !comparedAsText(value))
|
|
1673
1679
|
const clauses: string[] = []
|
|
1674
1680
|
const args: unknown[] = []
|
|
1675
1681
|
|
|
@@ -2096,6 +2102,14 @@ function orderedValueExpression(
|
|
|
2096
2102
|
return `NULLIF(${jsonValueExpression(propertyColumn)}, 'null'::jsonb)`
|
|
2097
2103
|
}
|
|
2098
2104
|
|
|
2105
|
+
/**
|
|
2106
|
+
* Strings whose equality index is on the JSON value, not on the text: preparation builds it, so a
|
|
2107
|
+
* filter has to compare the JSON value too, or PostgreSQL scans every row of the type.
|
|
2108
|
+
*/
|
|
2109
|
+
export function hasJsonValueIndex(scalarKind: QueryScalarKind | undefined): boolean {
|
|
2110
|
+
return scalarKind === "date" || scalarKind === "timestamp" || scalarKind === "uuid"
|
|
2111
|
+
}
|
|
2112
|
+
|
|
2099
2113
|
export function isTextScalarKind(scalarKind: QueryScalarKind | undefined): boolean {
|
|
2100
2114
|
return scalarKind === "string" || scalarKind === "uuid"
|
|
2101
2115
|
}
|