@sixb/pg 0.1.21 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/agents/agent-storage.d.ts +8 -2
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/approvals.d.ts +48 -0
- package/dist/agents/approvals.d.ts.map +1 -0
- package/dist/agents/checkpoints.d.ts +1 -0
- package/dist/agents/checkpoints.d.ts.map +1 -1
- package/dist/agents/index.d.ts +1 -0
- package/dist/agents/index.d.ts.map +1 -1
- package/dist/agents/memory.d.ts +19 -0
- package/dist/agents/memory.d.ts.map +1 -0
- package/dist/agents/messages.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +15 -1
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts +3 -1
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/agents/settings.d.ts +29 -0
- package/dist/agents/settings.d.ts.map +1 -0
- package/dist/agents/threads.d.ts +6 -1
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/ai-cost-groups.d.ts.map +1 -1
- package/dist/auth-storage/auth-storage.d.ts +3 -2
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/users.d.ts +8 -4
- package/dist/auth-storage/users.d.ts.map +1 -1
- package/dist/index.js +3720 -1594
- package/dist/index.js.map +37 -31
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/conditional-reader.d.ts +17 -0
- package/dist/objects/conditional-reader.d.ts.map +1 -0
- package/dist/objects/query-compiler.d.ts +27 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts +10 -1
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +2 -0
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/scoped-reader.d.ts +12 -0
- package/dist/objects/scoped-reader.d.ts.map +1 -0
- package/dist/objects/selected-reader.d.ts +1 -0
- package/dist/objects/selected-reader.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +7 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/ontology-storage/index.d.ts +2 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +6 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +15 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/unique-values.d.ts +60 -0
- package/dist/ontology-storage/unique-values.d.ts.map +1 -0
- package/dist/pg-ai-cost-storage.d.ts.map +1 -1
- package/dist/pg-ai-usage-storage.d.ts +8 -1
- package/dist/pg-ai-usage-storage.d.ts.map +1 -1
- package/dist/pg-artifact-storage.d.ts +8 -1
- package/dist/pg-artifact-storage.d.ts.map +1 -1
- package/dist/pg-execution-storage.d.ts +2 -1
- package/dist/pg-execution-storage.d.ts.map +1 -1
- package/dist/storage-errors.d.ts +2 -0
- package/dist/storage-errors.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/agent-storage.ts +11 -2
- package/src/agents/approvals.ts +400 -0
- package/src/agents/checkpoints.ts +8 -4
- package/src/agents/index.ts +1 -0
- package/src/agents/memory.ts +175 -0
- package/src/agents/messages.ts +14 -7
- package/src/agents/rows.ts +46 -1
- package/src/agents/runs.ts +154 -22
- package/src/agents/settings.ts +105 -0
- package/src/agents/threads.ts +356 -3
- package/src/ai-cost-groups.ts +11 -1
- package/src/auth-storage/auth-storage.ts +45 -13
- package/src/auth-storage/users.ts +105 -43
- package/src/migrations/069-artifact-kinds.sql +4 -0
- package/src/migrations/070-artifact-state.sql +16 -0
- package/src/migrations/071-agent-thread-participants.sql +16 -0
- package/src/migrations/072-forked-context-checkpoints.sql +23 -0
- package/src/migrations/073-unique-values.sql +25 -0
- package/src/migrations/074-generated-media-usage.sql +21 -0
- package/src/migrations/075-model-call-files.sql +11 -0
- package/src/migrations/076-agent-memory.sql +33 -0
- package/src/migrations/077-artifact-threads.sql +21 -0
- package/src/migrations/078-agent-approvals.sql +58 -0
- package/src/migrations/079-agent-approval-resume.sql +20 -0
- package/src/migrations/080-subagent-threads.sql +145 -0
- package/src/migrations/081-subagent-questions.sql +6 -0
- package/src/migrations.ts +34 -0
- package/src/objects/conditional-reader.ts +44 -0
- package/src/objects/prepare-queries.ts +1 -1
- package/src/objects/query-compiler.ts +96 -33
- package/src/objects/query-indexes.ts +29 -10
- package/src/objects/read-scope.ts +140 -25
- package/src/objects/reader.ts +50 -6
- package/src/objects/scoped-reader.ts +101 -0
- package/src/objects/selected-reader.ts +16 -76
- package/src/objects/storage.ts +24 -5
- package/src/objects/text-count-index.ts +1 -0
- package/src/ontology-storage/index.ts +3 -0
- package/src/ontology-storage/materialization-session.ts +8 -0
- package/src/ontology-storage/materialization-writer.ts +76 -1
- package/src/ontology-storage/materializations.ts +20 -1
- package/src/ontology-storage/unique-values.ts +274 -0
- package/src/pg-ai-cost-storage.ts +62 -12
- package/src/pg-ai-usage-storage.ts +90 -1
- package/src/pg-artifact-storage.ts +205 -9
- package/src/pg-execution-storage.ts +31 -0
- package/src/storage-errors.ts +7 -0
- package/src/transactions.ts +1 -1
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { ObjectQuery } from "@sixb/core"
|
|
2
|
+
import { hasVectorProfile } from "@sixb/core/internal/query"
|
|
3
|
+
import type {
|
|
4
|
+
CompiledConditionalObjectReadScope,
|
|
5
|
+
ObjectQueryCapabilities,
|
|
6
|
+
ObjectReadStorage,
|
|
7
|
+
} from "@sixb/core/storage"
|
|
8
|
+
import { type PgStoreClient, runPgRepeatableReadTransaction } from "../transactions"
|
|
9
|
+
import type { PgAggregateAdmission } from "./aggregate-admission"
|
|
10
|
+
import { compilePgConditionalObjectReadSource } from "./read-scope"
|
|
11
|
+
import { PgObjectReader } from "./reader"
|
|
12
|
+
import { createPgScopedReader } from "./scoped-reader"
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Read objects and links through relations that apply the scope's conditions as they are read.
|
|
16
|
+
*
|
|
17
|
+
* Reads run as unscoped ones do, under the same aggregate admission: on the pool, and a vector
|
|
18
|
+
* query on one repeatable-read snapshot for its candidate probe and its rows.
|
|
19
|
+
*/
|
|
20
|
+
export function createPgConditionalReader(
|
|
21
|
+
sql: PgStoreClient,
|
|
22
|
+
params: {
|
|
23
|
+
projectId: string
|
|
24
|
+
scope: CompiledConditionalObjectReadScope
|
|
25
|
+
},
|
|
26
|
+
options: {
|
|
27
|
+
aggregateAdmission?: PgAggregateAdmission
|
|
28
|
+
queryCapabilities: () => ObjectQueryCapabilities
|
|
29
|
+
}
|
|
30
|
+
): ObjectReadStorage {
|
|
31
|
+
const source = compilePgConditionalObjectReadSource(params.projectId, params.scope)
|
|
32
|
+
const reader = new PgObjectReader(sql, source, options.aggregateAdmission)
|
|
33
|
+
|
|
34
|
+
return createPgScopedReader(
|
|
35
|
+
{
|
|
36
|
+
projectId: params.projectId,
|
|
37
|
+
read: (run, query?: ObjectQuery) =>
|
|
38
|
+
query && hasVectorProfile(query)
|
|
39
|
+
? runPgRepeatableReadTransaction(sql, (tx) => run(new PgObjectReader(tx, source)))
|
|
40
|
+
: run(reader),
|
|
41
|
+
},
|
|
42
|
+
options.queryCapabilities
|
|
43
|
+
)
|
|
44
|
+
}
|
|
@@ -59,7 +59,7 @@ function preparationFingerprint(plan: ObjectQueryPreparationPlan): string {
|
|
|
59
59
|
.sort((a, b) => a.objectTypeId.localeCompare(b.objectTypeId))
|
|
60
60
|
|
|
61
61
|
// Bump the version when physical generation changes, even if the ontology stays the same.
|
|
62
|
-
const definition = { version:
|
|
62
|
+
const definition = { version: 2, projectId: plan.projectId, objectTypes }
|
|
63
63
|
return createHash("sha256").update(JSON.stringify(definition)).digest("hex")
|
|
64
64
|
}
|
|
65
65
|
|
|
@@ -181,6 +181,12 @@ export interface PgObjectQuerySource {
|
|
|
181
181
|
readonly objectsTable: string
|
|
182
182
|
readonly linksTable: string
|
|
183
183
|
readonly objectPropertyPermissionsTable?: string
|
|
184
|
+
/**
|
|
185
|
+
* Where to read one concrete type's objects, when the source can say it more precisely than
|
|
186
|
+
* `objectsTable`: a table and that type's own filter over its `properties`. A statement that
|
|
187
|
+
* reads a single type then holds the filter as a top-level condition, which an index can serve.
|
|
188
|
+
*/
|
|
189
|
+
objectsOfType?(objectTypeId: string): PgTypedObjects
|
|
184
190
|
/** Wrap a terminal SELECT without its own WITH clause; the selected source owns the only WITH. */
|
|
185
191
|
wrapStatement(
|
|
186
192
|
sql: string,
|
|
@@ -192,6 +198,26 @@ export interface PgObjectQuerySource {
|
|
|
192
198
|
wrapQuery(query: CompiledPgObjectQuery): CompiledPgObjectQuery
|
|
193
199
|
}
|
|
194
200
|
|
|
201
|
+
export interface PgTypedObjects {
|
|
202
|
+
readonly table: string
|
|
203
|
+
readonly filter?: { readonly sql: string; readonly args: readonly unknown[] }
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** The table and `AND ...` filter a statement reads one type's objects with. */
|
|
207
|
+
export function pgObjectsOfType(
|
|
208
|
+
source: PgObjectQuerySource,
|
|
209
|
+
objectTypeId: string
|
|
210
|
+
): { table: string; filter: string; args: readonly unknown[] } {
|
|
211
|
+
const typed = source.objectsOfType?.(objectTypeId)
|
|
212
|
+
if (!typed) return { table: source.objectsTable, filter: "", args: [] }
|
|
213
|
+
|
|
214
|
+
return {
|
|
215
|
+
table: typed.table,
|
|
216
|
+
filter: typed.filter ? `AND (${typed.filter.sql})` : "",
|
|
217
|
+
args: typed.filter?.args ?? [],
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
195
221
|
/** Prefix an optional query source, then assign PostgreSQL positional placeholders. */
|
|
196
222
|
export function compilePgObjectStatement(
|
|
197
223
|
sql: string,
|
|
@@ -462,10 +488,11 @@ function compileStart(
|
|
|
462
488
|
}
|
|
463
489
|
|
|
464
490
|
const order = compileOrder(identityOrderFields())
|
|
491
|
+
const objects = pgObjectsOfType(ctx.source, query.objectTypeId)
|
|
465
492
|
const sql = `
|
|
466
493
|
SELECT *, properties AS _cursor_properties
|
|
467
|
-
FROM ${
|
|
468
|
-
WHERE project_id = ? AND object_type_id = ?
|
|
494
|
+
FROM ${objects.table}
|
|
495
|
+
WHERE project_id = ? AND object_type_id = ? ${objects.filter}
|
|
469
496
|
${ctx.startPredicate ? `AND (${ctx.startPredicate})` : ""}
|
|
470
497
|
${ctx.unordered ? "" : `ORDER BY ${order.sql}`}
|
|
471
498
|
${ctx.isolateSortOverflow ? "OFFSET 0" : ""}
|
|
@@ -473,9 +500,9 @@ function compileStart(
|
|
|
473
500
|
|
|
474
501
|
return {
|
|
475
502
|
sql,
|
|
476
|
-
args: [projectId, query.objectTypeId, ...(ctx.unordered ? [] : order.args)],
|
|
477
|
-
totalSql: `SELECT COUNT(*)::bigint AS total FROM ${
|
|
478
|
-
totalArgs: [projectId, query.objectTypeId],
|
|
503
|
+
args: [projectId, query.objectTypeId, ...objects.args, ...(ctx.unordered ? [] : order.args)],
|
|
504
|
+
totalSql: `SELECT COUNT(*)::bigint AS total FROM ${objects.table} WHERE project_id = ? AND object_type_id = ? ${objects.filter}`,
|
|
505
|
+
totalArgs: [projectId, query.objectTypeId, ...objects.args],
|
|
479
506
|
order,
|
|
480
507
|
hasMore: () => false,
|
|
481
508
|
trimRows: identityRows,
|
|
@@ -1070,11 +1097,7 @@ function compileExpansionOrder(
|
|
|
1070
1097
|
const direction = field.direction === "desc" ? "DESC" : "ASC"
|
|
1071
1098
|
clauses.push(
|
|
1072
1099
|
`CASE WHEN ${jsonTypeExpression(propertyColumn)} IS NULL OR ${jsonTypeExpression(propertyColumn)} = 'null' THEN 1 ELSE 0 END ASC`,
|
|
1073
|
-
`${
|
|
1074
|
-
field.scalarKind === "decimal"
|
|
1075
|
-
? `(${jsonTextExpression(propertyColumn)})::numeric`
|
|
1076
|
-
: `NULLIF(${jsonValueExpression(propertyColumn)}, 'null'::jsonb)`
|
|
1077
|
-
} ${direction}`
|
|
1100
|
+
`${orderedValueExpression(field.scalarKind, propertyColumn)} ${direction}`
|
|
1078
1101
|
)
|
|
1079
1102
|
args.push(field.propertyId, field.propertyId, field.propertyId)
|
|
1080
1103
|
}
|
|
@@ -1239,13 +1262,14 @@ function compileAggregateStart(
|
|
|
1239
1262
|
throw new Error("[SixbPg] PostgreSQL object storage does not support start.includeSubtypes")
|
|
1240
1263
|
}
|
|
1241
1264
|
|
|
1265
|
+
const objects = pgObjectsOfType(source, query.objectTypeId)
|
|
1242
1266
|
return {
|
|
1243
1267
|
sql: `
|
|
1244
1268
|
SELECT project_id, object_type_id, primary_id, properties
|
|
1245
|
-
FROM ${
|
|
1246
|
-
WHERE project_id = ? AND object_type_id = ?
|
|
1269
|
+
FROM ${objects.table}
|
|
1270
|
+
WHERE project_id = ? AND object_type_id = ? ${objects.filter}
|
|
1247
1271
|
`,
|
|
1248
|
-
args: [projectId, query.objectTypeId],
|
|
1272
|
+
args: [projectId, query.objectTypeId, ...objects.args],
|
|
1249
1273
|
}
|
|
1250
1274
|
}
|
|
1251
1275
|
|
|
@@ -1452,7 +1476,8 @@ function compileProjectionExpression(properties: readonly string[]): CompiledPre
|
|
|
1452
1476
|
}
|
|
1453
1477
|
}
|
|
1454
1478
|
|
|
1455
|
-
|
|
1479
|
+
/** Compile a predicate over the unqualified `properties` column of the relation it filters. */
|
|
1480
|
+
export function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredicate {
|
|
1456
1481
|
switch (predicate.op) {
|
|
1457
1482
|
case "and":
|
|
1458
1483
|
case "or": {
|
|
@@ -1467,6 +1492,8 @@ function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredicate {
|
|
|
1467
1492
|
const item = compilePredicate(predicate.item)
|
|
1468
1493
|
return negatePredicate(item)
|
|
1469
1494
|
}
|
|
1495
|
+
case "constant":
|
|
1496
|
+
return { sql: predicate.value ? "true" : "false", args: [] }
|
|
1470
1497
|
case "eq":
|
|
1471
1498
|
return predicate.scalarKind === "decimal"
|
|
1472
1499
|
? compileDecimalEqualityPredicate(predicate.propertyId, "eq", predicate.value)
|
|
@@ -1486,6 +1513,14 @@ function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredicate {
|
|
|
1486
1513
|
args: [predicate.propertyId, predicate.propertyId, predicate.value],
|
|
1487
1514
|
}
|
|
1488
1515
|
}
|
|
1516
|
+
if (isTextScalarKind(predicate.scalarKind)) {
|
|
1517
|
+
// The null rank makes the range a prefix of the property's sort index keys.
|
|
1518
|
+
return {
|
|
1519
|
+
sql: `(${nullRankExpression()} = 0 AND ${orderedValueExpression(predicate.scalarKind)} ${op} ?::text)`,
|
|
1520
|
+
args: [predicate.propertyId, predicate.propertyId, predicate.propertyId, predicate.value],
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1489
1524
|
return {
|
|
1490
1525
|
sql: `${jsonValueExpression()} ${op} ?::text::jsonb`,
|
|
1491
1526
|
args: [predicate.propertyId, jsonbValue(predicate.value)],
|
|
@@ -1809,13 +1844,13 @@ function compileKeysetRanges(
|
|
|
1809
1844
|
})
|
|
1810
1845
|
if (value.nullish) return
|
|
1811
1846
|
const expression = compiledPropertyValueExpression(field, propertyColumn)
|
|
1812
|
-
const parameter = field.
|
|
1813
|
-
const args = [
|
|
1814
|
-
|
|
1815
|
-
field.
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
prefix.push({ sql: `${expression} = ${parameter}`, args })
|
|
1847
|
+
const parameter = compiledPropertyParameter(field, value.value)
|
|
1848
|
+
const args = [field.propertyId, parameter.value]
|
|
1849
|
+
addRange({
|
|
1850
|
+
sql: `${expression} ${field.direction === "desc" ? "<" : ">"} ${parameter.sql}`,
|
|
1851
|
+
args,
|
|
1852
|
+
})
|
|
1853
|
+
prefix.push({ sql: `${expression} = ${parameter.sql}`, args })
|
|
1819
1854
|
})
|
|
1820
1855
|
return ranges
|
|
1821
1856
|
}
|
|
@@ -1884,14 +1919,11 @@ function compileFieldEquality(
|
|
|
1884
1919
|
}
|
|
1885
1920
|
}
|
|
1886
1921
|
|
|
1922
|
+
const parameter = compiledPropertyParameter(field, cursor.value)
|
|
1923
|
+
|
|
1887
1924
|
return {
|
|
1888
|
-
sql: `(${jsonTypeExpression(propertyColumn)} IS NOT NULL AND ${jsonTypeExpression(propertyColumn)} != 'null' AND ${compiledPropertyValueExpression(field, propertyColumn)} = ${
|
|
1889
|
-
args: [
|
|
1890
|
-
field.propertyId,
|
|
1891
|
-
field.propertyId,
|
|
1892
|
-
field.propertyId,
|
|
1893
|
-
field.scalarKind === "decimal" ? cursor.value : jsonbValue(cursor.value),
|
|
1894
|
-
],
|
|
1925
|
+
sql: `(${jsonTypeExpression(propertyColumn)} IS NOT NULL AND ${jsonTypeExpression(propertyColumn)} != 'null' AND ${compiledPropertyValueExpression(field, propertyColumn)} = ${parameter.sql})`,
|
|
1926
|
+
args: [field.propertyId, field.propertyId, field.propertyId, parameter.value],
|
|
1895
1927
|
}
|
|
1896
1928
|
}
|
|
1897
1929
|
|
|
@@ -1911,16 +1943,17 @@ function compileFieldAfter(
|
|
|
1911
1943
|
|
|
1912
1944
|
const rankSql = `CASE WHEN ${jsonTypeExpression(propertyColumn)} IS NULL OR ${jsonTypeExpression(propertyColumn)} = 'null' THEN 1 ELSE 0 END`
|
|
1913
1945
|
const valueOperator = field.direction === "desc" ? "<" : ">"
|
|
1946
|
+
const parameter = compiledPropertyParameter(field, cursor.value)
|
|
1914
1947
|
|
|
1915
1948
|
return {
|
|
1916
|
-
sql: `(${rankSql} > 0 OR (${rankSql} = 0 AND ${compiledPropertyValueExpression(field, propertyColumn)} ${valueOperator} ${
|
|
1949
|
+
sql: `(${rankSql} > 0 OR (${rankSql} = 0 AND ${compiledPropertyValueExpression(field, propertyColumn)} ${valueOperator} ${parameter.sql}))`,
|
|
1917
1950
|
args: [
|
|
1918
1951
|
field.propertyId,
|
|
1919
1952
|
field.propertyId,
|
|
1920
1953
|
field.propertyId,
|
|
1921
1954
|
field.propertyId,
|
|
1922
1955
|
field.propertyId,
|
|
1923
|
-
|
|
1956
|
+
parameter.value,
|
|
1924
1957
|
],
|
|
1925
1958
|
}
|
|
1926
1959
|
}
|
|
@@ -2010,9 +2043,39 @@ function compiledPropertyValueExpression(
|
|
|
2010
2043
|
field: Extract<CompiledOrderField, { kind: "property" }>,
|
|
2011
2044
|
propertyColumn = "properties"
|
|
2012
2045
|
): string {
|
|
2013
|
-
return field.scalarKind
|
|
2014
|
-
|
|
2015
|
-
|
|
2046
|
+
return orderedValueExpression(field.scalarKind, propertyColumn)
|
|
2047
|
+
}
|
|
2048
|
+
|
|
2049
|
+
/** The parameter a cursor value binds to, typed like `compiledPropertyValueExpression`. */
|
|
2050
|
+
function compiledPropertyParameter(
|
|
2051
|
+
field: Extract<CompiledOrderField, { kind: "property" }>,
|
|
2052
|
+
value: unknown
|
|
2053
|
+
): { sql: string; value: unknown } {
|
|
2054
|
+
if (field.scalarKind === "decimal") return { sql: "?::numeric", value }
|
|
2055
|
+
if (isTextScalarKind(field.scalarKind)) return { sql: "?::text", value }
|
|
2056
|
+
return { sql: "?::text::jsonb", value: jsonbValue(value) }
|
|
2057
|
+
}
|
|
2058
|
+
|
|
2059
|
+
/**
|
|
2060
|
+
* The expression a property sorts and range-compares by. Strings compare by code point, as
|
|
2061
|
+
* SQLite and the in-memory store do: JSONB and the database's own collation would order case and
|
|
2062
|
+
* accents linguistically. Sort indexes repeat this expression, collation included.
|
|
2063
|
+
*/
|
|
2064
|
+
function orderedValueExpression(
|
|
2065
|
+
scalarKind: QueryScalarKind | undefined,
|
|
2066
|
+
propertyColumn = "properties"
|
|
2067
|
+
): string {
|
|
2068
|
+
if (scalarKind === "decimal") return `(${jsonTextExpression(propertyColumn)})::numeric`
|
|
2069
|
+
if (isTextScalarKind(scalarKind)) return `((${jsonTextExpression(propertyColumn)}) COLLATE "C")`
|
|
2070
|
+
return `NULLIF(${jsonValueExpression(propertyColumn)}, 'null'::jsonb)`
|
|
2071
|
+
}
|
|
2072
|
+
|
|
2073
|
+
export function isTextScalarKind(scalarKind: QueryScalarKind | undefined): boolean {
|
|
2074
|
+
return scalarKind === "string" || scalarKind === "uuid"
|
|
2075
|
+
}
|
|
2076
|
+
|
|
2077
|
+
function nullRankExpression(propertyColumn = "properties"): string {
|
|
2078
|
+
return `CASE WHEN ${jsonTypeExpression(propertyColumn)} IS NULL OR ${jsonTypeExpression(propertyColumn)} = 'null' THEN 1 ELSE 0 END`
|
|
2016
2079
|
}
|
|
2017
2080
|
|
|
2018
2081
|
function columnExpression(
|
|
@@ -4,6 +4,7 @@ import type { ReservedSQL, SQL, SQLClient } from "../pg-client"
|
|
|
4
4
|
import { isConnectionLost } from "../storage-errors"
|
|
5
5
|
import { withReservedPgConnection } from "../transactions"
|
|
6
6
|
import { lockPgPreparation } from "./preparation-lock"
|
|
7
|
+
import { isTextScalarKind } from "./query-compiler"
|
|
7
8
|
|
|
8
9
|
export interface PgObjectQueryIndexScope {
|
|
9
10
|
readonly projectId: string
|
|
@@ -110,6 +111,22 @@ async function createQueryIndex(
|
|
|
110
111
|
await connection.unsafe(
|
|
111
112
|
`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${identifier(`${name}_overflow`)} ON ${identifier(schema)}.objects (primary_id) WHERE ${scope} AND NOT (${sortIndexBound(definition.index)})`
|
|
112
113
|
)
|
|
114
|
+
|
|
115
|
+
// The same definition under another name is an earlier generation of this index, built
|
|
116
|
+
// with an expression queries no longer use. Writes would keep maintaining it.
|
|
117
|
+
const superseded = await connection<{ name: string }[]>`
|
|
118
|
+
SELECT c.relname AS name FROM pg_index i
|
|
119
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
120
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
121
|
+
WHERE n.nspname = ${schema} AND c.relname LIKE 'sixb_query_%' AND c.relname <> ${name}
|
|
122
|
+
AND obj_description(c.oid, 'pg_class') = ${JSON.stringify(definition.index)}`
|
|
123
|
+
for (const previous of superseded) {
|
|
124
|
+
for (const index of [previous.name, `${previous.name}_overflow`]) {
|
|
125
|
+
await connection.unsafe(
|
|
126
|
+
`DROP INDEX CONCURRENTLY IF EXISTS ${identifier(schema)}.${identifier(index)}`
|
|
127
|
+
)
|
|
128
|
+
}
|
|
129
|
+
}
|
|
113
130
|
}
|
|
114
131
|
return name
|
|
115
132
|
}
|
|
@@ -150,14 +167,7 @@ function statisticsExpressions(index: PgObjectQueryIndex): string[] {
|
|
|
150
167
|
if (index.kind === "value") expressions.push(`(properties -> ${literal(index.propertyId)})`)
|
|
151
168
|
if (index.kind === "text")
|
|
152
169
|
expressions.push(`lower(coalesce(properties ->> ${literal(index.propertyId)}, ''))`)
|
|
153
|
-
if (index.kind === "sort")
|
|
154
|
-
for (const field of index.fields) {
|
|
155
|
-
expressions.push(
|
|
156
|
-
field.scalarKind === "decimal"
|
|
157
|
-
? `((properties ->> ${literal(field.propertyId)})::numeric)`
|
|
158
|
-
: `(NULLIF(properties -> ${literal(field.propertyId)}, 'null'::jsonb))`
|
|
159
|
-
)
|
|
160
|
-
}
|
|
170
|
+
if (index.kind === "sort") for (const field of index.fields) expressions.push(sortValue(field))
|
|
161
171
|
return expressions
|
|
162
172
|
}
|
|
163
173
|
|
|
@@ -220,7 +230,7 @@ function compileIndex(index: PgObjectQueryIndex): string {
|
|
|
220
230
|
const property = `properties -> ${literal(field.propertyId)}`
|
|
221
231
|
fields.push(
|
|
222
232
|
`(CASE WHEN jsonb_typeof(${property}) IS NULL OR jsonb_typeof(${property}) = 'null' THEN 1 ELSE 0 END)`,
|
|
223
|
-
`${
|
|
233
|
+
`${sortValue(field)} ${field.direction === "desc" ? "DESC" : "ASC"}`
|
|
224
234
|
)
|
|
225
235
|
}
|
|
226
236
|
fields.push("primary_id")
|
|
@@ -228,6 +238,14 @@ function compileIndex(index: PgObjectQueryIndex): string {
|
|
|
228
238
|
return `(${fields.join(", ")}) WHERE ${predicate.join(" AND ")}`
|
|
229
239
|
}
|
|
230
240
|
|
|
241
|
+
/** The query compiler's ordered value expression; an index serves a sort only when they match. */
|
|
242
|
+
function sortValue(field: Extract<PgObjectQueryIndex, { kind: "sort" }>["fields"][number]): string {
|
|
243
|
+
const text = `(properties ->> ${literal(field.propertyId)})`
|
|
244
|
+
if (field.scalarKind === "decimal") return `(${text}::numeric)`
|
|
245
|
+
if (isTextScalarKind(field.scalarKind)) return `(${text} COLLATE "C")`
|
|
246
|
+
return `(NULLIF(properties -> ${literal(field.propertyId)}, 'null'::jsonb))`
|
|
247
|
+
}
|
|
248
|
+
|
|
231
249
|
function literal(value: string): string {
|
|
232
250
|
if (value.includes("\0"))
|
|
233
251
|
throw new Error("[SixbPg] Query index names and values cannot contain NUL")
|
|
@@ -337,7 +355,8 @@ export async function findPgSortIndexBound(
|
|
|
337
355
|
fields[i]?.kind === "property" &&
|
|
338
356
|
fields[i].propertyId === field.propertyId &&
|
|
339
357
|
(field.direction ?? "asc") === (fields[i].direction ?? "asc") &&
|
|
340
|
-
(field.scalarKind === "decimal") === (fields[i].scalarKind === "decimal")
|
|
358
|
+
(field.scalarKind === "decimal") === (fields[i].scalarKind === "decimal") &&
|
|
359
|
+
isTextScalarKind(field.scalarKind) === isTextScalarKind(fields[i].scalarKind)
|
|
341
360
|
)
|
|
342
361
|
)
|
|
343
362
|
continue
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
CompiledConditionalObjectReadScope,
|
|
3
|
+
CompiledSelectedObjectReadScope,
|
|
4
|
+
} from "@sixb/core/storage"
|
|
2
5
|
import {
|
|
3
6
|
type CompiledPgObjectQuery,
|
|
4
7
|
compilePgObjectStatement,
|
|
8
|
+
compilePredicate,
|
|
5
9
|
type PgObjectQuerySource,
|
|
6
10
|
} from "./query-compiler"
|
|
7
11
|
|
|
@@ -30,16 +34,78 @@ export function compilePgSelectedObjectReadSource(
|
|
|
30
34
|
objects: scope.objects,
|
|
31
35
|
steps: scope.steps,
|
|
32
36
|
})
|
|
33
|
-
const
|
|
37
|
+
const source: PgSelectedObjectReadSource = {
|
|
38
|
+
...scopedQuerySource(compileSelectedReadScopeCte(scopeJson, projectId)),
|
|
39
|
+
objectPropertyPermissionsTable: "_sixb_scope_object_permissions",
|
|
40
|
+
traversalProbe: { sql: "", args: [] },
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const traversalProbe = compilePgObjectStatement(
|
|
44
|
+
`
|
|
45
|
+
SELECT COUNT(*)::bigint AS total
|
|
46
|
+
FROM (
|
|
47
|
+
SELECT 1 AS traversal_fact
|
|
48
|
+
FROM _sixb_scope_root_facts
|
|
49
|
+
|
|
50
|
+
UNION ALL
|
|
51
|
+
|
|
52
|
+
SELECT 1 AS traversal_fact
|
|
53
|
+
FROM _sixb_scope_reached_links
|
|
54
|
+
|
|
55
|
+
LIMIT ?
|
|
56
|
+
) AS bounded_traversal_facts
|
|
57
|
+
`,
|
|
58
|
+
[BigInt(maxTraversalFacts) + 1n],
|
|
59
|
+
source
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
return Object.freeze({ ...source, traversalProbe })
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Prepare a conditional scope as PostgreSQL relations that filter stored rows as they are read.
|
|
67
|
+
*
|
|
68
|
+
* Both relations are NOT MATERIALIZED, so PostgreSQL inlines them into each statement: its
|
|
69
|
+
* filters, index scans and LIMIT reach the stored tables, and the conditions become one more
|
|
70
|
+
* filter. An object of a conditioned type is visible while its condition matches; a link while
|
|
71
|
+
* each of its conditioned endpoints is a visible object.
|
|
72
|
+
*/
|
|
73
|
+
export function compilePgConditionalObjectReadSource(
|
|
74
|
+
projectId: string,
|
|
75
|
+
scope: CompiledConditionalObjectReadScope
|
|
76
|
+
): PgObjectQuerySource {
|
|
77
|
+
const filters = new Map(
|
|
78
|
+
scope.conditions.map((condition) => [
|
|
79
|
+
condition.objectTypeId,
|
|
80
|
+
compilePredicate(condition.predicate),
|
|
81
|
+
])
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
return Object.freeze({
|
|
85
|
+
...scopedQuerySource(compileConditionalReadScopeCte(projectId, filters)),
|
|
86
|
+
// One type's visible objects are the stored ones its own condition matches.
|
|
87
|
+
objectsOfType: (objectTypeId: string) => {
|
|
88
|
+
const filter = filters.get(objectTypeId)
|
|
89
|
+
return filter ? { table: "objects", filter } : { table: "objects" }
|
|
90
|
+
},
|
|
91
|
+
})
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
interface CompiledReadScopeCte {
|
|
95
|
+
readonly sql: string
|
|
96
|
+
readonly args: readonly unknown[]
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Prefix every statement of a query with the scope's relations, ahead of its own arguments. */
|
|
100
|
+
function scopedQuerySource(compiled: CompiledReadScopeCte): PgObjectQuerySource {
|
|
34
101
|
const wrapStatement = (sql: string, args: readonly unknown[] = []) => ({
|
|
35
102
|
sql: `${compiled.sql}\n${sql}`,
|
|
36
103
|
args: [...compiled.args, ...args],
|
|
37
104
|
})
|
|
38
105
|
|
|
39
|
-
|
|
106
|
+
return {
|
|
40
107
|
objectsTable: "_sixb_scope_objects",
|
|
41
108
|
linksTable: "_sixb_scope_links",
|
|
42
|
-
objectPropertyPermissionsTable: "_sixb_scope_object_permissions",
|
|
43
109
|
wrapStatement,
|
|
44
110
|
wrapQuery: (query: CompiledPgObjectQuery): CompiledPgObjectQuery => ({
|
|
45
111
|
...query,
|
|
@@ -58,34 +124,83 @@ export function compilePgSelectedObjectReadSource(
|
|
|
58
124
|
}
|
|
59
125
|
: {}),
|
|
60
126
|
}),
|
|
61
|
-
traversalProbe: { sql: "", args: [] },
|
|
62
127
|
}
|
|
128
|
+
}
|
|
63
129
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
FROM _sixb_scope_root_facts
|
|
130
|
+
/** Conditions keyed by their concrete type, each compiled over that type's `properties`. */
|
|
131
|
+
type ConditionFilters = ReadonlyMap<
|
|
132
|
+
string,
|
|
133
|
+
{ readonly sql: string; readonly args: readonly unknown[] }
|
|
134
|
+
>
|
|
70
135
|
|
|
71
|
-
|
|
136
|
+
function compileConditionalReadScopeCte(
|
|
137
|
+
projectId: string,
|
|
138
|
+
filters: ConditionFilters
|
|
139
|
+
): CompiledReadScopeCte {
|
|
140
|
+
if (filters.size === 0) {
|
|
141
|
+
return {
|
|
142
|
+
sql: `
|
|
143
|
+
WITH
|
|
144
|
+
_sixb_scope_objects AS NOT MATERIALIZED (
|
|
145
|
+
SELECT * FROM objects WHERE project_id = ?
|
|
146
|
+
),
|
|
147
|
+
_sixb_scope_links AS NOT MATERIALIZED (
|
|
148
|
+
SELECT * FROM links WHERE project_id = ?
|
|
149
|
+
)
|
|
150
|
+
`,
|
|
151
|
+
args: [projectId, projectId],
|
|
152
|
+
}
|
|
153
|
+
}
|
|
72
154
|
|
|
73
|
-
|
|
74
|
-
|
|
155
|
+
const conditionedTypeArgs = [...filters.keys()]
|
|
156
|
+
const conditionedTypes = conditionedTypeArgs.map(() => "?::text").join(", ")
|
|
75
157
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
158
|
+
const matches: string[] = []
|
|
159
|
+
const matchArgs: unknown[] = []
|
|
160
|
+
for (const [objectTypeId, filter] of filters) {
|
|
161
|
+
matches.push(`(object_type_id = ?::text AND ${filter.sql})`)
|
|
162
|
+
matchArgs.push(objectTypeId, ...filter.args)
|
|
163
|
+
}
|
|
82
164
|
|
|
83
|
-
|
|
84
|
-
|
|
165
|
+
const visibleEndpoint = (end: "source" | "target") => `
|
|
166
|
+
(
|
|
167
|
+
stored.${end}_type_id NOT IN (${conditionedTypes})
|
|
168
|
+
OR EXISTS (
|
|
169
|
+
SELECT 1
|
|
170
|
+
FROM _sixb_scope_objects AS endpoint
|
|
171
|
+
WHERE endpoint.project_id = stored.project_id
|
|
172
|
+
AND endpoint.object_type_id = stored.${end}_type_id
|
|
173
|
+
AND endpoint.primary_id = stored.${end}_id
|
|
174
|
+
)
|
|
175
|
+
)
|
|
176
|
+
`
|
|
85
177
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
178
|
+
return {
|
|
179
|
+
sql: `
|
|
180
|
+
WITH
|
|
181
|
+
_sixb_scope_objects AS NOT MATERIALIZED (
|
|
182
|
+
SELECT *
|
|
183
|
+
FROM objects
|
|
184
|
+
WHERE project_id = ?
|
|
185
|
+
AND (object_type_id NOT IN (${conditionedTypes}) OR ${matches.join(" OR ")})
|
|
186
|
+
),
|
|
187
|
+
_sixb_scope_links AS NOT MATERIALIZED (
|
|
188
|
+
SELECT stored.*
|
|
189
|
+
FROM links AS stored
|
|
190
|
+
WHERE stored.project_id = ?
|
|
191
|
+
AND ${visibleEndpoint("source")}
|
|
192
|
+
AND ${visibleEndpoint("target")}
|
|
193
|
+
)
|
|
194
|
+
`,
|
|
195
|
+
args: [
|
|
196
|
+
projectId,
|
|
197
|
+
...conditionedTypeArgs,
|
|
198
|
+
...matchArgs,
|
|
199
|
+
projectId,
|
|
200
|
+
...conditionedTypeArgs,
|
|
201
|
+
...conditionedTypeArgs,
|
|
202
|
+
],
|
|
203
|
+
}
|
|
89
204
|
}
|
|
90
205
|
|
|
91
206
|
function compileSelectedReadScopeCte(scopeJson: string, projectId: string): CompiledReadScopeCte {
|
package/src/objects/reader.ts
CHANGED
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
compilePgObjectStatement,
|
|
31
31
|
DEFAULT_OBJECT_QUERY_SOURCE,
|
|
32
32
|
type PgObjectQuerySource,
|
|
33
|
+
pgObjectsOfType,
|
|
33
34
|
} from "./query-compiler"
|
|
34
35
|
import { findPgSortIndexBound } from "./query-indexes"
|
|
35
36
|
import type { PgSelectedObjectReadSource } from "./read-scope"
|
|
@@ -258,13 +259,14 @@ export class PgObjectReader {
|
|
|
258
259
|
}): Promise<ObjectRow | null> {
|
|
259
260
|
const sql = this.sql
|
|
260
261
|
const source = this.source
|
|
262
|
+
const objects = pgObjectsOfType(source, params.objectTypeId)
|
|
261
263
|
const statement = compilePgObjectStatement(
|
|
262
264
|
`
|
|
263
265
|
SELECT *
|
|
264
|
-
FROM ${
|
|
265
|
-
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
266
|
+
FROM ${objects.table}
|
|
267
|
+
WHERE project_id = ? AND object_type_id = ? AND primary_id = ? ${objects.filter}
|
|
266
268
|
`,
|
|
267
|
-
[params.projectId, params.objectTypeId, params.primaryId],
|
|
269
|
+
[params.projectId, params.objectTypeId, params.primaryId, ...objects.args],
|
|
268
270
|
source
|
|
269
271
|
)
|
|
270
272
|
const [row] = await sql.unsafe<ObjectDatabaseRow[]>(
|
|
@@ -275,6 +277,44 @@ export class PgObjectReader {
|
|
|
275
277
|
return row ? rowToObject(row) : null
|
|
276
278
|
}
|
|
277
279
|
|
|
280
|
+
/** On a selected source, the holder must be in scope and selected for the property. */
|
|
281
|
+
async getByUniqueValue(
|
|
282
|
+
params: Parameters<ObjectReadStorage["getByUniqueValue"]>[0]
|
|
283
|
+
): Promise<ObjectRow | null> {
|
|
284
|
+
const source = this.source
|
|
285
|
+
const selected = "objectPropertyPermissionsTable" in source
|
|
286
|
+
const permitted = selected
|
|
287
|
+
? `AND EXISTS (
|
|
288
|
+
SELECT 1 FROM ${source.objectPropertyPermissionsTable} AS permitted
|
|
289
|
+
WHERE permitted.project_id = objects.project_id
|
|
290
|
+
AND permitted.object_type_id = objects.object_type_id
|
|
291
|
+
AND permitted.primary_id = objects.primary_id
|
|
292
|
+
AND permitted.property_id = held.property_id
|
|
293
|
+
)`
|
|
294
|
+
: ""
|
|
295
|
+
const statement = compilePgObjectStatement(
|
|
296
|
+
`
|
|
297
|
+
SELECT objects.*
|
|
298
|
+
FROM object_unique_values AS held
|
|
299
|
+
JOIN ${source.objectsTable} AS objects
|
|
300
|
+
ON objects.project_id = held.project_id
|
|
301
|
+
AND objects.object_type_id = held.object_type_id
|
|
302
|
+
AND objects.primary_id = held.primary_id
|
|
303
|
+
WHERE held.project_id = ? AND held.object_type_id = ? AND held.property_id = ?
|
|
304
|
+
AND held.value_hash = sha256(convert_to(?, 'UTF8')) AND held.value_key = ?
|
|
305
|
+
${permitted}
|
|
306
|
+
`,
|
|
307
|
+
[params.projectId, params.objectTypeId, params.propertyId, params.valueKey, params.valueKey],
|
|
308
|
+
source
|
|
309
|
+
)
|
|
310
|
+
const [row] = await this.sql.unsafe<ObjectDatabaseRow[]>(
|
|
311
|
+
statement.sql,
|
|
312
|
+
statement.args as SqlParameter[]
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
return row ? rowToObject(row) : null
|
|
316
|
+
}
|
|
317
|
+
|
|
278
318
|
async selectsObjectProperties(
|
|
279
319
|
params: Parameters<ObjectReadStorage["selectsObjectProperties"]>[0]
|
|
280
320
|
): Promise<readonly boolean[]> {
|
|
@@ -592,11 +632,15 @@ export class PgObjectReader {
|
|
|
592
632
|
const order = params.order ?? "desc"
|
|
593
633
|
const orderColumn =
|
|
594
634
|
orderBy === "primaryId" ? "primary_id" : orderBy === "createdAt" ? "created_at" : "updated_at"
|
|
635
|
+
const typed =
|
|
636
|
+
typeof params.objectTypeId === "string"
|
|
637
|
+
? pgObjectsOfType(source, params.objectTypeId)
|
|
638
|
+
: { table: source.objectsTable, filter: "", args: [] }
|
|
595
639
|
const filters = ["project_id = ?"]
|
|
596
640
|
const args: unknown[] = [params.projectId]
|
|
597
641
|
if (typeof params.objectTypeId === "string") {
|
|
598
|
-
filters.push(
|
|
599
|
-
args.push(params.objectTypeId)
|
|
642
|
+
filters.push(`object_type_id = ? ${typed.filter}`)
|
|
643
|
+
args.push(params.objectTypeId, ...typed.args)
|
|
600
644
|
} else if (params.objectTypeId !== undefined) {
|
|
601
645
|
filters.push("object_type_id IN (SELECT jsonb_array_elements_text(?::text::jsonb))")
|
|
602
646
|
args.push(JSON.stringify(params.objectTypeId))
|
|
@@ -626,7 +670,7 @@ export class PgObjectReader {
|
|
|
626
670
|
args.push(params.createdBefore)
|
|
627
671
|
}
|
|
628
672
|
|
|
629
|
-
const objectsTable =
|
|
673
|
+
const objectsTable = typed.table
|
|
630
674
|
const whereSql = filters.join(" AND ")
|
|
631
675
|
const countStatement = compilePgObjectStatement(
|
|
632
676
|
`SELECT COUNT(*)::int AS total FROM ${objectsTable} WHERE ${whereSql}`,
|