@sixb/pg 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/dist/agents/rows.d.ts +3 -1
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/threads.d.ts +3 -2
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/auth-storage/access-tokens.d.ts.map +1 -1
- package/dist/auth-storage/device-authorizations.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +6 -3
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/sessions.d.ts +8 -1
- package/dist/auth-storage/sessions.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +917 -122
- package/dist/index.js.map +26 -23
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +8 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts +1 -0
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +1 -0
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/rows.d.ts +1 -0
- package/dist/objects/rows.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +1 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/ontology-storage/index.d.ts +4 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +1 -2
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +2 -1
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/vector-indexing.d.ts +20 -0
- package/dist/ontology-storage/vector-indexing.d.ts.map +1 -0
- package/dist/ontology-storage/vectors.d.ts +16 -0
- package/dist/ontology-storage/vectors.d.ts.map +1 -0
- package/dist/pg-ai-cost-storage.d.ts.map +1 -1
- package/dist/pg-file-upload-session-storage.d.ts +6 -0
- package/dist/pg-file-upload-session-storage.d.ts.map +1 -0
- package/dist/pg-workflow-intervention-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/rows.ts +21 -0
- package/src/agents/threads.ts +47 -2
- package/src/auth-storage/access-tokens.ts +10 -14
- package/src/auth-storage/auth-storage.ts +4 -6
- package/src/auth-storage/device-authorizations.ts +3 -4
- package/src/auth-storage/rows.ts +17 -7
- package/src/auth-storage/sessions.ts +82 -4
- package/src/auth-storage/shared.ts +8 -2
- package/src/index.ts +5 -0
- package/src/migrations/041-agent-thread-workspaces.sql +2 -0
- package/src/migrations/042-vector-profiles.sql +28 -0
- package/src/migrations/043-vector-indexing.sql +24 -0
- package/src/migrations/044-vector-batching.sql +3 -0
- package/src/migrations/045-agent-thread-sandbox-state.sql +2 -0
- package/src/migrations/046-workflow-intervention-principals.sql +15 -0
- package/src/migrations/047-ontology-commit-attribution.sql +50 -0
- package/src/migrations/048-file-upload-sessions.sql +34 -0
- package/src/migrations/049-native-sessions.sql +19 -0
- package/src/migrations.ts +26 -0
- package/src/objects/query-compiler.ts +102 -11
- package/src/objects/read-scope.ts +26 -25
- package/src/objects/reader.ts +33 -1
- package/src/objects/rows.ts +5 -1
- package/src/objects/storage.ts +22 -2
- package/src/ontology-storage/index.ts +15 -0
- package/src/ontology-storage/materializations.ts +28 -58
- package/src/ontology-storage/shared.ts +7 -3
- package/src/ontology-storage/vector-indexing.ts +168 -0
- package/src/ontology-storage/vectors.ts +172 -0
- package/src/pg-ai-cost-storage.ts +4 -0
- package/src/pg-file-upload-session-storage.ts +143 -0
- package/src/pg-workflow-intervention-storage.ts +11 -15
|
@@ -7,6 +7,12 @@ import {
|
|
|
7
7
|
type ObjectQuerySortField,
|
|
8
8
|
type QueryScalarKind,
|
|
9
9
|
} from "@sixb/core"
|
|
10
|
+
import {
|
|
11
|
+
hasVectorProfile,
|
|
12
|
+
isVectorProfileQuery,
|
|
13
|
+
normalizeVector,
|
|
14
|
+
vectorSearchCandidateLimit,
|
|
15
|
+
} from "@sixb/core/internal/query"
|
|
10
16
|
|
|
11
17
|
export interface PgObjectQueryPageRow {
|
|
12
18
|
object_type_id: string
|
|
@@ -16,6 +22,7 @@ export interface PgObjectQueryPageRow {
|
|
|
16
22
|
}
|
|
17
23
|
|
|
18
24
|
export interface CompiledPgObjectQuery {
|
|
25
|
+
vectorProbe?: { sql: string; args: unknown[]; limit: number }
|
|
19
26
|
sql: string
|
|
20
27
|
args: unknown[]
|
|
21
28
|
totalSql: string
|
|
@@ -74,7 +81,8 @@ type CompiledOrderField =
|
|
|
74
81
|
}
|
|
75
82
|
| {
|
|
76
83
|
kind: "column"
|
|
77
|
-
column: "object_type_id" | "primary_id"
|
|
84
|
+
column: "object_type_id" | "primary_id" | "_vector_score"
|
|
85
|
+
binary?: boolean
|
|
78
86
|
direction: "asc" | "desc"
|
|
79
87
|
}
|
|
80
88
|
|
|
@@ -99,6 +107,12 @@ export function compilePgObjectQuery(
|
|
|
99
107
|
query: ObjectQuery,
|
|
100
108
|
options: { includeTotal?: boolean; source?: PgObjectQuerySource } = {}
|
|
101
109
|
): CompiledPgObjectQuery {
|
|
110
|
+
if (hasVectorProfile(query) && !isVectorProfileQuery(query)) {
|
|
111
|
+
throw new ObjectQueryExecutionError(
|
|
112
|
+
"unsupported_vector_composition",
|
|
113
|
+
"Vector search supports one profile with filters before ranking and limit/project after."
|
|
114
|
+
)
|
|
115
|
+
}
|
|
102
116
|
const source = options.source ?? DEFAULT_OBJECT_QUERY_SOURCE
|
|
103
117
|
const compiled = source.wrapQuery(
|
|
104
118
|
compileObjectQueryInternal(projectId, query, {
|
|
@@ -110,6 +124,9 @@ export function compilePgObjectQuery(
|
|
|
110
124
|
...compiled,
|
|
111
125
|
sql: numberPlaceholders(compiled.sql),
|
|
112
126
|
totalSql: numberPlaceholders(compiled.totalSql),
|
|
127
|
+
vectorProbe: compiled.vectorProbe
|
|
128
|
+
? { ...compiled.vectorProbe, sql: numberPlaceholders(compiled.vectorProbe.sql) }
|
|
129
|
+
: undefined,
|
|
113
130
|
hasMoreProbe: compiled.hasMoreProbe
|
|
114
131
|
? {
|
|
115
132
|
...compiled.hasMoreProbe,
|
|
@@ -123,6 +140,7 @@ export function compilePgObjectQuery(
|
|
|
123
140
|
export interface PgObjectQuerySource {
|
|
124
141
|
readonly objectsTable: string
|
|
125
142
|
readonly linksTable: string
|
|
143
|
+
readonly objectPropertyPermissionsTable?: string
|
|
126
144
|
/** Wrap a terminal SELECT without its own WITH clause; the selected source owns the only WITH. */
|
|
127
145
|
wrapStatement(
|
|
128
146
|
sql: string,
|
|
@@ -263,9 +281,74 @@ function compileObjectQueryInternal(
|
|
|
263
281
|
case "expand":
|
|
264
282
|
return compileExpand(projectId, query.input, query.expansions, ctx)
|
|
265
283
|
case "vector":
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
284
|
+
return compileVector(projectId, query, ctx)
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function compileVector(
|
|
289
|
+
projectId: string,
|
|
290
|
+
query: Extract<ObjectQuery, { kind: "vector" }>,
|
|
291
|
+
ctx: CompileContext
|
|
292
|
+
): CompiledPgObjectQuery {
|
|
293
|
+
if (
|
|
294
|
+
typeof query.vector === "string" ||
|
|
295
|
+
!isVectorProfileQuery(query) ||
|
|
296
|
+
!query.configuration ||
|
|
297
|
+
!Number.isSafeInteger(query.k) ||
|
|
298
|
+
query.k < 1 ||
|
|
299
|
+
query.k > 1000 ||
|
|
300
|
+
query.vector.length < 1 ||
|
|
301
|
+
query.vector.length > 16000
|
|
302
|
+
) {
|
|
303
|
+
throw new ObjectQueryExecutionError(
|
|
304
|
+
"invalid_vector_query",
|
|
305
|
+
"A validated named vector profile is required."
|
|
306
|
+
)
|
|
307
|
+
}
|
|
308
|
+
const input = compileObjectQueryInternal(projectId, query.input, exactContext(ctx))
|
|
309
|
+
const permissions = ctx.source.objectPropertyPermissionsTable
|
|
310
|
+
const permissionJoin = permissions
|
|
311
|
+
? `JOIN (
|
|
312
|
+
SELECT project_id, object_type_id, primary_id, array_agg(property_id) AS property_ids
|
|
313
|
+
FROM ${permissions}
|
|
314
|
+
GROUP BY project_id, object_type_id, primary_id
|
|
315
|
+
) AS vector_permissions ON vector_permissions.project_id = vectors.project_id
|
|
316
|
+
AND vector_permissions.object_type_id = vectors.object_type_id
|
|
317
|
+
AND vector_permissions.primary_id = vectors.primary_id`
|
|
318
|
+
: ""
|
|
319
|
+
const authorized = permissions ? `AND vectors.source <@ vector_permissions.property_ids` : ""
|
|
320
|
+
const candidates = `FROM (${input.sql}) AS input
|
|
321
|
+
JOIN object_vectors AS vectors ON vectors.project_id = input.project_id
|
|
322
|
+
AND vectors.object_type_id = input.object_type_id AND vectors.primary_id = input.primary_id
|
|
323
|
+
${permissionJoin}
|
|
324
|
+
WHERE vectors.profile = ? AND vectors.configuration = ?
|
|
325
|
+
AND cardinality(vectors.embedding) = ? ${authorized}`
|
|
326
|
+
const candidateArgs = [...input.args, query.profile!, query.configuration, query.vector.length]
|
|
327
|
+
// Bound distance work, not just returned rows. Exceeding the envelope fails before ranking.
|
|
328
|
+
const limit = vectorSearchCandidateLimit(query.vector.length)
|
|
329
|
+
const values = normalizeVector(query.vector, query.vector.length)
|
|
330
|
+
const order = compileOrder([
|
|
331
|
+
{ kind: "column", column: "_vector_score", direction: "desc" },
|
|
332
|
+
...identityOrderFields().map((field) => ({ ...field, binary: true })),
|
|
333
|
+
])
|
|
334
|
+
const sql = `SELECT * FROM (SELECT input.*, 1 - (vectors.embedding::public.vector OPERATOR(public.<=>) ?::public.vector) AS _vector_score
|
|
335
|
+
${candidates}) AS ranked ORDER BY ${order.sql} LIMIT ?`
|
|
336
|
+
const args = [JSON.stringify(values), ...candidateArgs, query.k]
|
|
337
|
+
return {
|
|
338
|
+
sql,
|
|
339
|
+
args,
|
|
340
|
+
// Top-k cardinality depends on eligibility, not distances or ranking.
|
|
341
|
+
totalSql: `SELECT COUNT(*)::bigint AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS eligible`,
|
|
342
|
+
totalArgs: [...candidateArgs, query.k],
|
|
343
|
+
order,
|
|
344
|
+
vectorProbe: {
|
|
345
|
+
sql: `SELECT COUNT(*)::bigint AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS candidates`,
|
|
346
|
+
args: [...candidateArgs, limit + 1],
|
|
347
|
+
limit,
|
|
348
|
+
},
|
|
349
|
+
hasMore: () => false,
|
|
350
|
+
trimRows: identityRows,
|
|
351
|
+
nextPageToken: () => undefined,
|
|
269
352
|
}
|
|
270
353
|
}
|
|
271
354
|
|
|
@@ -377,6 +460,7 @@ function compileWhere(
|
|
|
377
460
|
WHERE ${predicate.sql}
|
|
378
461
|
`,
|
|
379
462
|
totalArgs: [...input.args, ...predicate.args],
|
|
463
|
+
vectorProbe: input.vectorProbe,
|
|
380
464
|
order: input.order,
|
|
381
465
|
hasMore: () => false,
|
|
382
466
|
trimRows: identityRows,
|
|
@@ -424,7 +508,7 @@ function compileSort(
|
|
|
424
508
|
input.created_at,
|
|
425
509
|
input.updated_at,
|
|
426
510
|
input.version,
|
|
427
|
-
input.last_commit_id
|
|
511
|
+
input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
|
|
428
512
|
FROM (${input.sql}) AS input
|
|
429
513
|
ORDER BY ${order.sql}
|
|
430
514
|
`,
|
|
@@ -462,6 +546,7 @@ function compileLimit(
|
|
|
462
546
|
FROM (${input.sql}) AS input
|
|
463
547
|
`,
|
|
464
548
|
totalArgs: input.args,
|
|
549
|
+
vectorProbe: input.vectorProbe,
|
|
465
550
|
order: input.order,
|
|
466
551
|
hasMore: (rowCount, total) =>
|
|
467
552
|
total === undefined ? ctx.probeLimit && rowCount > limit : limit < total,
|
|
@@ -499,6 +584,7 @@ function compilePage(
|
|
|
499
584
|
FROM (${input.sql}) AS input
|
|
500
585
|
`,
|
|
501
586
|
totalArgs: input.args,
|
|
587
|
+
vectorProbe: input.vectorProbe,
|
|
502
588
|
order: input.order,
|
|
503
589
|
hasMore: (rowCount) => rowCount > pageSize,
|
|
504
590
|
trimRows: (rows) => rows.slice(0, pageSize),
|
|
@@ -607,6 +693,7 @@ function compileExpand(
|
|
|
607
693
|
args: [...expand.args, ...input.args, ...inputOrder.args],
|
|
608
694
|
totalSql: input.totalSql,
|
|
609
695
|
totalArgs: input.totalArgs,
|
|
696
|
+
vectorProbe: input.vectorProbe,
|
|
610
697
|
order: input.order,
|
|
611
698
|
hasMoreProbe: input.hasMoreProbe,
|
|
612
699
|
hasMore: input.hasMore,
|
|
@@ -849,13 +936,14 @@ function compileProject(
|
|
|
849
936
|
input.created_at,
|
|
850
937
|
input.updated_at,
|
|
851
938
|
input.version,
|
|
852
|
-
input.last_commit_id
|
|
939
|
+
input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
|
|
853
940
|
FROM (${input.sql}) AS input
|
|
854
941
|
ORDER BY ${inputOrder.sql}
|
|
855
942
|
`,
|
|
856
943
|
args: [...projection.args, ...input.args, ...inputOrder.args],
|
|
857
944
|
totalSql: input.totalSql,
|
|
858
945
|
totalArgs: input.totalArgs,
|
|
946
|
+
vectorProbe: input.vectorProbe,
|
|
859
947
|
order: outputOrder,
|
|
860
948
|
hasMoreProbe: input.hasMoreProbe,
|
|
861
949
|
hasMore: input.hasMore,
|
|
@@ -909,9 +997,7 @@ function compileAggregateSource(
|
|
|
909
997
|
case "page":
|
|
910
998
|
return compileRowQueryAggregateSource(projectId, query, source)
|
|
911
999
|
case "vector":
|
|
912
|
-
|
|
913
|
-
`[SixbPg] PostgreSQL object storage does not support query node '${query.kind}'`
|
|
914
|
-
)
|
|
1000
|
+
return compileRowQueryAggregateSource(projectId, query, source)
|
|
915
1001
|
}
|
|
916
1002
|
}
|
|
917
1003
|
|
|
@@ -1416,7 +1502,9 @@ function compileOrder(
|
|
|
1416
1502
|
|
|
1417
1503
|
for (const field of fields) {
|
|
1418
1504
|
if (field.kind === "column") {
|
|
1419
|
-
clauses.push(
|
|
1505
|
+
clauses.push(
|
|
1506
|
+
`${columnExpression(field.column, qualifier)}${field.binary ? ' COLLATE "C"' : ""} ${field.direction.toUpperCase()}`
|
|
1507
|
+
)
|
|
1420
1508
|
continue
|
|
1421
1509
|
}
|
|
1422
1510
|
|
|
@@ -1637,7 +1725,10 @@ function compiledPropertyValueExpression(
|
|
|
1637
1725
|
: jsonValueExpression(propertyColumn)
|
|
1638
1726
|
}
|
|
1639
1727
|
|
|
1640
|
-
function columnExpression(
|
|
1728
|
+
function columnExpression(
|
|
1729
|
+
column: "object_type_id" | "primary_id" | "_vector_score",
|
|
1730
|
+
qualifier?: string
|
|
1731
|
+
): string {
|
|
1641
1732
|
return qualifier ? `${qualifier}.${column}` : column
|
|
1642
1733
|
}
|
|
1643
1734
|
|
|
@@ -18,6 +18,7 @@ export interface PgSelectedObjectReadSource extends PgObjectQuerySource {
|
|
|
18
18
|
*
|
|
19
19
|
* The complete compiled scope is serialized once. Every statement receives that one JSONB
|
|
20
20
|
* document and the bound project id, keeping the parameter count constant as selections grow.
|
|
21
|
+
* Group property grants after resolving paths so redaction reads only each object's own grants.
|
|
21
22
|
*/
|
|
22
23
|
export function compilePgSelectedObjectReadSource(
|
|
23
24
|
projectId: string,
|
|
@@ -43,6 +44,9 @@ export function compilePgSelectedObjectReadSource(
|
|
|
43
44
|
wrapQuery: (query: CompiledPgObjectQuery): CompiledPgObjectQuery => ({
|
|
44
45
|
...query,
|
|
45
46
|
...wrapStatement(query.sql, query.args),
|
|
47
|
+
vectorProbe: query.vectorProbe
|
|
48
|
+
? { ...query.vectorProbe, ...wrapStatement(query.vectorProbe.sql, query.vectorProbe.args) }
|
|
49
|
+
: undefined,
|
|
46
50
|
totalSql: `${compiled.sql}\n${query.totalSql}`,
|
|
47
51
|
totalArgs: [...compiled.args, ...query.totalArgs],
|
|
48
52
|
...(query.hasMoreProbe
|
|
@@ -177,32 +181,29 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
|
|
|
177
181
|
SELECT DISTINCT project_id, object_type_id, primary_id
|
|
178
182
|
FROM _sixb_scope_reachable
|
|
179
183
|
),
|
|
184
|
+
_sixb_scope_object_grants AS (
|
|
185
|
+
SELECT reachable.project_id, reachable.object_type_id, reachable.primary_id,
|
|
186
|
+
array_agg(DISTINCT selected_property.value) FILTER (WHERE selected_property.value IS NOT NULL) AS property_ids
|
|
187
|
+
FROM _sixb_scope_reachable AS reachable
|
|
188
|
+
LEFT JOIN _sixb_scope_object_selections AS selection
|
|
189
|
+
ON selection.node_id = reachable.node_id
|
|
190
|
+
AND selection.object_type_id = reachable.object_type_id
|
|
191
|
+
LEFT JOIN LATERAL jsonb_array_elements_text(selection.property_ids) AS selected_property(value) ON true
|
|
192
|
+
GROUP BY reachable.project_id, reachable.object_type_id, reachable.primary_id
|
|
193
|
+
),
|
|
180
194
|
_sixb_scope_objects AS (
|
|
181
|
-
SELECT
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
AND permission.property_id = property.key
|
|
194
|
-
),
|
|
195
|
-
'{}'::jsonb
|
|
196
|
-
) AS properties,
|
|
197
|
-
stored.created_at,
|
|
198
|
-
stored.updated_at,
|
|
199
|
-
stored.version,
|
|
200
|
-
stored.last_commit_id
|
|
201
|
-
FROM objects AS stored
|
|
202
|
-
JOIN _sixb_scope_object_ids AS visible
|
|
203
|
-
ON visible.project_id = stored.project_id
|
|
204
|
-
AND visible.object_type_id = stored.object_type_id
|
|
205
|
-
AND visible.primary_id = stored.primary_id
|
|
195
|
+
SELECT stored.project_id, stored.object_type_id, stored.primary_id,
|
|
196
|
+
COALESCE((
|
|
197
|
+
SELECT jsonb_object_agg(property.key, property.value)
|
|
198
|
+
FROM jsonb_each(stored.properties) AS property(key, value)
|
|
199
|
+
WHERE property.key = ANY(grants.property_ids)
|
|
200
|
+
), '{}'::jsonb) AS properties,
|
|
201
|
+
stored.created_at, stored.updated_at, stored.version, stored.last_commit_id
|
|
202
|
+
FROM _sixb_scope_object_grants AS grants
|
|
203
|
+
JOIN objects AS stored
|
|
204
|
+
ON stored.project_id = grants.project_id
|
|
205
|
+
AND stored.object_type_id = grants.object_type_id
|
|
206
|
+
AND stored.primary_id = grants.primary_id
|
|
206
207
|
),
|
|
207
208
|
_sixb_scope_reached_links(
|
|
208
209
|
project_id,
|
package/src/objects/reader.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type ObjectQuery, ObjectQueryExecutionError } from "@sixb/core"
|
|
2
|
+
import { hasVectorProfile } from "@sixb/core/internal/query"
|
|
2
3
|
import type {
|
|
3
4
|
CountObjectsInput,
|
|
4
5
|
CountObjectsResult,
|
|
@@ -51,7 +52,35 @@ export class PgObjectReader {
|
|
|
51
52
|
private readonly source: PgObjectQuerySource | PgSelectedObjectReadSource
|
|
52
53
|
) {}
|
|
53
54
|
|
|
55
|
+
private async assertVectorSearch(
|
|
56
|
+
params: QueryObjectsInput | CountObjectsInput | ExistsObjectsInput | FacetObjectsInput
|
|
57
|
+
): Promise<void> {
|
|
58
|
+
if (!hasVectorProfile(params.query)) return
|
|
59
|
+
const compiled = compilePgObjectQuery(params.projectId, params.query, { source: this.source })
|
|
60
|
+
if (!compiled.vectorProbe) return
|
|
61
|
+
const extensions = await this.sql.unsafe<{ ready: boolean }[]>(
|
|
62
|
+
"SELECT EXISTS (SELECT 1 FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace WHERE e.extname = 'vector' AND n.nspname = 'public') AS ready"
|
|
63
|
+
)
|
|
64
|
+
if (!extensions[0]?.ready)
|
|
65
|
+
throw new ObjectQueryExecutionError(
|
|
66
|
+
"vector_extension_missing",
|
|
67
|
+
"PostgreSQL vector search requires pgvector installed in schema public (CREATE EXTENSION vector WITH SCHEMA public)."
|
|
68
|
+
)
|
|
69
|
+
const probe = compiled.vectorProbe
|
|
70
|
+
const [row] = await this.sql.unsafe<{ total: string | number }[]>(
|
|
71
|
+
probe.sql,
|
|
72
|
+
probe.args as SqlParameter[]
|
|
73
|
+
)
|
|
74
|
+
if (Number(row?.total ?? 0) > probe.limit) {
|
|
75
|
+
throw new ObjectQueryExecutionError(
|
|
76
|
+
"vector_work_limit_exceeded",
|
|
77
|
+
`Exact vector search supports at most ${probe.limit} eligible vectors at this dimension. Narrow the query filters.`
|
|
78
|
+
)
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
54
82
|
async queryObjects(params: QueryObjectsInput): Promise<QueryObjectsResult> {
|
|
83
|
+
await this.assertVectorSearch(params)
|
|
55
84
|
const sql = this.sql
|
|
56
85
|
const source = this.source
|
|
57
86
|
const compiled = compilePgObjectQuery(params.projectId, params.query, {
|
|
@@ -85,6 +114,7 @@ export class PgObjectReader {
|
|
|
85
114
|
}
|
|
86
115
|
|
|
87
116
|
async countObjects(params: CountObjectsInput): Promise<CountObjectsResult> {
|
|
117
|
+
await this.assertVectorSearch(params)
|
|
88
118
|
const sql = this.sql
|
|
89
119
|
const source = this.source
|
|
90
120
|
const compiled = compilePgObjectCountQuery(params.projectId, stripOuterRowShape(params.query), {
|
|
@@ -98,6 +128,7 @@ export class PgObjectReader {
|
|
|
98
128
|
}
|
|
99
129
|
|
|
100
130
|
async existsObjects(params: ExistsObjectsInput): Promise<ExistsObjectsResult> {
|
|
131
|
+
await this.assertVectorSearch(params)
|
|
101
132
|
const sql = this.sql
|
|
102
133
|
const source = this.source
|
|
103
134
|
const compiled = compilePgObjectExistsQuery(
|
|
@@ -110,6 +141,7 @@ export class PgObjectReader {
|
|
|
110
141
|
}
|
|
111
142
|
|
|
112
143
|
async facetObjects(params: FacetObjectsInput): Promise<FacetObjectsResult> {
|
|
144
|
+
await this.assertVectorSearch(params)
|
|
113
145
|
const sql = this.sql
|
|
114
146
|
const source = this.source
|
|
115
147
|
const facets: ObjectFacetResult[] = []
|
package/src/objects/rows.ts
CHANGED
|
@@ -23,7 +23,10 @@ export function rowToObject(row: ObjectDatabaseRow): ObjectRow {
|
|
|
23
23
|
// base columns map as usual; `_expand` (a `jsonb_build_object`) is revived into
|
|
24
24
|
// the runtime link shape the core executor's fallback also produces.
|
|
25
25
|
export function queryRowToObject(row: ObjectQueryDatabaseRow): ObjectRow {
|
|
26
|
-
const base =
|
|
26
|
+
const base = {
|
|
27
|
+
...rowToObject(row),
|
|
28
|
+
...(row._vector_score === undefined ? {} : { score: row._vector_score }),
|
|
29
|
+
}
|
|
27
30
|
const links = reviveExpandedLinks(row._expand)
|
|
28
31
|
return links ? { ...base, links } : base
|
|
29
32
|
}
|
|
@@ -109,6 +112,7 @@ export interface ObjectDatabaseRow {
|
|
|
109
112
|
}
|
|
110
113
|
|
|
111
114
|
export interface ObjectQueryDatabaseRow extends ObjectDatabaseRow {
|
|
115
|
+
_vector_score?: number
|
|
112
116
|
_cursor_properties?: unknown
|
|
113
117
|
/** `jsonb_build_object(linkId, value, ...)` from an `expand` pushdown; absent otherwise. */
|
|
114
118
|
_expand?: unknown
|
package/src/objects/storage.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { hasVectorProfile } from "@sixb/core/internal/query"
|
|
1
2
|
import type {
|
|
2
3
|
CompiledSelectedObjectReadScope,
|
|
3
4
|
CountObjectsInput,
|
|
@@ -21,7 +22,7 @@ import type {
|
|
|
21
22
|
QueryObjectsResult,
|
|
22
23
|
} from "@sixb/core/storage"
|
|
23
24
|
import type { SQLClient, SqlParameter } from "../pg-client"
|
|
24
|
-
import type
|
|
25
|
+
import { type PgStoreClient, runPgRepeatableReadTransaction } from "../transactions"
|
|
25
26
|
import { DEFAULT_OBJECT_QUERY_SOURCE } from "./query-compiler"
|
|
26
27
|
import { assertLinkQueryLimit, PgObjectReader } from "./reader"
|
|
27
28
|
import {
|
|
@@ -43,6 +44,7 @@ const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
|
|
|
43
44
|
refs: true,
|
|
44
45
|
filter: true,
|
|
45
46
|
text: true,
|
|
47
|
+
vector: true,
|
|
46
48
|
sort: true,
|
|
47
49
|
limit: true,
|
|
48
50
|
page: true,
|
|
@@ -86,6 +88,8 @@ const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
|
|
|
86
88
|
decimal: { equality: true, ordering: true },
|
|
87
89
|
date: { equality: true, ordering: true },
|
|
88
90
|
timestamp: { equality: true, ordering: true },
|
|
91
|
+
// jsonb equality and containment compare both fields of `{ type, id }`, whatever key order.
|
|
92
|
+
userRef: { equality: true },
|
|
89
93
|
},
|
|
90
94
|
limits: {
|
|
91
95
|
totalCount: true,
|
|
@@ -95,7 +99,7 @@ const PG_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
|
|
|
95
99
|
"PostgreSQL object query pushdown supports start/refs/filter/text/sort/limit/page/traverse/set/project/expand over JSONB properties and object links.",
|
|
96
100
|
"expand hydrates linked objects in-database (top-N per parent via LATERAL + jsonb_agg); core resolves each expansion's cardinality before pushdown, and a mixed/unresolved one stays on the fallback.",
|
|
97
101
|
"Exact decimal predicates, ordering, and keyset pagination use PostgreSQL numeric casts.",
|
|
98
|
-
"Relevance sorting
|
|
102
|
+
"Relevance sorting and unresolved start.includeSubtypes remain planner fallback or rejection cases.",
|
|
99
103
|
],
|
|
100
104
|
}
|
|
101
105
|
|
|
@@ -120,18 +124,34 @@ export class PgObjectStorage implements ObjectStorage {
|
|
|
120
124
|
}
|
|
121
125
|
|
|
122
126
|
async queryObjects(params: QueryObjectsInput): Promise<QueryObjectsResult> {
|
|
127
|
+
if (hasVectorProfile(params.query))
|
|
128
|
+
return runPgRepeatableReadTransaction(this.sql, (tx) =>
|
|
129
|
+
new PgObjectReader(tx, DEFAULT_OBJECT_QUERY_SOURCE).queryObjects(params)
|
|
130
|
+
)
|
|
123
131
|
return this.reader.queryObjects(params)
|
|
124
132
|
}
|
|
125
133
|
|
|
126
134
|
async countObjects(params: CountObjectsInput): Promise<CountObjectsResult> {
|
|
135
|
+
if (hasVectorProfile(params.query))
|
|
136
|
+
return runPgRepeatableReadTransaction(this.sql, (tx) =>
|
|
137
|
+
new PgObjectReader(tx, DEFAULT_OBJECT_QUERY_SOURCE).countObjects(params)
|
|
138
|
+
)
|
|
127
139
|
return this.reader.countObjects(params)
|
|
128
140
|
}
|
|
129
141
|
|
|
130
142
|
async existsObjects(params: ExistsObjectsInput): Promise<ExistsObjectsResult> {
|
|
143
|
+
if (hasVectorProfile(params.query))
|
|
144
|
+
return runPgRepeatableReadTransaction(this.sql, (tx) =>
|
|
145
|
+
new PgObjectReader(tx, DEFAULT_OBJECT_QUERY_SOURCE).existsObjects(params)
|
|
146
|
+
)
|
|
131
147
|
return this.reader.existsObjects(params)
|
|
132
148
|
}
|
|
133
149
|
|
|
134
150
|
async facetObjects(params: FacetObjectsInput): Promise<FacetObjectsResult> {
|
|
151
|
+
if (hasVectorProfile(params.query))
|
|
152
|
+
return runPgRepeatableReadTransaction(this.sql, (tx) =>
|
|
153
|
+
new PgObjectReader(tx, DEFAULT_OBJECT_QUERY_SOURCE).facetObjects(params)
|
|
154
|
+
)
|
|
135
155
|
return this.reader.facetObjects(params)
|
|
136
156
|
}
|
|
137
157
|
|
|
@@ -6,12 +6,16 @@ import { PgOntologyMaterializationStorage } from "./materializations"
|
|
|
6
6
|
import { PgOntologyOutboxStorage } from "./outbox"
|
|
7
7
|
import type { PgRootOperation } from "./shared"
|
|
8
8
|
import { PgOntologySourceStorage } from "./sources"
|
|
9
|
+
import { PgVectorIndexingStorage } from "./vector-indexing"
|
|
10
|
+
import { PgOntologyVectorStorage } from "./vectors"
|
|
9
11
|
|
|
10
12
|
export class PgOntologyStorage implements OntologyStorage {
|
|
11
13
|
readonly commits: PgOntologyCommitStorage
|
|
12
14
|
readonly sources: PgOntologySourceStorage
|
|
13
15
|
readonly materializations: PgOntologyMaterializationStorage
|
|
14
16
|
readonly outbox: PgOntologyOutboxStorage
|
|
17
|
+
readonly vectorIndexing: PgVectorIndexingStorage
|
|
18
|
+
readonly vectors: PgOntologyVectorStorage
|
|
15
19
|
|
|
16
20
|
constructor(input: {
|
|
17
21
|
readonly sql: SQLClient
|
|
@@ -25,6 +29,17 @@ export class PgOntologyStorage implements OntologyStorage {
|
|
|
25
29
|
input.transactionContext
|
|
26
30
|
)
|
|
27
31
|
this.outbox = new PgOntologyOutboxStorage(input.runRootOperation)
|
|
32
|
+
this.vectorIndexing = new PgVectorIndexingStorage(
|
|
33
|
+
input.sql,
|
|
34
|
+
input.runRootOperation,
|
|
35
|
+
(session, projectId) => this.materializations.assertVectorSession(session, projectId)
|
|
36
|
+
)
|
|
37
|
+
this.vectors = new PgOntologyVectorStorage(
|
|
38
|
+
input.sql,
|
|
39
|
+
input.runRootOperation,
|
|
40
|
+
(session, projectId, commitId) =>
|
|
41
|
+
this.materializations.assertVectorSession(session, projectId, commitId)
|
|
42
|
+
)
|
|
28
43
|
}
|
|
29
44
|
|
|
30
45
|
deactivateSessions(): void {
|
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
EffectiveChangeCounts,
|
|
3
|
-
ExpectedLinkRevision,
|
|
4
|
-
ExpectedObjectRevision,
|
|
5
|
-
} from "@sixb/core/internal/materialization"
|
|
1
|
+
import type { EffectiveChangeCounts } from "@sixb/core/internal/materialization"
|
|
6
2
|
import {
|
|
7
3
|
assertPinnedDatasetWatermark,
|
|
8
4
|
linkRefKey,
|
|
9
5
|
linkRefSortKey,
|
|
10
6
|
linkScopeSortKey,
|
|
11
7
|
MaterializationConflictError,
|
|
8
|
+
MaterializationValidationError,
|
|
12
9
|
objectRefKey,
|
|
13
10
|
objectRefSortKey,
|
|
14
11
|
telemetryPointKey,
|
|
15
12
|
telemetryPointSortKey,
|
|
16
13
|
} from "@sixb/core/internal/materialization"
|
|
17
14
|
import {
|
|
15
|
+
assertExpectedLinkRevision,
|
|
16
|
+
assertExpectedLinkScopeRevision,
|
|
17
|
+
assertExpectedObjectRevision,
|
|
18
18
|
assertMaterializationFinalizationCorrelation,
|
|
19
19
|
assertMaterializationHeader,
|
|
20
20
|
assertMaterializationLaneCompletion,
|
|
@@ -100,6 +100,18 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
100
100
|
this.writer = new PgMaterializationWriter(sql)
|
|
101
101
|
}
|
|
102
102
|
|
|
103
|
+
assertVectorSession(session: MaterializationSession, projectId: string, commitId?: string): void {
|
|
104
|
+
const active = this.sessions.require(session)
|
|
105
|
+
if (
|
|
106
|
+
active.header.commit.projectId !== projectId ||
|
|
107
|
+
(commitId !== undefined && active.header.commit.id !== commitId)
|
|
108
|
+
) {
|
|
109
|
+
throw new MaterializationValidationError(
|
|
110
|
+
"Vector mutation does not belong to this materialization session."
|
|
111
|
+
)
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
103
115
|
async begin(input: MaterializationPlanHeader): Promise<MaterializationSession> {
|
|
104
116
|
assertMaterializationHeader(input)
|
|
105
117
|
const session = await this.sessions.create(input)
|
|
@@ -113,23 +125,21 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
113
125
|
true
|
|
114
126
|
)
|
|
115
127
|
for (const expected of input.expected.objects) {
|
|
116
|
-
|
|
128
|
+
assertExpectedObjectRevision(
|
|
129
|
+
objectRevisions.get(objectRefKey(expected.ref)) ?? null,
|
|
130
|
+
expected
|
|
131
|
+
)
|
|
117
132
|
}
|
|
118
133
|
const linkRevisions = await reader.effectiveLinkLastCommits(
|
|
119
134
|
input.expected.links.map((expected) => expected.ref),
|
|
120
135
|
true
|
|
121
136
|
)
|
|
122
137
|
for (const expected of input.expected.links) {
|
|
123
|
-
|
|
138
|
+
assertExpectedLinkRevision(linkRevisions.get(linkRefKey(expected.ref)), expected)
|
|
124
139
|
}
|
|
125
140
|
const linkScopeRevisions = await reader.linkScopeRevisions(input.expected.linkScopes)
|
|
126
141
|
for (const [index, expected] of input.expected.linkScopes.entries()) {
|
|
127
|
-
|
|
128
|
-
throw new MaterializationConflictError(
|
|
129
|
-
"effective-state",
|
|
130
|
-
`Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`
|
|
131
|
-
)
|
|
132
|
-
}
|
|
142
|
+
assertExpectedLinkScopeRevision(linkScopeRevisions[index]?.fingerprint, expected)
|
|
133
143
|
}
|
|
134
144
|
const points = await reader.exactPoints(input.expected.points, true)
|
|
135
145
|
const pointRevisions = new Map(
|
|
@@ -501,48 +511,6 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
501
511
|
}
|
|
502
512
|
}
|
|
503
513
|
|
|
504
|
-
private assertObject(
|
|
505
|
-
row: { readonly version: number; readonly lastCommitId: string | null } | null,
|
|
506
|
-
expected: ExpectedObjectRevision
|
|
507
|
-
): void {
|
|
508
|
-
if (!expected.exists) {
|
|
509
|
-
if (row) {
|
|
510
|
-
throw new MaterializationConflictError(
|
|
511
|
-
"effective-state",
|
|
512
|
-
`Expected object ${objectRefKey(expected.ref)} to be absent.`
|
|
513
|
-
)
|
|
514
|
-
}
|
|
515
|
-
return
|
|
516
|
-
}
|
|
517
|
-
if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId) {
|
|
518
|
-
throw new MaterializationConflictError(
|
|
519
|
-
"effective-state",
|
|
520
|
-
`Expected object ${objectRefKey(expected.ref)} changed.`
|
|
521
|
-
)
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
private assertLink(
|
|
526
|
-
lastCommitId: string | null | undefined,
|
|
527
|
-
expected: ExpectedLinkRevision
|
|
528
|
-
): void {
|
|
529
|
-
if (!expected.exists) {
|
|
530
|
-
if (lastCommitId !== undefined) {
|
|
531
|
-
throw new MaterializationConflictError(
|
|
532
|
-
"effective-state",
|
|
533
|
-
`Expected link ${linkRefKey(expected.ref)} to be absent.`
|
|
534
|
-
)
|
|
535
|
-
}
|
|
536
|
-
return
|
|
537
|
-
}
|
|
538
|
-
if (lastCommitId === undefined || lastCommitId !== expected.lastCommitId) {
|
|
539
|
-
throw new MaterializationConflictError(
|
|
540
|
-
"effective-state",
|
|
541
|
-
`Expected link ${linkRefKey(expected.ref)} changed.`
|
|
542
|
-
)
|
|
543
|
-
}
|
|
544
|
-
}
|
|
545
|
-
|
|
546
514
|
private async assertFinalization(
|
|
547
515
|
session: PgMaterializationSessionState,
|
|
548
516
|
input: FinalizeMaterializationInput
|
|
@@ -860,18 +828,20 @@ export class PgOntologyMaterializationStorage implements OntologyMaterialization
|
|
|
860
828
|
): Promise<OntologyCommitRecord> {
|
|
861
829
|
const { commit } = header
|
|
862
830
|
const origin = originColumns(commit.origin)
|
|
863
|
-
const
|
|
831
|
+
const requestedBy =
|
|
832
|
+
commit.requestedBy === undefined ? null : jsonParameter(this.sql, commit.requestedBy)
|
|
864
833
|
const rows = await this.sql<PgOntologyCommitRow[]>`
|
|
865
834
|
INSERT INTO ontology_commits (
|
|
866
835
|
project_id, id, idempotency_key, request_hash, execution_id,
|
|
867
|
-
origin_kind, origin_run_id, origin_batch_ordinal, origin,
|
|
836
|
+
origin_kind, origin_run_id, origin_batch_ordinal, origin, requested_by, executor,
|
|
868
837
|
ontology_revision, projection_revision, ownership_hash,
|
|
869
838
|
intent, result, committed_at
|
|
870
839
|
) VALUES (
|
|
871
840
|
${commit.projectId}, ${commit.id}, ${commit.idempotencyKey}, ${commit.requestHash},
|
|
872
841
|
${commit.executionId},
|
|
873
842
|
${origin.kind}, ${origin.runId}, ${origin.batchOrdinal},
|
|
874
|
-
${jsonParameter(this.sql, commit.origin)}, ${
|
|
843
|
+
${jsonParameter(this.sql, commit.origin)}, ${requestedBy},
|
|
844
|
+
${jsonParameter(this.sql, commit.executor)}, ${commit.ontologyRevision},
|
|
875
845
|
${commit.projectionRevision ?? null}, ${commit.ownershipHash ?? null},
|
|
876
846
|
${jsonParameter(this.sql, commit.intent)},
|
|
877
847
|
${jsonParameter(this.sql, input.finalization.result)}, ${commit.committedAt}
|
|
@@ -39,7 +39,8 @@ export interface PgOntologyCommitRow {
|
|
|
39
39
|
readonly origin_run_id: string | null
|
|
40
40
|
readonly origin_batch_ordinal: number | string | null
|
|
41
41
|
readonly origin: unknown
|
|
42
|
-
readonly
|
|
42
|
+
readonly requested_by: unknown | null
|
|
43
|
+
readonly executor: unknown
|
|
43
44
|
readonly ontology_revision: string
|
|
44
45
|
readonly projection_revision: string | null
|
|
45
46
|
readonly ownership_hash: string | null
|
|
@@ -187,11 +188,14 @@ export function commitRecord(row: PgOntologyCommitRow): OntologyCommitRecord {
|
|
|
187
188
|
requestHash: row.request_hash,
|
|
188
189
|
executionId: row.execution_id,
|
|
189
190
|
origin: structuredClone(row.origin) as OntologyCommitWrite["origin"],
|
|
190
|
-
...(row.
|
|
191
|
+
...(row.requested_by === null
|
|
191
192
|
? {}
|
|
192
193
|
: {
|
|
193
|
-
|
|
194
|
+
requestedBy: structuredClone(row.requested_by) as NonNullable<
|
|
195
|
+
OntologyCommitWrite["requestedBy"]
|
|
196
|
+
>,
|
|
194
197
|
}),
|
|
198
|
+
executor: structuredClone(row.executor) as OntologyCommitWrite["executor"],
|
|
195
199
|
ontologyRevision: row.ontology_revision,
|
|
196
200
|
...(row.projection_revision === null ? {} : { projectionRevision: row.projection_revision }),
|
|
197
201
|
...(row.ownership_hash === null ? {} : { ownershipHash: row.ownership_hash }),
|