@sixb/sqlite 0.1.11 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/agents/rows.d.ts +2 -0
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/threads.d.ts +2 -1
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/index.js +676 -61
- package/dist/index.js.map +18 -14
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +9 -2
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts +3 -0
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +1 -0
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/rows.d.ts +1 -0
- package/dist/objects/rows.d.ts.map +1 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/vector-extension.d.ts +4 -0
- package/dist/objects/vector-extension.d.ts.map +1 -0
- package/dist/ontology-storage/index.d.ts +4 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +1 -0
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/vector-indexing.d.ts +20 -0
- package/dist/ontology-storage/vector-indexing.d.ts.map +1 -0
- package/dist/ontology-storage/vectors.d.ts +16 -0
- package/dist/ontology-storage/vectors.d.ts.map +1 -0
- package/dist/vector-encoding.d.ts +3 -0
- package/dist/vector-encoding.d.ts.map +1 -0
- package/package.json +8 -4
- package/src/agents/rows.ts +12 -0
- package/src/agents/threads.ts +41 -2
- package/src/ai-cost-storage.ts +4 -0
- package/src/migrations/041-agent-thread-workspaces.sql +2 -0
- package/src/migrations/042-vector-profiles.sql +21 -0
- package/src/migrations/043-vector-indexing.sql +197 -0
- package/src/migrations/044-vector-batching.sql +3 -0
- package/src/migrations/045-agent-thread-sandbox-state.sql +2 -0
- package/src/migrations.ts +14 -0
- package/src/objects/query-compiler.ts +105 -7
- package/src/objects/read-scope.ts +34 -31
- package/src/objects/reader.ts +25 -1
- package/src/objects/rows.ts +5 -1
- package/src/objects/storage.ts +12 -1
- package/src/objects/vector-extension.ts +26 -0
- package/src/ontology-storage/index.ts +16 -0
- package/src/ontology-storage/materializations.ts +13 -0
- package/src/ontology-storage/vector-indexing.ts +196 -0
- package/src/ontology-storage/vectors.ts +182 -0
- package/src/vector-encoding.ts +9 -0
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
CREATE TABLE object_vector_indexing (
|
|
2
|
+
project_id TEXT NOT NULL,
|
|
3
|
+
id TEXT NOT NULL,
|
|
4
|
+
object_type_id TEXT NOT NULL,
|
|
5
|
+
primary_id TEXT NOT NULL,
|
|
6
|
+
profile TEXT NOT NULL,
|
|
7
|
+
request TEXT NOT NULL,
|
|
8
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'ready', 'failed')),
|
|
9
|
+
available_at TEXT NOT NULL,
|
|
10
|
+
dispatch_at TEXT NOT NULL,
|
|
11
|
+
values_json TEXT,
|
|
12
|
+
error TEXT,
|
|
13
|
+
PRIMARY KEY (project_id, id),
|
|
14
|
+
UNIQUE (project_id, object_type_id, primary_id, profile)
|
|
15
|
+
);
|
|
16
|
+
CREATE INDEX object_vector_indexing_due ON object_vector_indexing (project_id, dispatch_at, id)
|
|
17
|
+
WHERE status <> 'failed';
|
|
18
|
+
|
|
19
|
+
CREATE TABLE executions_v2 (
|
|
20
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
21
|
+
id TEXT NOT NULL CHECK (length(trim(id)) > 0),
|
|
22
|
+
executor_kind TEXT NOT NULL CHECK (
|
|
23
|
+
executor_kind IN (
|
|
24
|
+
'request', 'action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow',
|
|
25
|
+
'agent', 'kernel'
|
|
26
|
+
)
|
|
27
|
+
),
|
|
28
|
+
executor_id TEXT NOT NULL CHECK (length(trim(executor_id)) > 0),
|
|
29
|
+
source_kind TEXT NOT NULL CHECK (
|
|
30
|
+
source_kind IN ('http', 'webhook', 'schedule', 'event', 'datasetVersion', 'execution', 'ontologyCommit')
|
|
31
|
+
),
|
|
32
|
+
source_id TEXT NOT NULL CHECK (length(trim(source_id)) > 0),
|
|
33
|
+
requested_by_user_id TEXT,
|
|
34
|
+
requested_by_service_account_id TEXT,
|
|
35
|
+
correlation_id TEXT NOT NULL CHECK (length(trim(correlation_id)) > 0),
|
|
36
|
+
parent_execution_id TEXT,
|
|
37
|
+
authority_kind TEXT NOT NULL CHECK (
|
|
38
|
+
authority_kind IN ('principal', 'trustedPrimitive', 'delegated', 'kernel', 'disabled')
|
|
39
|
+
),
|
|
40
|
+
authority_user_id TEXT,
|
|
41
|
+
authority_service_account_id TEXT,
|
|
42
|
+
authority_session_id TEXT,
|
|
43
|
+
authority_access_token_id TEXT,
|
|
44
|
+
authority_primitive_kind TEXT CHECK (
|
|
45
|
+
authority_primitive_kind IS NULL
|
|
46
|
+
OR authority_primitive_kind IN (
|
|
47
|
+
'action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow'
|
|
48
|
+
)
|
|
49
|
+
),
|
|
50
|
+
authority_primitive_id TEXT,
|
|
51
|
+
authority_kernel_operation TEXT CHECK (
|
|
52
|
+
authority_kernel_operation IS NULL OR authority_kernel_operation IN ('ontology.recover', 'ontology.indexVectors')
|
|
53
|
+
),
|
|
54
|
+
authority_delegation_kind TEXT CHECK (
|
|
55
|
+
authority_delegation_kind IS NULL OR authority_delegation_kind = 'share'
|
|
56
|
+
),
|
|
57
|
+
authority_delegation_id TEXT,
|
|
58
|
+
authority_delegation_session_id TEXT,
|
|
59
|
+
requester_group_ids TEXT NOT NULL DEFAULT '[]',
|
|
60
|
+
created_at TEXT NOT NULL,
|
|
61
|
+
PRIMARY KEY (project_id, id),
|
|
62
|
+
FOREIGN KEY (project_id, parent_execution_id) REFERENCES executions (project_id, id),
|
|
63
|
+
FOREIGN KEY (project_id, requested_by_user_id) REFERENCES auth_users (project_id, id),
|
|
64
|
+
FOREIGN KEY (project_id, requested_by_service_account_id)
|
|
65
|
+
REFERENCES auth_service_accounts (project_id, id),
|
|
66
|
+
FOREIGN KEY (project_id, authority_user_id) REFERENCES auth_users (project_id, id),
|
|
67
|
+
FOREIGN KEY (project_id, authority_service_account_id)
|
|
68
|
+
REFERENCES auth_service_accounts (project_id, id),
|
|
69
|
+
FOREIGN KEY (project_id, authority_session_id) REFERENCES auth_sessions (project_id, id),
|
|
70
|
+
FOREIGN KEY (project_id, authority_access_token_id)
|
|
71
|
+
REFERENCES auth_access_tokens (project_id, id),
|
|
72
|
+
FOREIGN KEY (
|
|
73
|
+
project_id, authority_delegation_session_id, authority_delegation_id
|
|
74
|
+
) REFERENCES share_sessions (project_id, id, grant_id),
|
|
75
|
+
CHECK ((requested_by_user_id IS NOT NULL) + (requested_by_service_account_id IS NOT NULL) <= 1),
|
|
76
|
+
CHECK (
|
|
77
|
+
(source_kind = 'execution' AND parent_execution_id = source_id)
|
|
78
|
+
OR (source_kind <> 'execution' AND parent_execution_id IS NULL)
|
|
79
|
+
),
|
|
80
|
+
CHECK (
|
|
81
|
+
(
|
|
82
|
+
authority_kind = 'principal'
|
|
83
|
+
AND (authority_user_id IS NOT NULL) + (authority_service_account_id IS NOT NULL) = 1
|
|
84
|
+
AND (authority_session_id IS NOT NULL) + (authority_access_token_id IS NOT NULL) <= 1
|
|
85
|
+
AND (authority_session_id IS NULL OR authority_user_id IS NOT NULL)
|
|
86
|
+
AND authority_primitive_kind IS NULL
|
|
87
|
+
AND authority_primitive_id IS NULL
|
|
88
|
+
AND authority_kernel_operation IS NULL
|
|
89
|
+
AND authority_delegation_kind IS NULL
|
|
90
|
+
AND authority_delegation_id IS NULL
|
|
91
|
+
AND authority_delegation_session_id IS NULL
|
|
92
|
+
)
|
|
93
|
+
OR (
|
|
94
|
+
authority_kind = 'trustedPrimitive'
|
|
95
|
+
AND authority_user_id IS NULL
|
|
96
|
+
AND authority_service_account_id IS NULL
|
|
97
|
+
AND authority_session_id IS NULL
|
|
98
|
+
AND authority_access_token_id IS NULL
|
|
99
|
+
AND authority_primitive_kind IS NOT NULL
|
|
100
|
+
AND authority_primitive_id IS NOT NULL
|
|
101
|
+
AND length(trim(authority_primitive_id)) > 0
|
|
102
|
+
AND authority_kernel_operation IS NULL
|
|
103
|
+
AND authority_delegation_kind IS NULL
|
|
104
|
+
AND authority_delegation_id IS NULL
|
|
105
|
+
AND authority_delegation_session_id IS NULL
|
|
106
|
+
)
|
|
107
|
+
OR (
|
|
108
|
+
authority_kind = 'delegated'
|
|
109
|
+
AND authority_user_id IS NULL
|
|
110
|
+
AND authority_service_account_id IS NULL
|
|
111
|
+
AND authority_session_id IS NULL
|
|
112
|
+
AND authority_access_token_id IS NULL
|
|
113
|
+
AND authority_primitive_kind IS NULL
|
|
114
|
+
AND authority_primitive_id IS NULL
|
|
115
|
+
AND authority_kernel_operation IS NULL
|
|
116
|
+
AND authority_delegation_kind IS NOT NULL
|
|
117
|
+
AND authority_delegation_kind = 'share'
|
|
118
|
+
AND authority_delegation_id IS NOT NULL
|
|
119
|
+
AND length(trim(authority_delegation_id)) > 0
|
|
120
|
+
AND authority_delegation_session_id IS NOT NULL
|
|
121
|
+
AND length(trim(authority_delegation_session_id)) > 0
|
|
122
|
+
)
|
|
123
|
+
OR (
|
|
124
|
+
authority_kind = 'kernel'
|
|
125
|
+
AND authority_user_id IS NULL
|
|
126
|
+
AND authority_service_account_id IS NULL
|
|
127
|
+
AND authority_session_id IS NULL
|
|
128
|
+
AND authority_access_token_id IS NULL
|
|
129
|
+
AND authority_primitive_kind IS NULL
|
|
130
|
+
AND authority_primitive_id IS NULL
|
|
131
|
+
AND authority_kernel_operation IS NOT NULL
|
|
132
|
+
AND authority_delegation_kind IS NULL
|
|
133
|
+
AND authority_delegation_id IS NULL
|
|
134
|
+
AND authority_delegation_session_id IS NULL
|
|
135
|
+
)
|
|
136
|
+
OR (
|
|
137
|
+
authority_kind = 'disabled'
|
|
138
|
+
AND authority_user_id IS NULL
|
|
139
|
+
AND authority_service_account_id IS NULL
|
|
140
|
+
AND authority_session_id IS NULL
|
|
141
|
+
AND authority_access_token_id IS NULL
|
|
142
|
+
AND authority_primitive_kind IS NULL
|
|
143
|
+
AND authority_primitive_id IS NULL
|
|
144
|
+
AND authority_kernel_operation IS NULL
|
|
145
|
+
AND authority_delegation_kind IS NULL
|
|
146
|
+
AND authority_delegation_id IS NULL
|
|
147
|
+
AND authority_delegation_session_id IS NULL
|
|
148
|
+
)
|
|
149
|
+
),
|
|
150
|
+
CHECK (
|
|
151
|
+
(
|
|
152
|
+
executor_kind = 'request'
|
|
153
|
+
AND source_kind = 'http'
|
|
154
|
+
AND executor_id = source_id
|
|
155
|
+
AND (
|
|
156
|
+
(
|
|
157
|
+
authority_kind = 'principal'
|
|
158
|
+
AND requested_by_user_id IS authority_user_id
|
|
159
|
+
AND requested_by_service_account_id IS authority_service_account_id
|
|
160
|
+
)
|
|
161
|
+
OR (
|
|
162
|
+
authority_kind IN ('delegated', 'disabled')
|
|
163
|
+
AND requested_by_user_id IS NULL
|
|
164
|
+
AND requested_by_service_account_id IS NULL
|
|
165
|
+
)
|
|
166
|
+
)
|
|
167
|
+
)
|
|
168
|
+
OR (
|
|
169
|
+
executor_kind IN ('action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow')
|
|
170
|
+
AND authority_kind = 'trustedPrimitive'
|
|
171
|
+
AND executor_kind = authority_primitive_kind
|
|
172
|
+
)
|
|
173
|
+
OR (
|
|
174
|
+
executor_kind = 'agent'
|
|
175
|
+
AND source_kind = 'execution'
|
|
176
|
+
AND (
|
|
177
|
+
(
|
|
178
|
+
authority_kind = 'principal'
|
|
179
|
+
AND authority_service_account_id IS NOT NULL
|
|
180
|
+
AND authority_session_id IS NULL
|
|
181
|
+
AND authority_access_token_id IS NULL
|
|
182
|
+
)
|
|
183
|
+
OR (
|
|
184
|
+
authority_kind = 'principal'
|
|
185
|
+
AND authority_user_id IS NOT NULL
|
|
186
|
+
AND (authority_session_id IS NOT NULL) + (authority_access_token_id IS NOT NULL) = 1
|
|
187
|
+
)
|
|
188
|
+
OR authority_kind = 'disabled'
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
OR (executor_kind = 'kernel' AND authority_kind = 'kernel')
|
|
192
|
+
)
|
|
193
|
+
);
|
|
194
|
+
|
|
195
|
+
INSERT INTO executions_v2 (project_id, id, executor_kind, executor_id, source_kind, source_id, requested_by_user_id, requested_by_service_account_id, correlation_id, parent_execution_id, authority_kind, authority_user_id, authority_service_account_id, authority_session_id, authority_access_token_id, authority_primitive_kind, authority_primitive_id, authority_kernel_operation, authority_delegation_kind, authority_delegation_id, authority_delegation_session_id, requester_group_ids, created_at) SELECT project_id, id, executor_kind, executor_id, source_kind, source_id, requested_by_user_id, requested_by_service_account_id, correlation_id, parent_execution_id, authority_kind, authority_user_id, authority_service_account_id, authority_session_id, authority_access_token_id, authority_primitive_kind, authority_primitive_id, authority_kernel_operation, authority_delegation_kind, authority_delegation_id, authority_delegation_session_id, requester_group_ids, created_at FROM executions;
|
|
196
|
+
DROP TABLE executions;
|
|
197
|
+
ALTER TABLE executions_v2 RENAME TO executions;
|
package/src/migrations.ts
CHANGED
|
@@ -96,6 +96,15 @@ import projectionSourceRootsSql from "./migrations/038-projection-source-roots.s
|
|
|
96
96
|
import outboxPublicationOrderSql from "./migrations/040-outbox-publication-order.sql" with {
|
|
97
97
|
type: "text",
|
|
98
98
|
}
|
|
99
|
+
import agentThreadWorkspacesSql from "./migrations/041-agent-thread-workspaces.sql" with {
|
|
100
|
+
type: "text",
|
|
101
|
+
}
|
|
102
|
+
import vectorProfilesSql from "./migrations/042-vector-profiles.sql" with { type: "text" }
|
|
103
|
+
import vectorIndexingSql from "./migrations/043-vector-indexing.sql" with { type: "text" }
|
|
104
|
+
import vectorBatchingSql from "./migrations/044-vector-batching.sql" with { type: "text" }
|
|
105
|
+
import agentThreadSandboxStateSql from "./migrations/045-agent-thread-sandbox-state.sql" with {
|
|
106
|
+
type: "text",
|
|
107
|
+
}
|
|
99
108
|
|
|
100
109
|
const MIGRATIONS_TABLE_SQL = `
|
|
101
110
|
CREATE TABLE IF NOT EXISTS sixb_migrations (
|
|
@@ -191,6 +200,11 @@ export const sqliteStorageMigrations = defineMigrations({
|
|
|
191
200
|
sqliteSql("038-projection-source-roots", projectionSourceRootsSql),
|
|
192
201
|
sqliteSql("039-connector-optional-pkce", connectorOptionalPkceSql),
|
|
193
202
|
sqliteSql("040-outbox-publication-order", outboxPublicationOrderSql),
|
|
203
|
+
sqliteSql("041-agent-thread-workspaces", agentThreadWorkspacesSql),
|
|
204
|
+
sqliteSql("042-vector-profiles", vectorProfilesSql),
|
|
205
|
+
sqliteSql("043-vector-indexing", vectorIndexingSql),
|
|
206
|
+
sqliteSql("044-vector-batching", vectorBatchingSql),
|
|
207
|
+
sqliteSql("045-agent-thread-sandbox-state", agentThreadSandboxStateSql),
|
|
194
208
|
],
|
|
195
209
|
})
|
|
196
210
|
|
|
@@ -6,8 +6,15 @@ import {
|
|
|
6
6
|
type ObjectQuerySetOperation,
|
|
7
7
|
type ObjectQuerySortField,
|
|
8
8
|
} from "@sixb/core"
|
|
9
|
+
import {
|
|
10
|
+
hasVectorProfile,
|
|
11
|
+
isVectorProfileQuery,
|
|
12
|
+
normalizeVector,
|
|
13
|
+
vectorSearchCandidateLimit,
|
|
14
|
+
} from "@sixb/core/internal/query"
|
|
15
|
+
import { encodeSqliteVector } from "../vector-encoding"
|
|
9
16
|
|
|
10
|
-
export type SqliteValue = string | number | bigint | boolean | null
|
|
17
|
+
export type SqliteValue = string | number | bigint | boolean | null | Uint8Array
|
|
11
18
|
|
|
12
19
|
export interface SqliteObjectQueryPageRow {
|
|
13
20
|
object_type_id: string
|
|
@@ -17,6 +24,7 @@ export interface SqliteObjectQueryPageRow {
|
|
|
17
24
|
}
|
|
18
25
|
|
|
19
26
|
export interface CompiledObjectQuery {
|
|
27
|
+
vectorProbe?: { sql: string; args: SqliteValue[]; limit: number }
|
|
20
28
|
sql: string
|
|
21
29
|
args: SqliteValue[]
|
|
22
30
|
totalSql: string
|
|
@@ -59,7 +67,8 @@ type CompiledOrderField =
|
|
|
59
67
|
}
|
|
60
68
|
| {
|
|
61
69
|
kind: "column"
|
|
62
|
-
column: "object_type_id" | "primary_id"
|
|
70
|
+
column: "object_type_id" | "primary_id" | "_vector_score"
|
|
71
|
+
binary?: boolean
|
|
63
72
|
direction: "asc" | "desc"
|
|
64
73
|
}
|
|
65
74
|
|
|
@@ -86,6 +95,12 @@ export function compileObjectQuery(
|
|
|
86
95
|
query: ObjectQuery,
|
|
87
96
|
options: { includeTotal?: boolean; source?: SqliteObjectQuerySource } = {}
|
|
88
97
|
): CompiledObjectQuery {
|
|
98
|
+
if (hasVectorProfile(query) && !isVectorProfileQuery(query)) {
|
|
99
|
+
throw new ObjectQueryExecutionError(
|
|
100
|
+
"unsupported_vector_composition",
|
|
101
|
+
"Vector search supports one profile with filters before ranking and limit/project after."
|
|
102
|
+
)
|
|
103
|
+
}
|
|
89
104
|
const source = options.source ?? DEFAULT_OBJECT_QUERY_SOURCE
|
|
90
105
|
const ctx: CompileContext = { probeLimit: options.includeTotal === false, source }
|
|
91
106
|
return source.wrapQuery(compileObjectQueryInternal(projectId, query, ctx))
|
|
@@ -95,6 +110,7 @@ export function compileObjectQuery(
|
|
|
95
110
|
export interface SqliteObjectQuerySource {
|
|
96
111
|
readonly objectsTable: string
|
|
97
112
|
readonly linksTable: string
|
|
113
|
+
readonly objectPropertyPermissionsTable?: string
|
|
98
114
|
/** Wrap a terminal SELECT without its own WITH clause; the selected source owns the only WITH. */
|
|
99
115
|
wrapStatement(
|
|
100
116
|
sql: string,
|
|
@@ -160,7 +176,79 @@ function compileObjectQueryInternal(
|
|
|
160
176
|
case "expand":
|
|
161
177
|
return compileExpand(projectId, query.input, query.expansions, ctx)
|
|
162
178
|
case "vector":
|
|
163
|
-
|
|
179
|
+
return compileVector(projectId, query, ctx)
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function compileVector(
|
|
184
|
+
projectId: string,
|
|
185
|
+
query: Extract<ObjectQuery, { kind: "vector" }>,
|
|
186
|
+
ctx: CompileContext
|
|
187
|
+
): CompiledObjectQuery {
|
|
188
|
+
if (
|
|
189
|
+
typeof query.vector === "string" ||
|
|
190
|
+
!isVectorProfileQuery(query) ||
|
|
191
|
+
!query.configuration ||
|
|
192
|
+
!Number.isSafeInteger(query.k) ||
|
|
193
|
+
query.k < 1 ||
|
|
194
|
+
query.k > 1000 ||
|
|
195
|
+
query.vector.length < 1 ||
|
|
196
|
+
query.vector.length > 16000
|
|
197
|
+
) {
|
|
198
|
+
throw new ObjectQueryExecutionError(
|
|
199
|
+
"invalid_vector_query",
|
|
200
|
+
"A validated named vector profile is required."
|
|
201
|
+
)
|
|
202
|
+
}
|
|
203
|
+
const input = compileObjectQueryInternal(projectId, query.input, exactContext(ctx))
|
|
204
|
+
const permissions = ctx.source.objectPropertyPermissionsTable
|
|
205
|
+
const permissionJoin = permissions
|
|
206
|
+
? `CROSS JOIN (
|
|
207
|
+
SELECT project_id, object_type_id, primary_id, json_group_array(property_id) AS property_ids
|
|
208
|
+
FROM ${permissions}
|
|
209
|
+
GROUP BY project_id, object_type_id, primary_id
|
|
210
|
+
) AS vector_permissions ON vector_permissions.project_id = vectors.project_id
|
|
211
|
+
AND vector_permissions.object_type_id = vectors.object_type_id
|
|
212
|
+
AND vector_permissions.primary_id = vectors.primary_id`
|
|
213
|
+
: ""
|
|
214
|
+
const authorized = permissions
|
|
215
|
+
? `AND NOT EXISTS (
|
|
216
|
+
SELECT 1 FROM json_each(vectors.source) AS required
|
|
217
|
+
WHERE required.value NOT IN (SELECT value FROM json_each(vector_permissions.property_ids))
|
|
218
|
+
)`
|
|
219
|
+
: ""
|
|
220
|
+
const candidates = `FROM (${input.sql}) AS input
|
|
221
|
+
CROSS JOIN object_vectors AS vectors ON vectors.project_id = input.project_id
|
|
222
|
+
AND vectors.object_type_id = input.object_type_id AND vectors.primary_id = input.primary_id
|
|
223
|
+
${permissionJoin}
|
|
224
|
+
WHERE vectors.profile = ? AND vectors.configuration = ?
|
|
225
|
+
AND length(vectors.embedding) / 4 = ? ${authorized}`
|
|
226
|
+
const candidateArgs = [...input.args, query.profile!, query.configuration, query.vector.length]
|
|
227
|
+
// Bound distance work, not just returned rows. Exceeding the envelope fails before ranking.
|
|
228
|
+
const limit = vectorSearchCandidateLimit(query.vector.length)
|
|
229
|
+
const values = normalizeVector(query.vector, query.vector.length)
|
|
230
|
+
const order = compileOrder([
|
|
231
|
+
{ kind: "column", column: "_vector_score", direction: "desc" },
|
|
232
|
+
...identityOrderFields().map((field) => ({ ...field, binary: true })),
|
|
233
|
+
])
|
|
234
|
+
const sql = `SELECT * FROM (SELECT input.*, max(-1, min(1, 1 - vec_distance_cosine(vectors.embedding, ?))) AS _vector_score
|
|
235
|
+
${candidates}) AS ranked ORDER BY ${order.sql} LIMIT ?`
|
|
236
|
+
const args = [encodeSqliteVector(values), ...candidateArgs, query.k]
|
|
237
|
+
return {
|
|
238
|
+
sql,
|
|
239
|
+
args,
|
|
240
|
+
// Top-k cardinality depends on eligibility, not distances or ranking.
|
|
241
|
+
totalSql: `SELECT COUNT(*) AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS eligible`,
|
|
242
|
+
totalArgs: [...candidateArgs, query.k],
|
|
243
|
+
order,
|
|
244
|
+
vectorProbe: {
|
|
245
|
+
sql: `SELECT COUNT(*) AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS candidates`,
|
|
246
|
+
args: [...candidateArgs, limit + 1],
|
|
247
|
+
limit,
|
|
248
|
+
},
|
|
249
|
+
hasMore: () => false,
|
|
250
|
+
trimRows: identityRows,
|
|
251
|
+
nextPageToken: () => undefined,
|
|
164
252
|
}
|
|
165
253
|
}
|
|
166
254
|
|
|
@@ -274,6 +362,7 @@ function compileWhere(
|
|
|
274
362
|
WHERE ${predicate.sql}
|
|
275
363
|
`,
|
|
276
364
|
totalArgs: [...input.args, ...predicate.args],
|
|
365
|
+
vectorProbe: input.vectorProbe,
|
|
277
366
|
order: input.order,
|
|
278
367
|
hasMore: () => false,
|
|
279
368
|
trimRows: identityRows,
|
|
@@ -324,7 +413,7 @@ function compileSort(
|
|
|
324
413
|
input.created_at,
|
|
325
414
|
input.updated_at,
|
|
326
415
|
input.version,
|
|
327
|
-
input.last_commit_id
|
|
416
|
+
input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
|
|
328
417
|
FROM (${input.sql}) AS input
|
|
329
418
|
ORDER BY ${order.sql}
|
|
330
419
|
`,
|
|
@@ -362,6 +451,7 @@ function compileLimit(
|
|
|
362
451
|
FROM (${input.sql}) AS input
|
|
363
452
|
`,
|
|
364
453
|
totalArgs: input.args,
|
|
454
|
+
vectorProbe: input.vectorProbe,
|
|
365
455
|
order: input.order,
|
|
366
456
|
hasMore: (rowCount, total) =>
|
|
367
457
|
total === undefined ? ctx.probeLimit && rowCount > limit : limit < total,
|
|
@@ -399,6 +489,7 @@ function compilePage(
|
|
|
399
489
|
FROM (${input.sql}) AS input
|
|
400
490
|
`,
|
|
401
491
|
totalArgs: input.args,
|
|
492
|
+
vectorProbe: input.vectorProbe,
|
|
402
493
|
order: input.order,
|
|
403
494
|
hasMore: (rowCount) => rowCount > pageSize,
|
|
404
495
|
trimRows: (rows) => rows.slice(0, pageSize),
|
|
@@ -507,6 +598,7 @@ function compileExpand(
|
|
|
507
598
|
args: [...expand.args, ...input.args, ...inputOrder.args],
|
|
508
599
|
totalSql: input.totalSql,
|
|
509
600
|
totalArgs: input.totalArgs,
|
|
601
|
+
vectorProbe: input.vectorProbe,
|
|
510
602
|
order: input.order,
|
|
511
603
|
hasMoreProbe: input.hasMoreProbe,
|
|
512
604
|
hasMore: input.hasMore,
|
|
@@ -760,13 +852,14 @@ function compileProject(
|
|
|
760
852
|
input.created_at,
|
|
761
853
|
input.updated_at,
|
|
762
854
|
input.version,
|
|
763
|
-
input.last_commit_id
|
|
855
|
+
input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
|
|
764
856
|
FROM (${input.sql}) AS input
|
|
765
857
|
ORDER BY ${inputOrder.sql}
|
|
766
858
|
`,
|
|
767
859
|
args: [...projection.args, ...input.args, ...inputOrder.args],
|
|
768
860
|
totalSql: input.totalSql,
|
|
769
861
|
totalArgs: input.totalArgs,
|
|
862
|
+
vectorProbe: input.vectorProbe,
|
|
770
863
|
order: outputOrder,
|
|
771
864
|
hasMoreProbe: input.hasMoreProbe,
|
|
772
865
|
hasMore: input.hasMore,
|
|
@@ -1026,7 +1119,9 @@ function compileOrder(
|
|
|
1026
1119
|
|
|
1027
1120
|
for (const field of fields) {
|
|
1028
1121
|
if (field.kind === "column") {
|
|
1029
|
-
clauses.push(
|
|
1122
|
+
clauses.push(
|
|
1123
|
+
`${columnExpression(field.column, qualifier)}${field.binary ? " COLLATE BINARY" : ""} ${field.direction.toUpperCase()}`
|
|
1124
|
+
)
|
|
1030
1125
|
continue
|
|
1031
1126
|
}
|
|
1032
1127
|
|
|
@@ -1238,7 +1333,10 @@ function orderFieldKey(field: CompiledOrderField): string {
|
|
|
1238
1333
|
: `property:${field.propertyId}:${field.direction}`
|
|
1239
1334
|
}
|
|
1240
1335
|
|
|
1241
|
-
function columnExpression(
|
|
1336
|
+
function columnExpression(
|
|
1337
|
+
column: "object_type_id" | "primary_id" | "_vector_score",
|
|
1338
|
+
qualifier?: string
|
|
1339
|
+
): string {
|
|
1242
1340
|
return qualifier ? `${qualifier}.${column}` : column
|
|
1243
1341
|
}
|
|
1244
1342
|
|
|
@@ -19,6 +19,9 @@ export interface SqliteSelectedObjectReadSource extends SqliteObjectQuerySource
|
|
|
19
19
|
*
|
|
20
20
|
* The complete compiled scope is serialized once. Every statement receives that one JSON document
|
|
21
21
|
* and the bound project id, keeping the host-parameter count constant as selections grow.
|
|
22
|
+
* CROSS JOIN keeps authorized parents/grants before indexed edge/object lookups, even when
|
|
23
|
+
* statistics would otherwise make SQLite rescan all grants for every stored object.
|
|
24
|
+
* Property grants are grouped only after path-sensitive reachability has been resolved.
|
|
22
25
|
*/
|
|
23
26
|
export function compileSqliteSelectedObjectReadSource(
|
|
24
27
|
projectId: string,
|
|
@@ -44,6 +47,9 @@ export function compileSqliteSelectedObjectReadSource(
|
|
|
44
47
|
wrapQuery: (query: CompiledObjectQuery): CompiledObjectQuery => ({
|
|
45
48
|
...query,
|
|
46
49
|
...wrapStatement(query.sql, query.args),
|
|
50
|
+
vectorProbe: query.vectorProbe
|
|
51
|
+
? { ...query.vectorProbe, ...wrapStatement(query.vectorProbe.sql, query.vectorProbe.args) }
|
|
52
|
+
: undefined,
|
|
47
53
|
totalSql: `${compiled.sql}\n${query.totalSql}`,
|
|
48
54
|
totalArgs: [...compiled.args, ...query.totalArgs],
|
|
49
55
|
...(query.hasMoreProbe
|
|
@@ -140,16 +146,16 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
|
|
|
140
146
|
|
|
141
147
|
SELECT edge.project_id, selection.node_id, edge.target_type_id, edge.target_id
|
|
142
148
|
FROM _sixb_scope_reachable AS parent
|
|
143
|
-
JOIN _sixb_scope_link_selections AS selection
|
|
149
|
+
CROSS JOIN _sixb_scope_link_selections AS selection
|
|
144
150
|
ON selection.parent_node_id = parent.node_id
|
|
145
151
|
AND selection.source_object_type_id = parent.object_type_id
|
|
146
|
-
JOIN links AS edge
|
|
152
|
+
CROSS JOIN links AS edge
|
|
147
153
|
ON edge.project_id = parent.project_id
|
|
148
154
|
AND edge.source_type_id = parent.object_type_id
|
|
149
155
|
AND edge.source_id = parent.primary_id
|
|
150
156
|
AND edge.link_id = selection.link_id
|
|
151
157
|
AND edge.target_type_id = selection.target_object_type_id
|
|
152
|
-
JOIN objects AS target
|
|
158
|
+
CROSS JOIN objects AS target
|
|
153
159
|
ON target.project_id = edge.project_id
|
|
154
160
|
AND target.object_type_id = edge.target_type_id
|
|
155
161
|
AND target.primary_id = edge.target_id
|
|
@@ -170,32 +176,29 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
|
|
|
170
176
|
SELECT DISTINCT project_id, object_type_id, primary_id
|
|
171
177
|
FROM _sixb_scope_reachable
|
|
172
178
|
),
|
|
179
|
+
_sixb_scope_object_grants AS (
|
|
180
|
+
SELECT reachable.project_id, reachable.object_type_id, reachable.primary_id,
|
|
181
|
+
json_group_array(DISTINCT selected_property.value) FILTER (WHERE selected_property.value IS NOT NULL) AS property_ids
|
|
182
|
+
FROM _sixb_scope_reachable AS reachable
|
|
183
|
+
LEFT JOIN _sixb_scope_object_selections AS selection
|
|
184
|
+
ON selection.node_id = reachable.node_id
|
|
185
|
+
AND selection.object_type_id = reachable.object_type_id
|
|
186
|
+
LEFT JOIN json_each(selection.property_ids) AS selected_property ON true
|
|
187
|
+
GROUP BY reachable.project_id, reachable.object_type_id, reachable.primary_id
|
|
188
|
+
),
|
|
173
189
|
_sixb_scope_objects AS (
|
|
174
|
-
SELECT
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
AND permission.property_id = property.key
|
|
187
|
-
),
|
|
188
|
-
json('{}')
|
|
189
|
-
) AS properties,
|
|
190
|
-
stored.created_at,
|
|
191
|
-
stored.updated_at,
|
|
192
|
-
stored.version,
|
|
193
|
-
stored.last_commit_id
|
|
194
|
-
FROM objects AS stored
|
|
195
|
-
JOIN _sixb_scope_object_ids AS visible
|
|
196
|
-
ON visible.project_id = stored.project_id
|
|
197
|
-
AND visible.object_type_id = stored.object_type_id
|
|
198
|
-
AND visible.primary_id = stored.primary_id
|
|
190
|
+
SELECT stored.project_id, stored.object_type_id, stored.primary_id,
|
|
191
|
+
COALESCE((
|
|
192
|
+
SELECT json_group_object(property.key, ${sqliteJsonEachValue("property")})
|
|
193
|
+
FROM json_each(stored.properties) AS property
|
|
194
|
+
WHERE property.key IN (SELECT value FROM json_each(grants.property_ids))
|
|
195
|
+
), json('{}')) AS properties,
|
|
196
|
+
stored.created_at, stored.updated_at, stored.version, stored.last_commit_id
|
|
197
|
+
FROM _sixb_scope_object_grants AS grants
|
|
198
|
+
CROSS JOIN objects AS stored
|
|
199
|
+
ON stored.project_id = grants.project_id
|
|
200
|
+
AND stored.object_type_id = grants.object_type_id
|
|
201
|
+
AND stored.primary_id = grants.primary_id
|
|
199
202
|
),
|
|
200
203
|
_sixb_scope_reached_links(
|
|
201
204
|
project_id,
|
|
@@ -221,16 +224,16 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
|
|
|
221
224
|
edge.target_id,
|
|
222
225
|
selection.property_ids
|
|
223
226
|
FROM _sixb_scope_reachable AS parent
|
|
224
|
-
JOIN _sixb_scope_link_selections AS selection
|
|
227
|
+
CROSS JOIN _sixb_scope_link_selections AS selection
|
|
225
228
|
ON selection.parent_node_id = parent.node_id
|
|
226
229
|
AND selection.source_object_type_id = parent.object_type_id
|
|
227
|
-
JOIN links AS edge
|
|
230
|
+
CROSS JOIN links AS edge
|
|
228
231
|
ON edge.project_id = parent.project_id
|
|
229
232
|
AND edge.source_type_id = parent.object_type_id
|
|
230
233
|
AND edge.source_id = parent.primary_id
|
|
231
234
|
AND edge.link_id = selection.link_id
|
|
232
235
|
AND edge.target_type_id = selection.target_object_type_id
|
|
233
|
-
JOIN objects AS target
|
|
236
|
+
CROSS JOIN objects AS target
|
|
234
237
|
ON target.project_id = edge.project_id
|
|
235
238
|
AND target.object_type_id = edge.target_type_id
|
|
236
239
|
AND target.primary_id = edge.target_id
|
package/src/objects/reader.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite"
|
|
2
|
-
import type
|
|
2
|
+
import { type ObjectQuery, ObjectQueryExecutionError } from "@sixb/core"
|
|
3
|
+
import { hasVectorProfile } from "@sixb/core/internal/query"
|
|
3
4
|
import type {
|
|
4
5
|
CountObjectsInput,
|
|
5
6
|
CountObjectsResult,
|
|
@@ -24,6 +25,7 @@ import {
|
|
|
24
25
|
type CompiledObjectQuery,
|
|
25
26
|
compileObjectQuery,
|
|
26
27
|
type SqliteObjectQuerySource,
|
|
28
|
+
type SqliteValue,
|
|
27
29
|
} from "./query-compiler"
|
|
28
30
|
import type { SqliteSelectedObjectReadSource } from "./read-scope"
|
|
29
31
|
import {
|
|
@@ -34,6 +36,7 @@ import {
|
|
|
34
36
|
rowToLink,
|
|
35
37
|
rowToObject,
|
|
36
38
|
} from "./rows"
|
|
39
|
+
import { ensureSqliteVectorSearch } from "./vector-extension"
|
|
37
40
|
|
|
38
41
|
/** Synchronous SQL reads over one explicit source; transaction and budget ownership stay outside. */
|
|
39
42
|
export class SqliteObjectReader {
|
|
@@ -42,7 +45,25 @@ export class SqliteObjectReader {
|
|
|
42
45
|
private readonly source: SqliteObjectQuerySource | SqliteSelectedObjectReadSource
|
|
43
46
|
) {}
|
|
44
47
|
|
|
48
|
+
private assertVectorSearch(
|
|
49
|
+
params: QueryObjectsInput | CountObjectsInput | ExistsObjectsInput | FacetObjectsInput
|
|
50
|
+
): void {
|
|
51
|
+
if (!hasVectorProfile(params.query)) return
|
|
52
|
+
const compiled = compileObjectQuery(params.projectId, params.query, { source: this.source })
|
|
53
|
+
if (!compiled.vectorProbe) return
|
|
54
|
+
ensureSqliteVectorSearch(this.db)
|
|
55
|
+
const probe = compiled.vectorProbe
|
|
56
|
+
const row = this.db.query<{ total: number }, SqliteValue[]>(probe.sql).get(...probe.args)
|
|
57
|
+
if (Number(row?.total ?? 0) > probe.limit) {
|
|
58
|
+
throw new ObjectQueryExecutionError(
|
|
59
|
+
"vector_work_limit_exceeded",
|
|
60
|
+
`Exact vector search supports at most ${probe.limit} eligible vectors at this dimension. Narrow the query filters.`
|
|
61
|
+
)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
45
65
|
queryObjects(params: QueryObjectsInput): QueryObjectsResult {
|
|
66
|
+
this.assertVectorSearch(params)
|
|
46
67
|
const source = this.source
|
|
47
68
|
const compiled = compileObjectQuery(params.projectId, params.query, {
|
|
48
69
|
includeTotal: params.includeTotal,
|
|
@@ -67,6 +88,7 @@ export class SqliteObjectReader {
|
|
|
67
88
|
}
|
|
68
89
|
|
|
69
90
|
countObjects(params: CountObjectsInput): CountObjectsResult {
|
|
91
|
+
this.assertVectorSearch(params)
|
|
70
92
|
const source = this.source
|
|
71
93
|
return {
|
|
72
94
|
count: readTotal(
|
|
@@ -79,6 +101,7 @@ export class SqliteObjectReader {
|
|
|
79
101
|
}
|
|
80
102
|
|
|
81
103
|
existsObjects(params: ExistsObjectsInput): ExistsObjectsResult {
|
|
104
|
+
this.assertVectorSearch(params)
|
|
82
105
|
const source = this.source
|
|
83
106
|
const compiled = compileObjectQuery(params.projectId, existsProbeQuery(params.query), {
|
|
84
107
|
source,
|
|
@@ -87,6 +110,7 @@ export class SqliteObjectReader {
|
|
|
87
110
|
}
|
|
88
111
|
|
|
89
112
|
facetObjects(params: FacetObjectsInput): FacetObjectsResult {
|
|
113
|
+
this.assertVectorSearch(params)
|
|
90
114
|
const source = this.source
|
|
91
115
|
const compiled = compileObjectQuery(params.projectId, stripOuterRowShape(params.query), {
|
|
92
116
|
source,
|
package/src/objects/rows.ts
CHANGED
|
@@ -21,7 +21,10 @@ export function rowToObject(row: DatabaseRow): ObjectRow {
|
|
|
21
21
|
|
|
22
22
|
/** Decode an expanded query row with the same shape as Core link hydration. */
|
|
23
23
|
export function queryRowToObject(row: ObjectQueryDatabaseRow): ObjectRow {
|
|
24
|
-
const base =
|
|
24
|
+
const base = {
|
|
25
|
+
...rowToObject(row),
|
|
26
|
+
...(row._vector_score === undefined ? {} : { score: row._vector_score }),
|
|
27
|
+
}
|
|
25
28
|
const links = reviveExpandedLinks(parseExpandColumn(row._expand))
|
|
26
29
|
return links ? { ...base, links } : base
|
|
27
30
|
}
|
|
@@ -104,6 +107,7 @@ export interface DatabaseRow {
|
|
|
104
107
|
}
|
|
105
108
|
|
|
106
109
|
export interface ObjectQueryDatabaseRow extends DatabaseRow {
|
|
110
|
+
_vector_score?: number
|
|
107
111
|
_cursor_properties?: string
|
|
108
112
|
/** `json_object(linkId, value, ...)` serialized text from an `expand` pushdown; absent otherwise. */
|
|
109
113
|
_expand?: string | null
|