@sixb/sqlite 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/agents/rows.d.ts +2 -0
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/threads.d.ts +2 -1
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/auth-storage/access-tokens.d.ts +0 -1
- package/dist/auth-storage/access-tokens.d.ts.map +1 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/device-authorizations.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +6 -3
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/sessions.d.ts +8 -1
- package/dist/auth-storage/sessions.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/file-upload-session-storage.d.ts +6 -0
- package/dist/file-upload-session-storage.d.ts.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1062 -137
- package/dist/index.js.map +28 -23
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +9 -2
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts +3 -0
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +1 -0
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/rows.d.ts +1 -0
- package/dist/objects/rows.d.ts.map +1 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/vector-extension.d.ts +4 -0
- package/dist/objects/vector-extension.d.ts.map +1 -0
- package/dist/ontology-storage/index.d.ts +4 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +1 -2
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +2 -1
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/vector-indexing.d.ts +20 -0
- package/dist/ontology-storage/vector-indexing.d.ts.map +1 -0
- package/dist/ontology-storage/vectors.d.ts +16 -0
- package/dist/ontology-storage/vectors.d.ts.map +1 -0
- package/dist/vector-encoding.d.ts +3 -0
- package/dist/vector-encoding.d.ts.map +1 -0
- package/dist/workflow-intervention-storage.d.ts.map +1 -1
- package/package.json +8 -4
- package/src/agents/rows.ts +12 -0
- package/src/agents/threads.ts +41 -2
- package/src/ai-cost-storage.ts +4 -0
- package/src/auth-storage/access-tokens.ts +11 -18
- package/src/auth-storage/auth-storage.ts +5 -7
- package/src/auth-storage/device-authorizations.ts +3 -5
- package/src/auth-storage/rows.ts +17 -7
- package/src/auth-storage/sessions.ts +89 -4
- package/src/auth-storage/shared.ts +9 -3
- package/src/file-upload-session-storage.ts +136 -0
- package/src/index.ts +5 -0
- package/src/migrations/041-agent-thread-workspaces.sql +2 -0
- package/src/migrations/042-vector-profiles.sql +21 -0
- package/src/migrations/043-vector-indexing.sql +197 -0
- package/src/migrations/044-vector-batching.sql +3 -0
- package/src/migrations/045-agent-thread-sandbox-state.sql +2 -0
- package/src/migrations/046-workflow-intervention-principals.sql +15 -0
- package/src/migrations/047-ontology-commit-attribution.sql +67 -0
- package/src/migrations/048-file-upload-sessions.sql +35 -0
- package/src/migrations/049-native-sessions.sql +12 -0
- package/src/migrations.ts +26 -0
- package/src/objects/query-compiler.ts +164 -11
- package/src/objects/read-scope.ts +34 -31
- package/src/objects/reader.ts +29 -1
- package/src/objects/rows.ts +5 -1
- package/src/objects/storage.ts +13 -1
- package/src/objects/vector-extension.ts +26 -0
- package/src/ontology-storage/index.ts +16 -0
- package/src/ontology-storage/materializations.ts +29 -59
- package/src/ontology-storage/shared.ts +7 -3
- package/src/ontology-storage/vector-indexing.ts +196 -0
- package/src/ontology-storage/vectors.ts +182 -0
- package/src/vector-encoding.ts +9 -0
- package/src/workflow-intervention-storage.ts +9 -9
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
-- Commits and their events name who requested the write and what ran it, replacing the
|
|
2
|
+
-- authority-only `actor`. Both are copied from the commit's execution, which stays canonical and
|
|
3
|
+
-- is immutable, so the backfill is exact. SQLite cannot add a NOT NULL column without a default;
|
|
4
|
+
-- every insert supplies `executor`, and the UPDATE below replaces the placeholder on every row.
|
|
5
|
+
ALTER TABLE ontology_commits ADD COLUMN requested_by TEXT
|
|
6
|
+
CHECK (requested_by IS NULL OR json_valid(requested_by));
|
|
7
|
+
ALTER TABLE ontology_commits ADD COLUMN executor TEXT NOT NULL DEFAULT '{}'
|
|
8
|
+
CHECK (json_valid(executor));
|
|
9
|
+
|
|
10
|
+
UPDATE ontology_commits SET
|
|
11
|
+
requested_by = (
|
|
12
|
+
SELECT CASE
|
|
13
|
+
WHEN e.requested_by_user_id IS NOT NULL
|
|
14
|
+
THEN json_object('id', e.requested_by_user_id, 'type', 'user')
|
|
15
|
+
WHEN e.requested_by_service_account_id IS NOT NULL
|
|
16
|
+
THEN json_object('id', e.requested_by_service_account_id, 'type', 'serviceAccount')
|
|
17
|
+
END
|
|
18
|
+
FROM executions e
|
|
19
|
+
WHERE e.project_id = ontology_commits.project_id AND e.id = ontology_commits.execution_id
|
|
20
|
+
),
|
|
21
|
+
executor = (
|
|
22
|
+
SELECT CASE e.executor_kind
|
|
23
|
+
WHEN 'request' THEN json_object('requestId', e.executor_id, 'type', 'request')
|
|
24
|
+
WHEN 'agent' THEN json_object('runId', e.executor_id, 'type', 'agent')
|
|
25
|
+
WHEN 'kernel' THEN json_object(
|
|
26
|
+
'operation',
|
|
27
|
+
CASE e.authority_kernel_operation
|
|
28
|
+
WHEN 'ontology.recover'
|
|
29
|
+
THEN json_object('recoveryId', e.executor_id, 'type', 'ontology.recover')
|
|
30
|
+
ELSE json_object('indexingId', e.executor_id, 'type', 'ontology.indexVectors')
|
|
31
|
+
END,
|
|
32
|
+
'type', 'kernel'
|
|
33
|
+
)
|
|
34
|
+
ELSE json_object(
|
|
35
|
+
'id', e.authority_primitive_id,
|
|
36
|
+
'kind', e.executor_kind,
|
|
37
|
+
'runId', e.executor_id,
|
|
38
|
+
'type', 'primitive'
|
|
39
|
+
)
|
|
40
|
+
END
|
|
41
|
+
FROM executions e
|
|
42
|
+
WHERE e.project_id = ontology_commits.project_id AND e.id = ontology_commits.execution_id
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
-- Events not yet published would otherwise reach consumers in the retired shape.
|
|
46
|
+
UPDATE ontology_outbox SET envelope = (
|
|
47
|
+
SELECT json_remove(
|
|
48
|
+
CASE
|
|
49
|
+
WHEN c.requested_by IS NULL
|
|
50
|
+
THEN json_set(ontology_outbox.envelope, '$.executor', json(c.executor))
|
|
51
|
+
ELSE json_set(
|
|
52
|
+
ontology_outbox.envelope,
|
|
53
|
+
'$.executor', json(c.executor),
|
|
54
|
+
'$.requestedBy', json(c.requested_by)
|
|
55
|
+
)
|
|
56
|
+
END,
|
|
57
|
+
'$.actor'
|
|
58
|
+
)
|
|
59
|
+
FROM ontology_commits c
|
|
60
|
+
WHERE c.project_id = ontology_outbox.project_id AND c.id = ontology_outbox.commit_id
|
|
61
|
+
)
|
|
62
|
+
WHERE EXISTS (
|
|
63
|
+
SELECT 1 FROM ontology_commits c
|
|
64
|
+
WHERE c.project_id = ontology_outbox.project_id AND c.id = ontology_outbox.commit_id
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
ALTER TABLE ontology_commits DROP COLUMN actor;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
-- Durable staged-upload sessions, so an upload survives restarts and spans API replicas.
|
|
2
|
+
-- Timestamps are ISO-8601 UTC text and compare lexicographically.
|
|
3
|
+
CREATE TABLE file_upload_sessions (
|
|
4
|
+
id TEXT PRIMARY KEY CHECK (length(trim(id)) > 0),
|
|
5
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
6
|
+
owner_principal_type TEXT NOT NULL
|
|
7
|
+
CHECK (owner_principal_type IN ('user', 'serviceAccount', 'system')),
|
|
8
|
+
owner_principal_id TEXT NOT NULL CHECK (length(owner_principal_id) > 0),
|
|
9
|
+
strategy TEXT NOT NULL CHECK (strategy IN ('server', 'direct-put', 'multipart')),
|
|
10
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'aborted')),
|
|
11
|
+
file_name TEXT,
|
|
12
|
+
media_type TEXT,
|
|
13
|
+
logical_path TEXT,
|
|
14
|
+
expected_size_bytes INTEGER CHECK (expected_size_bytes IS NULL OR expected_size_bytes >= 0),
|
|
15
|
+
expected_digest TEXT,
|
|
16
|
+
provider_upload TEXT CHECK (provider_upload IS NULL OR json_type(provider_upload) = 'object'),
|
|
17
|
+
signed_parts TEXT NOT NULL CHECK (json_type(signed_parts) = 'array'),
|
|
18
|
+
file_ref TEXT CHECK (file_ref IS NULL OR json_type(file_ref) = 'object'),
|
|
19
|
+
created_at TEXT NOT NULL,
|
|
20
|
+
expires_at TEXT NOT NULL,
|
|
21
|
+
completed_at TEXT,
|
|
22
|
+
aborted_at TEXT,
|
|
23
|
+
-- Derived by core (fileUploadSessionReapAt); NULL while an abandoned upload awaits its abort.
|
|
24
|
+
reap_at TEXT,
|
|
25
|
+
CHECK ((status = 'completed') = (completed_at IS NOT NULL)),
|
|
26
|
+
CHECK ((status = 'aborted') = (aborted_at IS NOT NULL))
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
CREATE INDEX idx_file_upload_sessions_reap
|
|
30
|
+
ON file_upload_sessions (reap_at)
|
|
31
|
+
WHERE reap_at IS NOT NULL;
|
|
32
|
+
|
|
33
|
+
CREATE INDEX idx_file_upload_sessions_abandoned
|
|
34
|
+
ON file_upload_sessions (project_id, expires_at)
|
|
35
|
+
WHERE status = 'pending' AND provider_upload IS NOT NULL;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
-- Native clients, such as the CLI, hold their session as bearer tokens: a
|
|
2
|
+
-- short-lived access token, whose hash is `token_hash`, and a rotating refresh token. Browser
|
|
3
|
+
-- sessions leave these columns NULL; `refresh_token_hash IS NULL` is what marks a cookie session.
|
|
4
|
+
ALTER TABLE auth_sessions ADD COLUMN client_name TEXT;
|
|
5
|
+
ALTER TABLE auth_sessions ADD COLUMN access_expires_at TEXT;
|
|
6
|
+
ALTER TABLE auth_sessions ADD COLUMN refresh_token_hash TEXT;
|
|
7
|
+
ALTER TABLE auth_sessions ADD COLUMN previous_refresh_token_hash TEXT;
|
|
8
|
+
ALTER TABLE auth_sessions ADD COLUMN refreshed_at TEXT;
|
|
9
|
+
|
|
10
|
+
-- The device flow now starts a session instead of issuing a personal access token.
|
|
11
|
+
ALTER TABLE auth_device_authorizations DROP COLUMN token_name;
|
|
12
|
+
ALTER TABLE auth_device_authorizations DROP COLUMN token_expires_at;
|
package/src/migrations.ts
CHANGED
|
@@ -96,6 +96,22 @@ import projectionSourceRootsSql from "./migrations/038-projection-source-roots.s
|
|
|
96
96
|
import outboxPublicationOrderSql from "./migrations/040-outbox-publication-order.sql" with {
|
|
97
97
|
type: "text",
|
|
98
98
|
}
|
|
99
|
+
import agentThreadWorkspacesSql from "./migrations/041-agent-thread-workspaces.sql" with {
|
|
100
|
+
type: "text",
|
|
101
|
+
}
|
|
102
|
+
import vectorProfilesSql from "./migrations/042-vector-profiles.sql" with { type: "text" }
|
|
103
|
+
import vectorIndexingSql from "./migrations/043-vector-indexing.sql" with { type: "text" }
|
|
104
|
+
import vectorBatchingSql from "./migrations/044-vector-batching.sql" with { type: "text" }
|
|
105
|
+
import agentThreadSandboxStateSql from "./migrations/045-agent-thread-sandbox-state.sql" with {
|
|
106
|
+
type: "text",
|
|
107
|
+
}
|
|
108
|
+
import workflowInterventionPrincipalsSql from "./migrations/046-workflow-intervention-principals.sql" with {
|
|
109
|
+
type: "text",
|
|
110
|
+
}
|
|
111
|
+
import ontologyCommitAttributionSql from "./migrations/047-ontology-commit-attribution.sql" with {
|
|
112
|
+
type: "text",
|
|
113
|
+
}
|
|
114
|
+
import nativeSessionsSql from "./migrations/049-native-sessions.sql" with { type: "text" }
|
|
99
115
|
|
|
100
116
|
const MIGRATIONS_TABLE_SQL = `
|
|
101
117
|
CREATE TABLE IF NOT EXISTS sixb_migrations (
|
|
@@ -191,6 +207,15 @@ export const sqliteStorageMigrations = defineMigrations({
|
|
|
191
207
|
sqliteSql("038-projection-source-roots", projectionSourceRootsSql),
|
|
192
208
|
sqliteSql("039-connector-optional-pkce", connectorOptionalPkceSql),
|
|
193
209
|
sqliteSql("040-outbox-publication-order", outboxPublicationOrderSql),
|
|
210
|
+
sqliteSql("041-agent-thread-workspaces", agentThreadWorkspacesSql),
|
|
211
|
+
sqliteSql("042-vector-profiles", vectorProfilesSql),
|
|
212
|
+
sqliteSql("043-vector-indexing", vectorIndexingSql),
|
|
213
|
+
sqliteSql("044-vector-batching", vectorBatchingSql),
|
|
214
|
+
sqliteSql("045-agent-thread-sandbox-state", agentThreadSandboxStateSql),
|
|
215
|
+
sqliteSql("046-workflow-intervention-principals", workflowInterventionPrincipalsSql),
|
|
216
|
+
sqliteSql("047-ontology-commit-attribution", ontologyCommitAttributionSql),
|
|
217
|
+
sqliteSql("048-file-upload-sessions", fileUploadSessionsSql),
|
|
218
|
+
sqliteSql("049-native-sessions", nativeSessionsSql),
|
|
194
219
|
],
|
|
195
220
|
})
|
|
196
221
|
|
|
@@ -414,3 +439,4 @@ import executionRequesterGroupsSql from "./migrations/037-execution-requester-gr
|
|
|
414
439
|
import connectorOptionalPkceSql from "./migrations/039-connector-optional-pkce.sql" with {
|
|
415
440
|
type: "text",
|
|
416
441
|
}
|
|
442
|
+
import fileUploadSessionsSql from "./migrations/048-file-upload-sessions.sql" with { type: "text" }
|
|
@@ -6,8 +6,15 @@ import {
|
|
|
6
6
|
type ObjectQuerySetOperation,
|
|
7
7
|
type ObjectQuerySortField,
|
|
8
8
|
} from "@sixb/core"
|
|
9
|
+
import {
|
|
10
|
+
hasVectorProfile,
|
|
11
|
+
isVectorProfileQuery,
|
|
12
|
+
normalizeVector,
|
|
13
|
+
vectorSearchCandidateLimit,
|
|
14
|
+
} from "@sixb/core/internal/query"
|
|
15
|
+
import { encodeSqliteVector } from "../vector-encoding"
|
|
9
16
|
|
|
10
|
-
export type SqliteValue = string | number | bigint | boolean | null
|
|
17
|
+
export type SqliteValue = string | number | bigint | boolean | null | Uint8Array
|
|
11
18
|
|
|
12
19
|
export interface SqliteObjectQueryPageRow {
|
|
13
20
|
object_type_id: string
|
|
@@ -17,6 +24,7 @@ export interface SqliteObjectQueryPageRow {
|
|
|
17
24
|
}
|
|
18
25
|
|
|
19
26
|
export interface CompiledObjectQuery {
|
|
27
|
+
vectorProbe?: { sql: string; args: SqliteValue[]; limit: number }
|
|
20
28
|
sql: string
|
|
21
29
|
args: SqliteValue[]
|
|
22
30
|
totalSql: string
|
|
@@ -59,7 +67,8 @@ type CompiledOrderField =
|
|
|
59
67
|
}
|
|
60
68
|
| {
|
|
61
69
|
kind: "column"
|
|
62
|
-
column: "object_type_id" | "primary_id"
|
|
70
|
+
column: "object_type_id" | "primary_id" | "_vector_score"
|
|
71
|
+
binary?: boolean
|
|
63
72
|
direction: "asc" | "desc"
|
|
64
73
|
}
|
|
65
74
|
|
|
@@ -86,6 +95,12 @@ export function compileObjectQuery(
|
|
|
86
95
|
query: ObjectQuery,
|
|
87
96
|
options: { includeTotal?: boolean; source?: SqliteObjectQuerySource } = {}
|
|
88
97
|
): CompiledObjectQuery {
|
|
98
|
+
if (hasVectorProfile(query) && !isVectorProfileQuery(query)) {
|
|
99
|
+
throw new ObjectQueryExecutionError(
|
|
100
|
+
"unsupported_vector_composition",
|
|
101
|
+
"Vector search supports one profile with filters before ranking and limit/project after."
|
|
102
|
+
)
|
|
103
|
+
}
|
|
89
104
|
const source = options.source ?? DEFAULT_OBJECT_QUERY_SOURCE
|
|
90
105
|
const ctx: CompileContext = { probeLimit: options.includeTotal === false, source }
|
|
91
106
|
return source.wrapQuery(compileObjectQueryInternal(projectId, query, ctx))
|
|
@@ -95,6 +110,7 @@ export function compileObjectQuery(
|
|
|
95
110
|
export interface SqliteObjectQuerySource {
|
|
96
111
|
readonly objectsTable: string
|
|
97
112
|
readonly linksTable: string
|
|
113
|
+
readonly objectPropertyPermissionsTable?: string
|
|
98
114
|
/** Wrap a terminal SELECT without its own WITH clause; the selected source owns the only WITH. */
|
|
99
115
|
wrapStatement(
|
|
100
116
|
sql: string,
|
|
@@ -160,7 +176,79 @@ function compileObjectQueryInternal(
|
|
|
160
176
|
case "expand":
|
|
161
177
|
return compileExpand(projectId, query.input, query.expansions, ctx)
|
|
162
178
|
case "vector":
|
|
163
|
-
|
|
179
|
+
return compileVector(projectId, query, ctx)
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function compileVector(
|
|
184
|
+
projectId: string,
|
|
185
|
+
query: Extract<ObjectQuery, { kind: "vector" }>,
|
|
186
|
+
ctx: CompileContext
|
|
187
|
+
): CompiledObjectQuery {
|
|
188
|
+
if (
|
|
189
|
+
typeof query.vector === "string" ||
|
|
190
|
+
!isVectorProfileQuery(query) ||
|
|
191
|
+
!query.configuration ||
|
|
192
|
+
!Number.isSafeInteger(query.k) ||
|
|
193
|
+
query.k < 1 ||
|
|
194
|
+
query.k > 1000 ||
|
|
195
|
+
query.vector.length < 1 ||
|
|
196
|
+
query.vector.length > 16000
|
|
197
|
+
) {
|
|
198
|
+
throw new ObjectQueryExecutionError(
|
|
199
|
+
"invalid_vector_query",
|
|
200
|
+
"A validated named vector profile is required."
|
|
201
|
+
)
|
|
202
|
+
}
|
|
203
|
+
const input = compileObjectQueryInternal(projectId, query.input, exactContext(ctx))
|
|
204
|
+
const permissions = ctx.source.objectPropertyPermissionsTable
|
|
205
|
+
const permissionJoin = permissions
|
|
206
|
+
? `CROSS JOIN (
|
|
207
|
+
SELECT project_id, object_type_id, primary_id, json_group_array(property_id) AS property_ids
|
|
208
|
+
FROM ${permissions}
|
|
209
|
+
GROUP BY project_id, object_type_id, primary_id
|
|
210
|
+
) AS vector_permissions ON vector_permissions.project_id = vectors.project_id
|
|
211
|
+
AND vector_permissions.object_type_id = vectors.object_type_id
|
|
212
|
+
AND vector_permissions.primary_id = vectors.primary_id`
|
|
213
|
+
: ""
|
|
214
|
+
const authorized = permissions
|
|
215
|
+
? `AND NOT EXISTS (
|
|
216
|
+
SELECT 1 FROM json_each(vectors.source) AS required
|
|
217
|
+
WHERE required.value NOT IN (SELECT value FROM json_each(vector_permissions.property_ids))
|
|
218
|
+
)`
|
|
219
|
+
: ""
|
|
220
|
+
const candidates = `FROM (${input.sql}) AS input
|
|
221
|
+
CROSS JOIN object_vectors AS vectors ON vectors.project_id = input.project_id
|
|
222
|
+
AND vectors.object_type_id = input.object_type_id AND vectors.primary_id = input.primary_id
|
|
223
|
+
${permissionJoin}
|
|
224
|
+
WHERE vectors.profile = ? AND vectors.configuration = ?
|
|
225
|
+
AND length(vectors.embedding) / 4 = ? ${authorized}`
|
|
226
|
+
const candidateArgs = [...input.args, query.profile!, query.configuration, query.vector.length]
|
|
227
|
+
// Bound distance work, not just returned rows. Exceeding the envelope fails before ranking.
|
|
228
|
+
const limit = vectorSearchCandidateLimit(query.vector.length)
|
|
229
|
+
const values = normalizeVector(query.vector, query.vector.length)
|
|
230
|
+
const order = compileOrder([
|
|
231
|
+
{ kind: "column", column: "_vector_score", direction: "desc" },
|
|
232
|
+
...identityOrderFields().map((field) => ({ ...field, binary: true })),
|
|
233
|
+
])
|
|
234
|
+
const sql = `SELECT * FROM (SELECT input.*, max(-1, min(1, 1 - vec_distance_cosine(vectors.embedding, ?))) AS _vector_score
|
|
235
|
+
${candidates}) AS ranked ORDER BY ${order.sql} LIMIT ?`
|
|
236
|
+
const args = [encodeSqliteVector(values), ...candidateArgs, query.k]
|
|
237
|
+
return {
|
|
238
|
+
sql,
|
|
239
|
+
args,
|
|
240
|
+
// Top-k cardinality depends on eligibility, not distances or ranking.
|
|
241
|
+
totalSql: `SELECT COUNT(*) AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS eligible`,
|
|
242
|
+
totalArgs: [...candidateArgs, query.k],
|
|
243
|
+
order,
|
|
244
|
+
vectorProbe: {
|
|
245
|
+
sql: `SELECT COUNT(*) AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS candidates`,
|
|
246
|
+
args: [...candidateArgs, limit + 1],
|
|
247
|
+
limit,
|
|
248
|
+
},
|
|
249
|
+
hasMore: () => false,
|
|
250
|
+
trimRows: identityRows,
|
|
251
|
+
nextPageToken: () => undefined,
|
|
164
252
|
}
|
|
165
253
|
}
|
|
166
254
|
|
|
@@ -274,6 +362,7 @@ function compileWhere(
|
|
|
274
362
|
WHERE ${predicate.sql}
|
|
275
363
|
`,
|
|
276
364
|
totalArgs: [...input.args, ...predicate.args],
|
|
365
|
+
vectorProbe: input.vectorProbe,
|
|
277
366
|
order: input.order,
|
|
278
367
|
hasMore: () => false,
|
|
279
368
|
trimRows: identityRows,
|
|
@@ -324,7 +413,7 @@ function compileSort(
|
|
|
324
413
|
input.created_at,
|
|
325
414
|
input.updated_at,
|
|
326
415
|
input.version,
|
|
327
|
-
input.last_commit_id
|
|
416
|
+
input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
|
|
328
417
|
FROM (${input.sql}) AS input
|
|
329
418
|
ORDER BY ${order.sql}
|
|
330
419
|
`,
|
|
@@ -362,6 +451,7 @@ function compileLimit(
|
|
|
362
451
|
FROM (${input.sql}) AS input
|
|
363
452
|
`,
|
|
364
453
|
totalArgs: input.args,
|
|
454
|
+
vectorProbe: input.vectorProbe,
|
|
365
455
|
order: input.order,
|
|
366
456
|
hasMore: (rowCount, total) =>
|
|
367
457
|
total === undefined ? ctx.probeLimit && rowCount > limit : limit < total,
|
|
@@ -399,6 +489,7 @@ function compilePage(
|
|
|
399
489
|
FROM (${input.sql}) AS input
|
|
400
490
|
`,
|
|
401
491
|
totalArgs: input.args,
|
|
492
|
+
vectorProbe: input.vectorProbe,
|
|
402
493
|
order: input.order,
|
|
403
494
|
hasMore: (rowCount) => rowCount > pageSize,
|
|
404
495
|
trimRows: (rows) => rows.slice(0, pageSize),
|
|
@@ -507,6 +598,7 @@ function compileExpand(
|
|
|
507
598
|
args: [...expand.args, ...input.args, ...inputOrder.args],
|
|
508
599
|
totalSql: input.totalSql,
|
|
509
600
|
totalArgs: input.totalArgs,
|
|
601
|
+
vectorProbe: input.vectorProbe,
|
|
510
602
|
order: input.order,
|
|
511
603
|
hasMoreProbe: input.hasMoreProbe,
|
|
512
604
|
hasMore: input.hasMore,
|
|
@@ -760,13 +852,14 @@ function compileProject(
|
|
|
760
852
|
input.created_at,
|
|
761
853
|
input.updated_at,
|
|
762
854
|
input.version,
|
|
763
|
-
input.last_commit_id
|
|
855
|
+
input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
|
|
764
856
|
FROM (${input.sql}) AS input
|
|
765
857
|
ORDER BY ${inputOrder.sql}
|
|
766
858
|
`,
|
|
767
859
|
args: [...projection.args, ...input.args, ...inputOrder.args],
|
|
768
860
|
totalSql: input.totalSql,
|
|
769
861
|
totalArgs: input.totalArgs,
|
|
862
|
+
vectorProbe: input.vectorProbe,
|
|
770
863
|
order: outputOrder,
|
|
771
864
|
hasMoreProbe: input.hasMoreProbe,
|
|
772
865
|
hasMore: input.hasMore,
|
|
@@ -870,9 +963,13 @@ function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredicate {
|
|
|
870
963
|
return negatePredicate(item)
|
|
871
964
|
}
|
|
872
965
|
case "eq":
|
|
873
|
-
return
|
|
966
|
+
return predicate.scalarKind === "userRef"
|
|
967
|
+
? compileUserRefInPredicate(predicate.propertyId, [predicate.value])
|
|
968
|
+
: compileEqualityPredicate(predicate.propertyId, "eq", predicate.value)
|
|
874
969
|
case "neq":
|
|
875
|
-
return
|
|
970
|
+
return predicate.scalarKind === "userRef"
|
|
971
|
+
? negatePredicate(compileUserRefInPredicate(predicate.propertyId, [predicate.value]))
|
|
972
|
+
: compileEqualityPredicate(predicate.propertyId, "neq", predicate.value)
|
|
876
973
|
case "lt":
|
|
877
974
|
case "lte":
|
|
878
975
|
case "gt":
|
|
@@ -884,17 +981,68 @@ function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredicate {
|
|
|
884
981
|
}
|
|
885
982
|
}
|
|
886
983
|
case "in":
|
|
887
|
-
return
|
|
984
|
+
return predicate.scalarKind === "userRef"
|
|
985
|
+
? compileUserRefInPredicate(predicate.propertyId, predicate.values)
|
|
986
|
+
: compileInPredicate(predicate.propertyId, predicate.values)
|
|
888
987
|
case "exists": {
|
|
889
988
|
const sql = `${jsonTypeExpression()} IS NOT NULL`
|
|
890
989
|
const item: CompiledPredicate = { sql, args: [jsonPath(predicate.propertyId)] }
|
|
891
990
|
return predicate.value ? item : negatePredicate(item)
|
|
892
991
|
}
|
|
893
992
|
case "contains":
|
|
894
|
-
return
|
|
993
|
+
return predicate.scalarKind === "userRef"
|
|
994
|
+
? compileUserRefContainsPredicate(predicate.propertyId, predicate.value)
|
|
995
|
+
: compileContainsPredicate(predicate.propertyId, predicate.value)
|
|
895
996
|
}
|
|
896
997
|
}
|
|
897
998
|
|
|
999
|
+
/**
|
|
1000
|
+
* User references are JSON objects, so they match on both fields rather than on serialized text,
|
|
1001
|
+
* which would depend on key order.
|
|
1002
|
+
*/
|
|
1003
|
+
function compileUserRefInPredicate(
|
|
1004
|
+
propertyId: string,
|
|
1005
|
+
values: readonly unknown[]
|
|
1006
|
+
): CompiledPredicate {
|
|
1007
|
+
const path = jsonPath(propertyId)
|
|
1008
|
+
const ids = values.flatMap((value) => {
|
|
1009
|
+
const id = userRefId(value)
|
|
1010
|
+
return id === undefined ? [] : [id]
|
|
1011
|
+
})
|
|
1012
|
+
const clauses: string[] = []
|
|
1013
|
+
const args: SqliteValue[] = []
|
|
1014
|
+
|
|
1015
|
+
if (values.includes(null)) {
|
|
1016
|
+
clauses.push(`${jsonTypeExpression()} = 'null'`)
|
|
1017
|
+
args.push(path)
|
|
1018
|
+
}
|
|
1019
|
+
if (ids.length > 0) {
|
|
1020
|
+
clauses.push(
|
|
1021
|
+
`(${jsonValueExpression()} = 'user' AND ${jsonValueExpression()} IN (${ids.map(() => "?").join(", ")}))`
|
|
1022
|
+
)
|
|
1023
|
+
args.push(`${path}.type`, `${path}.id`, ...ids)
|
|
1024
|
+
}
|
|
1025
|
+
if (clauses.length === 0) return { sql: "0 = 1", args: [] }
|
|
1026
|
+
|
|
1027
|
+
return { sql: `(${clauses.join(" OR ")})`, args }
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
function compileUserRefContainsPredicate(propertyId: string, value: unknown): CompiledPredicate {
|
|
1031
|
+
const id = userRefId(value)
|
|
1032
|
+
if (id === undefined) return { sql: "0 = 1", args: [] }
|
|
1033
|
+
const path = jsonPath(propertyId)
|
|
1034
|
+
return {
|
|
1035
|
+
sql: `(${jsonTypeExpression()} = 'array' AND EXISTS (SELECT 1 FROM json_each(properties, ?) AS item WHERE item.type = 'object' AND json_extract(item.value, '$.type') = 'user' AND json_extract(item.value, '$.id') = ?))`,
|
|
1036
|
+
args: [path, path, id],
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
function userRefId(value: unknown): string | undefined {
|
|
1041
|
+
if (typeof value !== "object" || value === null) return undefined
|
|
1042
|
+
const { type, id } = value as { type?: unknown; id?: unknown }
|
|
1043
|
+
return type === "user" && typeof id === "string" ? id : undefined
|
|
1044
|
+
}
|
|
1045
|
+
|
|
898
1046
|
function negatePredicate(predicate: CompiledPredicate): CompiledPredicate {
|
|
899
1047
|
return {
|
|
900
1048
|
sql: `(NOT COALESCE((${predicate.sql}), 0))`,
|
|
@@ -1026,7 +1174,9 @@ function compileOrder(
|
|
|
1026
1174
|
|
|
1027
1175
|
for (const field of fields) {
|
|
1028
1176
|
if (field.kind === "column") {
|
|
1029
|
-
clauses.push(
|
|
1177
|
+
clauses.push(
|
|
1178
|
+
`${columnExpression(field.column, qualifier)}${field.binary ? " COLLATE BINARY" : ""} ${field.direction.toUpperCase()}`
|
|
1179
|
+
)
|
|
1030
1180
|
continue
|
|
1031
1181
|
}
|
|
1032
1182
|
|
|
@@ -1238,7 +1388,10 @@ function orderFieldKey(field: CompiledOrderField): string {
|
|
|
1238
1388
|
: `property:${field.propertyId}:${field.direction}`
|
|
1239
1389
|
}
|
|
1240
1390
|
|
|
1241
|
-
function columnExpression(
|
|
1391
|
+
function columnExpression(
|
|
1392
|
+
column: "object_type_id" | "primary_id" | "_vector_score",
|
|
1393
|
+
qualifier?: string
|
|
1394
|
+
): string {
|
|
1242
1395
|
return qualifier ? `${qualifier}.${column}` : column
|
|
1243
1396
|
}
|
|
1244
1397
|
|
|
@@ -19,6 +19,9 @@ export interface SqliteSelectedObjectReadSource extends SqliteObjectQuerySource
|
|
|
19
19
|
*
|
|
20
20
|
* The complete compiled scope is serialized once. Every statement receives that one JSON document
|
|
21
21
|
* and the bound project id, keeping the host-parameter count constant as selections grow.
|
|
22
|
+
* CROSS JOIN keeps authorized parents/grants before indexed edge/object lookups, even when
|
|
23
|
+
* statistics would otherwise make SQLite rescan all grants for every stored object.
|
|
24
|
+
* Property grants are grouped only after path-sensitive reachability has been resolved.
|
|
22
25
|
*/
|
|
23
26
|
export function compileSqliteSelectedObjectReadSource(
|
|
24
27
|
projectId: string,
|
|
@@ -44,6 +47,9 @@ export function compileSqliteSelectedObjectReadSource(
|
|
|
44
47
|
wrapQuery: (query: CompiledObjectQuery): CompiledObjectQuery => ({
|
|
45
48
|
...query,
|
|
46
49
|
...wrapStatement(query.sql, query.args),
|
|
50
|
+
vectorProbe: query.vectorProbe
|
|
51
|
+
? { ...query.vectorProbe, ...wrapStatement(query.vectorProbe.sql, query.vectorProbe.args) }
|
|
52
|
+
: undefined,
|
|
47
53
|
totalSql: `${compiled.sql}\n${query.totalSql}`,
|
|
48
54
|
totalArgs: [...compiled.args, ...query.totalArgs],
|
|
49
55
|
...(query.hasMoreProbe
|
|
@@ -140,16 +146,16 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
|
|
|
140
146
|
|
|
141
147
|
SELECT edge.project_id, selection.node_id, edge.target_type_id, edge.target_id
|
|
142
148
|
FROM _sixb_scope_reachable AS parent
|
|
143
|
-
JOIN _sixb_scope_link_selections AS selection
|
|
149
|
+
CROSS JOIN _sixb_scope_link_selections AS selection
|
|
144
150
|
ON selection.parent_node_id = parent.node_id
|
|
145
151
|
AND selection.source_object_type_id = parent.object_type_id
|
|
146
|
-
JOIN links AS edge
|
|
152
|
+
CROSS JOIN links AS edge
|
|
147
153
|
ON edge.project_id = parent.project_id
|
|
148
154
|
AND edge.source_type_id = parent.object_type_id
|
|
149
155
|
AND edge.source_id = parent.primary_id
|
|
150
156
|
AND edge.link_id = selection.link_id
|
|
151
157
|
AND edge.target_type_id = selection.target_object_type_id
|
|
152
|
-
JOIN objects AS target
|
|
158
|
+
CROSS JOIN objects AS target
|
|
153
159
|
ON target.project_id = edge.project_id
|
|
154
160
|
AND target.object_type_id = edge.target_type_id
|
|
155
161
|
AND target.primary_id = edge.target_id
|
|
@@ -170,32 +176,29 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
|
|
|
170
176
|
SELECT DISTINCT project_id, object_type_id, primary_id
|
|
171
177
|
FROM _sixb_scope_reachable
|
|
172
178
|
),
|
|
179
|
+
_sixb_scope_object_grants AS (
|
|
180
|
+
SELECT reachable.project_id, reachable.object_type_id, reachable.primary_id,
|
|
181
|
+
json_group_array(DISTINCT selected_property.value) FILTER (WHERE selected_property.value IS NOT NULL) AS property_ids
|
|
182
|
+
FROM _sixb_scope_reachable AS reachable
|
|
183
|
+
LEFT JOIN _sixb_scope_object_selections AS selection
|
|
184
|
+
ON selection.node_id = reachable.node_id
|
|
185
|
+
AND selection.object_type_id = reachable.object_type_id
|
|
186
|
+
LEFT JOIN json_each(selection.property_ids) AS selected_property ON true
|
|
187
|
+
GROUP BY reachable.project_id, reachable.object_type_id, reachable.primary_id
|
|
188
|
+
),
|
|
173
189
|
_sixb_scope_objects AS (
|
|
174
|
-
SELECT
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
AND permission.property_id = property.key
|
|
187
|
-
),
|
|
188
|
-
json('{}')
|
|
189
|
-
) AS properties,
|
|
190
|
-
stored.created_at,
|
|
191
|
-
stored.updated_at,
|
|
192
|
-
stored.version,
|
|
193
|
-
stored.last_commit_id
|
|
194
|
-
FROM objects AS stored
|
|
195
|
-
JOIN _sixb_scope_object_ids AS visible
|
|
196
|
-
ON visible.project_id = stored.project_id
|
|
197
|
-
AND visible.object_type_id = stored.object_type_id
|
|
198
|
-
AND visible.primary_id = stored.primary_id
|
|
190
|
+
SELECT stored.project_id, stored.object_type_id, stored.primary_id,
|
|
191
|
+
COALESCE((
|
|
192
|
+
SELECT json_group_object(property.key, ${sqliteJsonEachValue("property")})
|
|
193
|
+
FROM json_each(stored.properties) AS property
|
|
194
|
+
WHERE property.key IN (SELECT value FROM json_each(grants.property_ids))
|
|
195
|
+
), json('{}')) AS properties,
|
|
196
|
+
stored.created_at, stored.updated_at, stored.version, stored.last_commit_id
|
|
197
|
+
FROM _sixb_scope_object_grants AS grants
|
|
198
|
+
CROSS JOIN objects AS stored
|
|
199
|
+
ON stored.project_id = grants.project_id
|
|
200
|
+
AND stored.object_type_id = grants.object_type_id
|
|
201
|
+
AND stored.primary_id = grants.primary_id
|
|
199
202
|
),
|
|
200
203
|
_sixb_scope_reached_links(
|
|
201
204
|
project_id,
|
|
@@ -221,16 +224,16 @@ function compileSelectedReadScopeCte(scopeJson: string, projectId: string): Comp
|
|
|
221
224
|
edge.target_id,
|
|
222
225
|
selection.property_ids
|
|
223
226
|
FROM _sixb_scope_reachable AS parent
|
|
224
|
-
JOIN _sixb_scope_link_selections AS selection
|
|
227
|
+
CROSS JOIN _sixb_scope_link_selections AS selection
|
|
225
228
|
ON selection.parent_node_id = parent.node_id
|
|
226
229
|
AND selection.source_object_type_id = parent.object_type_id
|
|
227
|
-
JOIN links AS edge
|
|
230
|
+
CROSS JOIN links AS edge
|
|
228
231
|
ON edge.project_id = parent.project_id
|
|
229
232
|
AND edge.source_type_id = parent.object_type_id
|
|
230
233
|
AND edge.source_id = parent.primary_id
|
|
231
234
|
AND edge.link_id = selection.link_id
|
|
232
235
|
AND edge.target_type_id = selection.target_object_type_id
|
|
233
|
-
JOIN objects AS target
|
|
236
|
+
CROSS JOIN objects AS target
|
|
234
237
|
ON target.project_id = edge.project_id
|
|
235
238
|
AND target.object_type_id = edge.target_type_id
|
|
236
239
|
AND target.primary_id = edge.target_id
|
package/src/objects/reader.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite"
|
|
2
|
-
import type
|
|
2
|
+
import { type ObjectQuery, ObjectQueryExecutionError } from "@sixb/core"
|
|
3
|
+
import { hasVectorProfile } from "@sixb/core/internal/query"
|
|
3
4
|
import type {
|
|
4
5
|
CountObjectsInput,
|
|
5
6
|
CountObjectsResult,
|
|
@@ -24,6 +25,7 @@ import {
|
|
|
24
25
|
type CompiledObjectQuery,
|
|
25
26
|
compileObjectQuery,
|
|
26
27
|
type SqliteObjectQuerySource,
|
|
28
|
+
type SqliteValue,
|
|
27
29
|
} from "./query-compiler"
|
|
28
30
|
import type { SqliteSelectedObjectReadSource } from "./read-scope"
|
|
29
31
|
import {
|
|
@@ -34,6 +36,7 @@ import {
|
|
|
34
36
|
rowToLink,
|
|
35
37
|
rowToObject,
|
|
36
38
|
} from "./rows"
|
|
39
|
+
import { ensureSqliteVectorSearch } from "./vector-extension"
|
|
37
40
|
|
|
38
41
|
/** Synchronous SQL reads over one explicit source; transaction and budget ownership stay outside. */
|
|
39
42
|
export class SqliteObjectReader {
|
|
@@ -42,7 +45,25 @@ export class SqliteObjectReader {
|
|
|
42
45
|
private readonly source: SqliteObjectQuerySource | SqliteSelectedObjectReadSource
|
|
43
46
|
) {}
|
|
44
47
|
|
|
48
|
+
private assertVectorSearch(
|
|
49
|
+
params: QueryObjectsInput | CountObjectsInput | ExistsObjectsInput | FacetObjectsInput
|
|
50
|
+
): void {
|
|
51
|
+
if (!hasVectorProfile(params.query)) return
|
|
52
|
+
const compiled = compileObjectQuery(params.projectId, params.query, { source: this.source })
|
|
53
|
+
if (!compiled.vectorProbe) return
|
|
54
|
+
ensureSqliteVectorSearch(this.db)
|
|
55
|
+
const probe = compiled.vectorProbe
|
|
56
|
+
const row = this.db.query<{ total: number }, SqliteValue[]>(probe.sql).get(...probe.args)
|
|
57
|
+
if (Number(row?.total ?? 0) > probe.limit) {
|
|
58
|
+
throw new ObjectQueryExecutionError(
|
|
59
|
+
"vector_work_limit_exceeded",
|
|
60
|
+
`Exact vector search supports at most ${probe.limit} eligible vectors at this dimension. Narrow the query filters.`
|
|
61
|
+
)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
45
65
|
queryObjects(params: QueryObjectsInput): QueryObjectsResult {
|
|
66
|
+
this.assertVectorSearch(params)
|
|
46
67
|
const source = this.source
|
|
47
68
|
const compiled = compileObjectQuery(params.projectId, params.query, {
|
|
48
69
|
includeTotal: params.includeTotal,
|
|
@@ -67,6 +88,7 @@ export class SqliteObjectReader {
|
|
|
67
88
|
}
|
|
68
89
|
|
|
69
90
|
countObjects(params: CountObjectsInput): CountObjectsResult {
|
|
91
|
+
this.assertVectorSearch(params)
|
|
70
92
|
const source = this.source
|
|
71
93
|
return {
|
|
72
94
|
count: readTotal(
|
|
@@ -79,6 +101,7 @@ export class SqliteObjectReader {
|
|
|
79
101
|
}
|
|
80
102
|
|
|
81
103
|
existsObjects(params: ExistsObjectsInput): ExistsObjectsResult {
|
|
104
|
+
this.assertVectorSearch(params)
|
|
82
105
|
const source = this.source
|
|
83
106
|
const compiled = compileObjectQuery(params.projectId, existsProbeQuery(params.query), {
|
|
84
107
|
source,
|
|
@@ -87,6 +110,7 @@ export class SqliteObjectReader {
|
|
|
87
110
|
}
|
|
88
111
|
|
|
89
112
|
facetObjects(params: FacetObjectsInput): FacetObjectsResult {
|
|
113
|
+
this.assertVectorSearch(params)
|
|
90
114
|
const source = this.source
|
|
91
115
|
const compiled = compileObjectQuery(params.projectId, stripOuterRowShape(params.query), {
|
|
92
116
|
source,
|
|
@@ -488,6 +512,10 @@ function sqliteFacetValue(valueType: string | null, value: unknown): unknown {
|
|
|
488
512
|
if (valueType === "true") return true
|
|
489
513
|
if (valueType === "false") return false
|
|
490
514
|
if (valueType === "null") return null
|
|
515
|
+
// json_extract returns structured values, such as user references, as JSON text.
|
|
516
|
+
if ((valueType === "object" || valueType === "array") && typeof value === "string") {
|
|
517
|
+
return JSON.parse(value)
|
|
518
|
+
}
|
|
491
519
|
return value
|
|
492
520
|
}
|
|
493
521
|
|
package/src/objects/rows.ts
CHANGED
|
@@ -21,7 +21,10 @@ export function rowToObject(row: DatabaseRow): ObjectRow {
|
|
|
21
21
|
|
|
22
22
|
/** Decode an expanded query row with the same shape as Core link hydration. */
|
|
23
23
|
export function queryRowToObject(row: ObjectQueryDatabaseRow): ObjectRow {
|
|
24
|
-
const base =
|
|
24
|
+
const base = {
|
|
25
|
+
...rowToObject(row),
|
|
26
|
+
...(row._vector_score === undefined ? {} : { score: row._vector_score }),
|
|
27
|
+
}
|
|
25
28
|
const links = reviveExpandedLinks(parseExpandColumn(row._expand))
|
|
26
29
|
return links ? { ...base, links } : base
|
|
27
30
|
}
|
|
@@ -104,6 +107,7 @@ export interface DatabaseRow {
|
|
|
104
107
|
}
|
|
105
108
|
|
|
106
109
|
export interface ObjectQueryDatabaseRow extends DatabaseRow {
|
|
110
|
+
_vector_score?: number
|
|
107
111
|
_cursor_properties?: string
|
|
108
112
|
/** `json_object(linkId, value, ...)` serialized text from an `expand` pushdown; absent otherwise. */
|
|
109
113
|
_expand?: string | null
|