@sixb/sqlite 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/agents/rows.d.ts +2 -0
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/threads.d.ts +2 -1
- package/dist/agents/threads.d.ts.map +1 -1
- package/dist/auth-storage/access-tokens.d.ts +0 -1
- package/dist/auth-storage/access-tokens.d.ts.map +1 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/device-authorizations.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +6 -3
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/sessions.d.ts +8 -1
- package/dist/auth-storage/sessions.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/file-upload-session-storage.d.ts +6 -0
- package/dist/file-upload-session-storage.d.ts.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1062 -137
- package/dist/index.js.map +28 -23
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +9 -2
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts +3 -0
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +1 -0
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/rows.d.ts +1 -0
- package/dist/objects/rows.d.ts.map +1 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/vector-extension.d.ts +4 -0
- package/dist/objects/vector-extension.d.ts.map +1 -0
- package/dist/ontology-storage/index.d.ts +4 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +1 -2
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +2 -1
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/vector-indexing.d.ts +20 -0
- package/dist/ontology-storage/vector-indexing.d.ts.map +1 -0
- package/dist/ontology-storage/vectors.d.ts +16 -0
- package/dist/ontology-storage/vectors.d.ts.map +1 -0
- package/dist/vector-encoding.d.ts +3 -0
- package/dist/vector-encoding.d.ts.map +1 -0
- package/dist/workflow-intervention-storage.d.ts.map +1 -1
- package/package.json +8 -4
- package/src/agents/rows.ts +12 -0
- package/src/agents/threads.ts +41 -2
- package/src/ai-cost-storage.ts +4 -0
- package/src/auth-storage/access-tokens.ts +11 -18
- package/src/auth-storage/auth-storage.ts +5 -7
- package/src/auth-storage/device-authorizations.ts +3 -5
- package/src/auth-storage/rows.ts +17 -7
- package/src/auth-storage/sessions.ts +89 -4
- package/src/auth-storage/shared.ts +9 -3
- package/src/file-upload-session-storage.ts +136 -0
- package/src/index.ts +5 -0
- package/src/migrations/041-agent-thread-workspaces.sql +2 -0
- package/src/migrations/042-vector-profiles.sql +21 -0
- package/src/migrations/043-vector-indexing.sql +197 -0
- package/src/migrations/044-vector-batching.sql +3 -0
- package/src/migrations/045-agent-thread-sandbox-state.sql +2 -0
- package/src/migrations/046-workflow-intervention-principals.sql +15 -0
- package/src/migrations/047-ontology-commit-attribution.sql +67 -0
- package/src/migrations/048-file-upload-sessions.sql +35 -0
- package/src/migrations/049-native-sessions.sql +12 -0
- package/src/migrations.ts +26 -0
- package/src/objects/query-compiler.ts +164 -11
- package/src/objects/read-scope.ts +34 -31
- package/src/objects/reader.ts +29 -1
- package/src/objects/rows.ts +5 -1
- package/src/objects/storage.ts +13 -1
- package/src/objects/vector-extension.ts +26 -0
- package/src/ontology-storage/index.ts +16 -0
- package/src/ontology-storage/materializations.ts +29 -59
- package/src/ontology-storage/shared.ts +7 -3
- package/src/ontology-storage/vector-indexing.ts +196 -0
- package/src/ontology-storage/vectors.ts +182 -0
- package/src/vector-encoding.ts +9 -0
- package/src/workflow-intervention-storage.ts +9 -9
package/src/objects/storage.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite"
|
|
2
|
+
import { hasVectorProfile } from "@sixb/core/internal/query"
|
|
2
3
|
import type {
|
|
3
4
|
CompiledSelectedObjectReadScope,
|
|
4
5
|
CountObjectsInput,
|
|
@@ -25,6 +26,7 @@ import { installFreshSqliteSchema } from "../migrations"
|
|
|
25
26
|
import {
|
|
26
27
|
closeSqliteStoreConnection,
|
|
27
28
|
openSqliteStoreConnection,
|
|
29
|
+
runDeferredReadTransaction,
|
|
28
30
|
type SqliteStoreConnection,
|
|
29
31
|
} from "../transactions"
|
|
30
32
|
import { DEFAULT_OBJECT_QUERY_SOURCE } from "./query-compiler"
|
|
@@ -49,6 +51,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
|
|
|
49
51
|
refs: true,
|
|
50
52
|
filter: true,
|
|
51
53
|
text: true,
|
|
54
|
+
vector: true,
|
|
52
55
|
sort: true,
|
|
53
56
|
limit: true,
|
|
54
57
|
page: true,
|
|
@@ -92,6 +95,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
|
|
|
92
95
|
decimal: { equality: true },
|
|
93
96
|
date: { equality: true, ordering: true },
|
|
94
97
|
timestamp: { equality: true, ordering: true },
|
|
98
|
+
userRef: { equality: true },
|
|
95
99
|
},
|
|
96
100
|
limits: {
|
|
97
101
|
totalCount: true,
|
|
@@ -101,7 +105,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
|
|
|
101
105
|
"SQLite object query pushdown supports start/refs/filter/text/sort/limit/page/traverse/set/project/expand over JSON properties and object links.",
|
|
102
106
|
"expand hydrates linked objects in-database (top-N per parent via row_number() + json_group_array); core resolves each expansion's cardinality before pushdown, and a mixed/unresolved one stays on the fallback.",
|
|
103
107
|
"Ordered decimal predicates and sorting use the bounded core fallback because SQLite has no native exact decimal type; canonical decimal equality remains pushdown-safe.",
|
|
104
|
-
"Relevance sorting
|
|
108
|
+
"Relevance sorting and unresolved start.includeSubtypes remain planner fallback or rejection cases.",
|
|
105
109
|
],
|
|
106
110
|
}
|
|
107
111
|
|
|
@@ -139,18 +143,26 @@ export class SqliteObjectStorage implements ObjectStorage {
|
|
|
139
143
|
}
|
|
140
144
|
|
|
141
145
|
async queryObjects(params: QueryObjectsInput): Promise<QueryObjectsResult> {
|
|
146
|
+
if (hasVectorProfile(params.query))
|
|
147
|
+
return runDeferredReadTransaction(this.db, () => this.reader.queryObjects(params))
|
|
142
148
|
return this.reader.queryObjects(params)
|
|
143
149
|
}
|
|
144
150
|
|
|
145
151
|
async countObjects(params: CountObjectsInput): Promise<CountObjectsResult> {
|
|
152
|
+
if (hasVectorProfile(params.query))
|
|
153
|
+
return runDeferredReadTransaction(this.db, () => this.reader.countObjects(params))
|
|
146
154
|
return this.reader.countObjects(params)
|
|
147
155
|
}
|
|
148
156
|
|
|
149
157
|
async existsObjects(params: ExistsObjectsInput): Promise<ExistsObjectsResult> {
|
|
158
|
+
if (hasVectorProfile(params.query))
|
|
159
|
+
return runDeferredReadTransaction(this.db, () => this.reader.existsObjects(params))
|
|
150
160
|
return this.reader.existsObjects(params)
|
|
151
161
|
}
|
|
152
162
|
|
|
153
163
|
async facetObjects(params: FacetObjectsInput): Promise<FacetObjectsResult> {
|
|
164
|
+
if (hasVectorProfile(params.query))
|
|
165
|
+
return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params))
|
|
154
166
|
return this.reader.facetObjects(params)
|
|
155
167
|
}
|
|
156
168
|
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite"
|
|
2
|
+
import { ObjectQueryExecutionError } from "@sixb/core"
|
|
3
|
+
import { load } from "sqlite-vec"
|
|
4
|
+
|
|
5
|
+
const loaded = new WeakSet<Database>()
|
|
6
|
+
|
|
7
|
+
/** Load only for vector reads, leaving ordinary SQLite use independent of extension support. */
|
|
8
|
+
export function ensureSqliteVectorSearch(db: Database): void {
|
|
9
|
+
if (loaded.has(db)) return
|
|
10
|
+
const options = db.query<{ compile_options: string }, []>("PRAGMA compile_options").all()
|
|
11
|
+
if (options.some((option) => option.compile_options === "OMIT_LOAD_EXTENSION")) {
|
|
12
|
+
throw new ObjectQueryExecutionError(
|
|
13
|
+
"vector_extension_missing",
|
|
14
|
+
"SQLite vector search requires extension support. On macOS install SQLite (brew install sqlite), then call Database.setCustomSQLite with its library path before opening any database."
|
|
15
|
+
)
|
|
16
|
+
}
|
|
17
|
+
try {
|
|
18
|
+
load(db)
|
|
19
|
+
loaded.add(db)
|
|
20
|
+
} catch (cause) {
|
|
21
|
+
throw new Error(
|
|
22
|
+
"[SixbSqlite] Could not load sqlite-vec for vector search. Check that its native package supports your platform.",
|
|
23
|
+
{ cause }
|
|
24
|
+
)
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -7,10 +7,15 @@ import { SqliteOntologyOutboxStorage } from "./outbox"
|
|
|
7
7
|
import type { SqliteRootOperation } from "./shared"
|
|
8
8
|
import { SqliteOntologySourceStorage } from "./sources"
|
|
9
9
|
|
|
10
|
+
import { SqliteVectorIndexingStorage } from "./vector-indexing"
|
|
11
|
+
import { SqliteOntologyVectorStorage } from "./vectors"
|
|
12
|
+
|
|
10
13
|
export class SqliteOntologyStorage implements OntologyStorage {
|
|
11
14
|
readonly commits: SqliteOntologyCommitStorage
|
|
12
15
|
readonly sources: SqliteOntologySourceStorage
|
|
13
16
|
readonly materializations: SqliteOntologyMaterializationStorage
|
|
17
|
+
readonly vectorIndexing: SqliteVectorIndexingStorage
|
|
18
|
+
readonly vectors: SqliteOntologyVectorStorage
|
|
14
19
|
readonly outbox: SqliteOntologyOutboxStorage
|
|
15
20
|
|
|
16
21
|
constructor(input: {
|
|
@@ -24,6 +29,17 @@ export class SqliteOntologyStorage implements OntologyStorage {
|
|
|
24
29
|
input.db,
|
|
25
30
|
input.transactionContext
|
|
26
31
|
)
|
|
32
|
+
this.vectorIndexing = new SqliteVectorIndexingStorage(
|
|
33
|
+
input.db,
|
|
34
|
+
input.runRootOperation,
|
|
35
|
+
(session, projectId) => this.materializations.assertVectorSession(session, projectId)
|
|
36
|
+
)
|
|
37
|
+
this.vectors = new SqliteOntologyVectorStorage(
|
|
38
|
+
input.db,
|
|
39
|
+
input.runRootOperation,
|
|
40
|
+
(session, projectId, commitId) =>
|
|
41
|
+
this.materializations.assertVectorSession(session, projectId, commitId)
|
|
42
|
+
)
|
|
27
43
|
this.outbox = new SqliteOntologyOutboxStorage(input.db, input.runRootOperation)
|
|
28
44
|
}
|
|
29
45
|
|
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite"
|
|
2
|
-
import type {
|
|
3
|
-
ExpectedLinkRevision,
|
|
4
|
-
ExpectedObjectRevision,
|
|
5
|
-
} from "@sixb/core/internal/materialization"
|
|
6
2
|
import {
|
|
7
3
|
assertPinnedDatasetWatermark,
|
|
8
4
|
linkRefKey,
|
|
9
5
|
linkRefSortKey,
|
|
10
6
|
linkScopeSortKey,
|
|
11
7
|
MaterializationConflictError,
|
|
8
|
+
MaterializationValidationError,
|
|
12
9
|
objectRefKey,
|
|
13
10
|
objectRefSortKey,
|
|
14
11
|
telemetryPointKey,
|
|
15
12
|
telemetryPointSortKey,
|
|
16
13
|
} from "@sixb/core/internal/materialization"
|
|
17
14
|
import {
|
|
15
|
+
assertExpectedLinkRevision,
|
|
16
|
+
assertExpectedLinkScopeRevision,
|
|
17
|
+
assertExpectedObjectRevision,
|
|
18
18
|
assertMaterializationFinalizationCorrelation,
|
|
19
19
|
assertMaterializationHeader,
|
|
20
20
|
assertMaterializationLaneCompletion,
|
|
@@ -79,6 +79,18 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
|
|
|
79
79
|
this.writer = new SqliteMaterializationWriter(db)
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
assertVectorSession(session: MaterializationSession, projectId: string, commitId?: string): void {
|
|
83
|
+
const active = this.sessions.require(session)
|
|
84
|
+
if (
|
|
85
|
+
active.header.commit.projectId !== projectId ||
|
|
86
|
+
(commitId !== undefined && active.header.commit.id !== commitId)
|
|
87
|
+
) {
|
|
88
|
+
throw new MaterializationValidationError(
|
|
89
|
+
"Vector mutation does not belong to this materialization session."
|
|
90
|
+
)
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
82
94
|
async begin(input: MaterializationPlanHeader): Promise<MaterializationSession> {
|
|
83
95
|
assertMaterializationHeader(input)
|
|
84
96
|
const session = this.sessions.create(input)
|
|
@@ -87,16 +99,15 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
|
|
|
87
99
|
const reader = new SqliteMaterializationStateReader(this.db, input.commit.projectId)
|
|
88
100
|
for (const expected of input.expected.sources)
|
|
89
101
|
this.assertSource(expected, input.commit.projectId)
|
|
90
|
-
for (const expected of input.expected.objects)
|
|
91
|
-
|
|
102
|
+
for (const expected of input.expected.objects) {
|
|
103
|
+
assertExpectedObjectRevision(reader.effectiveObjectRevision(expected.ref), expected)
|
|
104
|
+
}
|
|
105
|
+
for (const expected of input.expected.links) {
|
|
106
|
+
assertExpectedLinkRevision(reader.effectiveLinkLastCommit(expected.ref), expected)
|
|
107
|
+
}
|
|
92
108
|
const linkScopeRevisions = reader.linkScopeRevisions(input.expected.linkScopes)
|
|
93
109
|
for (const [index, expected] of input.expected.linkScopes.entries()) {
|
|
94
|
-
|
|
95
|
-
throw new MaterializationConflictError(
|
|
96
|
-
"effective-state",
|
|
97
|
-
`Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`
|
|
98
|
-
)
|
|
99
|
-
}
|
|
110
|
+
assertExpectedLinkScopeRevision(linkScopeRevisions[index]?.fingerprint, expected)
|
|
100
111
|
}
|
|
101
112
|
for (const expected of input.expected.points) {
|
|
102
113
|
const point = reader.exactPoint(expected.series, expected.at)
|
|
@@ -436,50 +447,6 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
|
|
|
436
447
|
}
|
|
437
448
|
}
|
|
438
449
|
|
|
439
|
-
private assertObject(
|
|
440
|
-
reader: SqliteMaterializationStateReader,
|
|
441
|
-
expected: ExpectedObjectRevision
|
|
442
|
-
): void {
|
|
443
|
-
const row = reader.effectiveObjectRevision(expected.ref)
|
|
444
|
-
if (!expected.exists) {
|
|
445
|
-
if (row) {
|
|
446
|
-
throw new MaterializationConflictError(
|
|
447
|
-
"effective-state",
|
|
448
|
-
`Expected object ${objectRefKey(expected.ref)} to be absent.`
|
|
449
|
-
)
|
|
450
|
-
}
|
|
451
|
-
return
|
|
452
|
-
}
|
|
453
|
-
if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId) {
|
|
454
|
-
throw new MaterializationConflictError(
|
|
455
|
-
"effective-state",
|
|
456
|
-
`Expected object ${objectRefKey(expected.ref)} changed.`
|
|
457
|
-
)
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
private assertLink(
|
|
462
|
-
reader: SqliteMaterializationStateReader,
|
|
463
|
-
expected: ExpectedLinkRevision
|
|
464
|
-
): void {
|
|
465
|
-
const lastCommitId = reader.effectiveLinkLastCommit(expected.ref)
|
|
466
|
-
if (!expected.exists) {
|
|
467
|
-
if (lastCommitId !== undefined) {
|
|
468
|
-
throw new MaterializationConflictError(
|
|
469
|
-
"effective-state",
|
|
470
|
-
`Expected link ${linkRefKey(expected.ref)} to be absent.`
|
|
471
|
-
)
|
|
472
|
-
}
|
|
473
|
-
return
|
|
474
|
-
}
|
|
475
|
-
if (lastCommitId === undefined || lastCommitId !== expected.lastCommitId) {
|
|
476
|
-
throw new MaterializationConflictError(
|
|
477
|
-
"effective-state",
|
|
478
|
-
`Expected link ${linkRefKey(expected.ref)} changed.`
|
|
479
|
-
)
|
|
480
|
-
}
|
|
481
|
-
}
|
|
482
|
-
|
|
483
450
|
private async assertFinalization(
|
|
484
451
|
session: SqliteMaterializationSessionState,
|
|
485
452
|
input: FinalizeMaterializationInput
|
|
@@ -706,10 +673,12 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
|
|
|
706
673
|
`
|
|
707
674
|
INSERT INTO ontology_commits (
|
|
708
675
|
project_id, id, idempotency_key, request_hash, execution_id,
|
|
709
|
-
origin_kind, origin_run_id, origin_batch_ordinal, origin,
|
|
676
|
+
origin_kind, origin_run_id, origin_batch_ordinal, origin, requested_by, executor,
|
|
710
677
|
ontology_revision, projection_revision, ownership_hash,
|
|
711
678
|
intent, result, committed_at
|
|
712
|
-
) VALUES (
|
|
679
|
+
) VALUES (
|
|
680
|
+
?, ?, ?, ?, ?, ?, ?, ?, json(?), json(?), json(?), ?, ?, ?, json(?), json(?), ?
|
|
681
|
+
)
|
|
713
682
|
`
|
|
714
683
|
)
|
|
715
684
|
.run(
|
|
@@ -722,7 +691,8 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
|
|
|
722
691
|
origin.runId,
|
|
723
692
|
origin.batchOrdinal,
|
|
724
693
|
canonicalJson(commit.origin),
|
|
725
|
-
commit.
|
|
694
|
+
commit.requestedBy === undefined ? null : canonicalJson(commit.requestedBy),
|
|
695
|
+
canonicalJson(commit.executor),
|
|
726
696
|
commit.ontologyRevision,
|
|
727
697
|
commit.projectionRevision ?? null,
|
|
728
698
|
commit.ownershipHash ?? null,
|
|
@@ -37,7 +37,8 @@ export interface SqliteOntologyCommitRow {
|
|
|
37
37
|
readonly origin_run_id: string | null
|
|
38
38
|
readonly origin_batch_ordinal: number | null
|
|
39
39
|
readonly origin: string
|
|
40
|
-
readonly
|
|
40
|
+
readonly requested_by: string | null
|
|
41
|
+
readonly executor: string
|
|
41
42
|
readonly ontology_revision: string
|
|
42
43
|
readonly projection_revision: string | null
|
|
43
44
|
readonly ownership_hash: string | null
|
|
@@ -162,9 +163,12 @@ export function commitRecord(row: SqliteOntologyCommitRow): OntologyCommitRecord
|
|
|
162
163
|
requestHash: row.request_hash,
|
|
163
164
|
executionId: row.execution_id,
|
|
164
165
|
origin: parseJson<OntologyCommitWrite["origin"]>(row.origin),
|
|
165
|
-
...(row.
|
|
166
|
+
...(row.requested_by === null
|
|
166
167
|
? {}
|
|
167
|
-
: {
|
|
168
|
+
: {
|
|
169
|
+
requestedBy: parseJson<NonNullable<OntologyCommitWrite["requestedBy"]>>(row.requested_by),
|
|
170
|
+
}),
|
|
171
|
+
executor: parseJson<OntologyCommitWrite["executor"]>(row.executor),
|
|
168
172
|
ontologyRevision: row.ontology_revision,
|
|
169
173
|
...(row.projection_revision === null ? {} : { projectionRevision: row.projection_revision }),
|
|
170
174
|
...(row.ownership_hash === null ? {} : { ownershipHash: row.ownership_hash }),
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite"
|
|
2
|
+
import { parseSixbFailure } from "@sixb/core/internal/errors"
|
|
3
|
+
import {
|
|
4
|
+
type MaterializationSession,
|
|
5
|
+
type OntologyVectorIndexingStorage,
|
|
6
|
+
VECTOR_INDEXING_FAILURE_CODES,
|
|
7
|
+
type VectorIndexingRequest,
|
|
8
|
+
type VectorIndexingWork,
|
|
9
|
+
} from "@sixb/core/storage"
|
|
10
|
+
import type { SqliteRootOperation } from "./shared"
|
|
11
|
+
|
|
12
|
+
interface WorkRow {
|
|
13
|
+
batch_id: string | null
|
|
14
|
+
request: string
|
|
15
|
+
status: VectorIndexingWork["status"]
|
|
16
|
+
available_at: string
|
|
17
|
+
values_json: string | null
|
|
18
|
+
error: string | null
|
|
19
|
+
}
|
|
20
|
+
export class SqliteVectorIndexingStorage implements OntologyVectorIndexingStorage {
|
|
21
|
+
constructor(
|
|
22
|
+
private readonly db: Database,
|
|
23
|
+
private readonly run: SqliteRootOperation,
|
|
24
|
+
private readonly assertSession: (session: MaterializationSession, projectId: string) => void
|
|
25
|
+
) {}
|
|
26
|
+
async schedule(input: Parameters<OntologyVectorIndexingStorage["schedule"]>[0]) {
|
|
27
|
+
this.assertSession(input.session, input.projectId)
|
|
28
|
+
for (const ref of input.deleted)
|
|
29
|
+
this.db
|
|
30
|
+
.query(
|
|
31
|
+
`DELETE FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ? AND primary_id = ?`
|
|
32
|
+
)
|
|
33
|
+
.run(input.projectId, ref.objectTypeId, ref.primaryId)
|
|
34
|
+
const insert =
|
|
35
|
+
this.db.query(`INSERT INTO object_vector_indexing (project_id, id, object_type_id, primary_id, profile, batch_id, request, status, available_at, dispatch_at)
|
|
36
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)
|
|
37
|
+
ON CONFLICT (project_id, object_type_id, primary_id, profile) DO UPDATE SET id = excluded.id, batch_id = excluded.batch_id, request = excluded.request, status = 'pending', available_at = excluded.available_at, dispatch_at = excluded.dispatch_at, values_json = NULL, error = NULL`)
|
|
38
|
+
for (const { batchId, ...request } of input.requests)
|
|
39
|
+
insert.run(
|
|
40
|
+
input.projectId,
|
|
41
|
+
request.id,
|
|
42
|
+
request.ref.objectTypeId,
|
|
43
|
+
request.ref.primaryId,
|
|
44
|
+
request.profile,
|
|
45
|
+
batchId ?? null,
|
|
46
|
+
JSON.stringify(request),
|
|
47
|
+
input.availableAt,
|
|
48
|
+
input.availableAt
|
|
49
|
+
)
|
|
50
|
+
}
|
|
51
|
+
async complete(input: Parameters<OntologyVectorIndexingStorage["complete"]>[0]) {
|
|
52
|
+
this.assertSession(input.session, input.projectId)
|
|
53
|
+
const remove = this.db.query(
|
|
54
|
+
`DELETE FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ? AND primary_id = ? AND profile = ? AND json_extract(request, '$.configuration') = ? AND json_extract(request, '$.sourceFingerprint') = ?`
|
|
55
|
+
)
|
|
56
|
+
for (const entry of input.entries) {
|
|
57
|
+
remove.run(
|
|
58
|
+
input.projectId,
|
|
59
|
+
entry.ref.objectTypeId,
|
|
60
|
+
entry.ref.primaryId,
|
|
61
|
+
entry.profile,
|
|
62
|
+
entry.configuration,
|
|
63
|
+
entry.sourceFingerprint
|
|
64
|
+
)
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async dispatched(input: Parameters<OntologyVectorIndexingStorage["dispatched"]>[0]) {
|
|
69
|
+
await this.run(() => {
|
|
70
|
+
const update = this.db.query(
|
|
71
|
+
`UPDATE object_vector_indexing SET dispatch_at = ? WHERE project_id = ? AND id = ?`
|
|
72
|
+
)
|
|
73
|
+
for (const id of input.ids) update.run(input.nextDispatchAt, input.projectId, id)
|
|
74
|
+
})
|
|
75
|
+
}
|
|
76
|
+
async get(input: Parameters<OntologyVectorIndexingStorage["get"]>[0]) {
|
|
77
|
+
return this.run(() => {
|
|
78
|
+
const row = this.db
|
|
79
|
+
.query<WorkRow, [string, string]>(
|
|
80
|
+
`SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing WHERE project_id = ? AND id = ?`
|
|
81
|
+
)
|
|
82
|
+
.get(input.projectId, input.id)
|
|
83
|
+
return row ? decode(row) : null
|
|
84
|
+
})
|
|
85
|
+
}
|
|
86
|
+
async getBatch(input: Parameters<OntologyVectorIndexingStorage["getBatch"]>[0]) {
|
|
87
|
+
return this.run(() =>
|
|
88
|
+
this.db
|
|
89
|
+
.query<WorkRow, [string, string]>(`
|
|
90
|
+
SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing
|
|
91
|
+
WHERE project_id = ? AND batch_id = ? ORDER BY id`)
|
|
92
|
+
.all(input.projectId, input.batchId)
|
|
93
|
+
.map(decode)
|
|
94
|
+
)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async getBatchMember(input: Parameters<OntologyVectorIndexingStorage["getBatchMember"]>[0]) {
|
|
98
|
+
return this.run(() => {
|
|
99
|
+
const row = this.db
|
|
100
|
+
.query<WorkRow, [string, string, string, string, string]>(`
|
|
101
|
+
SELECT batch_id, request, status, available_at, values_json, error
|
|
102
|
+
FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ?
|
|
103
|
+
AND primary_id = ? AND profile = ? AND batch_id = ?`)
|
|
104
|
+
.get(
|
|
105
|
+
input.projectId,
|
|
106
|
+
input.ref.objectTypeId,
|
|
107
|
+
input.ref.primaryId,
|
|
108
|
+
input.profile,
|
|
109
|
+
input.batchId
|
|
110
|
+
)
|
|
111
|
+
return row ? decode(row) : null
|
|
112
|
+
})
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
async updateBatch(input: Parameters<OntologyVectorIndexingStorage["updateBatch"]>[0]) {
|
|
116
|
+
return this.run(() =>
|
|
117
|
+
this.db.transaction(() => {
|
|
118
|
+
const status = this.db.query<{ status: string }, [string, string]>(
|
|
119
|
+
"SELECT status FROM object_vector_indexing WHERE project_id = ? AND id = ?"
|
|
120
|
+
)
|
|
121
|
+
if (
|
|
122
|
+
input.requireAll &&
|
|
123
|
+
input.updates.some(
|
|
124
|
+
(update) => status.get(input.projectId, update.id)?.status !== update.expectedStatus
|
|
125
|
+
)
|
|
126
|
+
)
|
|
127
|
+
return false
|
|
128
|
+
const updateRow = this.db.query(`UPDATE object_vector_indexing
|
|
129
|
+
SET status = ?, available_at = ?, values_json = COALESCE(?, values_json), error = ?
|
|
130
|
+
WHERE project_id = ? AND id = ? AND status = ?`)
|
|
131
|
+
for (const update of input.updates)
|
|
132
|
+
updateRow.run(
|
|
133
|
+
update.status,
|
|
134
|
+
update.availableAt,
|
|
135
|
+
update.values ? JSON.stringify(update.values) : null,
|
|
136
|
+
update.error ? JSON.stringify(update.error) : null,
|
|
137
|
+
input.projectId,
|
|
138
|
+
update.id,
|
|
139
|
+
update.expectedStatus
|
|
140
|
+
)
|
|
141
|
+
return true
|
|
142
|
+
})()
|
|
143
|
+
)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async listDue(input: Parameters<OntologyVectorIndexingStorage["listDue"]>[0]) {
|
|
147
|
+
return this.run(() =>
|
|
148
|
+
this.db
|
|
149
|
+
.query<WorkRow, [string, string, number]>(
|
|
150
|
+
`SELECT batch_id, request, status, available_at, NULL AS values_json, error FROM object_vector_indexing WHERE project_id = ? AND status <> 'failed' AND dispatch_at <= ? ORDER BY dispatch_at, id LIMIT ?`
|
|
151
|
+
)
|
|
152
|
+
.all(input.projectId, input.now, input.limit)
|
|
153
|
+
.map(decode)
|
|
154
|
+
)
|
|
155
|
+
}
|
|
156
|
+
async update(input: Parameters<OntologyVectorIndexingStorage["update"]>[0]) {
|
|
157
|
+
return this.run(
|
|
158
|
+
() =>
|
|
159
|
+
this.db
|
|
160
|
+
.query(
|
|
161
|
+
`UPDATE object_vector_indexing SET status = ?, available_at = ?, values_json = COALESCE(?, values_json), error = ? WHERE project_id = ? AND id = ? AND status = ?`
|
|
162
|
+
)
|
|
163
|
+
.run(
|
|
164
|
+
input.status,
|
|
165
|
+
input.availableAt,
|
|
166
|
+
input.values ? JSON.stringify(input.values) : null,
|
|
167
|
+
input.error ? JSON.stringify(input.error) : null,
|
|
168
|
+
input.projectId,
|
|
169
|
+
input.id,
|
|
170
|
+
input.expectedStatus
|
|
171
|
+
).changes === 1
|
|
172
|
+
)
|
|
173
|
+
}
|
|
174
|
+
async remove(input: Parameters<OntologyVectorIndexingStorage["remove"]>[0]) {
|
|
175
|
+
await this.run(() => {
|
|
176
|
+
this.db
|
|
177
|
+
.query(`DELETE FROM object_vector_indexing WHERE project_id = ? AND id = ?`)
|
|
178
|
+
.run(input.projectId, input.id)
|
|
179
|
+
})
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
function decode(row: WorkRow): VectorIndexingWork {
|
|
183
|
+
const request = JSON.parse(row.request) as VectorIndexingRequest
|
|
184
|
+
return {
|
|
185
|
+
...request,
|
|
186
|
+
...(row.batch_id ? { batchId: row.batch_id } : {}),
|
|
187
|
+
status: row.status,
|
|
188
|
+
availableAt: row.available_at,
|
|
189
|
+
...(row.values_json ? { values: JSON.parse(row.values_json) as number[] } : {}),
|
|
190
|
+
...(row.error
|
|
191
|
+
? {
|
|
192
|
+
error: parseSixbFailure(row.error, VECTOR_INDEXING_FAILURE_CODES),
|
|
193
|
+
}
|
|
194
|
+
: {}),
|
|
195
|
+
}
|
|
196
|
+
}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite"
|
|
2
|
+
import { MaterializationConflictError } from "@sixb/core/internal/materialization"
|
|
3
|
+
import type {
|
|
4
|
+
MaterializationSession,
|
|
5
|
+
ObjectVectorState,
|
|
6
|
+
OntologyVectorStorage,
|
|
7
|
+
} from "@sixb/core/storage"
|
|
8
|
+
import { encodeSqliteVector } from "../vector-encoding"
|
|
9
|
+
import type { SqliteRootOperation } from "./shared"
|
|
10
|
+
|
|
11
|
+
interface VectorMetadataRow {
|
|
12
|
+
readonly profile: string
|
|
13
|
+
readonly configuration: string
|
|
14
|
+
readonly source: string
|
|
15
|
+
readonly source_fingerprint: string
|
|
16
|
+
readonly last_commit_id: string
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export class SqliteOntologyVectorStorage implements OntologyVectorStorage {
|
|
20
|
+
constructor(
|
|
21
|
+
private readonly db: Database,
|
|
22
|
+
private readonly runRootOperation: SqliteRootOperation,
|
|
23
|
+
private readonly assertSession: (
|
|
24
|
+
session: MaterializationSession,
|
|
25
|
+
projectId: string,
|
|
26
|
+
commitId?: string
|
|
27
|
+
) => void
|
|
28
|
+
) {}
|
|
29
|
+
|
|
30
|
+
async list(
|
|
31
|
+
input: Parameters<OntologyVectorStorage["list"]>[0]
|
|
32
|
+
): Promise<readonly ObjectVectorState[]> {
|
|
33
|
+
return this.runRootOperation(() => {
|
|
34
|
+
const rows = this.db
|
|
35
|
+
.query<VectorMetadataRow, [string, string, string]>(`
|
|
36
|
+
SELECT profile, configuration, source, source_fingerprint, last_commit_id
|
|
37
|
+
FROM object_vectors
|
|
38
|
+
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
39
|
+
ORDER BY profile
|
|
40
|
+
`)
|
|
41
|
+
.all(input.projectId, input.ref.objectTypeId, input.ref.primaryId)
|
|
42
|
+
return rows.map((row) => ({
|
|
43
|
+
ref: { ...input.ref },
|
|
44
|
+
profile: row.profile,
|
|
45
|
+
configuration: row.configuration,
|
|
46
|
+
source: JSON.parse(row.source) as string[],
|
|
47
|
+
sourceFingerprint: row.source_fingerprint,
|
|
48
|
+
lastCommitId: row.last_commit_id,
|
|
49
|
+
}))
|
|
50
|
+
})
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async listBatch(
|
|
54
|
+
input: Parameters<OntologyVectorStorage["listBatch"]>[0]
|
|
55
|
+
): Promise<readonly ObjectVectorState[]> {
|
|
56
|
+
if (!input.refs.length) return []
|
|
57
|
+
return this.runRootOperation(() => {
|
|
58
|
+
// Keep the bounded request set first so SQLite probes the composite object index.
|
|
59
|
+
const rows = this.db
|
|
60
|
+
.query<
|
|
61
|
+
VectorMetadataRow & { object_type_id: string; primary_id: string },
|
|
62
|
+
[string, string]
|
|
63
|
+
>(`
|
|
64
|
+
WITH requested AS (
|
|
65
|
+
SELECT DISTINCT json_extract(value, '$.objectTypeId') AS object_type_id,
|
|
66
|
+
json_extract(value, '$.primaryId') AS primary_id FROM json_each(?)
|
|
67
|
+
)
|
|
68
|
+
SELECT stored.object_type_id, stored.primary_id, stored.profile, stored.configuration,
|
|
69
|
+
stored.source, stored.source_fingerprint, stored.last_commit_id
|
|
70
|
+
FROM requested CROSS JOIN object_vectors AS stored
|
|
71
|
+
WHERE stored.project_id = ? AND stored.object_type_id = requested.object_type_id
|
|
72
|
+
AND stored.primary_id = requested.primary_id
|
|
73
|
+
ORDER BY stored.object_type_id, stored.primary_id, stored.profile
|
|
74
|
+
`)
|
|
75
|
+
.all(JSON.stringify(input.refs), input.projectId)
|
|
76
|
+
return rows.map((row) => ({
|
|
77
|
+
ref: { objectTypeId: row.object_type_id, primaryId: row.primary_id },
|
|
78
|
+
profile: row.profile,
|
|
79
|
+
configuration: row.configuration,
|
|
80
|
+
source: JSON.parse(row.source) as string[],
|
|
81
|
+
sourceFingerprint: row.source_fingerprint,
|
|
82
|
+
lastCommitId: row.last_commit_id,
|
|
83
|
+
}))
|
|
84
|
+
})
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async removeBatch(input: Parameters<OntologyVectorStorage["removeBatch"]>[0]): Promise<void> {
|
|
88
|
+
this.assertSession(input.session, input.projectId)
|
|
89
|
+
if (!input.entries.length) return
|
|
90
|
+
const { changes } = this.db
|
|
91
|
+
.query(`
|
|
92
|
+
DELETE FROM object_vectors WHERE rowid IN (
|
|
93
|
+
SELECT stored.rowid FROM json_each(?) AS requested CROSS JOIN object_vectors AS stored
|
|
94
|
+
WHERE stored.project_id = ? AND stored.object_type_id = json_extract(requested.value, '$.ref.objectTypeId')
|
|
95
|
+
AND stored.primary_id = json_extract(requested.value, '$.ref.primaryId')
|
|
96
|
+
AND stored.profile = json_extract(requested.value, '$.profile')
|
|
97
|
+
AND stored.last_commit_id = json_extract(requested.value, '$.expectedCommitId')
|
|
98
|
+
)
|
|
99
|
+
`)
|
|
100
|
+
.run(JSON.stringify(input.entries), input.projectId)
|
|
101
|
+
if (changes !== input.entries.length) throw vectorConflict()
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async write(input: Parameters<OntologyVectorStorage["write"]>[0]): Promise<void> {
|
|
105
|
+
const { value } = input
|
|
106
|
+
this.assertSession(input.session, input.projectId, value.lastCommitId)
|
|
107
|
+
// The Materializer validates and normalizes float32 values before this boundary.
|
|
108
|
+
const embedding = encodeSqliteVector(value.values)
|
|
109
|
+
const changes =
|
|
110
|
+
input.expectedCommitId === null
|
|
111
|
+
? this.db
|
|
112
|
+
.query(`
|
|
113
|
+
INSERT INTO object_vectors (
|
|
114
|
+
project_id, object_type_id, primary_id, profile,
|
|
115
|
+
configuration, source, source_fingerprint, embedding, last_commit_id
|
|
116
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
117
|
+
ON CONFLICT (project_id, object_type_id, primary_id, profile) DO NOTHING
|
|
118
|
+
`)
|
|
119
|
+
.run(
|
|
120
|
+
input.projectId,
|
|
121
|
+
value.ref.objectTypeId,
|
|
122
|
+
value.ref.primaryId,
|
|
123
|
+
value.profile,
|
|
124
|
+
value.configuration,
|
|
125
|
+
JSON.stringify(value.source),
|
|
126
|
+
value.sourceFingerprint,
|
|
127
|
+
embedding,
|
|
128
|
+
value.lastCommitId
|
|
129
|
+
).changes
|
|
130
|
+
: this.db
|
|
131
|
+
.query(`
|
|
132
|
+
UPDATE object_vectors
|
|
133
|
+
SET configuration = ?, source = ?, source_fingerprint = ?, embedding = ?, last_commit_id = ?
|
|
134
|
+
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
135
|
+
AND profile = ? AND last_commit_id = ?
|
|
136
|
+
`)
|
|
137
|
+
.run(
|
|
138
|
+
value.configuration,
|
|
139
|
+
JSON.stringify(value.source),
|
|
140
|
+
value.sourceFingerprint,
|
|
141
|
+
embedding,
|
|
142
|
+
value.lastCommitId,
|
|
143
|
+
input.projectId,
|
|
144
|
+
value.ref.objectTypeId,
|
|
145
|
+
value.ref.primaryId,
|
|
146
|
+
value.profile,
|
|
147
|
+
input.expectedCommitId
|
|
148
|
+
).changes
|
|
149
|
+
if (changes !== 1) throw vectorConflict()
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async writeBatch(input: Parameters<OntologyVectorStorage["writeBatch"]>[0]): Promise<void> {
|
|
153
|
+
for (const entry of input.entries) {
|
|
154
|
+
await this.write({ session: input.session, projectId: input.projectId, ...entry })
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
async remove(input: Parameters<OntologyVectorStorage["remove"]>[0]): Promise<void> {
|
|
159
|
+
this.assertSession(input.session, input.projectId)
|
|
160
|
+
const { changes } = this.db
|
|
161
|
+
.query(`
|
|
162
|
+
DELETE FROM object_vectors
|
|
163
|
+
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
164
|
+
AND profile = ? AND last_commit_id = ?
|
|
165
|
+
`)
|
|
166
|
+
.run(
|
|
167
|
+
input.projectId,
|
|
168
|
+
input.ref.objectTypeId,
|
|
169
|
+
input.ref.primaryId,
|
|
170
|
+
input.profile,
|
|
171
|
+
input.expectedCommitId
|
|
172
|
+
)
|
|
173
|
+
if (changes !== 1) throw vectorConflict()
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function vectorConflict(): MaterializationConflictError {
|
|
178
|
+
return new MaterializationConflictError(
|
|
179
|
+
"effective-state",
|
|
180
|
+
"Vector changed since it was prepared."
|
|
181
|
+
)
|
|
182
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Shared little-endian float32 encoding for stored vectors and sqlite-vec query bindings. */
|
|
2
|
+
export function encodeSqliteVector(values: readonly number[]): Uint8Array {
|
|
3
|
+
const bytes = new Uint8Array(values.length * 4)
|
|
4
|
+
const view = new DataView(bytes.buffer)
|
|
5
|
+
for (let index = 0; index < values.length; index++) {
|
|
6
|
+
view.setFloat32(index * 4, values[index]!, true)
|
|
7
|
+
}
|
|
8
|
+
return bytes
|
|
9
|
+
}
|