@sixb/sqlite 0.1.10 → 0.1.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +32 -0
  2. package/dist/agents/rows.d.ts +2 -0
  3. package/dist/agents/rows.d.ts.map +1 -1
  4. package/dist/agents/threads.d.ts +2 -1
  5. package/dist/agents/threads.d.ts.map +1 -1
  6. package/dist/index.js +717 -74
  7. package/dist/index.js.map +20 -16
  8. package/dist/migrations.d.ts.map +1 -1
  9. package/dist/objects/query-compiler.d.ts +9 -2
  10. package/dist/objects/query-compiler.d.ts.map +1 -1
  11. package/dist/objects/read-scope.d.ts +3 -0
  12. package/dist/objects/read-scope.d.ts.map +1 -1
  13. package/dist/objects/reader.d.ts +1 -0
  14. package/dist/objects/reader.d.ts.map +1 -1
  15. package/dist/objects/rows.d.ts +1 -0
  16. package/dist/objects/rows.d.ts.map +1 -1
  17. package/dist/objects/storage.d.ts.map +1 -1
  18. package/dist/objects/vector-extension.d.ts +4 -0
  19. package/dist/objects/vector-extension.d.ts.map +1 -0
  20. package/dist/ontology-storage/index.d.ts +4 -0
  21. package/dist/ontology-storage/index.d.ts.map +1 -1
  22. package/dist/ontology-storage/materializations.d.ts +1 -0
  23. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  24. package/dist/ontology-storage/outbox.d.ts.map +1 -1
  25. package/dist/ontology-storage/vector-indexing.d.ts +20 -0
  26. package/dist/ontology-storage/vector-indexing.d.ts.map +1 -0
  27. package/dist/ontology-storage/vectors.d.ts +16 -0
  28. package/dist/ontology-storage/vectors.d.ts.map +1 -0
  29. package/dist/vector-encoding.d.ts +3 -0
  30. package/dist/vector-encoding.d.ts.map +1 -0
  31. package/package.json +8 -4
  32. package/src/agents/rows.ts +12 -0
  33. package/src/agents/threads.ts +41 -2
  34. package/src/ai-cost-storage.ts +4 -0
  35. package/src/connector-connection-storage.ts +10 -0
  36. package/src/migrations/040-outbox-publication-order.sql +8 -0
  37. package/src/migrations/041-agent-thread-workspaces.sql +2 -0
  38. package/src/migrations/042-vector-profiles.sql +21 -0
  39. package/src/migrations/043-vector-indexing.sql +197 -0
  40. package/src/migrations/044-vector-batching.sql +3 -0
  41. package/src/migrations/045-agent-thread-sandbox-state.sql +2 -0
  42. package/src/migrations.ts +18 -0
  43. package/src/objects/query-compiler.ts +105 -7
  44. package/src/objects/read-scope.ts +34 -31
  45. package/src/objects/reader.ts +25 -1
  46. package/src/objects/rows.ts +5 -1
  47. package/src/objects/storage.ts +12 -1
  48. package/src/objects/vector-extension.ts +26 -0
  49. package/src/ontology-storage/index.ts +16 -0
  50. package/src/ontology-storage/materializations.ts +13 -0
  51. package/src/ontology-storage/outbox.ts +33 -16
  52. package/src/ontology-storage/vector-indexing.ts +196 -0
  53. package/src/ontology-storage/vectors.ts +182 -0
  54. package/src/vector-encoding.ts +9 -0
@@ -1,4 +1,5 @@
1
1
  import type { Database } from "bun:sqlite"
2
+ import { hasVectorProfile } from "@sixb/core/internal/query"
2
3
  import type {
3
4
  CompiledSelectedObjectReadScope,
4
5
  CountObjectsInput,
@@ -25,6 +26,7 @@ import { installFreshSqliteSchema } from "../migrations"
25
26
  import {
26
27
  closeSqliteStoreConnection,
27
28
  openSqliteStoreConnection,
29
+ runDeferredReadTransaction,
28
30
  type SqliteStoreConnection,
29
31
  } from "../transactions"
30
32
  import { DEFAULT_OBJECT_QUERY_SOURCE } from "./query-compiler"
@@ -49,6 +51,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
49
51
  refs: true,
50
52
  filter: true,
51
53
  text: true,
54
+ vector: true,
52
55
  sort: true,
53
56
  limit: true,
54
57
  page: true,
@@ -101,7 +104,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
101
104
  "SQLite object query pushdown supports start/refs/filter/text/sort/limit/page/traverse/set/project/expand over JSON properties and object links.",
102
105
  "expand hydrates linked objects in-database (top-N per parent via row_number() + json_group_array); core resolves each expansion's cardinality before pushdown, and a mixed/unresolved one stays on the fallback.",
103
106
  "Ordered decimal predicates and sorting use the bounded core fallback because SQLite has no native exact decimal type; canonical decimal equality remains pushdown-safe.",
104
- "Relevance sorting, vector search, and unresolved start.includeSubtypes remain planner fallback or rejection cases.",
107
+ "Relevance sorting and unresolved start.includeSubtypes remain planner fallback or rejection cases.",
105
108
  ],
106
109
  }
107
110
 
@@ -139,18 +142,26 @@ export class SqliteObjectStorage implements ObjectStorage {
139
142
  }
140
143
 
141
144
  async queryObjects(params: QueryObjectsInput): Promise<QueryObjectsResult> {
145
+ if (hasVectorProfile(params.query))
146
+ return runDeferredReadTransaction(this.db, () => this.reader.queryObjects(params))
142
147
  return this.reader.queryObjects(params)
143
148
  }
144
149
 
145
150
  async countObjects(params: CountObjectsInput): Promise<CountObjectsResult> {
151
+ if (hasVectorProfile(params.query))
152
+ return runDeferredReadTransaction(this.db, () => this.reader.countObjects(params))
146
153
  return this.reader.countObjects(params)
147
154
  }
148
155
 
149
156
  async existsObjects(params: ExistsObjectsInput): Promise<ExistsObjectsResult> {
157
+ if (hasVectorProfile(params.query))
158
+ return runDeferredReadTransaction(this.db, () => this.reader.existsObjects(params))
150
159
  return this.reader.existsObjects(params)
151
160
  }
152
161
 
153
162
  async facetObjects(params: FacetObjectsInput): Promise<FacetObjectsResult> {
163
+ if (hasVectorProfile(params.query))
164
+ return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params))
154
165
  return this.reader.facetObjects(params)
155
166
  }
156
167
 
@@ -0,0 +1,26 @@
1
+ import type { Database } from "bun:sqlite"
2
+ import { ObjectQueryExecutionError } from "@sixb/core"
3
+ import { load } from "sqlite-vec"
4
+
5
+ const loaded = new WeakSet<Database>()
6
+
7
+ /** Load only for vector reads, leaving ordinary SQLite use independent of extension support. */
8
+ export function ensureSqliteVectorSearch(db: Database): void {
9
+ if (loaded.has(db)) return
10
+ const options = db.query<{ compile_options: string }, []>("PRAGMA compile_options").all()
11
+ if (options.some((option) => option.compile_options === "OMIT_LOAD_EXTENSION")) {
12
+ throw new ObjectQueryExecutionError(
13
+ "vector_extension_missing",
14
+ "SQLite vector search requires extension support. On macOS install SQLite (brew install sqlite), then call Database.setCustomSQLite with its library path before opening any database."
15
+ )
16
+ }
17
+ try {
18
+ load(db)
19
+ loaded.add(db)
20
+ } catch (cause) {
21
+ throw new Error(
22
+ "[SixbSqlite] Could not load sqlite-vec for vector search. Check that its native package supports your platform.",
23
+ { cause }
24
+ )
25
+ }
26
+ }
@@ -7,10 +7,15 @@ import { SqliteOntologyOutboxStorage } from "./outbox"
7
7
  import type { SqliteRootOperation } from "./shared"
8
8
  import { SqliteOntologySourceStorage } from "./sources"
9
9
 
10
+ import { SqliteVectorIndexingStorage } from "./vector-indexing"
11
+ import { SqliteOntologyVectorStorage } from "./vectors"
12
+
10
13
  export class SqliteOntologyStorage implements OntologyStorage {
11
14
  readonly commits: SqliteOntologyCommitStorage
12
15
  readonly sources: SqliteOntologySourceStorage
13
16
  readonly materializations: SqliteOntologyMaterializationStorage
17
+ readonly vectorIndexing: SqliteVectorIndexingStorage
18
+ readonly vectors: SqliteOntologyVectorStorage
14
19
  readonly outbox: SqliteOntologyOutboxStorage
15
20
 
16
21
  constructor(input: {
@@ -24,6 +29,17 @@ export class SqliteOntologyStorage implements OntologyStorage {
24
29
  input.db,
25
30
  input.transactionContext
26
31
  )
32
+ this.vectorIndexing = new SqliteVectorIndexingStorage(
33
+ input.db,
34
+ input.runRootOperation,
35
+ (session, projectId) => this.materializations.assertVectorSession(session, projectId)
36
+ )
37
+ this.vectors = new SqliteOntologyVectorStorage(
38
+ input.db,
39
+ input.runRootOperation,
40
+ (session, projectId, commitId) =>
41
+ this.materializations.assertVectorSession(session, projectId, commitId)
42
+ )
27
43
  this.outbox = new SqliteOntologyOutboxStorage(input.db, input.runRootOperation)
28
44
  }
29
45
 
@@ -9,6 +9,7 @@ import {
9
9
  linkRefSortKey,
10
10
  linkScopeSortKey,
11
11
  MaterializationConflictError,
12
+ MaterializationValidationError,
12
13
  objectRefKey,
13
14
  objectRefSortKey,
14
15
  telemetryPointKey,
@@ -79,6 +80,18 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
79
80
  this.writer = new SqliteMaterializationWriter(db)
80
81
  }
81
82
 
83
+ assertVectorSession(session: MaterializationSession, projectId: string, commitId?: string): void {
84
+ const active = this.sessions.require(session)
85
+ if (
86
+ active.header.commit.projectId !== projectId ||
87
+ (commitId !== undefined && active.header.commit.id !== commitId)
88
+ ) {
89
+ throw new MaterializationValidationError(
90
+ "Vector mutation does not belong to this materialization session."
91
+ )
92
+ }
93
+ }
94
+
82
95
  async begin(input: MaterializationPlanHeader): Promise<MaterializationSession> {
83
96
  assertMaterializationHeader(input)
84
97
  const session = this.sessions.create(input)
@@ -43,19 +43,37 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
43
43
  )
44
44
  }
45
45
 
46
+ // Probe at most one page plus a sentinel through the eligibility index. Sparse retries
47
+ // avoid scanning delayed rows; overflowing backlogs use publication order without a sort.
48
+ const eligible = this.db
49
+ .query(
50
+ `SELECT CAST(rowid AS TEXT) AS rowId
51
+ FROM ontology_outbox INDEXED BY idx_ontology_outbox_claim
52
+ WHERE project_id = ? AND published_at IS NULL AND available_at <= ?
53
+ AND (lease_expires_at IS NULL OR lease_expires_at <= ?)
54
+ LIMIT ?`
55
+ )
56
+ .all(input.projectId, input.now, input.now, input.limit + 1) as { rowId: string }[]
57
+ if (eligible.length === 0) return []
58
+ const selected =
59
+ eligible.length <= input.limit
60
+ ? eligible
61
+ : (this.db
62
+ .query(
63
+ `SELECT CAST(rowid AS TEXT) AS rowId
64
+ FROM ontology_outbox INDEXED BY idx_ontology_outbox_publication_order
65
+ WHERE project_id = ? AND published_at IS NULL AND available_at <= ?
66
+ AND (lease_expires_at IS NULL OR lease_expires_at <= ?)
67
+ ORDER BY created_at, commit_id, commit_ordinal
68
+ LIMIT ?`
69
+ )
70
+ .all(input.projectId, input.now, input.now, input.limit) as { rowId: string }[])
46
71
  const rows = this.db
47
72
  .query(
48
73
  `
49
74
  UPDATE ontology_outbox
50
75
  SET attempts = attempts + 1, lease_id = ?, lease_expires_at = ?
51
- WHERE rowid IN (
52
- SELECT rowid
53
- FROM ontology_outbox
54
- WHERE project_id = ? AND published_at IS NULL AND available_at <= ?
55
- AND (lease_expires_at IS NULL OR lease_expires_at <= ?)
56
- ORDER BY created_at, commit_id, commit_ordinal
57
- LIMIT ?
58
- )
76
+ WHERE rowid IN (SELECT CAST(value AS INTEGER) FROM json_each(?))
59
77
  RETURNING envelope, available_at, attempts, lease_id, lease_expires_at,
60
78
  published_at, last_failure, created_at
61
79
  `
@@ -63,10 +81,7 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
63
81
  .all(
64
82
  input.leaseId,
65
83
  input.leaseExpiresAt,
66
- input.projectId,
67
- input.now,
68
- input.now,
69
- input.limit
84
+ JSON.stringify(selected.map((row) => row.rowId))
70
85
  ) as SqliteOntologyOutboxRow[]
71
86
  return rows
72
87
  .map((row) => outboxRecord(row))
@@ -98,8 +113,9 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
98
113
  WITH requested(id) AS MATERIALIZED (
99
114
  SELECT CAST(value AS TEXT) FROM json_each(?)
100
115
  ), leased(id) AS MATERIALIZED (
101
- SELECT outbox.id FROM ontology_outbox AS outbox
102
- JOIN requested USING (id)
116
+ -- Drive validation from the bounded ID list even before ANALYZE.
117
+ SELECT outbox.id FROM requested
118
+ CROSS JOIN ontology_outbox AS outbox USING (id)
103
119
  WHERE outbox.project_id = ? AND outbox.lease_id = ?
104
120
  AND outbox.lease_expires_at IS NOT NULL
105
121
  ), eligible(id) AS (
@@ -139,8 +155,9 @@ export class SqliteOntologyOutboxStorage implements OntologyOutboxStorage {
139
155
  WITH requested(id) AS MATERIALIZED (
140
156
  SELECT CAST(value AS TEXT) FROM json_each(?)
141
157
  ), leased(id) AS MATERIALIZED (
142
- SELECT outbox.id FROM ontology_outbox AS outbox
143
- JOIN requested USING (id)
158
+ -- Drive validation from the bounded ID list even before ANALYZE.
159
+ SELECT outbox.id FROM requested
160
+ CROSS JOIN ontology_outbox AS outbox USING (id)
144
161
  WHERE outbox.project_id = ? AND outbox.lease_id = ?
145
162
  AND outbox.lease_expires_at IS NOT NULL
146
163
  ), eligible(id) AS (
@@ -0,0 +1,196 @@
1
+ import type { Database } from "bun:sqlite"
2
+ import { parseSixbFailure } from "@sixb/core/internal/errors"
3
+ import {
4
+ type MaterializationSession,
5
+ type OntologyVectorIndexingStorage,
6
+ VECTOR_INDEXING_FAILURE_CODES,
7
+ type VectorIndexingRequest,
8
+ type VectorIndexingWork,
9
+ } from "@sixb/core/storage"
10
+ import type { SqliteRootOperation } from "./shared"
11
+
12
+ interface WorkRow {
13
+ batch_id: string | null
14
+ request: string
15
+ status: VectorIndexingWork["status"]
16
+ available_at: string
17
+ values_json: string | null
18
+ error: string | null
19
+ }
20
+ export class SqliteVectorIndexingStorage implements OntologyVectorIndexingStorage {
21
+ constructor(
22
+ private readonly db: Database,
23
+ private readonly run: SqliteRootOperation,
24
+ private readonly assertSession: (session: MaterializationSession, projectId: string) => void
25
+ ) {}
26
+ async schedule(input: Parameters<OntologyVectorIndexingStorage["schedule"]>[0]) {
27
+ this.assertSession(input.session, input.projectId)
28
+ for (const ref of input.deleted)
29
+ this.db
30
+ .query(
31
+ `DELETE FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ? AND primary_id = ?`
32
+ )
33
+ .run(input.projectId, ref.objectTypeId, ref.primaryId)
34
+ const insert =
35
+ this.db.query(`INSERT INTO object_vector_indexing (project_id, id, object_type_id, primary_id, profile, batch_id, request, status, available_at, dispatch_at)
36
+ VALUES (?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)
37
+ ON CONFLICT (project_id, object_type_id, primary_id, profile) DO UPDATE SET id = excluded.id, batch_id = excluded.batch_id, request = excluded.request, status = 'pending', available_at = excluded.available_at, dispatch_at = excluded.dispatch_at, values_json = NULL, error = NULL`)
38
+ for (const { batchId, ...request } of input.requests)
39
+ insert.run(
40
+ input.projectId,
41
+ request.id,
42
+ request.ref.objectTypeId,
43
+ request.ref.primaryId,
44
+ request.profile,
45
+ batchId ?? null,
46
+ JSON.stringify(request),
47
+ input.availableAt,
48
+ input.availableAt
49
+ )
50
+ }
51
+ async complete(input: Parameters<OntologyVectorIndexingStorage["complete"]>[0]) {
52
+ this.assertSession(input.session, input.projectId)
53
+ const remove = this.db.query(
54
+ `DELETE FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ? AND primary_id = ? AND profile = ? AND json_extract(request, '$.configuration') = ? AND json_extract(request, '$.sourceFingerprint') = ?`
55
+ )
56
+ for (const entry of input.entries) {
57
+ remove.run(
58
+ input.projectId,
59
+ entry.ref.objectTypeId,
60
+ entry.ref.primaryId,
61
+ entry.profile,
62
+ entry.configuration,
63
+ entry.sourceFingerprint
64
+ )
65
+ }
66
+ }
67
+
68
+ async dispatched(input: Parameters<OntologyVectorIndexingStorage["dispatched"]>[0]) {
69
+ await this.run(() => {
70
+ const update = this.db.query(
71
+ `UPDATE object_vector_indexing SET dispatch_at = ? WHERE project_id = ? AND id = ?`
72
+ )
73
+ for (const id of input.ids) update.run(input.nextDispatchAt, input.projectId, id)
74
+ })
75
+ }
76
+ async get(input: Parameters<OntologyVectorIndexingStorage["get"]>[0]) {
77
+ return this.run(() => {
78
+ const row = this.db
79
+ .query<WorkRow, [string, string]>(
80
+ `SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing WHERE project_id = ? AND id = ?`
81
+ )
82
+ .get(input.projectId, input.id)
83
+ return row ? decode(row) : null
84
+ })
85
+ }
86
+ async getBatch(input: Parameters<OntologyVectorIndexingStorage["getBatch"]>[0]) {
87
+ return this.run(() =>
88
+ this.db
89
+ .query<WorkRow, [string, string]>(`
90
+ SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing
91
+ WHERE project_id = ? AND batch_id = ? ORDER BY id`)
92
+ .all(input.projectId, input.batchId)
93
+ .map(decode)
94
+ )
95
+ }
96
+
97
+ async getBatchMember(input: Parameters<OntologyVectorIndexingStorage["getBatchMember"]>[0]) {
98
+ return this.run(() => {
99
+ const row = this.db
100
+ .query<WorkRow, [string, string, string, string, string]>(`
101
+ SELECT batch_id, request, status, available_at, values_json, error
102
+ FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ?
103
+ AND primary_id = ? AND profile = ? AND batch_id = ?`)
104
+ .get(
105
+ input.projectId,
106
+ input.ref.objectTypeId,
107
+ input.ref.primaryId,
108
+ input.profile,
109
+ input.batchId
110
+ )
111
+ return row ? decode(row) : null
112
+ })
113
+ }
114
+
115
+ async updateBatch(input: Parameters<OntologyVectorIndexingStorage["updateBatch"]>[0]) {
116
+ return this.run(() =>
117
+ this.db.transaction(() => {
118
+ const status = this.db.query<{ status: string }, [string, string]>(
119
+ "SELECT status FROM object_vector_indexing WHERE project_id = ? AND id = ?"
120
+ )
121
+ if (
122
+ input.requireAll &&
123
+ input.updates.some(
124
+ (update) => status.get(input.projectId, update.id)?.status !== update.expectedStatus
125
+ )
126
+ )
127
+ return false
128
+ const updateRow = this.db.query(`UPDATE object_vector_indexing
129
+ SET status = ?, available_at = ?, values_json = COALESCE(?, values_json), error = ?
130
+ WHERE project_id = ? AND id = ? AND status = ?`)
131
+ for (const update of input.updates)
132
+ updateRow.run(
133
+ update.status,
134
+ update.availableAt,
135
+ update.values ? JSON.stringify(update.values) : null,
136
+ update.error ? JSON.stringify(update.error) : null,
137
+ input.projectId,
138
+ update.id,
139
+ update.expectedStatus
140
+ )
141
+ return true
142
+ })()
143
+ )
144
+ }
145
+
146
+ async listDue(input: Parameters<OntologyVectorIndexingStorage["listDue"]>[0]) {
147
+ return this.run(() =>
148
+ this.db
149
+ .query<WorkRow, [string, string, number]>(
150
+ `SELECT batch_id, request, status, available_at, NULL AS values_json, error FROM object_vector_indexing WHERE project_id = ? AND status <> 'failed' AND dispatch_at <= ? ORDER BY dispatch_at, id LIMIT ?`
151
+ )
152
+ .all(input.projectId, input.now, input.limit)
153
+ .map(decode)
154
+ )
155
+ }
156
+ async update(input: Parameters<OntologyVectorIndexingStorage["update"]>[0]) {
157
+ return this.run(
158
+ () =>
159
+ this.db
160
+ .query(
161
+ `UPDATE object_vector_indexing SET status = ?, available_at = ?, values_json = COALESCE(?, values_json), error = ? WHERE project_id = ? AND id = ? AND status = ?`
162
+ )
163
+ .run(
164
+ input.status,
165
+ input.availableAt,
166
+ input.values ? JSON.stringify(input.values) : null,
167
+ input.error ? JSON.stringify(input.error) : null,
168
+ input.projectId,
169
+ input.id,
170
+ input.expectedStatus
171
+ ).changes === 1
172
+ )
173
+ }
174
+ async remove(input: Parameters<OntologyVectorIndexingStorage["remove"]>[0]) {
175
+ await this.run(() => {
176
+ this.db
177
+ .query(`DELETE FROM object_vector_indexing WHERE project_id = ? AND id = ?`)
178
+ .run(input.projectId, input.id)
179
+ })
180
+ }
181
+ }
182
+ function decode(row: WorkRow): VectorIndexingWork {
183
+ const request = JSON.parse(row.request) as VectorIndexingRequest
184
+ return {
185
+ ...request,
186
+ ...(row.batch_id ? { batchId: row.batch_id } : {}),
187
+ status: row.status,
188
+ availableAt: row.available_at,
189
+ ...(row.values_json ? { values: JSON.parse(row.values_json) as number[] } : {}),
190
+ ...(row.error
191
+ ? {
192
+ error: parseSixbFailure(row.error, VECTOR_INDEXING_FAILURE_CODES),
193
+ }
194
+ : {}),
195
+ }
196
+ }
@@ -0,0 +1,182 @@
1
+ import type { Database } from "bun:sqlite"
2
+ import { MaterializationConflictError } from "@sixb/core/internal/materialization"
3
+ import type {
4
+ MaterializationSession,
5
+ ObjectVectorState,
6
+ OntologyVectorStorage,
7
+ } from "@sixb/core/storage"
8
+ import { encodeSqliteVector } from "../vector-encoding"
9
+ import type { SqliteRootOperation } from "./shared"
10
+
11
+ interface VectorMetadataRow {
12
+ readonly profile: string
13
+ readonly configuration: string
14
+ readonly source: string
15
+ readonly source_fingerprint: string
16
+ readonly last_commit_id: string
17
+ }
18
+
19
+ export class SqliteOntologyVectorStorage implements OntologyVectorStorage {
20
+ constructor(
21
+ private readonly db: Database,
22
+ private readonly runRootOperation: SqliteRootOperation,
23
+ private readonly assertSession: (
24
+ session: MaterializationSession,
25
+ projectId: string,
26
+ commitId?: string
27
+ ) => void
28
+ ) {}
29
+
30
+ async list(
31
+ input: Parameters<OntologyVectorStorage["list"]>[0]
32
+ ): Promise<readonly ObjectVectorState[]> {
33
+ return this.runRootOperation(() => {
34
+ const rows = this.db
35
+ .query<VectorMetadataRow, [string, string, string]>(`
36
+ SELECT profile, configuration, source, source_fingerprint, last_commit_id
37
+ FROM object_vectors
38
+ WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
39
+ ORDER BY profile
40
+ `)
41
+ .all(input.projectId, input.ref.objectTypeId, input.ref.primaryId)
42
+ return rows.map((row) => ({
43
+ ref: { ...input.ref },
44
+ profile: row.profile,
45
+ configuration: row.configuration,
46
+ source: JSON.parse(row.source) as string[],
47
+ sourceFingerprint: row.source_fingerprint,
48
+ lastCommitId: row.last_commit_id,
49
+ }))
50
+ })
51
+ }
52
+
53
+ async listBatch(
54
+ input: Parameters<OntologyVectorStorage["listBatch"]>[0]
55
+ ): Promise<readonly ObjectVectorState[]> {
56
+ if (!input.refs.length) return []
57
+ return this.runRootOperation(() => {
58
+ // Keep the bounded request set first so SQLite probes the composite object index.
59
+ const rows = this.db
60
+ .query<
61
+ VectorMetadataRow & { object_type_id: string; primary_id: string },
62
+ [string, string]
63
+ >(`
64
+ WITH requested AS (
65
+ SELECT DISTINCT json_extract(value, '$.objectTypeId') AS object_type_id,
66
+ json_extract(value, '$.primaryId') AS primary_id FROM json_each(?)
67
+ )
68
+ SELECT stored.object_type_id, stored.primary_id, stored.profile, stored.configuration,
69
+ stored.source, stored.source_fingerprint, stored.last_commit_id
70
+ FROM requested CROSS JOIN object_vectors AS stored
71
+ WHERE stored.project_id = ? AND stored.object_type_id = requested.object_type_id
72
+ AND stored.primary_id = requested.primary_id
73
+ ORDER BY stored.object_type_id, stored.primary_id, stored.profile
74
+ `)
75
+ .all(JSON.stringify(input.refs), input.projectId)
76
+ return rows.map((row) => ({
77
+ ref: { objectTypeId: row.object_type_id, primaryId: row.primary_id },
78
+ profile: row.profile,
79
+ configuration: row.configuration,
80
+ source: JSON.parse(row.source) as string[],
81
+ sourceFingerprint: row.source_fingerprint,
82
+ lastCommitId: row.last_commit_id,
83
+ }))
84
+ })
85
+ }
86
+
87
+ async removeBatch(input: Parameters<OntologyVectorStorage["removeBatch"]>[0]): Promise<void> {
88
+ this.assertSession(input.session, input.projectId)
89
+ if (!input.entries.length) return
90
+ const { changes } = this.db
91
+ .query(`
92
+ DELETE FROM object_vectors WHERE rowid IN (
93
+ SELECT stored.rowid FROM json_each(?) AS requested CROSS JOIN object_vectors AS stored
94
+ WHERE stored.project_id = ? AND stored.object_type_id = json_extract(requested.value, '$.ref.objectTypeId')
95
+ AND stored.primary_id = json_extract(requested.value, '$.ref.primaryId')
96
+ AND stored.profile = json_extract(requested.value, '$.profile')
97
+ AND stored.last_commit_id = json_extract(requested.value, '$.expectedCommitId')
98
+ )
99
+ `)
100
+ .run(JSON.stringify(input.entries), input.projectId)
101
+ if (changes !== input.entries.length) throw vectorConflict()
102
+ }
103
+
104
+ async write(input: Parameters<OntologyVectorStorage["write"]>[0]): Promise<void> {
105
+ const { value } = input
106
+ this.assertSession(input.session, input.projectId, value.lastCommitId)
107
+ // The Materializer validates and normalizes float32 values before this boundary.
108
+ const embedding = encodeSqliteVector(value.values)
109
+ const changes =
110
+ input.expectedCommitId === null
111
+ ? this.db
112
+ .query(`
113
+ INSERT INTO object_vectors (
114
+ project_id, object_type_id, primary_id, profile,
115
+ configuration, source, source_fingerprint, embedding, last_commit_id
116
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
117
+ ON CONFLICT (project_id, object_type_id, primary_id, profile) DO NOTHING
118
+ `)
119
+ .run(
120
+ input.projectId,
121
+ value.ref.objectTypeId,
122
+ value.ref.primaryId,
123
+ value.profile,
124
+ value.configuration,
125
+ JSON.stringify(value.source),
126
+ value.sourceFingerprint,
127
+ embedding,
128
+ value.lastCommitId
129
+ ).changes
130
+ : this.db
131
+ .query(`
132
+ UPDATE object_vectors
133
+ SET configuration = ?, source = ?, source_fingerprint = ?, embedding = ?, last_commit_id = ?
134
+ WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
135
+ AND profile = ? AND last_commit_id = ?
136
+ `)
137
+ .run(
138
+ value.configuration,
139
+ JSON.stringify(value.source),
140
+ value.sourceFingerprint,
141
+ embedding,
142
+ value.lastCommitId,
143
+ input.projectId,
144
+ value.ref.objectTypeId,
145
+ value.ref.primaryId,
146
+ value.profile,
147
+ input.expectedCommitId
148
+ ).changes
149
+ if (changes !== 1) throw vectorConflict()
150
+ }
151
+
152
+ async writeBatch(input: Parameters<OntologyVectorStorage["writeBatch"]>[0]): Promise<void> {
153
+ for (const entry of input.entries) {
154
+ await this.write({ session: input.session, projectId: input.projectId, ...entry })
155
+ }
156
+ }
157
+
158
+ async remove(input: Parameters<OntologyVectorStorage["remove"]>[0]): Promise<void> {
159
+ this.assertSession(input.session, input.projectId)
160
+ const { changes } = this.db
161
+ .query(`
162
+ DELETE FROM object_vectors
163
+ WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
164
+ AND profile = ? AND last_commit_id = ?
165
+ `)
166
+ .run(
167
+ input.projectId,
168
+ input.ref.objectTypeId,
169
+ input.ref.primaryId,
170
+ input.profile,
171
+ input.expectedCommitId
172
+ )
173
+ if (changes !== 1) throw vectorConflict()
174
+ }
175
+ }
176
+
177
+ function vectorConflict(): MaterializationConflictError {
178
+ return new MaterializationConflictError(
179
+ "effective-state",
180
+ "Vector changed since it was prepared."
181
+ )
182
+ }
@@ -0,0 +1,9 @@
1
+ /** Shared little-endian float32 encoding for stored vectors and sqlite-vec query bindings. */
2
+ export function encodeSqliteVector(values: readonly number[]): Uint8Array {
3
+ const bytes = new Uint8Array(values.length * 4)
4
+ const view = new DataView(bytes.buffer)
5
+ for (let index = 0; index < values.length; index++) {
6
+ view.setFloat32(index * 4, values[index]!, true)
7
+ }
8
+ return bytes
9
+ }