@sixb/sqlite 0.1.11 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +32 -0
  2. package/dist/agents/rows.d.ts +2 -0
  3. package/dist/agents/rows.d.ts.map +1 -1
  4. package/dist/agents/threads.d.ts +2 -1
  5. package/dist/agents/threads.d.ts.map +1 -1
  6. package/dist/auth-storage/access-tokens.d.ts +0 -1
  7. package/dist/auth-storage/access-tokens.d.ts.map +1 -1
  8. package/dist/auth-storage/auth-storage.d.ts.map +1 -1
  9. package/dist/auth-storage/device-authorizations.d.ts.map +1 -1
  10. package/dist/auth-storage/rows.d.ts +6 -3
  11. package/dist/auth-storage/rows.d.ts.map +1 -1
  12. package/dist/auth-storage/sessions.d.ts +8 -1
  13. package/dist/auth-storage/sessions.d.ts.map +1 -1
  14. package/dist/auth-storage/shared.d.ts.map +1 -1
  15. package/dist/file-upload-session-storage.d.ts +6 -0
  16. package/dist/file-upload-session-storage.d.ts.map +1 -0
  17. package/dist/index.d.ts +2 -0
  18. package/dist/index.d.ts.map +1 -1
  19. package/dist/index.js +1062 -137
  20. package/dist/index.js.map +28 -23
  21. package/dist/migrations.d.ts.map +1 -1
  22. package/dist/objects/query-compiler.d.ts +9 -2
  23. package/dist/objects/query-compiler.d.ts.map +1 -1
  24. package/dist/objects/read-scope.d.ts +3 -0
  25. package/dist/objects/read-scope.d.ts.map +1 -1
  26. package/dist/objects/reader.d.ts +1 -0
  27. package/dist/objects/reader.d.ts.map +1 -1
  28. package/dist/objects/rows.d.ts +1 -0
  29. package/dist/objects/rows.d.ts.map +1 -1
  30. package/dist/objects/storage.d.ts.map +1 -1
  31. package/dist/objects/vector-extension.d.ts +4 -0
  32. package/dist/objects/vector-extension.d.ts.map +1 -0
  33. package/dist/ontology-storage/index.d.ts +4 -0
  34. package/dist/ontology-storage/index.d.ts.map +1 -1
  35. package/dist/ontology-storage/materializations.d.ts +1 -2
  36. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  37. package/dist/ontology-storage/shared.d.ts +2 -1
  38. package/dist/ontology-storage/shared.d.ts.map +1 -1
  39. package/dist/ontology-storage/vector-indexing.d.ts +20 -0
  40. package/dist/ontology-storage/vector-indexing.d.ts.map +1 -0
  41. package/dist/ontology-storage/vectors.d.ts +16 -0
  42. package/dist/ontology-storage/vectors.d.ts.map +1 -0
  43. package/dist/vector-encoding.d.ts +3 -0
  44. package/dist/vector-encoding.d.ts.map +1 -0
  45. package/dist/workflow-intervention-storage.d.ts.map +1 -1
  46. package/package.json +8 -4
  47. package/src/agents/rows.ts +12 -0
  48. package/src/agents/threads.ts +41 -2
  49. package/src/ai-cost-storage.ts +4 -0
  50. package/src/auth-storage/access-tokens.ts +11 -18
  51. package/src/auth-storage/auth-storage.ts +5 -7
  52. package/src/auth-storage/device-authorizations.ts +3 -5
  53. package/src/auth-storage/rows.ts +17 -7
  54. package/src/auth-storage/sessions.ts +89 -4
  55. package/src/auth-storage/shared.ts +9 -3
  56. package/src/file-upload-session-storage.ts +136 -0
  57. package/src/index.ts +5 -0
  58. package/src/migrations/041-agent-thread-workspaces.sql +2 -0
  59. package/src/migrations/042-vector-profiles.sql +21 -0
  60. package/src/migrations/043-vector-indexing.sql +197 -0
  61. package/src/migrations/044-vector-batching.sql +3 -0
  62. package/src/migrations/045-agent-thread-sandbox-state.sql +2 -0
  63. package/src/migrations/046-workflow-intervention-principals.sql +15 -0
  64. package/src/migrations/047-ontology-commit-attribution.sql +67 -0
  65. package/src/migrations/048-file-upload-sessions.sql +35 -0
  66. package/src/migrations/049-native-sessions.sql +12 -0
  67. package/src/migrations.ts +26 -0
  68. package/src/objects/query-compiler.ts +164 -11
  69. package/src/objects/read-scope.ts +34 -31
  70. package/src/objects/reader.ts +29 -1
  71. package/src/objects/rows.ts +5 -1
  72. package/src/objects/storage.ts +13 -1
  73. package/src/objects/vector-extension.ts +26 -0
  74. package/src/ontology-storage/index.ts +16 -0
  75. package/src/ontology-storage/materializations.ts +29 -59
  76. package/src/ontology-storage/shared.ts +7 -3
  77. package/src/ontology-storage/vector-indexing.ts +196 -0
  78. package/src/ontology-storage/vectors.ts +182 -0
  79. package/src/vector-encoding.ts +9 -0
  80. package/src/workflow-intervention-storage.ts +9 -9
@@ -1,4 +1,5 @@
1
1
  import type { Database } from "bun:sqlite"
2
+ import { hasVectorProfile } from "@sixb/core/internal/query"
2
3
  import type {
3
4
  CompiledSelectedObjectReadScope,
4
5
  CountObjectsInput,
@@ -25,6 +26,7 @@ import { installFreshSqliteSchema } from "../migrations"
25
26
  import {
26
27
  closeSqliteStoreConnection,
27
28
  openSqliteStoreConnection,
29
+ runDeferredReadTransaction,
28
30
  type SqliteStoreConnection,
29
31
  } from "../transactions"
30
32
  import { DEFAULT_OBJECT_QUERY_SOURCE } from "./query-compiler"
@@ -49,6 +51,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
49
51
  refs: true,
50
52
  filter: true,
51
53
  text: true,
54
+ vector: true,
52
55
  sort: true,
53
56
  limit: true,
54
57
  page: true,
@@ -92,6 +95,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
92
95
  decimal: { equality: true },
93
96
  date: { equality: true, ordering: true },
94
97
  timestamp: { equality: true, ordering: true },
98
+ userRef: { equality: true },
95
99
  },
96
100
  limits: {
97
101
  totalCount: true,
@@ -101,7 +105,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
101
105
  "SQLite object query pushdown supports start/refs/filter/text/sort/limit/page/traverse/set/project/expand over JSON properties and object links.",
102
106
  "expand hydrates linked objects in-database (top-N per parent via row_number() + json_group_array); core resolves each expansion's cardinality before pushdown, and a mixed/unresolved one stays on the fallback.",
103
107
  "Ordered decimal predicates and sorting use the bounded core fallback because SQLite has no native exact decimal type; canonical decimal equality remains pushdown-safe.",
104
- "Relevance sorting, vector search, and unresolved start.includeSubtypes remain planner fallback or rejection cases.",
108
+ "Relevance sorting and unresolved start.includeSubtypes remain planner fallback or rejection cases.",
105
109
  ],
106
110
  }
107
111
 
@@ -139,18 +143,26 @@ export class SqliteObjectStorage implements ObjectStorage {
139
143
  }
140
144
 
141
145
  async queryObjects(params: QueryObjectsInput): Promise<QueryObjectsResult> {
146
+ if (hasVectorProfile(params.query))
147
+ return runDeferredReadTransaction(this.db, () => this.reader.queryObjects(params))
142
148
  return this.reader.queryObjects(params)
143
149
  }
144
150
 
145
151
  async countObjects(params: CountObjectsInput): Promise<CountObjectsResult> {
152
+ if (hasVectorProfile(params.query))
153
+ return runDeferredReadTransaction(this.db, () => this.reader.countObjects(params))
146
154
  return this.reader.countObjects(params)
147
155
  }
148
156
 
149
157
  async existsObjects(params: ExistsObjectsInput): Promise<ExistsObjectsResult> {
158
+ if (hasVectorProfile(params.query))
159
+ return runDeferredReadTransaction(this.db, () => this.reader.existsObjects(params))
150
160
  return this.reader.existsObjects(params)
151
161
  }
152
162
 
153
163
  async facetObjects(params: FacetObjectsInput): Promise<FacetObjectsResult> {
164
+ if (hasVectorProfile(params.query))
165
+ return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params))
154
166
  return this.reader.facetObjects(params)
155
167
  }
156
168
 
@@ -0,0 +1,26 @@
1
+ import type { Database } from "bun:sqlite"
2
+ import { ObjectQueryExecutionError } from "@sixb/core"
3
+ import { load } from "sqlite-vec"
4
+
5
+ const loaded = new WeakSet<Database>()
6
+
7
+ /** Load only for vector reads, leaving ordinary SQLite use independent of extension support. */
8
+ export function ensureSqliteVectorSearch(db: Database): void {
9
+ if (loaded.has(db)) return
10
+ const options = db.query<{ compile_options: string }, []>("PRAGMA compile_options").all()
11
+ if (options.some((option) => option.compile_options === "OMIT_LOAD_EXTENSION")) {
12
+ throw new ObjectQueryExecutionError(
13
+ "vector_extension_missing",
14
+ "SQLite vector search requires extension support. On macOS install SQLite (brew install sqlite), then call Database.setCustomSQLite with its library path before opening any database."
15
+ )
16
+ }
17
+ try {
18
+ load(db)
19
+ loaded.add(db)
20
+ } catch (cause) {
21
+ throw new Error(
22
+ "[SixbSqlite] Could not load sqlite-vec for vector search. Check that its native package supports your platform.",
23
+ { cause }
24
+ )
25
+ }
26
+ }
@@ -7,10 +7,15 @@ import { SqliteOntologyOutboxStorage } from "./outbox"
7
7
  import type { SqliteRootOperation } from "./shared"
8
8
  import { SqliteOntologySourceStorage } from "./sources"
9
9
 
10
+ import { SqliteVectorIndexingStorage } from "./vector-indexing"
11
+ import { SqliteOntologyVectorStorage } from "./vectors"
12
+
10
13
  export class SqliteOntologyStorage implements OntologyStorage {
11
14
  readonly commits: SqliteOntologyCommitStorage
12
15
  readonly sources: SqliteOntologySourceStorage
13
16
  readonly materializations: SqliteOntologyMaterializationStorage
17
+ readonly vectorIndexing: SqliteVectorIndexingStorage
18
+ readonly vectors: SqliteOntologyVectorStorage
14
19
  readonly outbox: SqliteOntologyOutboxStorage
15
20
 
16
21
  constructor(input: {
@@ -24,6 +29,17 @@ export class SqliteOntologyStorage implements OntologyStorage {
24
29
  input.db,
25
30
  input.transactionContext
26
31
  )
32
+ this.vectorIndexing = new SqliteVectorIndexingStorage(
33
+ input.db,
34
+ input.runRootOperation,
35
+ (session, projectId) => this.materializations.assertVectorSession(session, projectId)
36
+ )
37
+ this.vectors = new SqliteOntologyVectorStorage(
38
+ input.db,
39
+ input.runRootOperation,
40
+ (session, projectId, commitId) =>
41
+ this.materializations.assertVectorSession(session, projectId, commitId)
42
+ )
27
43
  this.outbox = new SqliteOntologyOutboxStorage(input.db, input.runRootOperation)
28
44
  }
29
45
 
@@ -1,20 +1,20 @@
1
1
  import type { Database } from "bun:sqlite"
2
- import type {
3
- ExpectedLinkRevision,
4
- ExpectedObjectRevision,
5
- } from "@sixb/core/internal/materialization"
6
2
  import {
7
3
  assertPinnedDatasetWatermark,
8
4
  linkRefKey,
9
5
  linkRefSortKey,
10
6
  linkScopeSortKey,
11
7
  MaterializationConflictError,
8
+ MaterializationValidationError,
12
9
  objectRefKey,
13
10
  objectRefSortKey,
14
11
  telemetryPointKey,
15
12
  telemetryPointSortKey,
16
13
  } from "@sixb/core/internal/materialization"
17
14
  import {
15
+ assertExpectedLinkRevision,
16
+ assertExpectedLinkScopeRevision,
17
+ assertExpectedObjectRevision,
18
18
  assertMaterializationFinalizationCorrelation,
19
19
  assertMaterializationHeader,
20
20
  assertMaterializationLaneCompletion,
@@ -79,6 +79,18 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
79
79
  this.writer = new SqliteMaterializationWriter(db)
80
80
  }
81
81
 
82
+ assertVectorSession(session: MaterializationSession, projectId: string, commitId?: string): void {
83
+ const active = this.sessions.require(session)
84
+ if (
85
+ active.header.commit.projectId !== projectId ||
86
+ (commitId !== undefined && active.header.commit.id !== commitId)
87
+ ) {
88
+ throw new MaterializationValidationError(
89
+ "Vector mutation does not belong to this materialization session."
90
+ )
91
+ }
92
+ }
93
+
82
94
  async begin(input: MaterializationPlanHeader): Promise<MaterializationSession> {
83
95
  assertMaterializationHeader(input)
84
96
  const session = this.sessions.create(input)
@@ -87,16 +99,15 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
87
99
  const reader = new SqliteMaterializationStateReader(this.db, input.commit.projectId)
88
100
  for (const expected of input.expected.sources)
89
101
  this.assertSource(expected, input.commit.projectId)
90
- for (const expected of input.expected.objects) this.assertObject(reader, expected)
91
- for (const expected of input.expected.links) this.assertLink(reader, expected)
102
+ for (const expected of input.expected.objects) {
103
+ assertExpectedObjectRevision(reader.effectiveObjectRevision(expected.ref), expected)
104
+ }
105
+ for (const expected of input.expected.links) {
106
+ assertExpectedLinkRevision(reader.effectiveLinkLastCommit(expected.ref), expected)
107
+ }
92
108
  const linkScopeRevisions = reader.linkScopeRevisions(input.expected.linkScopes)
93
109
  for (const [index, expected] of input.expected.linkScopes.entries()) {
94
- if (linkScopeRevisions[index]?.fingerprint !== expected.fingerprint) {
95
- throw new MaterializationConflictError(
96
- "effective-state",
97
- `Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`
98
- )
99
- }
110
+ assertExpectedLinkScopeRevision(linkScopeRevisions[index]?.fingerprint, expected)
100
111
  }
101
112
  for (const expected of input.expected.points) {
102
113
  const point = reader.exactPoint(expected.series, expected.at)
@@ -436,50 +447,6 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
436
447
  }
437
448
  }
438
449
 
439
- private assertObject(
440
- reader: SqliteMaterializationStateReader,
441
- expected: ExpectedObjectRevision
442
- ): void {
443
- const row = reader.effectiveObjectRevision(expected.ref)
444
- if (!expected.exists) {
445
- if (row) {
446
- throw new MaterializationConflictError(
447
- "effective-state",
448
- `Expected object ${objectRefKey(expected.ref)} to be absent.`
449
- )
450
- }
451
- return
452
- }
453
- if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId) {
454
- throw new MaterializationConflictError(
455
- "effective-state",
456
- `Expected object ${objectRefKey(expected.ref)} changed.`
457
- )
458
- }
459
- }
460
-
461
- private assertLink(
462
- reader: SqliteMaterializationStateReader,
463
- expected: ExpectedLinkRevision
464
- ): void {
465
- const lastCommitId = reader.effectiveLinkLastCommit(expected.ref)
466
- if (!expected.exists) {
467
- if (lastCommitId !== undefined) {
468
- throw new MaterializationConflictError(
469
- "effective-state",
470
- `Expected link ${linkRefKey(expected.ref)} to be absent.`
471
- )
472
- }
473
- return
474
- }
475
- if (lastCommitId === undefined || lastCommitId !== expected.lastCommitId) {
476
- throw new MaterializationConflictError(
477
- "effective-state",
478
- `Expected link ${linkRefKey(expected.ref)} changed.`
479
- )
480
- }
481
- }
482
-
483
450
  private async assertFinalization(
484
451
  session: SqliteMaterializationSessionState,
485
452
  input: FinalizeMaterializationInput
@@ -706,10 +673,12 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
706
673
  `
707
674
  INSERT INTO ontology_commits (
708
675
  project_id, id, idempotency_key, request_hash, execution_id,
709
- origin_kind, origin_run_id, origin_batch_ordinal, origin, actor,
676
+ origin_kind, origin_run_id, origin_batch_ordinal, origin, requested_by, executor,
710
677
  ontology_revision, projection_revision, ownership_hash,
711
678
  intent, result, committed_at
712
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, json(?), json(?), ?, ?, ?, json(?), json(?), ?)
679
+ ) VALUES (
680
+ ?, ?, ?, ?, ?, ?, ?, ?, json(?), json(?), json(?), ?, ?, ?, json(?), json(?), ?
681
+ )
713
682
  `
714
683
  )
715
684
  .run(
@@ -722,7 +691,8 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
722
691
  origin.runId,
723
692
  origin.batchOrdinal,
724
693
  canonicalJson(commit.origin),
725
- commit.actor === undefined ? null : canonicalJson(commit.actor),
694
+ commit.requestedBy === undefined ? null : canonicalJson(commit.requestedBy),
695
+ canonicalJson(commit.executor),
726
696
  commit.ontologyRevision,
727
697
  commit.projectionRevision ?? null,
728
698
  commit.ownershipHash ?? null,
@@ -37,7 +37,8 @@ export interface SqliteOntologyCommitRow {
37
37
  readonly origin_run_id: string | null
38
38
  readonly origin_batch_ordinal: number | null
39
39
  readonly origin: string
40
- readonly actor: string | null
40
+ readonly requested_by: string | null
41
+ readonly executor: string
41
42
  readonly ontology_revision: string
42
43
  readonly projection_revision: string | null
43
44
  readonly ownership_hash: string | null
@@ -162,9 +163,12 @@ export function commitRecord(row: SqliteOntologyCommitRow): OntologyCommitRecord
162
163
  requestHash: row.request_hash,
163
164
  executionId: row.execution_id,
164
165
  origin: parseJson<OntologyCommitWrite["origin"]>(row.origin),
165
- ...(row.actor === null
166
+ ...(row.requested_by === null
166
167
  ? {}
167
- : { actor: parseJson<NonNullable<OntologyCommitWrite["actor"]>>(row.actor) }),
168
+ : {
169
+ requestedBy: parseJson<NonNullable<OntologyCommitWrite["requestedBy"]>>(row.requested_by),
170
+ }),
171
+ executor: parseJson<OntologyCommitWrite["executor"]>(row.executor),
168
172
  ontologyRevision: row.ontology_revision,
169
173
  ...(row.projection_revision === null ? {} : { projectionRevision: row.projection_revision }),
170
174
  ...(row.ownership_hash === null ? {} : { ownershipHash: row.ownership_hash }),
@@ -0,0 +1,196 @@
1
+ import type { Database } from "bun:sqlite"
2
+ import { parseSixbFailure } from "@sixb/core/internal/errors"
3
+ import {
4
+ type MaterializationSession,
5
+ type OntologyVectorIndexingStorage,
6
+ VECTOR_INDEXING_FAILURE_CODES,
7
+ type VectorIndexingRequest,
8
+ type VectorIndexingWork,
9
+ } from "@sixb/core/storage"
10
+ import type { SqliteRootOperation } from "./shared"
11
+
12
+ interface WorkRow {
13
+ batch_id: string | null
14
+ request: string
15
+ status: VectorIndexingWork["status"]
16
+ available_at: string
17
+ values_json: string | null
18
+ error: string | null
19
+ }
20
+ export class SqliteVectorIndexingStorage implements OntologyVectorIndexingStorage {
21
+ constructor(
22
+ private readonly db: Database,
23
+ private readonly run: SqliteRootOperation,
24
+ private readonly assertSession: (session: MaterializationSession, projectId: string) => void
25
+ ) {}
26
+ async schedule(input: Parameters<OntologyVectorIndexingStorage["schedule"]>[0]) {
27
+ this.assertSession(input.session, input.projectId)
28
+ for (const ref of input.deleted)
29
+ this.db
30
+ .query(
31
+ `DELETE FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ? AND primary_id = ?`
32
+ )
33
+ .run(input.projectId, ref.objectTypeId, ref.primaryId)
34
+ const insert =
35
+ this.db.query(`INSERT INTO object_vector_indexing (project_id, id, object_type_id, primary_id, profile, batch_id, request, status, available_at, dispatch_at)
36
+ VALUES (?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)
37
+ ON CONFLICT (project_id, object_type_id, primary_id, profile) DO UPDATE SET id = excluded.id, batch_id = excluded.batch_id, request = excluded.request, status = 'pending', available_at = excluded.available_at, dispatch_at = excluded.dispatch_at, values_json = NULL, error = NULL`)
38
+ for (const { batchId, ...request } of input.requests)
39
+ insert.run(
40
+ input.projectId,
41
+ request.id,
42
+ request.ref.objectTypeId,
43
+ request.ref.primaryId,
44
+ request.profile,
45
+ batchId ?? null,
46
+ JSON.stringify(request),
47
+ input.availableAt,
48
+ input.availableAt
49
+ )
50
+ }
51
+ async complete(input: Parameters<OntologyVectorIndexingStorage["complete"]>[0]) {
52
+ this.assertSession(input.session, input.projectId)
53
+ const remove = this.db.query(
54
+ `DELETE FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ? AND primary_id = ? AND profile = ? AND json_extract(request, '$.configuration') = ? AND json_extract(request, '$.sourceFingerprint') = ?`
55
+ )
56
+ for (const entry of input.entries) {
57
+ remove.run(
58
+ input.projectId,
59
+ entry.ref.objectTypeId,
60
+ entry.ref.primaryId,
61
+ entry.profile,
62
+ entry.configuration,
63
+ entry.sourceFingerprint
64
+ )
65
+ }
66
+ }
67
+
68
+ async dispatched(input: Parameters<OntologyVectorIndexingStorage["dispatched"]>[0]) {
69
+ await this.run(() => {
70
+ const update = this.db.query(
71
+ `UPDATE object_vector_indexing SET dispatch_at = ? WHERE project_id = ? AND id = ?`
72
+ )
73
+ for (const id of input.ids) update.run(input.nextDispatchAt, input.projectId, id)
74
+ })
75
+ }
76
+ async get(input: Parameters<OntologyVectorIndexingStorage["get"]>[0]) {
77
+ return this.run(() => {
78
+ const row = this.db
79
+ .query<WorkRow, [string, string]>(
80
+ `SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing WHERE project_id = ? AND id = ?`
81
+ )
82
+ .get(input.projectId, input.id)
83
+ return row ? decode(row) : null
84
+ })
85
+ }
86
+ async getBatch(input: Parameters<OntologyVectorIndexingStorage["getBatch"]>[0]) {
87
+ return this.run(() =>
88
+ this.db
89
+ .query<WorkRow, [string, string]>(`
90
+ SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing
91
+ WHERE project_id = ? AND batch_id = ? ORDER BY id`)
92
+ .all(input.projectId, input.batchId)
93
+ .map(decode)
94
+ )
95
+ }
96
+
97
+ async getBatchMember(input: Parameters<OntologyVectorIndexingStorage["getBatchMember"]>[0]) {
98
+ return this.run(() => {
99
+ const row = this.db
100
+ .query<WorkRow, [string, string, string, string, string]>(`
101
+ SELECT batch_id, request, status, available_at, values_json, error
102
+ FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ?
103
+ AND primary_id = ? AND profile = ? AND batch_id = ?`)
104
+ .get(
105
+ input.projectId,
106
+ input.ref.objectTypeId,
107
+ input.ref.primaryId,
108
+ input.profile,
109
+ input.batchId
110
+ )
111
+ return row ? decode(row) : null
112
+ })
113
+ }
114
+
115
+ async updateBatch(input: Parameters<OntologyVectorIndexingStorage["updateBatch"]>[0]) {
116
+ return this.run(() =>
117
+ this.db.transaction(() => {
118
+ const status = this.db.query<{ status: string }, [string, string]>(
119
+ "SELECT status FROM object_vector_indexing WHERE project_id = ? AND id = ?"
120
+ )
121
+ if (
122
+ input.requireAll &&
123
+ input.updates.some(
124
+ (update) => status.get(input.projectId, update.id)?.status !== update.expectedStatus
125
+ )
126
+ )
127
+ return false
128
+ const updateRow = this.db.query(`UPDATE object_vector_indexing
129
+ SET status = ?, available_at = ?, values_json = COALESCE(?, values_json), error = ?
130
+ WHERE project_id = ? AND id = ? AND status = ?`)
131
+ for (const update of input.updates)
132
+ updateRow.run(
133
+ update.status,
134
+ update.availableAt,
135
+ update.values ? JSON.stringify(update.values) : null,
136
+ update.error ? JSON.stringify(update.error) : null,
137
+ input.projectId,
138
+ update.id,
139
+ update.expectedStatus
140
+ )
141
+ return true
142
+ })()
143
+ )
144
+ }
145
+
146
+ async listDue(input: Parameters<OntologyVectorIndexingStorage["listDue"]>[0]) {
147
+ return this.run(() =>
148
+ this.db
149
+ .query<WorkRow, [string, string, number]>(
150
+ `SELECT batch_id, request, status, available_at, NULL AS values_json, error FROM object_vector_indexing WHERE project_id = ? AND status <> 'failed' AND dispatch_at <= ? ORDER BY dispatch_at, id LIMIT ?`
151
+ )
152
+ .all(input.projectId, input.now, input.limit)
153
+ .map(decode)
154
+ )
155
+ }
156
+ async update(input: Parameters<OntologyVectorIndexingStorage["update"]>[0]) {
157
+ return this.run(
158
+ () =>
159
+ this.db
160
+ .query(
161
+ `UPDATE object_vector_indexing SET status = ?, available_at = ?, values_json = COALESCE(?, values_json), error = ? WHERE project_id = ? AND id = ? AND status = ?`
162
+ )
163
+ .run(
164
+ input.status,
165
+ input.availableAt,
166
+ input.values ? JSON.stringify(input.values) : null,
167
+ input.error ? JSON.stringify(input.error) : null,
168
+ input.projectId,
169
+ input.id,
170
+ input.expectedStatus
171
+ ).changes === 1
172
+ )
173
+ }
174
+ async remove(input: Parameters<OntologyVectorIndexingStorage["remove"]>[0]) {
175
+ await this.run(() => {
176
+ this.db
177
+ .query(`DELETE FROM object_vector_indexing WHERE project_id = ? AND id = ?`)
178
+ .run(input.projectId, input.id)
179
+ })
180
+ }
181
+ }
182
+ function decode(row: WorkRow): VectorIndexingWork {
183
+ const request = JSON.parse(row.request) as VectorIndexingRequest
184
+ return {
185
+ ...request,
186
+ ...(row.batch_id ? { batchId: row.batch_id } : {}),
187
+ status: row.status,
188
+ availableAt: row.available_at,
189
+ ...(row.values_json ? { values: JSON.parse(row.values_json) as number[] } : {}),
190
+ ...(row.error
191
+ ? {
192
+ error: parseSixbFailure(row.error, VECTOR_INDEXING_FAILURE_CODES),
193
+ }
194
+ : {}),
195
+ }
196
+ }
@@ -0,0 +1,182 @@
1
+ import type { Database } from "bun:sqlite"
2
+ import { MaterializationConflictError } from "@sixb/core/internal/materialization"
3
+ import type {
4
+ MaterializationSession,
5
+ ObjectVectorState,
6
+ OntologyVectorStorage,
7
+ } from "@sixb/core/storage"
8
+ import { encodeSqliteVector } from "../vector-encoding"
9
+ import type { SqliteRootOperation } from "./shared"
10
+
11
+ interface VectorMetadataRow {
12
+ readonly profile: string
13
+ readonly configuration: string
14
+ readonly source: string
15
+ readonly source_fingerprint: string
16
+ readonly last_commit_id: string
17
+ }
18
+
19
+ export class SqliteOntologyVectorStorage implements OntologyVectorStorage {
20
+ constructor(
21
+ private readonly db: Database,
22
+ private readonly runRootOperation: SqliteRootOperation,
23
+ private readonly assertSession: (
24
+ session: MaterializationSession,
25
+ projectId: string,
26
+ commitId?: string
27
+ ) => void
28
+ ) {}
29
+
30
+ async list(
31
+ input: Parameters<OntologyVectorStorage["list"]>[0]
32
+ ): Promise<readonly ObjectVectorState[]> {
33
+ return this.runRootOperation(() => {
34
+ const rows = this.db
35
+ .query<VectorMetadataRow, [string, string, string]>(`
36
+ SELECT profile, configuration, source, source_fingerprint, last_commit_id
37
+ FROM object_vectors
38
+ WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
39
+ ORDER BY profile
40
+ `)
41
+ .all(input.projectId, input.ref.objectTypeId, input.ref.primaryId)
42
+ return rows.map((row) => ({
43
+ ref: { ...input.ref },
44
+ profile: row.profile,
45
+ configuration: row.configuration,
46
+ source: JSON.parse(row.source) as string[],
47
+ sourceFingerprint: row.source_fingerprint,
48
+ lastCommitId: row.last_commit_id,
49
+ }))
50
+ })
51
+ }
52
+
53
+ async listBatch(
54
+ input: Parameters<OntologyVectorStorage["listBatch"]>[0]
55
+ ): Promise<readonly ObjectVectorState[]> {
56
+ if (!input.refs.length) return []
57
+ return this.runRootOperation(() => {
58
+ // Keep the bounded request set first so SQLite probes the composite object index.
59
+ const rows = this.db
60
+ .query<
61
+ VectorMetadataRow & { object_type_id: string; primary_id: string },
62
+ [string, string]
63
+ >(`
64
+ WITH requested AS (
65
+ SELECT DISTINCT json_extract(value, '$.objectTypeId') AS object_type_id,
66
+ json_extract(value, '$.primaryId') AS primary_id FROM json_each(?)
67
+ )
68
+ SELECT stored.object_type_id, stored.primary_id, stored.profile, stored.configuration,
69
+ stored.source, stored.source_fingerprint, stored.last_commit_id
70
+ FROM requested CROSS JOIN object_vectors AS stored
71
+ WHERE stored.project_id = ? AND stored.object_type_id = requested.object_type_id
72
+ AND stored.primary_id = requested.primary_id
73
+ ORDER BY stored.object_type_id, stored.primary_id, stored.profile
74
+ `)
75
+ .all(JSON.stringify(input.refs), input.projectId)
76
+ return rows.map((row) => ({
77
+ ref: { objectTypeId: row.object_type_id, primaryId: row.primary_id },
78
+ profile: row.profile,
79
+ configuration: row.configuration,
80
+ source: JSON.parse(row.source) as string[],
81
+ sourceFingerprint: row.source_fingerprint,
82
+ lastCommitId: row.last_commit_id,
83
+ }))
84
+ })
85
+ }
86
+
87
+ async removeBatch(input: Parameters<OntologyVectorStorage["removeBatch"]>[0]): Promise<void> {
88
+ this.assertSession(input.session, input.projectId)
89
+ if (!input.entries.length) return
90
+ const { changes } = this.db
91
+ .query(`
92
+ DELETE FROM object_vectors WHERE rowid IN (
93
+ SELECT stored.rowid FROM json_each(?) AS requested CROSS JOIN object_vectors AS stored
94
+ WHERE stored.project_id = ? AND stored.object_type_id = json_extract(requested.value, '$.ref.objectTypeId')
95
+ AND stored.primary_id = json_extract(requested.value, '$.ref.primaryId')
96
+ AND stored.profile = json_extract(requested.value, '$.profile')
97
+ AND stored.last_commit_id = json_extract(requested.value, '$.expectedCommitId')
98
+ )
99
+ `)
100
+ .run(JSON.stringify(input.entries), input.projectId)
101
+ if (changes !== input.entries.length) throw vectorConflict()
102
+ }
103
+
104
+ async write(input: Parameters<OntologyVectorStorage["write"]>[0]): Promise<void> {
105
+ const { value } = input
106
+ this.assertSession(input.session, input.projectId, value.lastCommitId)
107
+ // The Materializer validates and normalizes float32 values before this boundary.
108
+ const embedding = encodeSqliteVector(value.values)
109
+ const changes =
110
+ input.expectedCommitId === null
111
+ ? this.db
112
+ .query(`
113
+ INSERT INTO object_vectors (
114
+ project_id, object_type_id, primary_id, profile,
115
+ configuration, source, source_fingerprint, embedding, last_commit_id
116
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
117
+ ON CONFLICT (project_id, object_type_id, primary_id, profile) DO NOTHING
118
+ `)
119
+ .run(
120
+ input.projectId,
121
+ value.ref.objectTypeId,
122
+ value.ref.primaryId,
123
+ value.profile,
124
+ value.configuration,
125
+ JSON.stringify(value.source),
126
+ value.sourceFingerprint,
127
+ embedding,
128
+ value.lastCommitId
129
+ ).changes
130
+ : this.db
131
+ .query(`
132
+ UPDATE object_vectors
133
+ SET configuration = ?, source = ?, source_fingerprint = ?, embedding = ?, last_commit_id = ?
134
+ WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
135
+ AND profile = ? AND last_commit_id = ?
136
+ `)
137
+ .run(
138
+ value.configuration,
139
+ JSON.stringify(value.source),
140
+ value.sourceFingerprint,
141
+ embedding,
142
+ value.lastCommitId,
143
+ input.projectId,
144
+ value.ref.objectTypeId,
145
+ value.ref.primaryId,
146
+ value.profile,
147
+ input.expectedCommitId
148
+ ).changes
149
+ if (changes !== 1) throw vectorConflict()
150
+ }
151
+
152
+ async writeBatch(input: Parameters<OntologyVectorStorage["writeBatch"]>[0]): Promise<void> {
153
+ for (const entry of input.entries) {
154
+ await this.write({ session: input.session, projectId: input.projectId, ...entry })
155
+ }
156
+ }
157
+
158
+ async remove(input: Parameters<OntologyVectorStorage["remove"]>[0]): Promise<void> {
159
+ this.assertSession(input.session, input.projectId)
160
+ const { changes } = this.db
161
+ .query(`
162
+ DELETE FROM object_vectors
163
+ WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
164
+ AND profile = ? AND last_commit_id = ?
165
+ `)
166
+ .run(
167
+ input.projectId,
168
+ input.ref.objectTypeId,
169
+ input.ref.primaryId,
170
+ input.profile,
171
+ input.expectedCommitId
172
+ )
173
+ if (changes !== 1) throw vectorConflict()
174
+ }
175
+ }
176
+
177
+ function vectorConflict(): MaterializationConflictError {
178
+ return new MaterializationConflictError(
179
+ "effective-state",
180
+ "Vector changed since it was prepared."
181
+ )
182
+ }
@@ -0,0 +1,9 @@
1
+ /** Shared little-endian float32 encoding for stored vectors and sqlite-vec query bindings. */
2
+ export function encodeSqliteVector(values: readonly number[]): Uint8Array {
3
+ const bytes = new Uint8Array(values.length * 4)
4
+ const view = new DataView(bytes.buffer)
5
+ for (let index = 0; index < values.length; index++) {
6
+ view.setFloat32(index * 4, values[index]!, true)
7
+ }
8
+ return bytes
9
+ }