@sixb/sqlite 0.1.11 → 0.1.12

package/src/objects/storage.ts

@@ -1,4 +1,5 @@
 import type { Database } from "bun:sqlite"
+import { hasVectorProfile } from "@sixb/core/internal/query"
 import type {
   CompiledSelectedObjectReadScope,
   CountObjectsInput,
@@ -25,6 +26,7 @@ import { installFreshSqliteSchema } from "../migrations"
 import {
   closeSqliteStoreConnection,
   openSqliteStoreConnection,
+  runDeferredReadTransaction,
   type SqliteStoreConnection,
 } from "../transactions"
 import { DEFAULT_OBJECT_QUERY_SOURCE } from "./query-compiler"
@@ -49,6 +51,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
     refs: true,
     filter: true,
     text: true,
+    vector: true,
     sort: true,
     limit: true,
     page: true,
@@ -101,7 +104,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
     "SQLite object query pushdown supports start/refs/filter/text/sort/limit/page/traverse/set/project/expand over JSON properties and object links.",
     "expand hydrates linked objects in-database (top-N per parent via row_number() + json_group_array); core resolves each expansion's cardinality before pushdown, and a mixed/unresolved one stays on the fallback.",
     "Ordered decimal predicates and sorting use the bounded core fallback because SQLite has no native exact decimal type; canonical decimal equality remains pushdown-safe.",
-    "Relevance sorting, vector search, and unresolved start.includeSubtypes remain planner fallback or rejection cases.",
+    "Relevance sorting and unresolved start.includeSubtypes remain planner fallback or rejection cases.",
   ],
 }
 
@@ -139,18 +142,26 @@ export class SqliteObjectStorage implements ObjectStorage {
   }
 
   async queryObjects(params: QueryObjectsInput): Promise<QueryObjectsResult> {
+    if (hasVectorProfile(params.query))
+      return runDeferredReadTransaction(this.db, () => this.reader.queryObjects(params))
     return this.reader.queryObjects(params)
   }
 
   async countObjects(params: CountObjectsInput): Promise<CountObjectsResult> {
+    if (hasVectorProfile(params.query))
+      return runDeferredReadTransaction(this.db, () => this.reader.countObjects(params))
     return this.reader.countObjects(params)
   }
 
   async existsObjects(params: ExistsObjectsInput): Promise<ExistsObjectsResult> {
+    if (hasVectorProfile(params.query))
+      return runDeferredReadTransaction(this.db, () => this.reader.existsObjects(params))
     return this.reader.existsObjects(params)
   }
 
   async facetObjects(params: FacetObjectsInput): Promise<FacetObjectsResult> {
+    if (hasVectorProfile(params.query))
+      return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params))
     return this.reader.facetObjects(params)
   }
 

package/src/objects/vector-extension.ts

@@ -0,0 +1,26 @@
+import type { Database } from "bun:sqlite"
+import { ObjectQueryExecutionError } from "@sixb/core"
+import { load } from "sqlite-vec"
+
+const loaded = new WeakSet<Database>()
+
+/** Load only for vector reads, leaving ordinary SQLite use independent of extension support. */
+export function ensureSqliteVectorSearch(db: Database): void {
+  if (loaded.has(db)) return
+  const options = db.query<{ compile_options: string }, []>("PRAGMA compile_options").all()
+  if (options.some((option) => option.compile_options === "OMIT_LOAD_EXTENSION")) {
+    throw new ObjectQueryExecutionError(
+      "vector_extension_missing",
+      "SQLite vector search requires extension support. On macOS install SQLite (brew install sqlite), then call Database.setCustomSQLite with its library path before opening any database."
+    )
+  }
+  try {
+    load(db)
+    loaded.add(db)
+  } catch (cause) {
+    throw new Error(
+      "[SixbSqlite] Could not load sqlite-vec for vector search. Check that its native package supports your platform.",
+      { cause }
+    )
+  }
+}

package/src/ontology-storage/index.ts

@@ -7,10 +7,15 @@ import { SqliteOntologyOutboxStorage } from "./outbox"
 import type { SqliteRootOperation } from "./shared"
 import { SqliteOntologySourceStorage } from "./sources"
 
+import { SqliteVectorIndexingStorage } from "./vector-indexing"
+import { SqliteOntologyVectorStorage } from "./vectors"
+
 export class SqliteOntologyStorage implements OntologyStorage {
   readonly commits: SqliteOntologyCommitStorage
   readonly sources: SqliteOntologySourceStorage
   readonly materializations: SqliteOntologyMaterializationStorage
+  readonly vectorIndexing: SqliteVectorIndexingStorage
+  readonly vectors: SqliteOntologyVectorStorage
   readonly outbox: SqliteOntologyOutboxStorage
 
   constructor(input: {
@@ -24,6 +29,17 @@ export class SqliteOntologyStorage implements OntologyStorage {
       input.db,
       input.transactionContext
     )
+    this.vectorIndexing = new SqliteVectorIndexingStorage(
+      input.db,
+      input.runRootOperation,
+      (session, projectId) => this.materializations.assertVectorSession(session, projectId)
+    )
+    this.vectors = new SqliteOntologyVectorStorage(
+      input.db,
+      input.runRootOperation,
+      (session, projectId, commitId) =>
+        this.materializations.assertVectorSession(session, projectId, commitId)
+    )
     this.outbox = new SqliteOntologyOutboxStorage(input.db, input.runRootOperation)
   }
 

package/src/ontology-storage/materializations.ts

@@ -9,6 +9,7 @@ import {
   linkRefSortKey,
   linkScopeSortKey,
   MaterializationConflictError,
+  MaterializationValidationError,
   objectRefKey,
   objectRefSortKey,
   telemetryPointKey,
@@ -79,6 +80,18 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
     this.writer = new SqliteMaterializationWriter(db)
   }
 
+  assertVectorSession(session: MaterializationSession, projectId: string, commitId?: string): void {
+    const active = this.sessions.require(session)
+    if (
+      active.header.commit.projectId !== projectId ||
+      (commitId !== undefined && active.header.commit.id !== commitId)
+    ) {
+      throw new MaterializationValidationError(
+        "Vector mutation does not belong to this materialization session."
+      )
+    }
+  }
+
   async begin(input: MaterializationPlanHeader): Promise<MaterializationSession> {
     assertMaterializationHeader(input)
     const session = this.sessions.create(input)

package/src/ontology-storage/vector-indexing.ts

@@ -0,0 +1,196 @@
+import type { Database } from "bun:sqlite"
+import { parseSixbFailure } from "@sixb/core/internal/errors"
+import {
+  type MaterializationSession,
+  type OntologyVectorIndexingStorage,
+  VECTOR_INDEXING_FAILURE_CODES,
+  type VectorIndexingRequest,
+  type VectorIndexingWork,
+} from "@sixb/core/storage"
+import type { SqliteRootOperation } from "./shared"
+
+interface WorkRow {
+  batch_id: string | null
+  request: string
+  status: VectorIndexingWork["status"]
+  available_at: string
+  values_json: string | null
+  error: string | null
+}
+export class SqliteVectorIndexingStorage implements OntologyVectorIndexingStorage {
+  constructor(
+    private readonly db: Database,
+    private readonly run: SqliteRootOperation,
+    private readonly assertSession: (session: MaterializationSession, projectId: string) => void
+  ) {}
+  async schedule(input: Parameters<OntologyVectorIndexingStorage["schedule"]>[0]) {
+    this.assertSession(input.session, input.projectId)
+    for (const ref of input.deleted)
+      this.db
+        .query(
+          `DELETE FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ? AND primary_id = ?`
+        )
+        .run(input.projectId, ref.objectTypeId, ref.primaryId)
+    const insert =
+      this.db.query(`INSERT INTO object_vector_indexing (project_id, id, object_type_id, primary_id, profile, batch_id, request, status, available_at, dispatch_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)
+      ON CONFLICT (project_id, object_type_id, primary_id, profile) DO UPDATE SET id = excluded.id, batch_id = excluded.batch_id, request = excluded.request, status = 'pending', available_at = excluded.available_at, dispatch_at = excluded.dispatch_at, values_json = NULL, error = NULL`)
+    for (const { batchId, ...request } of input.requests)
+      insert.run(
+        input.projectId,
+        request.id,
+        request.ref.objectTypeId,
+        request.ref.primaryId,
+        request.profile,
+        batchId ?? null,
+        JSON.stringify(request),
+        input.availableAt,
+        input.availableAt
+      )
+  }
+  async complete(input: Parameters<OntologyVectorIndexingStorage["complete"]>[0]) {
+    this.assertSession(input.session, input.projectId)
+    const remove = this.db.query(
+      `DELETE FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ? AND primary_id = ? AND profile = ? AND json_extract(request, '$.configuration') = ? AND json_extract(request, '$.sourceFingerprint') = ?`
+    )
+    for (const entry of input.entries) {
+      remove.run(
+        input.projectId,
+        entry.ref.objectTypeId,
+        entry.ref.primaryId,
+        entry.profile,
+        entry.configuration,
+        entry.sourceFingerprint
+      )
+    }
+  }
+
+  async dispatched(input: Parameters<OntologyVectorIndexingStorage["dispatched"]>[0]) {
+    await this.run(() => {
+      const update = this.db.query(
+        `UPDATE object_vector_indexing SET dispatch_at = ? WHERE project_id = ? AND id = ?`
+      )
+      for (const id of input.ids) update.run(input.nextDispatchAt, input.projectId, id)
+    })
+  }
+  async get(input: Parameters<OntologyVectorIndexingStorage["get"]>[0]) {
+    return this.run(() => {
+      const row = this.db
+        .query<WorkRow, [string, string]>(
+          `SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing WHERE project_id = ? AND id = ?`
+        )
+        .get(input.projectId, input.id)
+      return row ? decode(row) : null
+    })
+  }
+  async getBatch(input: Parameters<OntologyVectorIndexingStorage["getBatch"]>[0]) {
+    return this.run(() =>
+      this.db
+        .query<WorkRow, [string, string]>(`
+      SELECT batch_id, request, status, available_at, values_json, error FROM object_vector_indexing
+      WHERE project_id = ? AND batch_id = ? ORDER BY id`)
+        .all(input.projectId, input.batchId)
+        .map(decode)
+    )
+  }
+
+  async getBatchMember(input: Parameters<OntologyVectorIndexingStorage["getBatchMember"]>[0]) {
+    return this.run(() => {
+      const row = this.db
+        .query<WorkRow, [string, string, string, string, string]>(`
+        SELECT batch_id, request, status, available_at, values_json, error
+        FROM object_vector_indexing WHERE project_id = ? AND object_type_id = ?
+        AND primary_id = ? AND profile = ? AND batch_id = ?`)
+        .get(
+          input.projectId,
+          input.ref.objectTypeId,
+          input.ref.primaryId,
+          input.profile,
+          input.batchId
+        )
+      return row ? decode(row) : null
+    })
+  }
+
+  async updateBatch(input: Parameters<OntologyVectorIndexingStorage["updateBatch"]>[0]) {
+    return this.run(() =>
+      this.db.transaction(() => {
+        const status = this.db.query<{ status: string }, [string, string]>(
+          "SELECT status FROM object_vector_indexing WHERE project_id = ? AND id = ?"
+        )
+        if (
+          input.requireAll &&
+          input.updates.some(
+            (update) => status.get(input.projectId, update.id)?.status !== update.expectedStatus
+          )
+        )
+          return false
+        const updateRow = this.db.query(`UPDATE object_vector_indexing
+        SET status = ?, available_at = ?, values_json = COALESCE(?, values_json), error = ?
+        WHERE project_id = ? AND id = ? AND status = ?`)
+        for (const update of input.updates)
+          updateRow.run(
+            update.status,
+            update.availableAt,
+            update.values ? JSON.stringify(update.values) : null,
+            update.error ? JSON.stringify(update.error) : null,
+            input.projectId,
+            update.id,
+            update.expectedStatus
+          )
+        return true
+      })()
+    )
+  }
+
+  async listDue(input: Parameters<OntologyVectorIndexingStorage["listDue"]>[0]) {
+    return this.run(() =>
+      this.db
+        .query<WorkRow, [string, string, number]>(
+          `SELECT batch_id, request, status, available_at, NULL AS values_json, error FROM object_vector_indexing WHERE project_id = ? AND status <> 'failed' AND dispatch_at <= ? ORDER BY dispatch_at, id LIMIT ?`
+        )
+        .all(input.projectId, input.now, input.limit)
+        .map(decode)
+    )
+  }
+  async update(input: Parameters<OntologyVectorIndexingStorage["update"]>[0]) {
+    return this.run(
+      () =>
+        this.db
+          .query(
+            `UPDATE object_vector_indexing SET status = ?, available_at = ?, values_json = COALESCE(?, values_json), error = ? WHERE project_id = ? AND id = ? AND status = ?`
+          )
+          .run(
+            input.status,
+            input.availableAt,
+            input.values ? JSON.stringify(input.values) : null,
+            input.error ? JSON.stringify(input.error) : null,
+            input.projectId,
+            input.id,
+            input.expectedStatus
+          ).changes === 1
+    )
+  }
+  async remove(input: Parameters<OntologyVectorIndexingStorage["remove"]>[0]) {
+    await this.run(() => {
+      this.db
+        .query(`DELETE FROM object_vector_indexing WHERE project_id = ? AND id = ?`)
+        .run(input.projectId, input.id)
+    })
+  }
+}
+function decode(row: WorkRow): VectorIndexingWork {
+  const request = JSON.parse(row.request) as VectorIndexingRequest
+  return {
+    ...request,
+    ...(row.batch_id ? { batchId: row.batch_id } : {}),
+    status: row.status,
+    availableAt: row.available_at,
+    ...(row.values_json ? { values: JSON.parse(row.values_json) as number[] } : {}),
+    ...(row.error
+      ? {
+          error: parseSixbFailure(row.error, VECTOR_INDEXING_FAILURE_CODES),
+        }
+      : {}),
+  }
+}

package/src/ontology-storage/vectors.ts

@@ -0,0 +1,182 @@
+import type { Database } from "bun:sqlite"
+import { MaterializationConflictError } from "@sixb/core/internal/materialization"
+import type {
+  MaterializationSession,
+  ObjectVectorState,
+  OntologyVectorStorage,
+} from "@sixb/core/storage"
+import { encodeSqliteVector } from "../vector-encoding"
+import type { SqliteRootOperation } from "./shared"
+
+interface VectorMetadataRow {
+  readonly profile: string
+  readonly configuration: string
+  readonly source: string
+  readonly source_fingerprint: string
+  readonly last_commit_id: string
+}
+
+export class SqliteOntologyVectorStorage implements OntologyVectorStorage {
+  constructor(
+    private readonly db: Database,
+    private readonly runRootOperation: SqliteRootOperation,
+    private readonly assertSession: (
+      session: MaterializationSession,
+      projectId: string,
+      commitId?: string
+    ) => void
+  ) {}
+
+  async list(
+    input: Parameters<OntologyVectorStorage["list"]>[0]
+  ): Promise<readonly ObjectVectorState[]> {
+    return this.runRootOperation(() => {
+      const rows = this.db
+        .query<VectorMetadataRow, [string, string, string]>(`
+        SELECT profile, configuration, source, source_fingerprint, last_commit_id
+        FROM object_vectors
+        WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
+        ORDER BY profile
+      `)
+        .all(input.projectId, input.ref.objectTypeId, input.ref.primaryId)
+      return rows.map((row) => ({
+        ref: { ...input.ref },
+        profile: row.profile,
+        configuration: row.configuration,
+        source: JSON.parse(row.source) as string[],
+        sourceFingerprint: row.source_fingerprint,
+        lastCommitId: row.last_commit_id,
+      }))
+    })
+  }
+
+  async listBatch(
+    input: Parameters<OntologyVectorStorage["listBatch"]>[0]
+  ): Promise<readonly ObjectVectorState[]> {
+    if (!input.refs.length) return []
+    return this.runRootOperation(() => {
+      // Keep the bounded request set first so SQLite probes the composite object index.
+      const rows = this.db
+        .query<
+          VectorMetadataRow & { object_type_id: string; primary_id: string },
+          [string, string]
+        >(`
+        WITH requested AS (
+          SELECT DISTINCT json_extract(value, '$.objectTypeId') AS object_type_id,
+            json_extract(value, '$.primaryId') AS primary_id FROM json_each(?)
+        )
+        SELECT stored.object_type_id, stored.primary_id, stored.profile, stored.configuration,
+          stored.source, stored.source_fingerprint, stored.last_commit_id
+        FROM requested CROSS JOIN object_vectors AS stored
+        WHERE stored.project_id = ? AND stored.object_type_id = requested.object_type_id
+          AND stored.primary_id = requested.primary_id
+        ORDER BY stored.object_type_id, stored.primary_id, stored.profile
+      `)
+        .all(JSON.stringify(input.refs), input.projectId)
+      return rows.map((row) => ({
+        ref: { objectTypeId: row.object_type_id, primaryId: row.primary_id },
+        profile: row.profile,
+        configuration: row.configuration,
+        source: JSON.parse(row.source) as string[],
+        sourceFingerprint: row.source_fingerprint,
+        lastCommitId: row.last_commit_id,
+      }))
+    })
+  }
+
+  async removeBatch(input: Parameters<OntologyVectorStorage["removeBatch"]>[0]): Promise<void> {
+    this.assertSession(input.session, input.projectId)
+    if (!input.entries.length) return
+    const { changes } = this.db
+      .query(`
+      DELETE FROM object_vectors WHERE rowid IN (
+        SELECT stored.rowid FROM json_each(?) AS requested CROSS JOIN object_vectors AS stored
+        WHERE stored.project_id = ? AND stored.object_type_id = json_extract(requested.value, '$.ref.objectTypeId')
+          AND stored.primary_id = json_extract(requested.value, '$.ref.primaryId')
+          AND stored.profile = json_extract(requested.value, '$.profile')
+          AND stored.last_commit_id = json_extract(requested.value, '$.expectedCommitId')
+      )
+    `)
+      .run(JSON.stringify(input.entries), input.projectId)
+    if (changes !== input.entries.length) throw vectorConflict()
+  }
+
+  async write(input: Parameters<OntologyVectorStorage["write"]>[0]): Promise<void> {
+    const { value } = input
+    this.assertSession(input.session, input.projectId, value.lastCommitId)
+    // The Materializer validates and normalizes float32 values before this boundary.
+    const embedding = encodeSqliteVector(value.values)
+    const changes =
+      input.expectedCommitId === null
+        ? this.db
+            .query(`
+          INSERT INTO object_vectors (
+            project_id, object_type_id, primary_id, profile,
+            configuration, source, source_fingerprint, embedding, last_commit_id
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+          ON CONFLICT (project_id, object_type_id, primary_id, profile) DO NOTHING
+        `)
+            .run(
+              input.projectId,
+              value.ref.objectTypeId,
+              value.ref.primaryId,
+              value.profile,
+              value.configuration,
+              JSON.stringify(value.source),
+              value.sourceFingerprint,
+              embedding,
+              value.lastCommitId
+            ).changes
+        : this.db
+            .query(`
+          UPDATE object_vectors
+          SET configuration = ?, source = ?, source_fingerprint = ?, embedding = ?, last_commit_id = ?
+          WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
+            AND profile = ? AND last_commit_id = ?
+        `)
+            .run(
+              value.configuration,
+              JSON.stringify(value.source),
+              value.sourceFingerprint,
+              embedding,
+              value.lastCommitId,
+              input.projectId,
+              value.ref.objectTypeId,
+              value.ref.primaryId,
+              value.profile,
+              input.expectedCommitId
+            ).changes
+    if (changes !== 1) throw vectorConflict()
+  }
+
+  async writeBatch(input: Parameters<OntologyVectorStorage["writeBatch"]>[0]): Promise<void> {
+    for (const entry of input.entries) {
+      await this.write({ session: input.session, projectId: input.projectId, ...entry })
+    }
+  }
+
+  async remove(input: Parameters<OntologyVectorStorage["remove"]>[0]): Promise<void> {
+    this.assertSession(input.session, input.projectId)
+    const { changes } = this.db
+      .query(`
+      DELETE FROM object_vectors
+      WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
+        AND profile = ? AND last_commit_id = ?
+    `)
+      .run(
+        input.projectId,
+        input.ref.objectTypeId,
+        input.ref.primaryId,
+        input.profile,
+        input.expectedCommitId
+      )
+    if (changes !== 1) throw vectorConflict()
+  }
+}
+
+function vectorConflict(): MaterializationConflictError {
+  return new MaterializationConflictError(
+    "effective-state",
+    "Vector changed since it was prepared."
+  )
+}

package/src/vector-encoding.ts

@@ -0,0 +1,9 @@
+/** Shared little-endian float32 encoding for stored vectors and sqlite-vec query bindings. */
+export function encodeSqliteVector(values: readonly number[]): Uint8Array {
+  const bytes = new Uint8Array(values.length * 4)
+  const view = new DataView(bytes.buffer)
+  for (let index = 0; index < values.length; index++) {
+    view.setFloat32(index * 4, values[index]!, true)
+  }
+  return bytes
+}