@fgv/ts-agent-memory-sqlite-vec 5.1.0-44 → 5.1.0-46

package/src/packlets/sqlite-vec-index/sqliteVecFragmentIndex.ts

@@ -9,6 +9,7 @@ import { Result, captureResult, fail, succeed } from '@fgv/ts-utils';
 import {
   IEdgeTarget,
   IEmbeddedFragment,
+  IFragmentLocator,
   IFragmentVectorIndex,
   IVectorQueryHit,
   MemoryId,
@@ -23,20 +24,49 @@ const DEFAULT_TABLE_NAME: string = 'memory_fragments';
 /** A simple SQL identifier — the only shape allowed for the table name (it is interpolated into DDL). */
 const IDENTIFIER_RE: RegExp = /^[A-Za-z_][A-Za-z0-9_]*$/;
 
+/**
+ * The auxiliary (`+`-prefixed) columns this version of the index writes. A table
+ * created by an earlier version carries a different set; see
+ * {@link SqliteVecFragmentIndex._readExistingDimension} for why that has to be
+ * detected explicitly rather than migrated.
+ */
+const AUXILIARY_COLUMNS: ReadonlyArray<string> = ['start_off', 'end_off', 'fragment_id'];
+
+/**
+ * Matches one `+name` auxiliary-column declaration in a `vec0` `CREATE VIRTUAL TABLE`
+ * statement. Only ever consumed via `String.matchAll`, which iterates a clone rather
+ * than advancing this instance's `lastIndex`, so the shared `/g` regex is reusable.
+ */
+const AUXILIARY_COLUMN_RE: RegExp = /\+\s*([A-Za-z_][A-Za-z0-9_]*)/g;
+
 /**
  * One KNN row as returned by the fragment `vec0` MATCH query. The offset columns are
  * typed `number | bigint` because `better-sqlite3` returns integer columns as
  * `bigint` when a consumer enables its safe-integer mode (`defaultSafeIntegers`);
  * {@link SqliteVecFragmentIndex._toOffset} coerces them to a plain `number` (and
  * fails loudly on an out-of-safe-range value) before they reach the public locator.
+ * All three identity columns are nullable: a fragment stored without a locator has
+ * `NULL` offsets, and one stored without a `fragmentId` has a `NULL` `fragment_id`.
  */
 interface IKnnRow {
   readonly target_key: string;
-  readonly start_off: number | bigint;
-  readonly end_off: number | bigint;
+  // eslint-disable-next-line @rushstack/no-new-null -- SQLite returns NULL (not undefined) for an absent locator offset
+  readonly start_off: number | bigint | null;
+  // eslint-disable-next-line @rushstack/no-new-null -- SQLite returns NULL (not undefined) for an absent locator offset
+  readonly end_off: number | bigint | null;
+  // eslint-disable-next-line @rushstack/no-new-null -- SQLite returns NULL (not undefined) for an absent fragment id
+  readonly fragment_id: string | null;
   readonly distance: number;
 }
 
+/**
+ * The identity fields of a fragment hit, in `IVectorQueryHit` shape: a field the
+ * stored fragment did not carry is *absent*, never present-but-`undefined`, so a hit
+ * for a fragment stored without a `fragmentId` is structurally identical to one this
+ * index produced before `fragment_id` existed.
+ */
+type FragmentIdentity = Pick<IVectorQueryHit, 'locator' | 'fragmentId'>;
+
 /**
  * A persistent, `sqlite-vec`-backed `IFragmentVectorIndex` (from
  * `@fgv/ts-agent-memory`) — the fragment-granular sibling of
@@ -46,12 +76,25 @@ interface IKnnRow {
  * @remarks
  * Where {@link SqliteVecVectorIndex} keys one vector per record on a
  * `target_key` primary key, this index holds **many** vectors per record — one per
- * in-record `[start, end)` span — so it keys the `vec0` table on `target_key` as a
- * **`PARTITION KEY`** (many rows may share it) and stores each fragment's locator
- * offsets in two auxiliary columns (`+start_off`, `+end_off`) that ride alongside
- * the vector and are returned on query but never filtered. A query is a brute-force
- * `vec0` KNN scan across all partitions returning per-fragment hits, each carrying
- * its record `target` and the matched `locator`.
+ * fragment — so it keys the `vec0` table on `target_key` as a **`PARTITION KEY`**
+ * (many rows may share it) and stores each fragment's identity in three auxiliary
+ * columns (`+start_off`, `+end_off`, `+fragment_id`) that ride alongside the vector
+ * and are returned on query but never filtered — in particular `fragment_id` is
+ * stored and returned verbatim, never parsed and never part of the query path. A
+ * query is a brute-force `vec0` KNN scan across all partitions returning per-fragment
+ * hits, each carrying its record `target` plus whichever identity fields the stored
+ * fragment was added with (a fragment must carry at least one).
+ *
+ * **`vec0` schema changes require a drop-and-re-index.** A
+ * `CREATE VIRTUAL TABLE IF NOT EXISTS` is a no-op against an existing table (SQLite
+ * does not compare schemas) and `vec0` has no `ALTER TABLE ADD COLUMN`, so a database written by an
+ * earlier version of this package keeps its old auxiliary columns. `create` detects
+ * that by parsing the stored `CREATE VIRTUAL TABLE` SQL and fails with an actionable
+ * message naming the expected and found columns, rather than letting a widened
+ * `INSERT` surface an opaque `no such column` at statement-prepare time. There are no
+ * in-place migrations: drop the table (or use a fresh `tableName`) and re-index.
+ * Fragment vectors are re-derivable from the records, so this costs embedding time,
+ * never data.
  *
  * Semantics match `InMemoryFragmentCosineIndex` exactly: `addFragments` is
  * whole-record-replace (a single transaction deletes every prior fragment of the
@@ -104,12 +147,15 @@ export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
   /**
    * Family-convention factory. Loads the `sqlite-vec` extension onto the supplied
    * `better-sqlite3` connection and, if the fragment table already exists (a
-   * reopened persistent file), recovers its established dimension so no
-   * re-embedding is needed on open.
+   * reopened persistent file), verifies its auxiliary-column set matches this
+   * version's and recovers its established dimension so no re-embedding is needed on
+   * open.
    *
    * @param params - See {@link ISqliteVecFragmentIndexCreateParams}.
    * @returns `Success` with the index, or `Failure` if the table name is not a
-   * simple identifier or the extension fails to load.
+   * simple identifier, the extension fails to load, or the existing table was
+   * written by a version with a different auxiliary-column set (which requires a
+   * drop-and-re-index — `vec0` cannot be altered in place).
    */
   public static create(params: ISqliteVecFragmentIndexCreateParams): Promise<Result<SqliteVecFragmentIndex>> {
     const table: string = params.tableName ?? DEFAULT_TABLE_NAME;
@@ -146,11 +192,25 @@ export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
       if (fragment.vector.length === 0) {
         return Promise.resolve(fail(`fragment index: cannot add '${key}': empty fragment vector`));
       }
+      // A fragment carrying neither identity cannot be resolved back to anything by a
+      // consumer holding the hit — the same invariant `embeddedFragmentConverter`
+      // enforces at the untyped boundary, re-checked here at the index seam.
+      if (fragment.locator === undefined && fragment.fragmentId === undefined) {
+        return Promise.resolve(
+          fail(
+            `fragment index: cannot add '${key}': fragment requires at least one of 'locator' or 'fragmentId'`
+          )
+        );
+      }
       // Locator offsets are persisted as SQLite integers (bound via BigInt). Reject a
       // non-safe-integer offset up front with a clear message, rather than letting
       // `BigInt(nonInteger)` throw cryptically inside the write transaction OR storing
       // a value the read-side `_toOffset` guard would later reject on every query.
-      if (!Number.isSafeInteger(fragment.locator.start) || !Number.isSafeInteger(fragment.locator.end)) {
+      // An absent locator persists as a NULL offset pair and skips the check.
+      if (
+        fragment.locator !== undefined &&
+        (!Number.isSafeInteger(fragment.locator.start) || !Number.isSafeInteger(fragment.locator.end))
+      ) {
         return Promise.resolve(
           fail(
             `fragment index: cannot add '${key}': locator [${fragment.locator.start}, ${fragment.locator.end}) offsets must be safe integers`
@@ -256,10 +316,7 @@ export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
           hits.push({
             target: SqliteVecFragmentIndex._parseKey(key),
             score: 1 - row.distance,
-            locator: {
-              start: SqliteVecFragmentIndex._toOffset(row.start_off, key),
-              end: SqliteVecFragmentIndex._toOffset(row.end_off, key)
-            }
+            ...SqliteVecFragmentIndex._toIdentity(row, key)
           });
         }
         return hits;
@@ -267,12 +324,16 @@ export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
     );
   }
 
-  /** Create the fragment `vec0` virtual table with the established dimension. */
+  /**
+   * Create the fragment `vec0` virtual table with the established dimension. The
+   * auxiliary columns must stay in sync with `AUXILIARY_COLUMNS`, which
+   * `create` compares against an existing table's stored DDL.
+   */
   private _createTable(dimension: number): void {
     this._db.exec(
       `CREATE VIRTUAL TABLE IF NOT EXISTS "${this._table}" USING vec0(` +
         `target_key TEXT PARTITION KEY, embedding float[${dimension}] distance_metric=cosine, ` +
-        `+start_off integer, +end_off integer)`
+        `+start_off integer, +end_off integer, +fragment_id text)`
     );
   }
 
@@ -282,7 +343,8 @@ export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
       `DELETE FROM "${this._table}" WHERE target_key = ?`
     );
     const ins: BetterSqlite3.Statement = this._db.prepare(
-      `INSERT INTO "${this._table}"(target_key, embedding, start_off, end_off) VALUES (?, ?, ?, ?)`
+      `INSERT INTO "${this._table}"(target_key, embedding, start_off, end_off, fragment_id) ` +
+        `VALUES (?, ?, ?, ?, ?)`
     );
     // Whole-record replace: drop every prior fragment of the target, then insert the
     // new set, atomically. An empty set collapses to a pure delete.
@@ -294,9 +356,13 @@ export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
         ins.run(
           key,
           SqliteVecFragmentIndex._toBlob(fragment.vector),
-          // vec0 typed columns reject a JS float; bind the offsets as integers.
-          BigInt(fragment.locator.start),
-          BigInt(fragment.locator.end)
+          // vec0 typed columns reject a JS float; bind the offsets as integers. An
+          // absent locator binds the pair as NULL — never a partial pair, so the read
+          // side can treat a half-NULL pair as corruption rather than a legal shape.
+          fragment.locator === undefined ? null : BigInt(fragment.locator.start),
+          fragment.locator === undefined ? null : BigInt(fragment.locator.end),
+          // Stored verbatim and never parsed; absent binds as NULL.
+          fragment.fragmentId ?? null
         );
       }
     });
@@ -306,7 +372,8 @@ export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
         replaceTxn(key, fragments);
       },
       query: this._db.prepare(
-        `SELECT target_key, start_off, end_off, distance FROM "${this._table}" WHERE embedding MATCH ? AND k = ?`
+        `SELECT target_key, start_off, end_off, fragment_id, distance FROM "${this._table}" ` +
+          `WHERE embedding MATCH ? AND k = ?`
       ),
       fragmentCount: this._db.prepare(`SELECT count(*) AS c FROM "${this._table}"`),
       recordCount: this._db.prepare(`SELECT count(DISTINCT target_key) AS c FROM "${this._table}"`)
@@ -315,8 +382,15 @@ export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
 
   /**
    * Recover the established dimension of an existing fragment `vec0` table from its
-   * stored `CREATE VIRTUAL TABLE` SQL (`float[<n>]`). Returns `undefined` when the
+   * stored `CREATE VIRTUAL TABLE` SQL (`float[<n>]`), after checking that the table's
+   * auxiliary columns match `AUXILIARY_COLUMNS`. Returns `undefined` when the
    * table does not exist yet (a fresh database — dimension is set by the first add).
+   *
+   * Throws when a table of that name exists but is not a usable fragment index (a
+   * mismatched auxiliary-column set, or no `vec0` embedding column); the caller runs
+   * this inside `captureResult`, so it surfaces as a loud `Failure` from `create`.
+   * The same stored DDL answers every one of those questions, so the checks cost
+   * nothing extra.
    */
   private static _readExistingDimension(db: BetterSqlite3.Database, table: string): number | undefined {
     const row: { sql: string } | undefined = db
@@ -325,8 +399,93 @@ export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
     if (row === undefined) {
       return undefined;
     }
+    SqliteVecFragmentIndex._verifyAuxiliaryColumns(row.sql, table);
     const match: RegExpMatchArray | null = row.sql.match(/float\[(\d+)\]/);
-    return match === null ? undefined : Number(match[1]);
+    if (match === null) {
+      // The auxiliary columns matched but there is no `float[<n>]` embedding column,
+      // so this is not a usable fragment index table. Same remedy as a column
+      // mismatch — and failing here beats handing back a dimensionless index whose
+      // first add would `CREATE VIRTUAL TABLE IF NOT EXISTS` into a no-op.
+      throw new Error(
+        `existing table '${table}' has no vec0 embedding column, so it is not a usable fragment ` +
+          `index table. Drop it (or pass a fresh tableName) and re-add every fragment.`
+      );
+    }
+    return Number(match[1]);
+  }
+
+  /**
+   * Compare an existing table's auxiliary columns against `AUXILIARY_COLUMNS`.
+   *
+   * `CREATE VIRTUAL TABLE IF NOT EXISTS` is a no-op against an existing table (SQLite
+   * never compares schemas) and `vec0` has no `ALTER TABLE ADD COLUMN`, so a table
+   * written by an earlier version of this package silently keeps its old columns and
+   * only fails later — as an opaque `no such column` when the widened `INSERT` is
+   * prepared. Detect it here instead and say what to do about it. Order is not
+   * compared: every statement names its columns explicitly, so only the set matters.
+   */
+  private static _verifyAuxiliaryColumns(sql: string, table: string): void {
+    const found: string[] = Array.from(sql.matchAll(AUXILIARY_COLUMN_RE), (m) => m[1]);
+    const expected: ReadonlyArray<string> = AUXILIARY_COLUMNS;
+    const matches: boolean =
+      found.length === expected.length && expected.every((column) => found.includes(column));
+    if (!matches) {
+      throw new Error(
+        `existing table '${table}' has auxiliary columns [${found.join(', ')}] but this index ` +
+          `requires [${expected.join(', ')}] — it was written by a different version of ` +
+          `@fgv/ts-agent-memory-sqlite-vec, or it is not a fragment index table at all. vec0 virtual ` +
+          `tables cannot be altered in place, so this requires a drop-and-re-index: DROP TABLE ` +
+          `"${table}" (or pass a fresh tableName) and re-add every fragment. Fragment vectors are ` +
+          `re-derivable from the records, so this costs embedding time, never data.`
+      );
+    }
+  }
+
+  /**
+   * Rebuild the identity fields of a hit from a persisted row, omitting each field
+   * the stored fragment did not carry (so a hit is structurally identical to one this
+   * index produced before `fragment_id` existed).
+   *
+   * A row carrying neither identity violates the write-side invariant and could not
+   * be resolved by the caller, so it fails loudly instead of yielding an anonymous
+   * hit.
+   */
+  private static _toIdentity(row: IKnnRow, key: string): FragmentIdentity {
+    const locator: IFragmentLocator | undefined = SqliteVecFragmentIndex._toLocator(row, key);
+    if (locator === undefined && row.fragment_id === null) {
+      throw new Error(
+        `fragment '${key}': row carries neither a locator nor a fragment id (corrupt persisted data)`
+      );
+    }
+    return {
+      ...(locator !== undefined ? { locator } : {}),
+      ...(row.fragment_id !== null ? { fragmentId: row.fragment_id } : {})
+    };
+  }
+
+  /**
+   * Rebuild a fragment's locator from its persisted offsets, or `undefined` when the
+   * fragment was stored without one (both offsets `NULL`).
+   *
+   * The pair is written all-or-nothing, so a half-`NULL` pair can only come from
+   * corrupt / externally-edited data. Throw rather than coerce — `Number(null)` is
+   * `0`, which would silently fabricate a span starting at the top of the body.
+   */
+  private static _toLocator(row: IKnnRow, key: string): IFragmentLocator | undefined {
+    const start: number | bigint | null = row.start_off;
+    const end: number | bigint | null = row.end_off;
+    if (start === null && end === null) {
+      return undefined;
+    }
+    if (start === null || end === null) {
+      throw new Error(
+        `fragment '${key}': locator has only one of its start/end offsets (corrupt persisted data)`
+      );
+    }
+    return {
+      start: SqliteVecFragmentIndex._toOffset(start, key),
+      end: SqliteVecFragmentIndex._toOffset(end, key)
+    };
   }
 
   /** Pack a `Float32Array` as the little-endian byte blob `vec0` stores. Copies, so the caller may reuse its buffer. */

package/src/test/unit/sqliteVecFragmentIndex.test.ts

@@ -6,6 +6,7 @@
 import '@fgv/ts-utils-jest';
 
 import BetterSqlite3 from 'better-sqlite3';
+import { load as loadSqliteVec } from 'sqlite-vec';
 import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
@@ -68,14 +69,77 @@ describe('SqliteVecFragmentIndex', () => {
       expect(await SqliteVecFragmentIndex.create({ database: closed })).toFailWith(/failed to initialize/i);
     });
 
-    test('recovers no dimension from a pre-existing non-vec0 table of the same name', async () => {
+    test('rejects a pre-existing non-vec0 table of the same name rather than adopting it', async () => {
       db.exec('CREATE TABLE memory_fragments (foo TEXT)');
-      expect(await SqliteVecFragmentIndex.create({ database: db })).toSucceedAndSatisfy(
-        (index: SqliteVecFragmentIndex) => {
-          expect(index.recordCount).toBe(0);
-        }
+      expect(await SqliteVecFragmentIndex.create({ database: db })).toFailWith(
+        /auxiliary columns \[\] but this index requires \[start_off, end_off, fragment_id\]/i
       );
     });
+
+    describe('auxiliary-column schema detection', () => {
+      // `CREATE VIRTUAL TABLE IF NOT EXISTS` is a no-op against an existing table and
+      // vec0 has no `ALTER TABLE ADD COLUMN`, so a database written by an earlier
+      // version keeps its narrower column set. Without detection the widened INSERT
+      // surfaces an opaque `no such column: fragment_id` at statement-prepare time.
+      function createLegacyTable(database: BetterSqlite3.Database): void {
+        loadSqliteVec(database);
+        database.exec(
+          'CREATE VIRTUAL TABLE memory_fragments USING vec0(' +
+            'target_key TEXT PARTITION KEY, embedding float[2] distance_metric=cosine, ' +
+            '+start_off integer, +end_off integer)'
+        );
+      }
+
+      test('fails with an actionable message naming expected and found columns', async () => {
+        createLegacyTable(db);
+        expect(await SqliteVecFragmentIndex.create({ database: db })).toFailWith(
+          /auxiliary columns \[start_off, end_off\] but this index requires \[start_off, end_off, fragment_id\]/i
+        );
+      });
+
+      test('states that a drop-and-re-index is required and that no data is lost', async () => {
+        createLegacyTable(db);
+        expect(await SqliteVecFragmentIndex.create({ database: db })).toFailWith(
+          /cannot be altered in place[\s\S]*drop-and-re-index[\s\S]*DROP TABLE "memory_fragments"[\s\S]*costs embedding time, never data/i
+        );
+      });
+
+      test('does not surface the opaque sqlite "no such column" error', async () => {
+        createLegacyTable(db);
+        const created = await SqliteVecFragmentIndex.create({ database: db });
+        expect(created).toFail();
+        expect(created.isFailure() && created.message).not.toMatch(/no such column/i);
+      });
+
+      test('accepts a table whose auxiliary columns match, regardless of declaration order', async () => {
+        loadSqliteVec(db);
+        db.exec(
+          'CREATE VIRTUAL TABLE memory_fragments USING vec0(' +
+            'target_key TEXT PARTITION KEY, embedding float[3] distance_metric=cosine, ' +
+            '+fragment_id text, +end_off integer, +start_off integer)'
+        );
+        expect(await SqliteVecFragmentIndex.create({ database: db })).toSucceed();
+      });
+
+      test('rejects a table that mimics the auxiliary columns but has no vec0 embedding column', async () => {
+        db.exec(
+          'CREATE TABLE memory_fragments ("+start_off" integer, "+end_off" integer, "+fragment_id" text)'
+        );
+        expect(await SqliteVecFragmentIndex.create({ database: db })).toFailWith(/no vec0 embedding column/i);
+      });
+
+      test('rejects a table with the right number of columns but a different one', async () => {
+        loadSqliteVec(db);
+        db.exec(
+          'CREATE VIRTUAL TABLE memory_fragments USING vec0(' +
+            'target_key TEXT PARTITION KEY, embedding float[2] distance_metric=cosine, ' +
+            '+start_off integer, +end_off integer, +frag_id text)'
+        );
+        expect(await SqliteVecFragmentIndex.create({ database: db })).toFailWith(
+          /auxiliary columns \[start_off, end_off, frag_id\]/i
+        );
+      });
+    });
   });
 
   describe('addFragments', () => {
@@ -179,13 +243,11 @@ describe('SqliteVecFragmentIndex', () => {
 
     test('fails loudly (never silently corrupts) when the table name collides with a non-vec0 table', async () => {
       // A plain table already occupies the default name. `CREATE VIRTUAL TABLE IF NOT
-      // EXISTS` no-ops against it, so preparing the vector INSERT against the missing
-      // columns must fail loudly rather than corrupt state.
+      // EXISTS` no-ops against it, so the collision must fail loudly rather than
+      // corrupt state — `create` now catches it up front via the auxiliary-column
+      // check, so the index is never handed out at all.
       db.exec('CREATE TABLE memory_fragments (foo TEXT)');
-      const index = (await SqliteVecFragmentIndex.create({ database: db })).orThrow();
-      expect(await index.addFragments(target('knowledge', 'doc-a'), [frag(0, 5, 1, 0)])).toFailWith(
-        /cannot add 'knowledge\0doc-a'/i
-      );
+      expect(await SqliteVecFragmentIndex.create({ database: db })).toFailWith(/auxiliary columns/i);
     });
   });
 
@@ -366,6 +428,46 @@ describe('SqliteVecFragmentIndex', () => {
       ).run(key, toBlob(...vec), start, end);
     }
 
+    /** Insert a row with arbitrary (possibly NULL) identity columns, bypassing the write-side guards. */
+    function insertRawIdentity(
+      key: string,
+      start: bigint | null,
+      end: bigint | null,
+      fragmentId: string | null,
+      ...vec: number[]
+    ): void {
+      db.prepare(
+        'INSERT INTO memory_fragments(target_key, embedding, start_off, end_off, fragment_id) ' +
+          'VALUES (?, ?, ?, ?, ?)'
+      ).run(key, toBlob(...vec), start, end, fragmentId);
+    }
+
+    test('fails loudly when a stored locator has a start offset but no end offset', async () => {
+      // `Number(null)` is 0, so coercing a half-NULL pair would silently fabricate a
+      // span rather than surface the corruption.
+      const index = await seededWithRow();
+      insertRawIdentity('knowledge\0doc-b', BigInt(3), null, 'frag-1', 0, 1);
+      expect(await index.query(Float32Array.from([0, 1]), 5)).toFailWith(
+        /locator has only one of its start\/end offsets/i
+      );
+    });
+
+    test('fails loudly when a stored locator has an end offset but no start offset', async () => {
+      const index = await seededWithRow();
+      insertRawIdentity('knowledge\0doc-b', null, BigInt(9), 'frag-1', 0, 1);
+      expect(await index.query(Float32Array.from([0, 1]), 5)).toFailWith(
+        /locator has only one of its start\/end offsets/i
+      );
+    });
+
+    test('fails loudly when a stored row carries neither a locator nor a fragment id', async () => {
+      const index = await seededWithRow();
+      insertRawIdentity('knowledge\0doc-b', null, null, null, 0, 1);
+      expect(await index.query(Float32Array.from([0, 1]), 5)).toFailWith(
+        /carries neither a locator nor a fragment id/i
+      );
+    });
+
     test('coerces bigint offsets (better-sqlite3 safe-integer mode) to number locators', async () => {
       const index = await seededWithRow();
       // Under safe-integer mode every integer column comes back as a bigint.
@@ -398,6 +500,129 @@ describe('SqliteVecFragmentIndex', () => {
     });
   });
 
+  describe('fragment identity', () => {
+    /** A fragment identified only by an opaque id — no honest body span. */
+    function idFrag(fragmentId: string, ...values: number[]): IEmbeddedFragment {
+      return { fragmentId, vector: Float32Array.from(values) };
+    }
+
+    test('rejects a fragment carrying neither a locator nor a fragmentId', async () => {
+      const index = await makeIndex();
+      expect(
+        await index.addFragments(target('knowledge', 'doc-a'), [{ vector: Float32Array.from([1, 0]) }])
+      ).toFailWith(/at least one of 'locator' or 'fragmentId'/i);
+      expect(index.fragmentCount).toBe(0);
+    });
+
+    test('carries an opaque fragmentId through to the query hit verbatim', async () => {
+      const index = await makeIndex();
+      const opaque = 'urn:frag:9f8e::{not-parsed}';
+      (await index.addFragments(target('knowledge', 'doc-a'), [idFrag(opaque, 1, 0)])).orThrow();
+      expect(await index.query(Float32Array.from([1, 0]), 1)).toSucceedAndSatisfy(
+        (hits: ReadonlyArray<IVectorQueryHit>) => {
+          expect(hits[0].fragmentId).toBe(opaque);
+          expect(hits[0].locator).toBeUndefined();
+        }
+      );
+    });
+
+    test('carries both identities when a fragment supplies both', async () => {
+      const index = await makeIndex();
+      (
+        await index.addFragments(target('knowledge', 'doc-a'), [
+          { ...frag(2, 8, 1, 0), fragmentId: 'frag-1' }
+        ])
+      ).orThrow();
+      expect(await index.query(Float32Array.from([1, 0]), 1)).toSucceedAndSatisfy(
+        (hits: ReadonlyArray<IVectorQueryHit>) => {
+          expect(hits[0].locator).toEqual(loc(2, 8));
+          expect(hits[0].fragmentId).toBe('frag-1');
+        }
+      );
+    });
+
+    test('a locator-only fragment produces a hit with no fragmentId key at all', async () => {
+      // Byte-identical to what this index produced before `fragment_id` existed: the
+      // key is absent, not present-and-undefined, so an existing caller's structural
+      // comparisons are unaffected by the addition.
+      const index = await makeIndex();
+      (await index.addFragments(target('knowledge', 'doc-a'), [frag(0, 5, 1, 0)])).orThrow();
+      expect(await index.query(Float32Array.from([1, 0]), 1)).toSucceedAndSatisfy(
+        (hits: ReadonlyArray<IVectorQueryHit>) => {
+          expect(Object.keys(hits[0]).sort()).toEqual(['locator', 'score', 'target']);
+          expect(hits[0]).toStrictEqual({
+            target: target('knowledge', 'doc-a'),
+            score: hits[0].score,
+            locator: loc(0, 5)
+          });
+        }
+      );
+    });
+
+    test('an id-only fragment produces a hit with no locator key at all', async () => {
+      const index = await makeIndex();
+      (await index.addFragments(target('knowledge', 'doc-a'), [idFrag('frag-1', 1, 0)])).orThrow();
+      expect(await index.query(Float32Array.from([1, 0]), 1)).toSucceedAndSatisfy(
+        (hits: ReadonlyArray<IVectorQueryHit>) => {
+          expect(Object.keys(hits[0]).sort()).toEqual(['fragmentId', 'score', 'target']);
+        }
+      );
+    });
+
+    test('skips the safe-integer offset check for a fragment with no locator', async () => {
+      // The write-side offset validation must not fire (or throw on `BigInt(undefined)`)
+      // when there is no locator to validate.
+      const index = await makeIndex();
+      expect(await index.addFragments(target('knowledge', 'doc-a'), [idFrag('frag-1', 1, 0)])).toSucceedWith(
+        1
+      );
+    });
+
+    test('still rejects a non-safe-integer offset when a locator IS supplied', async () => {
+      const index = await makeIndex();
+      expect(
+        await index.addFragments(target('knowledge', 'doc-a'), [
+          { locator: loc(0.5, 5), fragmentId: 'frag-1', vector: Float32Array.from([1, 0]) }
+        ])
+      ).toFailWith(/offsets must be safe integers/i);
+    });
+
+    test('round-trips every identity shape across a close + reopen', async () => {
+      const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'svfragid-'));
+      const dbPath = path.join(dir, 'fragments.db');
+      const first = new BetterSqlite3(dbPath);
+      try {
+        const writeIndex = (await SqliteVecFragmentIndex.create({ database: first })).orThrow();
+        (
+          await writeIndex.addFragments(target('knowledge', 'doc-a'), [
+            frag(0, 5, 1, 0),
+            idFrag('frag-rewritten', 0, 1),
+            { ...frag(5, 9, 0.9, 0.1), fragmentId: 'frag-both' }
+          ])
+        ).orThrow();
+      } finally {
+        first.close();
+      }
+
+      const second = new BetterSqlite3(dbPath);
+      try {
+        const reopened = (await SqliteVecFragmentIndex.create({ database: second })).orThrow();
+        expect(reopened.fragmentCount).toBe(3);
+        expect(await reopened.query(Float32Array.from([1, 0]), 3)).toSucceedAndSatisfy(
+          (hits: ReadonlyArray<IVectorQueryHit>) => {
+            const byShape = new Map(hits.map((h) => [h.fragmentId ?? '<none>', h]));
+            expect(byShape.get('<none>')?.locator).toEqual(loc(0, 5));
+            expect(byShape.get('frag-rewritten')?.locator).toBeUndefined();
+            expect(byShape.get('frag-both')?.locator).toEqual(loc(5, 9));
+          }
+        );
+      } finally {
+        second.close();
+        fs.rmSync(dir, { recursive: true, force: true });
+      }
+    });
+  });
+
   describe('custom table name', () => {
     test('two fragment indexes on distinct tables in one database are independent', async () => {
       const a = (await SqliteVecFragmentIndex.create({ database: db, tableName: 'frag_a' })).orThrow();

package/temp/build/lint/_eslint-5eVG3S6w.json

@@ -11,7 +11,7 @@
     ],
     [
       "packlets/sqlite-vec-index/sqliteVecFragmentIndex.ts",
-      "88a8d56b74db20fe3fe2afbd23b47f5b244b53d8c62ea2ad8d824b842eb5e96d_B592XbdjJPfwmjGGtzeNtHae18Y="
+      "033c31d058e5ab9a24ba21bcd70971fe3d757071fd5019e030c9be1313110cd4_B592XbdjJPfwmjGGtzeNtHae18Y="
     ],
     [
       "packlets/sqlite-vec-index/index.ts",
@@ -23,7 +23,7 @@
     ],
     [
       "test/unit/sqliteVecFragmentIndex.test.ts",
-      "9a66a8ef6e5ee615b7efdaf537f36c9ec06161c36a047285da6fba5815cfccfa_tjT+aXFBH4Gc6UgYzDdGTpLJ+SY="
+      "f99104ddbd95b2673f19035335ee47a4117727d60cfb7a7578e2207f001e6a84_tjT+aXFBH4Gc6UgYzDdGTpLJ+SY="
     ],
     [
       "test/unit/sqliteVecVectorIndex.test.ts",