@fgv/ts-agent-memory 5.1.0-45 → 5.1.0-47

package/lib/packlets/vector/vectorIndex.d.ts

@@ -2,10 +2,24 @@ import { Result } from '@fgv/ts-utils';
 import { IEdgeTarget, IMemoryRecord } from '../types';
 /**
  * A half-open `[start, end)` span into a record's body — the in-record locator a
- * {@link IFragmentVectorIndex} carries on each fragment hit. `start` is inclusive,
+ * {@link IFragmentVectorIndex} may carry on a fragment. `start` is inclusive,
  * `end` exclusive. The unit (character / byte / token offsets) is the consumer's
  * choice: the index stores the two integers opaquely and never interprets them,
  * so they line up with whatever locator the consumer's own read side uses.
+ *
+ * @remarks
+ * **The span is advisory.** It names the region of the body a fragment was
+ * *derived from*; it is NOT a slice guaranteed to reproduce the fragment's text.
+ * `body.slice(start, end)` round-trips only under a segmenter that merely chooses
+ * boundaries. Under a **rewriting** segmenter — one that turns a span into a
+ * curated block, an increasingly common ingestion shape when a model both selects
+ * and rewrites — the fragment text is not a substring of the body at all, and the
+ * fragmentation is not re-derivable from the body. Treat the span as a pointer for
+ * locating context, never as an extraction recipe.
+ *
+ * A fragment whose provenance cannot honestly be expressed as a body span should
+ * omit the locator entirely and carry an {@link IEmbeddedFragment.fragmentId}
+ * instead.
  * @public
  */
 export interface IFragmentLocator {
@@ -28,9 +42,27 @@ export interface IFragmentLocator {
  * that share a stem. The caller re-resolves the hit against the record index by
  * the same scoped address.
  *
- * `locator` is present only on hits from a {@link IFragmentVectorIndex} — it
- * identifies WHICH fragment of the record matched. Record-granular
- * {@link IVectorIndex} hits omit it.
+ * **No single field discriminates a fragment hit from a record-granular hit.** A
+ * record hit carries neither `locator` nor `fragmentId`; a fragment hit carries at
+ * least one of the two, but not necessarily any particular one — a fragment with a
+ * body span but no consumer-minted id, and a fragment with an id but no honest span,
+ * are both legal. Testing one field for presence therefore cannot tell you which
+ * kind of hit you hold.
+ *
+ * That "at least one" requirement is enforced on the upsert side by
+ * {@link embeddedFragmentConverter} — a different boundary from this type — and is
+ * deliberately NOT offered here as a discriminator either. A caller keyed off it
+ * would be coupled to an invariant this type does not own, and would fail silently
+ * if the invariant were ever relaxed.
+ *
+ * **The robust rule is that fragment-ness is determined by which index produced the
+ * hit**: {@link IFragmentVectorIndex.query} returns fragment hits and
+ * {@link IVectorIndex.query} returns record hits. The caller chose the index it
+ * queried, so it already knows which kind it is holding.
+ *
+ * Note in particular that an absent `locator` now carries **two** distinct meanings
+ * — a record-granular hit, or a fragment with no honest body span (see
+ * {@link IFragmentLocator}) — which is precisely why presence-branching is unsafe.
  * @public
  */
 export interface IVectorQueryHit {
@@ -38,8 +70,18 @@ export interface IVectorQueryHit {
     readonly target: IEdgeTarget;
     /** Backend similarity score; higher is more similar. */
     readonly score: number;
-    /** The matched fragment's in-record span; present only for fragment-index hits. */
+    /**
+     * The advisory in-record span the matched fragment was derived from, when the
+     * producing fragment carried one. Absent on record-granular hits AND on fragment
+     * hits with no honest span — see the remarks above; do not branch on its presence.
+     */
     readonly locator?: IFragmentLocator;
+    /**
+     * The opaque identity the producing fragment was stored with, carried back
+     * verbatim. Absent on record-granular hits AND on fragment hits stored without
+     * one — see the remarks above; do not branch on its presence.
+     */
+    readonly fragmentId?: string;
 }
 /**
  * The vector-index seam an embedding backend implements to make
@@ -77,31 +119,63 @@ export interface IVectorIndex {
     query(vector: Float32Array, topK: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
 }
 /**
- * One embedded fragment of a record: its in-record {@link IFragmentLocator | span}
- * and the vector for that span. Produced by a {@link FragmentEmbedder} and stored
- * via {@link IFragmentVectorIndex.addFragments}.
+ * One embedded fragment of a record: the fragment's vector, plus at least one of the
+ * two ways to identify it — its advisory in-record {@link IFragmentLocator | span}
+ * and/or an opaque consumer-minted {@link IEmbeddedFragment.fragmentId | fragmentId}.
+ * Produced by a {@link FragmentEmbedder} and stored via
+ * {@link IFragmentVectorIndex.addFragments}.
+ *
+ * @remarks
+ * Both identity fields are optional **in the type**, but the "at least one"
+ * requirement is real — a fragment carrying neither is unidentifiable at the read
+ * side. It is enforced by {@link embeddedFragmentConverter} (and re-checked by the
+ * in-package index implementations) rather than by a conditional-required union
+ * (`{ locator; fragmentId? } | { locator?; fragmentId }`), which was considered and
+ * declined: the union costs at every construction site and buys nothing at the read
+ * site, where each field reads as `… | undefined` either way.
  * @public
  */
 export interface IEmbeddedFragment {
-    /** The fragment's in-record span. */
-    readonly locator: IFragmentLocator;
-    /** The embedding vector for that span. */
+    /**
+     * The region of the record body this fragment was derived from, when one can be
+     * stated honestly. Advisory — see {@link IFragmentLocator}; it is NOT a slice that
+     * reproduces the fragment text. Omit it for a fragment with no honest body span (a
+     * rewriting segmenter), in which case `fragmentId` must be supplied.
+     */
+    readonly locator?: IFragmentLocator;
+    /**
+     * An opaque, consumer-minted identity for this fragment, carried verbatim through
+     * the index and returned on the corresponding {@link IVectorQueryHit}. The index
+     * **never parses it, never filters on it, and never assigns meaning to it** — it is
+     * a bytestring, not part of the query path. It exists so a fragment stays
+     * identifiable when its text is not re-derivable from the record body.
+     *
+     * The guarantee is "we never parse it", NOT "we keep it stable". Because
+     * `addFragments` is whole-record-replace, an updated record re-emits its entire
+     * fragment set, so **any stability of a fragment id across re-embeds is the
+     * consumer's responsibility**, not the index's.
+     */
+    readonly fragmentId?: string;
+    /** The embedding vector for this fragment. */
     readonly vector: Float32Array;
 }
 /**
  * The fragment-granular sibling of {@link IVectorIndex}: instead of one vector per
- * record it holds many vectors per record, each tagged with an in-record
- * {@link IFragmentLocator}, and its `query` returns per-fragment hits carrying that
- * locator. This is the seam behind sub-document semantic search — the "discovery"
- * half of a search-then-read contract, where a hit's `(target, locator)` tells the
- * consumer which record AND which span to read.
+ * record it holds many vectors per record, each tagged with the identity its
+ * {@link IEmbeddedFragment} carried, and its `query` returns per-fragment hits
+ * carrying that identity back. This is the seam behind sub-document semantic search
+ * — the "discovery" half of a search-then-read contract, where a hit tells the
+ * consumer which record AND which fragment of it to read.
  *
  * @remarks
- * Deliberately NOT `extends IVectorIndex`: an index keyed by `(target, locator)`
- * has no well-defined single-vector `add(target, vector)`. It is a parallel
- * contract with three operations — `addFragments`, `remove`, `query` — reusing
- * {@link IVectorQueryHit} (whose `locator` is always populated here). Kept distinct
- * from the record-granular index per the consumer contract: memory recall stays
+ * Deliberately NOT `extends IVectorIndex`: an index holding many vectors per record
+ * has no well-defined single-vector `add(target, vector)`. It is a parallel contract
+ * with three operations — `addFragments`, `remove`, `query` — reusing
+ * {@link IVectorQueryHit}, on which both `locator` and `fragmentId` are optional. A
+ * fragment hit populates whichever of the two its stored fragment carried; see
+ * {@link IVectorQueryHit} for why that is not a discriminator and why fragment-ness
+ * is determined by the index queried, not by field presence. Kept distinct from the
+ * record-granular index per the consumer contract: memory recall stays
  * record-granular; sub-document knowledge uses a separate fragment index.
  * @public
  */
@@ -120,7 +194,8 @@ export interface IFragmentVectorIndex {
     remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
     /**
      * Return the `topK` nearest fragments to `vector`, in descending score order,
-     * each hit carrying its record `target` and fragment `locator`. When
+     * each hit carrying its record `target` plus whichever of `locator` /
+     * `fragmentId` the stored fragment was added with. When
      * `maxPerRecord` is supplied, no more than that many fragments of any single
      * record appear in the result — the cap is applied during selection (before the
      * `topK` cut) so one long document cannot crowd out others.

package/lib/packlets/vector/vectorIndex.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vectorIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/vector/vectorIndex.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEtD;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAC/B,mDAAmD;IACnD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,iDAAiD;IACjD,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,eAAe;IAC9B,yDAAyD;IACzD,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,wDAAwD;IACxD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,mFAAmF;IACnF,QAAQ,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAExE;;;;OAIG;IACH,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;IAE1D;;OAEG;IACH,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;CAC5F;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IAChC,qCAAqC;IACrC,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;IACnC,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;OAKG;IACH,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAExG;;;OAGG;IACH,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;IAE1D;;;;;;OAMG;IACH,KAAK,CACH,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;CACpD;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;AAE/F;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAC7B,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,KAC3B,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;AAEvD;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IAClC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,iDAAiD;IACjD,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC;CACzC;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,mBAAmB;IAClC,2EAA2E;IAC3E,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAC7D"}
+{"version":3,"file":"vectorIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/vector/vectorIndex.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEtD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,mDAAmD;IACnD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,iDAAiD;IACjD,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAM,WAAW,eAAe;IAC9B,yDAAyD;IACzD,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,wDAAwD;IACxD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC;IACpC;;;;OAIG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAExE;;;;OAIG;IACH,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;IAE1D;;OAEG;IACH,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;CAC5F;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,iBAAiB;IAChC;;;;;OAKG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC;IACpC;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,8CAA8C;IAC9C,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;OAKG;IACH,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAExG;;;OAGG;IACH,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;IAE1D;;;;;;;OAOG;IACH,KAAK,CACH,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;CACpD;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;AAE/F;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAC7B,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,KAC3B,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;AAEvD;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IAClC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,iDAAiD;IACjD,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC;CACzC;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,mBAAmB;IAClC,2EAA2E;IAC3E,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAC7D"}

package/lib/packlets/vector/vectorIndex.js.map

@@ -1 +1 @@
-{"version":3,"file":"vectorIndex.js","sourceRoot":"","sources":["../../../src/packlets/vector/vectorIndex.ts"],"names":[],"mappings":";AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result } from '@fgv/ts-utils';\nimport { IEdgeTarget, IMemoryRecord } from '../types';\n\n/**\n * A half-open `[start, end)` span into a record's body — the in-record locator a\n * {@link IFragmentVectorIndex} carries on each fragment hit. `start` is inclusive,\n * `end` exclusive. The unit (character / byte / token offsets) is the consumer's\n * choice: the index stores the two integers opaquely and never interprets them,\n * so they line up with whatever locator the consumer's own read side uses.\n * @public\n */\nexport interface IFragmentLocator {\n  /** Inclusive start offset into the record body. */\n  readonly start: number;\n  /** Exclusive end offset into the record body. */\n  readonly end: number;\n}\n\n/**\n * A single hit returned by {@link IVectorIndex.query} (or\n * {@link IFragmentVectorIndex.query}): the matched record's scope-qualified\n * {@link IEdgeTarget | address} and the backend's similarity score (higher = more\n * similar; the exact scale is backend-defined). Hits are returned in descending\n * score order.\n *\n * @remarks\n * The address is a `(scope, id)` pair, NOT a bare {@link MemoryId} — per-scope\n * codecs (e.g. the medium-term codec's `turn-<n>` stems) legally mint the same\n * stem under different scopes, so a bare id could not disambiguate two records\n * that share a stem. The caller re-resolves the hit against the record index by\n * the same scoped address.\n *\n * `locator` is present only on hits from a {@link IFragmentVectorIndex} — it\n * identifies WHICH fragment of the record matched. Record-granular\n * {@link IVectorIndex} hits omit it.\n * @public\n */\nexport interface IVectorQueryHit {\n  /** The scope-qualified address of the matched record. */\n  readonly target: IEdgeTarget;\n  /** Backend similarity score; higher is more similar. */\n  readonly score: number;\n  /** The matched fragment's in-record span; present only for fragment-index hits. */\n  readonly locator?: IFragmentLocator;\n}\n\n/**\n * The vector-index seam an embedding backend implements to make\n * {@link SemanticRetriever | semantic recall} operational.\n *\n * @remarks\n * Vectors cross this seam as `Float32Array` (the in-memory representation an\n * embedding model produces); `number[]` is reserved for the JSON-wire edges\n * (e.g. a provider's embedding response). The in-package brute-force cosine\n * implementation is {@link InMemoryCosineIndex}; a consumer can swap an external\n * ANN backend behind the same seam once N grows beyond the in-memory regime.\n *\n * Every operation returns a `Result` (async, since a real backend does I/O) so\n * failure is explicit and never throws across the seam.\n * @public\n */\nexport interface IVectorIndex {\n  /**\n   * Add (or replace) the embedding for the scope-qualified `target`. Returns the\n   * opaque {@link IMemoryEnvelope.embeddingRef | embeddingRef} the store stamps\n   * onto the envelope so a later read knows the record is embedded. Keying on the\n   * `(scope, id)` address (not a bare id) is load-bearing: two records that share\n   * a filename stem across scopes must not clobber each other's embedding.\n   */\n  add(target: IEdgeTarget, vector: Float32Array): Promise<Result<string>>;\n\n  /**\n   * Remove the embedding for the scope-qualified `target`. Returns the removed\n   * target. Idempotent — removing a target with no embedding still succeeds\n   * (returns the target).\n   */\n  remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;\n\n  /**\n   * Return the `topK` nearest records to `vector`, in descending score order.\n   */\n  query(vector: Float32Array, topK: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;\n}\n\n/**\n * One embedded fragment of a record: its in-record {@link IFragmentLocator | span}\n * and the vector for that span. Produced by a {@link FragmentEmbedder} and stored\n * via {@link IFragmentVectorIndex.addFragments}.\n * @public\n */\nexport interface IEmbeddedFragment {\n  /** The fragment's in-record span. */\n  readonly locator: IFragmentLocator;\n  /** The embedding vector for that span. */\n  readonly vector: Float32Array;\n}\n\n/**\n * The fragment-granular sibling of {@link IVectorIndex}: instead of one vector per\n * record it holds many vectors per record, each tagged with an in-record\n * {@link IFragmentLocator}, and its `query` returns per-fragment hits carrying that\n * locator. This is the seam behind sub-document semantic search — the \"discovery\"\n * half of a search-then-read contract, where a hit's `(target, locator)` tells the\n * consumer which record AND which span to read.\n *\n * @remarks\n * Deliberately NOT `extends IVectorIndex`: an index keyed by `(target, locator)`\n * has no well-defined single-vector `add(target, vector)`. It is a parallel\n * contract with three operations — `addFragments`, `remove`, `query` — reusing\n * {@link IVectorQueryHit} (whose `locator` is always populated here). Kept distinct\n * from the record-granular index per the consumer contract: memory recall stays\n * record-granular; sub-document knowledge uses a separate fragment index.\n * @public\n */\nexport interface IFragmentVectorIndex {\n  /**\n   * Add (or replace) all fragments for the scope-qualified `target`. Whole-record\n   * semantics: every fragment previously held for `target` is dropped and replaced\n   * by `fragments`, so a re-authored document never leaves stale fragments behind.\n   * Returns the number of fragments now held for the record.\n   */\n  addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;\n\n  /**\n   * Remove every fragment for the scope-qualified `target`. Returns the removed\n   * target. Idempotent — removing a target with no fragments still succeeds.\n   */\n  remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;\n\n  /**\n   * Return the `topK` nearest fragments to `vector`, in descending score order,\n   * each hit carrying its record `target` and fragment `locator`. When\n   * `maxPerRecord` is supplied, no more than that many fragments of any single\n   * record appear in the result — the cap is applied during selection (before the\n   * `topK` cut) so one long document cannot crowd out others.\n   */\n  query(\n    vector: Float32Array,\n    topK: number,\n    maxPerRecord?: number\n  ): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;\n}\n\n/**\n * Embeds a complete record into a vector for the store's embed-on-write hook.\n * Async and `Result`-returning, since a real embedder does a network call (cloud\n * provider) or in-process model inference. The consumer wires this — the core\n * package never calls an embedding provider directly, staying embedder-agnostic.\n * @public\n */\nexport type MemoryEmbedder = (record: IMemoryRecord<unknown>) => Promise<Result<Float32Array>>;\n\n/**\n * The fragment-granular sibling of {@link MemoryEmbedder}: chunks a record's body\n * and embeds each chunk, returning one {@link IEmbeddedFragment} per chunk. The\n * chunking policy (window size, overlap) lives entirely in the consumer's embedder\n * — the core stays chunking-agnostic, exactly as it stays embedder-agnostic for\n * the record-granular path. Used by the store's fragment-embed-on-write hook.\n * @public\n */\nexport type FragmentEmbedder = (\n  record: IMemoryRecord<unknown>\n) => Promise<Result<ReadonlyArray<IEmbeddedFragment>>>;\n\n/**\n * A record paired with its scope-qualified {@link IEdgeTarget | address}, as\n * yielded by {@link IMemoryRecordSource.list}. The address is required because\n * {@link InMemoryCosineIndex.rebuild} keys each re-embedded entry on the\n * scope-qualified target, not a bare {@link MemoryId} — two records that share a\n * filename stem across scopes must not collide when the whole vault is re-indexed.\n * @public\n */\nexport interface IScopedMemoryRecord {\n  /** The record's scope-qualified `(scope, id)` address. */\n  readonly target: IEdgeTarget;\n  /** The record itself, passed to the embedder. */\n  readonly record: IMemoryRecord<unknown>;\n}\n\n/**\n * The minimal record-source surface {@link InMemoryCosineIndex.rebuild} reads to\n * re-embed an entire vault. Each entry carries the record's scope-qualified\n * address (see {@link IScopedMemoryRecord}) so the rebuild keys the vector index\n * exactly as the incremental embed-on-write path does. A consumer backs this with\n * the store's scoped index — the vector packlet does not import the store packlet\n * (which depends on the vector packlet for {@link IVectorIndex}, so the reverse\n * import would be a cycle).\n * @public\n */\nexport interface IMemoryRecordSource {\n  /** List every record in the vault, each paired with its scoped address. */\n  list(): Promise<Result<ReadonlyArray<IScopedMemoryRecord>>>;\n}\n"]}
+{"version":3,"file":"vectorIndex.js","sourceRoot":"","sources":["../../../src/packlets/vector/vectorIndex.ts"],"names":[],"mappings":";AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result } from '@fgv/ts-utils';\nimport { IEdgeTarget, IMemoryRecord } from '../types';\n\n/**\n * A half-open `[start, end)` span into a record's body — the in-record locator a\n * {@link IFragmentVectorIndex} may carry on a fragment. `start` is inclusive,\n * `end` exclusive. The unit (character / byte / token offsets) is the consumer's\n * choice: the index stores the two integers opaquely and never interprets them,\n * so they line up with whatever locator the consumer's own read side uses.\n *\n * @remarks\n * **The span is advisory.** It names the region of the body a fragment was\n * *derived from*; it is NOT a slice guaranteed to reproduce the fragment's text.\n * `body.slice(start, end)` round-trips only under a segmenter that merely chooses\n * boundaries. Under a **rewriting** segmenter — one that turns a span into a\n * curated block, an increasingly common ingestion shape when a model both selects\n * and rewrites — the fragment text is not a substring of the body at all, and the\n * fragmentation is not re-derivable from the body. Treat the span as a pointer for\n * locating context, never as an extraction recipe.\n *\n * A fragment whose provenance cannot honestly be expressed as a body span should\n * omit the locator entirely and carry an {@link IEmbeddedFragment.fragmentId}\n * instead.\n * @public\n */\nexport interface IFragmentLocator {\n  /** Inclusive start offset into the record body. */\n  readonly start: number;\n  /** Exclusive end offset into the record body. */\n  readonly end: number;\n}\n\n/**\n * A single hit returned by {@link IVectorIndex.query} (or\n * {@link IFragmentVectorIndex.query}): the matched record's scope-qualified\n * {@link IEdgeTarget | address} and the backend's similarity score (higher = more\n * similar; the exact scale is backend-defined). Hits are returned in descending\n * score order.\n *\n * @remarks\n * The address is a `(scope, id)` pair, NOT a bare {@link MemoryId} — per-scope\n * codecs (e.g. the medium-term codec's `turn-<n>` stems) legally mint the same\n * stem under different scopes, so a bare id could not disambiguate two records\n * that share a stem. The caller re-resolves the hit against the record index by\n * the same scoped address.\n *\n * **No single field discriminates a fragment hit from a record-granular hit.** A\n * record hit carries neither `locator` nor `fragmentId`; a fragment hit carries at\n * least one of the two, but not necessarily any particular one — a fragment with a\n * body span but no consumer-minted id, and a fragment with an id but no honest span,\n * are both legal. Testing one field for presence therefore cannot tell you which\n * kind of hit you hold.\n *\n * That \"at least one\" requirement is enforced on the upsert side by\n * {@link embeddedFragmentConverter} — a different boundary from this type — and is\n * deliberately NOT offered here as a discriminator either. A caller keyed off it\n * would be coupled to an invariant this type does not own, and would fail silently\n * if the invariant were ever relaxed.\n *\n * **The robust rule is that fragment-ness is determined by which index produced the\n * hit**: {@link IFragmentVectorIndex.query} returns fragment hits and\n * {@link IVectorIndex.query} returns record hits. The caller chose the index it\n * queried, so it already knows which kind it is holding.\n *\n * Note in particular that an absent `locator` now carries **two** distinct meanings\n * — a record-granular hit, or a fragment with no honest body span (see\n * {@link IFragmentLocator}) — which is precisely why presence-branching is unsafe.\n * @public\n */\nexport interface IVectorQueryHit {\n  /** The scope-qualified address of the matched record. */\n  readonly target: IEdgeTarget;\n  /** Backend similarity score; higher is more similar. */\n  readonly score: number;\n  /**\n   * The advisory in-record span the matched fragment was derived from, when the\n   * producing fragment carried one. Absent on record-granular hits AND on fragment\n   * hits with no honest span — see the remarks above; do not branch on its presence.\n   */\n  readonly locator?: IFragmentLocator;\n  /**\n   * The opaque identity the producing fragment was stored with, carried back\n   * verbatim. Absent on record-granular hits AND on fragment hits stored without\n   * one — see the remarks above; do not branch on its presence.\n   */\n  readonly fragmentId?: string;\n}\n\n/**\n * The vector-index seam an embedding backend implements to make\n * {@link SemanticRetriever | semantic recall} operational.\n *\n * @remarks\n * Vectors cross this seam as `Float32Array` (the in-memory representation an\n * embedding model produces); `number[]` is reserved for the JSON-wire edges\n * (e.g. a provider's embedding response). The in-package brute-force cosine\n * implementation is {@link InMemoryCosineIndex}; a consumer can swap an external\n * ANN backend behind the same seam once N grows beyond the in-memory regime.\n *\n * Every operation returns a `Result` (async, since a real backend does I/O) so\n * failure is explicit and never throws across the seam.\n * @public\n */\nexport interface IVectorIndex {\n  /**\n   * Add (or replace) the embedding for the scope-qualified `target`. Returns the\n   * opaque {@link IMemoryEnvelope.embeddingRef | embeddingRef} the store stamps\n   * onto the envelope so a later read knows the record is embedded. Keying on the\n   * `(scope, id)` address (not a bare id) is load-bearing: two records that share\n   * a filename stem across scopes must not clobber each other's embedding.\n   */\n  add(target: IEdgeTarget, vector: Float32Array): Promise<Result<string>>;\n\n  /**\n   * Remove the embedding for the scope-qualified `target`. Returns the removed\n   * target. Idempotent — removing a target with no embedding still succeeds\n   * (returns the target).\n   */\n  remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;\n\n  /**\n   * Return the `topK` nearest records to `vector`, in descending score order.\n   */\n  query(vector: Float32Array, topK: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;\n}\n\n/**\n * One embedded fragment of a record: the fragment's vector, plus at least one of the\n * two ways to identify it — its advisory in-record {@link IFragmentLocator | span}\n * and/or an opaque consumer-minted {@link IEmbeddedFragment.fragmentId | fragmentId}.\n * Produced by a {@link FragmentEmbedder} and stored via\n * {@link IFragmentVectorIndex.addFragments}.\n *\n * @remarks\n * Both identity fields are optional **in the type**, but the \"at least one\"\n * requirement is real — a fragment carrying neither is unidentifiable at the read\n * side. It is enforced by {@link embeddedFragmentConverter} (and re-checked by the\n * in-package index implementations) rather than by a conditional-required union\n * (`{ locator; fragmentId? } | { locator?; fragmentId }`), which was considered and\n * declined: the union costs at every construction site and buys nothing at the read\n * site, where each field reads as `… | undefined` either way.\n * @public\n */\nexport interface IEmbeddedFragment {\n  /**\n   * The region of the record body this fragment was derived from, when one can be\n   * stated honestly. Advisory — see {@link IFragmentLocator}; it is NOT a slice that\n   * reproduces the fragment text. Omit it for a fragment with no honest body span (a\n   * rewriting segmenter), in which case `fragmentId` must be supplied.\n   */\n  readonly locator?: IFragmentLocator;\n  /**\n   * An opaque, consumer-minted identity for this fragment, carried verbatim through\n   * the index and returned on the corresponding {@link IVectorQueryHit}. The index\n   * **never parses it, never filters on it, and never assigns meaning to it** — it is\n   * a bytestring, not part of the query path. It exists so a fragment stays\n   * identifiable when its text is not re-derivable from the record body.\n   *\n   * The guarantee is \"we never parse it\", NOT \"we keep it stable\". Because\n   * `addFragments` is whole-record-replace, an updated record re-emits its entire\n   * fragment set, so **any stability of a fragment id across re-embeds is the\n   * consumer's responsibility**, not the index's.\n   */\n  readonly fragmentId?: string;\n  /** The embedding vector for this fragment. */\n  readonly vector: Float32Array;\n}\n\n/**\n * The fragment-granular sibling of {@link IVectorIndex}: instead of one vector per\n * record it holds many vectors per record, each tagged with the identity its\n * {@link IEmbeddedFragment} carried, and its `query` returns per-fragment hits\n * carrying that identity back. This is the seam behind sub-document semantic search\n * — the \"discovery\" half of a search-then-read contract, where a hit tells the\n * consumer which record AND which fragment of it to read.\n *\n * @remarks\n * Deliberately NOT `extends IVectorIndex`: an index holding many vectors per record\n * has no well-defined single-vector `add(target, vector)`. It is a parallel contract\n * with three operations — `addFragments`, `remove`, `query` — reusing\n * {@link IVectorQueryHit}, on which both `locator` and `fragmentId` are optional. A\n * fragment hit populates whichever of the two its stored fragment carried; see\n * {@link IVectorQueryHit} for why that is not a discriminator and why fragment-ness\n * is determined by the index queried, not by field presence. Kept distinct from the\n * record-granular index per the consumer contract: memory recall stays\n * record-granular; sub-document knowledge uses a separate fragment index.\n * @public\n */\nexport interface IFragmentVectorIndex {\n  /**\n   * Add (or replace) all fragments for the scope-qualified `target`. Whole-record\n   * semantics: every fragment previously held for `target` is dropped and replaced\n   * by `fragments`, so a re-authored document never leaves stale fragments behind.\n   * Returns the number of fragments now held for the record.\n   */\n  addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;\n\n  /**\n   * Remove every fragment for the scope-qualified `target`. Returns the removed\n   * target. Idempotent — removing a target with no fragments still succeeds.\n   */\n  remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;\n\n  /**\n   * Return the `topK` nearest fragments to `vector`, in descending score order,\n   * each hit carrying its record `target` plus whichever of `locator` /\n   * `fragmentId` the stored fragment was added with. When\n   * `maxPerRecord` is supplied, no more than that many fragments of any single\n   * record appear in the result — the cap is applied during selection (before the\n   * `topK` cut) so one long document cannot crowd out others.\n   */\n  query(\n    vector: Float32Array,\n    topK: number,\n    maxPerRecord?: number\n  ): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;\n}\n\n/**\n * Embeds a complete record into a vector for the store's embed-on-write hook.\n * Async and `Result`-returning, since a real embedder does a network call (cloud\n * provider) or in-process model inference. The consumer wires this — the core\n * package never calls an embedding provider directly, staying embedder-agnostic.\n * @public\n */\nexport type MemoryEmbedder = (record: IMemoryRecord<unknown>) => Promise<Result<Float32Array>>;\n\n/**\n * The fragment-granular sibling of {@link MemoryEmbedder}: chunks a record's body\n * and embeds each chunk, returning one {@link IEmbeddedFragment} per chunk. The\n * chunking policy (window size, overlap) lives entirely in the consumer's embedder\n * — the core stays chunking-agnostic, exactly as it stays embedder-agnostic for\n * the record-granular path. Used by the store's fragment-embed-on-write hook.\n * @public\n */\nexport type FragmentEmbedder = (\n  record: IMemoryRecord<unknown>\n) => Promise<Result<ReadonlyArray<IEmbeddedFragment>>>;\n\n/**\n * A record paired with its scope-qualified {@link IEdgeTarget | address}, as\n * yielded by {@link IMemoryRecordSource.list}. The address is required because\n * {@link InMemoryCosineIndex.rebuild} keys each re-embedded entry on the\n * scope-qualified target, not a bare {@link MemoryId} — two records that share a\n * filename stem across scopes must not collide when the whole vault is re-indexed.\n * @public\n */\nexport interface IScopedMemoryRecord {\n  /** The record's scope-qualified `(scope, id)` address. */\n  readonly target: IEdgeTarget;\n  /** The record itself, passed to the embedder. */\n  readonly record: IMemoryRecord<unknown>;\n}\n\n/**\n * The minimal record-source surface {@link InMemoryCosineIndex.rebuild} reads to\n * re-embed an entire vault. Each entry carries the record's scope-qualified\n * address (see {@link IScopedMemoryRecord}) so the rebuild keys the vector index\n * exactly as the incremental embed-on-write path does. A consumer backs this with\n * the store's scoped index — the vector packlet does not import the store packlet\n * (which depends on the vector packlet for {@link IVectorIndex}, so the reverse\n * import would be a cycle).\n * @public\n */\nexport interface IMemoryRecordSource {\n  /** List every record in the vault, each paired with its scoped address. */\n  list(): Promise<Result<ReadonlyArray<IScopedMemoryRecord>>>;\n}\n"]}

package/lib/test/unit/ingest/dedupScope.test.d.ts

@@ -0,0 +1,2 @@
+import '@fgv/ts-utils-jest';
+//# sourceMappingURL=dedupScope.test.d.ts.map

package/lib/test/unit/ingest/dedupScope.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedupScope.test.d.ts","sourceRoot":"","sources":["../../../../src/test/unit/ingest/dedupScope.test.ts"],"names":[],"mappings":"AAKA,OAAO,oBAAoB,CAAC"}