@fgv/ts-agent-memory 5.1.0-46 → 5.1.0-48

package/src/packlets/vector/vectorIndex.ts

@@ -1,273 +0,0 @@
-/*
- * Copyright (c) 2026 Erik Fortune
- * SPDX-License-Identifier: MIT
- */
-
-import { Result } from '@fgv/ts-utils';
-import { IEdgeTarget, IMemoryRecord } from '../types';
-
-/**
- * A half-open `[start, end)` span into a record's body — the in-record locator a
- * {@link IFragmentVectorIndex} may carry on a fragment. `start` is inclusive,
- * `end` exclusive. The unit (character / byte / token offsets) is the consumer's
- * choice: the index stores the two integers opaquely and never interprets them,
- * so they line up with whatever locator the consumer's own read side uses.
- *
- * @remarks
- * **The span is advisory.** It names the region of the body a fragment was
- * *derived from*; it is NOT a slice guaranteed to reproduce the fragment's text.
- * `body.slice(start, end)` round-trips only under a segmenter that merely chooses
- * boundaries. Under a **rewriting** segmenter — one that turns a span into a
- * curated block, an increasingly common ingestion shape when a model both selects
- * and rewrites — the fragment text is not a substring of the body at all, and the
- * fragmentation is not re-derivable from the body. Treat the span as a pointer for
- * locating context, never as an extraction recipe.
- *
- * A fragment whose provenance cannot honestly be expressed as a body span should
- * omit the locator entirely and carry an {@link IEmbeddedFragment.fragmentId}
- * instead.
- * @public
- */
-export interface IFragmentLocator {
-  /** Inclusive start offset into the record body. */
-  readonly start: number;
-  /** Exclusive end offset into the record body. */
-  readonly end: number;
-}
-
-/**
- * A single hit returned by {@link IVectorIndex.query} (or
- * {@link IFragmentVectorIndex.query}): the matched record's scope-qualified
- * {@link IEdgeTarget | address} and the backend's similarity score (higher = more
- * similar; the exact scale is backend-defined). Hits are returned in descending
- * score order.
- *
- * @remarks
- * The address is a `(scope, id)` pair, NOT a bare {@link MemoryId} — per-scope
- * codecs (e.g. the medium-term codec's `turn-<n>` stems) legally mint the same
- * stem under different scopes, so a bare id could not disambiguate two records
- * that share a stem. The caller re-resolves the hit against the record index by
- * the same scoped address.
- *
- * **No single field discriminates a fragment hit from a record-granular hit.** A
- * record hit carries neither `locator` nor `fragmentId`; a fragment hit carries at
- * least one of the two, but not necessarily any particular one — a fragment with a
- * body span but no consumer-minted id, and a fragment with an id but no honest span,
- * are both legal. Testing one field for presence therefore cannot tell you which
- * kind of hit you hold.
- *
- * That "at least one" requirement is enforced on the upsert side by
- * {@link embeddedFragmentConverter} — a different boundary from this type — and is
- * deliberately NOT offered here as a discriminator either. A caller keyed off it
- * would be coupled to an invariant this type does not own, and would fail silently
- * if the invariant were ever relaxed.
- *
- * **The robust rule is that fragment-ness is determined by which index produced the
- * hit**: {@link IFragmentVectorIndex.query} returns fragment hits and
- * {@link IVectorIndex.query} returns record hits. The caller chose the index it
- * queried, so it already knows which kind it is holding.
- *
- * Note in particular that an absent `locator` now carries **two** distinct meanings
- * — a record-granular hit, or a fragment with no honest body span (see
- * {@link IFragmentLocator}) — which is precisely why presence-branching is unsafe.
- * @public
- */
-export interface IVectorQueryHit {
-  /** The scope-qualified address of the matched record. */
-  readonly target: IEdgeTarget;
-  /** Backend similarity score; higher is more similar. */
-  readonly score: number;
-  /**
-   * The advisory in-record span the matched fragment was derived from, when the
-   * producing fragment carried one. Absent on record-granular hits AND on fragment
-   * hits with no honest span — see the remarks above; do not branch on its presence.
-   */
-  readonly locator?: IFragmentLocator;
-  /**
-   * The opaque identity the producing fragment was stored with, carried back
-   * verbatim. Absent on record-granular hits AND on fragment hits stored without
-   * one — see the remarks above; do not branch on its presence.
-   */
-  readonly fragmentId?: string;
-}
-
-/**
- * The vector-index seam an embedding backend implements to make
- * {@link SemanticRetriever | semantic recall} operational.
- *
- * @remarks
- * Vectors cross this seam as `Float32Array` (the in-memory representation an
- * embedding model produces); `number[]` is reserved for the JSON-wire edges
- * (e.g. a provider's embedding response). The in-package brute-force cosine
- * implementation is {@link InMemoryCosineIndex}; a consumer can swap an external
- * ANN backend behind the same seam once N grows beyond the in-memory regime.
- *
- * Every operation returns a `Result` (async, since a real backend does I/O) so
- * failure is explicit and never throws across the seam.
- * @public
- */
-export interface IVectorIndex {
-  /**
-   * Add (or replace) the embedding for the scope-qualified `target`. Returns the
-   * opaque {@link IMemoryEnvelope.embeddingRef | embeddingRef} the store stamps
-   * onto the envelope so a later read knows the record is embedded. Keying on the
-   * `(scope, id)` address (not a bare id) is load-bearing: two records that share
-   * a filename stem across scopes must not clobber each other's embedding.
-   */
-  add(target: IEdgeTarget, vector: Float32Array): Promise<Result<string>>;
-
-  /**
-   * Remove the embedding for the scope-qualified `target`. Returns the removed
-   * target. Idempotent — removing a target with no embedding still succeeds
-   * (returns the target).
-   */
-  remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
-
-  /**
-   * Return the `topK` nearest records to `vector`, in descending score order.
-   */
-  query(vector: Float32Array, topK: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
-}
-
-/**
- * One embedded fragment of a record: the fragment's vector, plus at least one of the
- * two ways to identify it — its advisory in-record {@link IFragmentLocator | span}
- * and/or an opaque consumer-minted {@link IEmbeddedFragment.fragmentId | fragmentId}.
- * Produced by a {@link FragmentEmbedder} and stored via
- * {@link IFragmentVectorIndex.addFragments}.
- *
- * @remarks
- * Both identity fields are optional **in the type**, but the "at least one"
- * requirement is real — a fragment carrying neither is unidentifiable at the read
- * side. It is enforced by {@link embeddedFragmentConverter} (and re-checked by the
- * in-package index implementations) rather than by a conditional-required union
- * (`{ locator; fragmentId? } | { locator?; fragmentId }`), which was considered and
- * declined: the union costs at every construction site and buys nothing at the read
- * site, where each field reads as `… | undefined` either way.
- * @public
- */
-export interface IEmbeddedFragment {
-  /**
-   * The region of the record body this fragment was derived from, when one can be
-   * stated honestly. Advisory — see {@link IFragmentLocator}; it is NOT a slice that
-   * reproduces the fragment text. Omit it for a fragment with no honest body span (a
-   * rewriting segmenter), in which case `fragmentId` must be supplied.
-   */
-  readonly locator?: IFragmentLocator;
-  /**
-   * An opaque, consumer-minted identity for this fragment, carried verbatim through
-   * the index and returned on the corresponding {@link IVectorQueryHit}. The index
-   * **never parses it, never filters on it, and never assigns meaning to it** — it is
-   * a bytestring, not part of the query path. It exists so a fragment stays
-   * identifiable when its text is not re-derivable from the record body.
-   *
-   * The guarantee is "we never parse it", NOT "we keep it stable". Because
-   * `addFragments` is whole-record-replace, an updated record re-emits its entire
-   * fragment set, so **any stability of a fragment id across re-embeds is the
-   * consumer's responsibility**, not the index's.
-   */
-  readonly fragmentId?: string;
-  /** The embedding vector for this fragment. */
-  readonly vector: Float32Array;
-}
-
-/**
- * The fragment-granular sibling of {@link IVectorIndex}: instead of one vector per
- * record it holds many vectors per record, each tagged with the identity its
- * {@link IEmbeddedFragment} carried, and its `query` returns per-fragment hits
- * carrying that identity back. This is the seam behind sub-document semantic search
- * — the "discovery" half of a search-then-read contract, where a hit tells the
- * consumer which record AND which fragment of it to read.
- *
- * @remarks
- * Deliberately NOT `extends IVectorIndex`: an index holding many vectors per record
- * has no well-defined single-vector `add(target, vector)`. It is a parallel contract
- * with three operations — `addFragments`, `remove`, `query` — reusing
- * {@link IVectorQueryHit}, on which both `locator` and `fragmentId` are optional. A
- * fragment hit populates whichever of the two its stored fragment carried; see
- * {@link IVectorQueryHit} for why that is not a discriminator and why fragment-ness
- * is determined by the index queried, not by field presence. Kept distinct from the
- * record-granular index per the consumer contract: memory recall stays
- * record-granular; sub-document knowledge uses a separate fragment index.
- * @public
- */
-export interface IFragmentVectorIndex {
-  /**
-   * Add (or replace) all fragments for the scope-qualified `target`. Whole-record
-   * semantics: every fragment previously held for `target` is dropped and replaced
-   * by `fragments`, so a re-authored document never leaves stale fragments behind.
-   * Returns the number of fragments now held for the record.
-   */
-  addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;
-
-  /**
-   * Remove every fragment for the scope-qualified `target`. Returns the removed
-   * target. Idempotent — removing a target with no fragments still succeeds.
-   */
-  remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
-
-  /**
-   * Return the `topK` nearest fragments to `vector`, in descending score order,
-   * each hit carrying its record `target` plus whichever of `locator` /
-   * `fragmentId` the stored fragment was added with. When
-   * `maxPerRecord` is supplied, no more than that many fragments of any single
-   * record appear in the result — the cap is applied during selection (before the
-   * `topK` cut) so one long document cannot crowd out others.
-   */
-  query(
-    vector: Float32Array,
-    topK: number,
-    maxPerRecord?: number
-  ): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
-}
-
-/**
- * Embeds a complete record into a vector for the store's embed-on-write hook.
- * Async and `Result`-returning, since a real embedder does a network call (cloud
- * provider) or in-process model inference. The consumer wires this — the core
- * package never calls an embedding provider directly, staying embedder-agnostic.
- * @public
- */
-export type MemoryEmbedder = (record: IMemoryRecord<unknown>) => Promise<Result<Float32Array>>;
-
-/**
- * The fragment-granular sibling of {@link MemoryEmbedder}: chunks a record's body
- * and embeds each chunk, returning one {@link IEmbeddedFragment} per chunk. The
- * chunking policy (window size, overlap) lives entirely in the consumer's embedder
- * — the core stays chunking-agnostic, exactly as it stays embedder-agnostic for
- * the record-granular path. Used by the store's fragment-embed-on-write hook.
- * @public
- */
-export type FragmentEmbedder = (
-  record: IMemoryRecord<unknown>
-) => Promise<Result<ReadonlyArray<IEmbeddedFragment>>>;
-
-/**
- * A record paired with its scope-qualified {@link IEdgeTarget | address}, as
- * yielded by {@link IMemoryRecordSource.list}. The address is required because
- * {@link InMemoryCosineIndex.rebuild} keys each re-embedded entry on the
- * scope-qualified target, not a bare {@link MemoryId} — two records that share a
- * filename stem across scopes must not collide when the whole vault is re-indexed.
- * @public
- */
-export interface IScopedMemoryRecord {
-  /** The record's scope-qualified `(scope, id)` address. */
-  readonly target: IEdgeTarget;
-  /** The record itself, passed to the embedder. */
-  readonly record: IMemoryRecord<unknown>;
-}
-
-/**
- * The minimal record-source surface {@link InMemoryCosineIndex.rebuild} reads to
- * re-embed an entire vault. Each entry carries the record's scope-qualified
- * address (see {@link IScopedMemoryRecord}) so the rebuild keys the vector index
- * exactly as the incremental embed-on-write path does. A consumer backs this with
- * the store's scoped index — the vector packlet does not import the store packlet
- * (which depends on the vector packlet for {@link IVectorIndex}, so the reverse
- * import would be a cycle).
- * @public
- */
-export interface IMemoryRecordSource {
-  /** List every record in the vault, each paired with its scoped address. */
-  list(): Promise<Result<ReadonlyArray<IScopedMemoryRecord>>>;
-}

package/src/test/unit/converters/antagonistRoundTrip.test.ts

@@ -1,110 +0,0 @@
-/*
- * Copyright (c) 2026 Erik Fortune
- * SPDX-License-Identifier: MIT
- */
-
-/**
- * Antagonist torture test — convert/validate round-trip symmetry (target class
- * 7): every OPTIONAL field the envelope/edge/provenance shapes carry must
- * survive a full on-disk round-trip (serialize → YAML text → parse), not just a
- * single `Converter.convert` pass in memory. This exercises the exact path a
- * corrupted or field-dropping serializer/parser pair would break, mirroring the
- * `aiClientToolConfig`/`annotations` field-drop class named in the brief.
- */
-
-import '@fgv/ts-utils-jest';
-import { Converters } from '@fgv/ts-utils';
-import {
-  BodyConverterRegistry,
-  IBodyConverterRegistry,
-  IMemoryEnvelope,
-  parseMemoryFile,
-  serializeMemoryFile
-} from '../../../index';
-
-const kind = 'note' as IMemoryEnvelope['kind'];
-
-function registry(): IBodyConverterRegistry {
-  const reg = BodyConverterRegistry.create().orThrow();
-  reg.register(kind, Converters.string);
-  return reg;
-}
-
-describe('antagonist — full on-disk round-trip preserves every optional field', () => {
-  // Wrong impl this catches: a serializer/parser pair where one side silently
-  // drops an optional field (the exact class of bug the brief calls out for
-  // `aiClientToolConfig`/`annotations`-shaped converters) — e.g. omitting
-  // `temporal.invalid_at: null`, an edge's `valid_at`/`invalid_at`/`provenance`,
-  // a `null` `embeddingRef`, or a provenance extension key, because the author
-  // forgot to thread it through both the YAML emit AND the envelope Converter.
-  test('every optional envelope/edge/provenance field set simultaneously survives a full YAML round-trip', () => {
-    const envelope: IMemoryEnvelope = {
-      id: 'doc-1' as IMemoryEnvelope['id'],
-      entityId: 'doc-1' as IMemoryEnvelope['entityId'],
-      kind,
-      tags: ['t1', 't2'] as unknown as IMemoryEnvelope['tags'],
-      links: [
-        {
-          type: 'rel' as never,
-          target: { scope: 'knowledge', id: 'doc-2' } as never,
-          confidence: 0.42,
-          provenance: { source: 'agent', by: 'curator', extra: { nested: true } },
-          valid_at: 111,
-
-          invalid_at: null
-        }
-      ],
-      created: 1000,
-      updated: 2000,
-      seq: 7,
-      contentHash: 'abc123',
-      provenance: {
-        source: 'host-ingest',
-        by: 'erik',
-        model: 'gpt-5',
-        confidence: 0.87,
-        derivedFrom: { scope: 'conversations/c1', id: 'turn-3' } as never,
-        // Opaque extension keys (per IProvenance's `[key: string]: unknown` arm).
-        sentiment: { score: 0.5 },
-        epistemic: 'belief'
-      },
-
-      temporal: { valid_at: 500, invalid_at: null },
-
-      embeddingRef: null
-    };
-
-    const raw = serializeMemoryFile(envelope, 'the body text').orThrow();
-    expect(parseMemoryFile(raw, registry())).toSucceedAndSatisfy((record) => {
-      // Deep-equal the ENTIRE envelope, not field-by-field — a partial assertion
-      // list is exactly how a single dropped field slips through review.
-      expect(record.envelope).toEqual(envelope);
-      expect(record.body).toBe('the body text');
-    });
-  });
-
-  test('an absent temporal/embeddingRef/edge-optionals round-trips to fully absent (no null-vs-undefined drift)', () => {
-    const envelope: IMemoryEnvelope = {
-      id: 'doc-2' as IMemoryEnvelope['id'],
-      entityId: 'doc-2' as IMemoryEnvelope['entityId'],
-      kind,
-      tags: [],
-      links: [{ type: 'rel' as never, target: { scope: 'knowledge', id: 'doc-3' } as never }],
-      created: 0,
-      updated: 0,
-      seq: 0,
-      contentHash: '',
-      provenance: { source: 'agent' }
-    };
-    const raw = serializeMemoryFile(envelope, 'body').orThrow();
-    expect(parseMemoryFile(raw, registry())).toSucceedAndSatisfy((record) => {
-      expect(record.envelope.temporal).toBeUndefined();
-      expect(record.envelope.embeddingRef).toBeUndefined();
-      expect(record.envelope.links[0].confidence).toBeUndefined();
-      expect(record.envelope.links[0].provenance).toBeUndefined();
-      expect(record.envelope.links[0].valid_at).toBeUndefined();
-      expect(record.envelope.links[0].invalid_at).toBeUndefined();
-      expect(record.envelope).toEqual(envelope);
-    });
-  });
-});

package/src/test/unit/converters/bodyConverterRegistry.test.ts

@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2026 Erik Fortune
- * SPDX-License-Identifier: MIT
- */
-
-import '@fgv/ts-utils-jest';
-import { Converters } from '@fgv/ts-utils';
-import { JsonSchema } from '@fgv/ts-json-base';
-import { BodyConverterRegistry } from '../../../packlets/converters';
-import { Kind } from '../../../packlets/types';
-
-const knowledgeKind: Kind = 'knowledge' as Kind;
-const profileKind: Kind = 'profile' as Kind;
-
-describe('BodyConverterRegistry', () => {
-  let registry: BodyConverterRegistry;
-
-  beforeEach(() => {
-    registry = BodyConverterRegistry.create().orThrow();
-  });
-
-  test('create succeeds', () => {
-    expect(BodyConverterRegistry.create()).toSucceed();
-  });
-
-  describe('register / has / convert', () => {
-    test('registers a Converter and converts a matching body', () => {
-      registry.register(knowledgeKind, Converters.string);
-      expect(registry.has(knowledgeKind)).toBe(true);
-      expect(registry.convert(knowledgeKind, 'a knowledge doc')).toSucceedWith('a knowledge doc');
-    });
-
-    test('a registered Converter rejects an invalid body', () => {
-      registry.register(knowledgeKind, Converters.string);
-      expect(registry.convert(knowledgeKind, 42)).toFail();
-    });
-
-    test('has returns false for an unregistered kind', () => {
-      expect(registry.has(knowledgeKind)).toBe(false);
-    });
-
-    test('convert fails for an unregistered kind', () => {
-      expect(registry.convert(knowledgeKind, 'x')).toFailWith(
-        /no converter registered for kind 'knowledge'/i
-      );
-    });
-
-    test('register replaces a prior registration for the same kind', () => {
-      registry.register(knowledgeKind, Converters.string);
-      registry.register(knowledgeKind, Converters.number);
-      expect(registry.convert(knowledgeKind, 7)).toSucceedWith(7);
-      expect(registry.convert(knowledgeKind, 'x')).toFail();
-    });
-  });
-
-  describe('getConverter', () => {
-    test('returns the registered converter', () => {
-      registry.register(knowledgeKind, Converters.string);
-      expect(registry.getConverter(knowledgeKind)).toSucceedAndSatisfy((converter) => {
-        expect(converter.convert('hello')).toSucceedWith('hello');
-      });
-    });
-
-    test('fails for an unregistered kind', () => {
-      expect(registry.getConverter(knowledgeKind)).toFailWith(/no converter registered for kind/i);
-    });
-  });
-
-  describe('registerSchema', () => {
-    const profileSchema = JsonSchema.object({
-      name: JsonSchema.string(),
-      age: JsonSchema.integer()
-    });
-
-    test('registers a JsonSchema validator and validates a matching body', () => {
-      registry.registerSchema(profileKind, profileSchema);
-      expect(registry.has(profileKind)).toBe(true);
-      expect(registry.convert(profileKind, { name: 'Ada', age: 36 })).toSucceedWith({
-        name: 'Ada',
-        age: 36
-      });
-    });
-
-    test('a registered schema rejects an invalid body', () => {
-      registry.registerSchema(profileKind, profileSchema);
-      expect(registry.convert(profileKind, { name: 'Ada' })).toFail();
-    });
-  });
-});