@graphorin/memory 0.6.0 → 0.7.0

package/src/search/query-transform.ts

@@ -0,0 +1,285 @@
+/**
+ * Query transformation for hybrid retrieval (P2-3) - **multi-query /
+ * RAG-Fusion** and optional **HyDE**.
+ *
+ * Auto-recall is a regex trigger and the default search is single-shot:
+ * the user's phrasing rarely matches how a memory was stored, so a
+ * relevant fact is missed at the retrieval step. This module fans a
+ * query into a few reworded variants (one cheap LLM call) and/or writes
+ * a hypothetical answer passage to embed (HyDE, arXiv:2212.10496); the
+ * caller retrieves each and fuses the lists through the existing RRF
+ * reranker. Variant generation is the only thing that touches a
+ * provider, so it is **opt-in**: with no transformer configured the
+ * default search path stays fully offline and single-shot.
+ *
+ * The module is provider-agnostic - it imports only `@graphorin/core`
+ * types and never performs I/O itself. The concrete `Provider` is
+ * injected by the caller via {@link createProviderQueryTransformer},
+ * which is resilient: any provider error or unparseable output degrades
+ * to "no variants" / "no hypothetical" rather than throwing into the
+ * search hot path.
+ *
+ * @packageDocumentation
+ */
+
+import type { Provider, ProviderRequest } from '@graphorin/core';
+import { stripFence } from '../internal/llm-json.js';
+
+/** Default ceiling on reworded variants a provider-backed transformer requests. */
+export const DEFAULT_MAX_QUERY_VARIANTS = 5;
+
+/** Default output-token ceiling for a single transform call. */
+const DEFAULT_TRANSFORM_MAX_TOKENS = 256;
+
+/**
+ * Pluggable query-transformation seam consumed by
+ * `SemanticMemory.search(..., { multiQuery, hyde })`. The built-in
+ * provider-backed implementation lives in
+ * {@link createProviderQueryTransformer}; advanced callers can supply a
+ * bespoke one (e.g. a deterministic synonym expander) to
+ * `new SemanticMemory({ queryTransformer })`.
+ *
+ * Implementations MUST degrade gracefully - return `[]` / `null` rather
+ * than throw - so a transform failure never breaks recall.
+ *
+ * @stable
+ */
+export interface QueryTransformer {
+  /**
+   * Rewrite a query into up to `count` **additional** reworded variants
+   * (the original query is retained by the caller). Returns `[]` when
+   * transformation is unavailable or the model returns nothing usable.
+   */
+  expand(
+    query: string,
+    count: number,
+    options?: QueryTransformOptions,
+  ): Promise<ReadonlyArray<string>>;
+  /**
+   * Generate a single short hypothetical-answer passage to embed (HyDE),
+   * or `null` when unavailable. The caller embeds the passage and fuses
+   * its vector neighbours into the result.
+   */
+  hypothetical(query: string, options?: QueryTransformOptions): Promise<string | null>;
+}
+
+/**
+ * Per-call options for a {@link QueryTransformer}.
+ *
+ * @stable
+ */
+export interface QueryTransformOptions {
+  /** Cancellation signal forwarded to the underlying provider call. */
+  readonly signal?: AbortSignal;
+}
+
+/**
+ * System prompt for multi-query variant generation. Asks for a bare
+ * JSON array of standalone rephrasings that preserve the original
+ * intent - {@link parseQueryVariants} also tolerates a chatty model.
+ *
+ * @internal
+ */
+export const QUERY_EXPANSION_SYSTEM_PROMPT =
+  'You rewrite a search query into alternative phrasings to improve recall over a ' +
+  'personal memory store. Preserve the original intent; vary vocabulary, specificity, ' +
+  'and perspective so different wordings of the same stored memory are matched. ' +
+  'Return ONLY a JSON array of strings - each a standalone query, no numbering, no ' +
+  'commentary, no duplicates of the original.';
+
+/**
+ * System prompt for HyDE. Asks for a short, plausible hypothetical
+ * answer (not a question rephrase) whose embedding sits near the
+ * passage that would actually answer the query.
+ *
+ * @internal
+ */
+export const HYDE_SYSTEM_PROMPT =
+  'Write a short, plausible hypothetical answer to the question, phrased as a ' +
+  'first-person statement of fact as it might be recorded in a memory store. One to ' +
+  'three sentences. Output only the passage - no preamble, no caveats, no markdown.';
+
+/**
+ * Build the multi-query expansion request. Pure - no I/O. A higher
+ * temperature is used deliberately so the variants diverge (the
+ * downstream retrieval + RRF fusion stays deterministic).
+ *
+ * @stable
+ */
+export function buildExpansionRequest(
+  query: string,
+  count: number,
+  options: { readonly maxTokens?: number; readonly signal?: AbortSignal } = {},
+): ProviderRequest {
+  return {
+    messages: [
+      {
+        role: 'user',
+        content: `Original query: ${query}\nReturn up to ${count} alternative phrasings as a JSON array of strings.`,
+      },
+    ],
+    systemMessage: QUERY_EXPANSION_SYSTEM_PROMPT,
+    temperature: 0.7,
+    maxTokens: options.maxTokens ?? DEFAULT_TRANSFORM_MAX_TOKENS,
+    ...(options.signal !== undefined ? { signal: options.signal } : {}),
+    outputType: { kind: 'structured' },
+  };
+}
+
+/**
+ * Build the HyDE pseudo-document request. Pure - no I/O.
+ *
+ * @stable
+ */
+export function buildHydeRequest(
+  query: string,
+  options: { readonly maxTokens?: number; readonly signal?: AbortSignal } = {},
+): ProviderRequest {
+  return {
+    messages: [{ role: 'user', content: query }],
+    systemMessage: HYDE_SYSTEM_PROMPT,
+    temperature: 0.3,
+    maxTokens: options.maxTokens ?? DEFAULT_TRANSFORM_MAX_TOKENS,
+    ...(options.signal !== undefined ? { signal: options.signal } : {}),
+  };
+}
+
+/**
+ * Parse the variant-generation model output into a deduped, capped list
+ * of reworded queries. Tolerates a JSON array, a `{ "variants": [...] }`
+ * / `{ "queries": [...] }` wrapper, fenced blocks, and (as a last
+ * resort) a newline / numbered list - so a chatty model never breaks
+ * recall. Empty strings and case-insensitive duplicates are dropped;
+ * the result is capped at `max`.
+ *
+ * @stable
+ */
+export function parseQueryVariants(text: string | undefined, max: number): ReadonlyArray<string> {
+  if (max <= 0 || text === undefined) return [];
+  const candidate = stripFence(text).trim();
+  if (candidate.length === 0) return [];
+  const raw = extractVariantStrings(candidate);
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const item of raw) {
+    const trimmed = item.trim();
+    if (trimmed.length === 0) continue;
+    const key = trimmed.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(trimmed);
+    if (out.length >= max) break;
+  }
+  return out;
+}
+
+/**
+ * Parse the HyDE model output into a single passage, or `null` when the
+ * model returned nothing usable. Strips a fenced block and trims.
+ *
+ * @stable
+ */
+export function parseHypothetical(text: string | undefined): string | null {
+  if (text === undefined) return null;
+  const trimmed = stripFence(text).trim();
+  return trimmed.length === 0 ? null : trimmed;
+}
+
+/**
+ * Wrap a {@link Provider} as a {@link QueryTransformer}. Both methods
+ * are **resilient**: a provider error or unparseable response degrades
+ * to `[]` / `null` so transformation never throws into `search`. The
+ * `maxVariants` ceiling caps the variants requested regardless of the
+ * caller's `multiQuery` value (a latency guardrail).
+ *
+ * @stable
+ */
+export function createProviderQueryTransformer(
+  provider: Provider,
+  options: { readonly maxVariants?: number; readonly maxTokens?: number } = {},
+): QueryTransformer {
+  const maxVariants = Math.max(0, options.maxVariants ?? DEFAULT_MAX_QUERY_VARIANTS);
+  const maxTokens = options.maxTokens ?? DEFAULT_TRANSFORM_MAX_TOKENS;
+  return {
+    async expand(query, count, opts = {}): Promise<ReadonlyArray<string>> {
+      const want = Math.min(Math.max(0, count), maxVariants);
+      if (want === 0 || query.trim().length === 0) return [];
+      try {
+        const request = buildExpansionRequest(query, want, {
+          maxTokens,
+          ...(opts.signal !== undefined ? { signal: opts.signal } : {}),
+        });
+        const response = await provider.generate(request);
+        return parseQueryVariants(response.text, want);
+      } catch {
+        return [];
+      }
+    },
+    async hypothetical(query, opts = {}): Promise<string | null> {
+      if (query.trim().length === 0) return null;
+      try {
+        const request = buildHydeRequest(query, {
+          maxTokens,
+          ...(opts.signal !== undefined ? { signal: opts.signal } : {}),
+        });
+        const response = await provider.generate(request);
+        return parseHypothetical(response.text);
+      } catch {
+        return null;
+      }
+    },
+  };
+}
+
+/** Extract the raw (pre-dedup) variant strings from tolerant model output. */
+function extractVariantStrings(candidate: string): ReadonlyArray<string> {
+  const parsed = tryParseJson(candidate);
+  if (parsed !== undefined) {
+    const arr = Array.isArray(parsed) ? parsed : isRecord(parsed) ? pickArray(parsed) : null;
+    if (arr !== null) {
+      return arr.filter((v): v is string => typeof v === 'string');
+    }
+  }
+  // Fallback - split a newline / numbered list and strip list markers.
+  return candidate.split('\n').map((line) =>
+    line
+      .replace(/^\s*(?:[-*•]|\d+[.)])\s*/u, '')
+      .replace(/^["']|["']$/gu, '')
+      .trim(),
+  );
+}
+
+function pickArray(record: Record<string, unknown>): ReadonlyArray<unknown> | null {
+  if (Array.isArray(record.variants)) return record.variants;
+  if (Array.isArray(record.queries)) return record.queries;
+  return null;
+}
+
+function tryParseJson(text: string): unknown {
+  try {
+    return JSON.parse(text);
+  } catch {
+    const slice = sliceJson(text);
+    if (slice === null) return undefined;
+    try {
+      return JSON.parse(slice);
+    } catch {
+      return undefined;
+    }
+  }
+}
+
+/** Slice the widest `[...]` array or `{...}` object out of chatty text. */
+function sliceJson(text: string): string | null {
+  const arrStart = text.indexOf('[');
+  const arrEnd = text.lastIndexOf(']');
+  if (arrStart >= 0 && arrEnd > arrStart) return text.slice(arrStart, arrEnd + 1);
+  const objStart = text.indexOf('{');
+  const objEnd = text.lastIndexOf('}');
+  if (objStart >= 0 && objEnd > objStart) return text.slice(objStart, objEnd + 1);
+  return null;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object';
+}

package/src/search/rrf.ts

@@ -0,0 +1,225 @@
+import type { MemoryHit, MemoryRecord } from '@graphorin/core';
+import type { ReRanker, ReRankOptions } from './types.js';
+
+/**
+ * Industry-standard Reciprocal Rank Fusion constant. Lower `k` makes
+ * top-ranked items dominate; higher `k` smooths the contribution of
+ * lower-ranked items. `k = 60` is the value popularised by Cormack,
+ * Clarke, and Büttcher (2009) and is the framework default.
+ *
+ * @stable
+ */
+export const RRF_DEFAULT_K = 60;
+
+/**
+ * Built-in Reciprocal Rank Fusion reranker. Combines multiple ranked
+ * lists (vector hits, FTS5 hits, optional entity boost) into a single
+ * fused ranking by summing `1 / (k + rank)` per list each item appears
+ * in.
+ *
+ * Properties (verified by the property-based test suite):
+ *
+ *  - **Deterministic.** Identical input lists yield identical output.
+ *  - **Stable under permutation of input lists.** The fusion is
+ *    independent of the order in which the input lists are passed in.
+ *  - **Tie-broken by stable record id.** Records with equal RRF
+ *    scores keep their first-seen order.
+ *
+ * @stable
+ */
+export class RRFReranker implements ReRanker {
+  readonly id = 'rrf' as const;
+  readonly k: number;
+
+  constructor(k: number = RRF_DEFAULT_K) {
+    if (!Number.isFinite(k) || k <= 0) {
+      throw new TypeError(
+        `[graphorin/memory] RRFReranker: 'k' must be a positive finite number, got ${String(k)}.`,
+      );
+    }
+    this.k = k;
+  }
+
+  async rerank<TRecord extends MemoryRecord>(
+    query: string,
+    lists: ReadonlyArray<ReadonlyArray<MemoryHit<TRecord>>>,
+    options: ReRankOptions = {},
+  ): Promise<ReadonlyArray<MemoryHit<TRecord>>> {
+    void query;
+    if (options.signal?.aborted === true) {
+      throw new DOMException('RRFReranker aborted', 'AbortError');
+    }
+    const fused = fuseRrf(lists, this.k, options.labels);
+    const topK = options.topK ?? 10;
+    return fused.slice(0, Math.max(0, topK));
+  }
+}
+
+/**
+ * Weighted-RRF reranker (X-2). Fuses parallel ranked lists through
+ * {@link fuseWeighted}, applying a per-list `weights[i]` so a caller who
+ * has calibrated list reliability against labels (the P0-1 eval harness)
+ * can trust one retriever over another - e.g. up-weight dense vector
+ * hits over lexical FTS hits. `weights` is **positional**: aligned to the
+ * input `lists` order (the built-in hybrid search passes FTS first, then
+ * vector). At equal weights it is identical to {@link RRFReranker}; RRF
+ * stays the framework default.
+ *
+ * Like the RRF reranker it is deterministic and tie-broken by stable
+ * record id, and it preserves the upstream `signals` on each hit.
+ *
+ * @stable
+ */
+export class WeightedRRFReranker implements ReRanker {
+  readonly id = 'weighted-rrf' as const;
+  readonly k: number;
+  readonly weights: ReadonlyArray<number>;
+
+  constructor(args: { weights: ReadonlyArray<number>; k?: number }) {
+    const k = args.k ?? RRF_DEFAULT_K;
+    if (!Number.isFinite(k) || k <= 0) {
+      throw new TypeError(
+        `[graphorin/memory] WeightedRRFReranker: 'k' must be a positive finite number, got ${String(k)}.`,
+      );
+    }
+    for (const w of args.weights) {
+      if (!Number.isFinite(w) || w < 0) {
+        throw new TypeError(
+          `[graphorin/memory] WeightedRRFReranker: every weight must be a non-negative finite number, got ${String(w)}.`,
+        );
+      }
+    }
+    this.k = k;
+    this.weights = args.weights;
+  }
+
+  async rerank<TRecord extends MemoryRecord>(
+    query: string,
+    lists: ReadonlyArray<ReadonlyArray<MemoryHit<TRecord>>>,
+    options: ReRankOptions = {},
+  ): Promise<ReadonlyArray<MemoryHit<TRecord>>> {
+    void query;
+    if (options.signal?.aborted === true) {
+      throw new DOMException('WeightedRRFReranker aborted', 'AbortError');
+    }
+    const fused = fuseWeighted(lists, this.weights, this.k, options.labels);
+    const topK = options.topK ?? 10;
+    return fused.slice(0, Math.max(0, topK));
+  }
+}
+
+/**
+ * Pure functional core of the RRF reranker - the equal-weight case of
+ * {@link fuseWeighted}. Exported separately so the test suite (and the
+ * property-based fuzzer) can exercise the math without the `Promise<…>`
+ * wrapping of the public surface.
+ *
+ * @stable
+ */
+export function fuseRrf<TRecord extends MemoryRecord>(
+  lists: ReadonlyArray<ReadonlyArray<MemoryHit<TRecord>>>,
+  k: number,
+  labels?: ReadonlyArray<string>,
+): ReadonlyArray<MemoryHit<TRecord>> {
+  return fuseWeighted(lists, undefined, k, labels);
+}
+
+/**
+ * Weighted / convex generalization of {@link fuseRrf} (X-2). Each input
+ * list `i` contributes `weights[i] · 1 / (k + rank)` to a record's fused
+ * score instead of the flat `1 / (k + rank)`, so a caller who has
+ * calibrated list reliability against labels (the P0-1 eval harness) can
+ * trust one retriever over another. RRF stays the zero-tuning default: a
+ * `weights` of `undefined` (or all-`1`) is byte-for-byte identical to
+ * {@link fuseRrf}. A missing, non-finite, or negative entry falls back to
+ * the neutral weight `1`, so a partial / malformed `weights` never throws
+ * or poisons the ranking.
+ *
+ * Like {@link fuseRrf} the fusion is deterministic, tie-broken by
+ * first-seen order, and preserves any upstream `signals` (FTS `bm25`,
+ * vector similarity) the hits carried in - the `rrf` / `rrf.list_N`
+ * signals it records are the *weighted* contributions, so the fused
+ * `score` still equals their sum. Note that - unlike RRF - the result
+ * depends on input list *order*, because each weight is bound to a list
+ * position.
+ *
+ * @stable
+ */
+export function fuseWeighted<TRecord extends MemoryRecord>(
+  lists: ReadonlyArray<ReadonlyArray<MemoryHit<TRecord>>>,
+  weights: ReadonlyArray<number> | undefined,
+  k: number,
+  labels?: ReadonlyArray<string>,
+): ReadonlyArray<MemoryHit<TRecord>> {
+  type Aggregate = {
+    record: TRecord;
+    score: number;
+    signals: Record<string, number>;
+    firstSeenOrder: number;
+  };
+  const aggregates = new Map<string, Aggregate>();
+  let nextOrder = 0;
+  for (let listIdx = 0; listIdx < lists.length; listIdx++) {
+    const list = lists[listIdx] ?? [];
+    const weight = resolveWeight(weights, listIdx);
+    // MRET-13: explanation signals key by the caller's retriever-kind
+    // label when supplied - `rrf.list_N` is an ephemeral position that
+    // changes meaning whenever a leg (vector / HyDE / graph) is
+    // conditionally absent from the fan-out.
+    const signalKey = `rrf.${labels?.[listIdx] ?? `list_${listIdx}`}`;
+    for (let i = 0; i < list.length; i++) {
+      const hit = list[i];
+      if (hit === undefined) continue;
+      const rank = i + 1;
+      const contribution = weight / (k + rank);
+      const id = hit.record.id;
+      const existing = aggregates.get(id);
+      if (existing === undefined) {
+        const signals: Record<string, number> = {
+          ...(hit.signals ?? {}),
+          rrf: contribution,
+          [signalKey]: contribution,
+        };
+        aggregates.set(id, {
+          record: hit.record,
+          score: contribution,
+          signals,
+          firstSeenOrder: nextOrder++,
+        });
+      } else {
+        existing.score += contribution;
+        existing.signals[signalKey] = contribution;
+        existing.signals.rrf = (existing.signals.rrf ?? 0) + contribution;
+        for (const [key, value] of Object.entries(hit.signals ?? {})) {
+          if (key in existing.signals) continue;
+          existing.signals[key] = value;
+        }
+      }
+    }
+  }
+  const fused: MemoryHit<TRecord>[] = [];
+  for (const a of aggregates.values()) {
+    fused.push({
+      record: a.record,
+      score: a.score,
+      signals: Object.freeze({ ...a.signals }),
+    });
+  }
+  fused.sort((a, b) => {
+    if (b.score !== a.score) return b.score - a.score;
+    const orderA = aggregates.get(a.record.id)?.firstSeenOrder ?? 0;
+    const orderB = aggregates.get(b.record.id)?.firstSeenOrder ?? 0;
+    return orderA - orderB;
+  });
+  return fused;
+}
+
+/**
+ * Resolve list `index`'s fusion weight, defaulting a missing / non-finite
+ * / negative entry to the neutral `1` (so an absent or malformed
+ * `weights` degrades to plain RRF rather than poisoning the ranking).
+ */
+function resolveWeight(weights: ReadonlyArray<number> | undefined, index: number): number {
+  const raw = weights?.[index];
+  return typeof raw === 'number' && Number.isFinite(raw) && raw >= 0 ? raw : 1;
+}

package/src/search/trust.ts

@@ -0,0 +1,55 @@
+/**
+ * C5: trust-aware retrieval ranking (pairs memory-retrieval-01's ranking
+ * leg / security-04). The provenance + quarantine discounts that the
+ * eviction-time `salience()` composite already applies now also apply at
+ * search SCORING time, so a poisoned or foreign-origin memory cannot
+ * outrank the user's own words purely on lexical/vector similarity - the
+ * direct defense against MINJA-class memory poisoning (query-only
+ * injection >95% ASR without rank-time trust).
+ *
+ * The factor reuses the {@link SalienceWeights} semantics verbatim:
+ * quarantined -> `1 - weights.quarantine` (default 0.3x), foreign
+ * provenance -> `1 - weights.foreignProvenance` (default 0.8x), first
+ * party -> `1` (neutral: default rankings of ordinary user/extraction
+ * facts are byte-identical).
+ *
+ * @packageDocumentation
+ */
+
+import { DEFAULT_SALIENCE_WEIGHTS, type SalienceWeights } from '../consolidator/decay.js';
+
+/**
+ * `true` for provenance that did not originate first-party (P1-4) -
+ * mirrors the eviction-path classification: `null` (legacy / direct
+ * write), `'user'`, and `'extraction'` (the consolidator distilling the
+ * user's own session) are first-party; `'tool'`, `'imported'`,
+ * `'reflection'`, and `'induction'` are foreign.
+ *
+ * @stable
+ */
+export function isForeignProvenance(provenance: string | null | undefined): boolean {
+  return (
+    provenance !== null &&
+    provenance !== undefined &&
+    provenance !== 'user' &&
+    provenance !== 'extraction'
+  );
+}
+
+/**
+ * Rank-time trust multiplier for one record. `1` for first-party active
+ * facts (the common case - zero ranking change), `1 - quarantine` for
+ * quarantined-but-included rows, `1 - foreignProvenance` for foreign
+ * origin. Quarantine wins when both apply (same branch order as
+ * `salience()`).
+ *
+ * @stable
+ */
+export function trustDiscount(
+  record: { readonly status?: string; readonly provenance?: string | null },
+  weights: SalienceWeights = DEFAULT_SALIENCE_WEIGHTS,
+): number {
+  if (record.status === 'quarantined') return Math.max(0, 1 - weights.quarantine);
+  if (isForeignProvenance(record.provenance)) return Math.max(0, 1 - weights.foreignProvenance);
+  return 1;
+}

package/src/search/types.ts

@@ -0,0 +1,50 @@
+import type { MemoryHit, MemoryRecord } from '@graphorin/core';
+
+/**
+ * Pluggable reranker contract. Concrete implementations live in
+ * `@graphorin/memory/search` (the built-in `RRFReranker`) and the
+ * Phase 16 optional packages (`@graphorin/reranker-transformersjs`,
+ * `@graphorin/reranker-llm`).
+ *
+ * The reranker accepts one or more parallel ranked lists (vector +
+ * FTS5 + optional entity boost) and produces a single fused ranking.
+ * Implementations MUST be pure (no I/O outside `signal`-aware
+ * dependencies) so the agent runtime can call them mid-stream without
+ * deadlocking the main loop.
+ *
+ * @stable
+ */
+export interface ReRanker {
+  /** Stable lowercase identifier surfaced on every span. */
+  readonly id: string;
+  /**
+   * Rerank one or more parallel ranked lists and return the fused
+   * top-K (default `topK = 10`). Each input list must already be
+   * sorted by `score` descending.
+   */
+  rerank<TRecord extends MemoryRecord>(
+    query: string,
+    lists: ReadonlyArray<ReadonlyArray<MemoryHit<TRecord>>>,
+    options?: ReRankOptions,
+  ): Promise<ReadonlyArray<MemoryHit<TRecord>>>;
+}
+
+/**
+ * Per-call reranker options. `topK` defaults to `10`; `signal` is
+ * propagated to any async work the reranker performs.
+ *
+ * @stable
+ */
+export interface ReRankOptions {
+  readonly topK?: number;
+  readonly signal?: AbortSignal;
+  /**
+   * Stable per-list labels for the explanation signals (MRET-13) -
+   * `rrf.<label>` instead of the ephemeral `rrf.list_<index>`. Callers
+   * that fan lists out conditionally (multi-query, HyDE, graph) pass
+   * retriever-kind labels so X-3 consumers can identify which leg a
+   * contribution came from across calls. Missing entries fall back to
+   * the positional key.
+   */
+  readonly labels?: ReadonlyArray<string>;
+}