@graphorin/memory 0.6.1 → 0.7.0

package/src/conflict/stages/helpers.ts

@@ -0,0 +1,65 @@
+/**
+ * Internal helpers shared across the five pipeline stages.
+ *
+ * @packageDocumentation
+ * @internal
+ */
+
+import { md5 } from '@graphorin/core';
+import type { LocalePack } from '../locale-packs/index.js';
+
+/**
+ * Lowercase + collapse whitespace + trim. Used by Stage 1 (exact
+ * dedup) to compute the canonical fact body that feeds the MD5
+ * comparison.
+ */
+export function canonicaliseFactBody(text: string): string {
+  return text.replace(/\s+/g, ' ').trim().toLowerCase();
+}
+
+/**
+ * Stable hash for a canonical fact body. We re-use the framework's
+ * MD5 helper to stay consistent with the storage layer's existing
+ * dedup hash column (see `facts.hash`).
+ */
+export function fastFactHash(text: string): string {
+  return md5(text);
+}
+
+/** Strip leading articles + trim a candidate subject token. */
+export function normaliseSubject(input: string, pack: LocalePack): string {
+  const tokens = canonicaliseFactBody(input).split(' ');
+  const stop = new Set(pack.subjectStopWords.map((s) => s.toLowerCase()));
+  return tokens.filter((token) => token.length > 0 && !stop.has(token)).join(' ');
+}
+
+/**
+ * Naive subject / predicate / object split. The locale pack supplies
+ * the verb tokens used as predicate normalisers - the first matching
+ * verb in the sentence becomes the predicate and splits the rest
+ * into subject (left side) and object (right side). The split is
+ * intentionally simple: anything more clever lives behind the entity-
+ * linking layer (post-MVP).
+ */
+export function splitSubjectPredicateObject(
+  text: string,
+  pack: LocalePack,
+): { readonly subject: string; readonly predicate: string; readonly object: string } | null {
+  const canonical = canonicaliseFactBody(text);
+  if (canonical.length === 0) return null;
+  const tokens = canonical.split(' ');
+  const verbs = new Set(pack.predicateNormalisers.map((v) => v.toLowerCase()));
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i];
+    if (token === undefined) continue;
+    const stripped = token.replace(/[.,!?;:]/g, '');
+    if (verbs.has(stripped)) {
+      const subject = normaliseSubject(tokens.slice(0, i).join(' '), pack);
+      const predicate = stripped;
+      const object = canonicaliseFactBody(tokens.slice(i + 1).join(' '));
+      if (subject.length === 0 || object.length === 0) continue;
+      return { subject, predicate, object };
+    }
+  }
+  return null;
+}

package/src/conflict/stages/stage1-exact-dedup.ts

@@ -0,0 +1,40 @@
+/**
+ * Stage 1 - Exact dedup. Computes an MD5 hash on the canonical fact
+ * body (lowercase + trimmed + collapsed whitespace) and short-circuits
+ * to `dedup` when an existing fact in the supplied vector hit list
+ * shares the same canonical form.
+ *
+ * Stage 1 only inspects `existing` when Stage 2 already populated it
+ * for the orchestrator (re-using the embedding round-trip). When the
+ * orchestrator calls Stage 1 with an empty list the stage falls
+ * through to `'continue'` - Stage 2 will gather candidates and Stage 1
+ * is folded into Stage 2's HOT zone via the same canonical-form
+ * comparison.
+ *
+ * @packageDocumentation
+ */
+
+import type { PipelineStage } from '../types.js';
+import { canonicaliseFactBody, fastFactHash } from './helpers.js';
+
+export const stage1ExactDedup: PipelineStage = {
+  id: 'exact-dedup',
+  async evaluate(ctx) {
+    const candidateHash = fastFactHash(canonicaliseFactBody(ctx.candidate.text));
+    if (ctx.existing.length === 0) {
+      return { kind: 'continue' };
+    }
+    for (const hit of ctx.existing) {
+      const existingHash = fastFactHash(canonicaliseFactBody(hit.record.text));
+      if (existingHash === candidateHash) {
+        return {
+          kind: 'dedup',
+          existingId: hit.record.id,
+          similarity: 1,
+          reason: 'exact-hash-match',
+        };
+      }
+    }
+    return { kind: 'continue' };
+  },
+};

package/src/conflict/stages/stage2-embedding-three-zone.ts

@@ -0,0 +1,54 @@
+/**
+ * Stage 2 - Embedding three-zone classifier (RB-02 §8.1).
+ *
+ * Inspects the candidate similarity against the top-K nearest
+ * neighbours surfaced by the semantic vector store and routes the
+ * decision into one of four zones:
+ *
+ *  - HOT (`sim ≥ thresholds.hot`): silent dedup - the existing fact is
+ *    semantically identical, no audit-worthy conflict.
+ *  - NEAR-DUP (`thresholds.nearDup ≤ sim < thresholds.hot`): same
+ *    fact phrased differently - dedup with `near-dup` zone label so
+ *    operators can spot the case in `fact_conflicts`.
+ *  - CONFLICT-CHECK (`thresholds.cold < sim < thresholds.nearDup`):
+ *    candidate sounds related; defer decision to Stages 3 → 4 → 5.
+ *  - COLD (`sim ≤ thresholds.cold`): no conflict - admit the candidate.
+ *
+ * @packageDocumentation
+ */
+
+import type { PipelineStage, StageOutcome } from '../types.js';
+
+export const stage2EmbeddingThreeZone: PipelineStage = {
+  id: 'embedding-three-zone',
+  async evaluate(ctx): Promise<StageOutcome> {
+    if (ctx.existing.length === 0) {
+      return { kind: 'admit', reason: 'no-existing-candidates' };
+    }
+    const top = ctx.existing[0];
+    if (top === undefined) {
+      return { kind: 'admit', reason: 'no-existing-candidates' };
+    }
+    const similarity = top.score;
+    if (similarity >= ctx.thresholds.hot) {
+      return {
+        kind: 'dedup',
+        existingId: top.record.id,
+        similarity,
+        reason: 'embedding-hot-zone',
+      };
+    }
+    if (similarity >= ctx.thresholds.nearDup) {
+      return {
+        kind: 'dedup',
+        existingId: top.record.id,
+        similarity,
+        reason: 'embedding-near-dup-zone',
+      };
+    }
+    if (similarity > ctx.thresholds.cold) {
+      return { kind: 'continue' };
+    }
+    return { kind: 'admit', reason: 'embedding-cold-zone' };
+  },
+};

package/src/conflict/stages/stage3-heuristic-regex.ts

@@ -0,0 +1,51 @@
+/**
+ * Stage 3 - Heuristic regex (per-locale, pluggable).
+ *
+ * Inspects the candidate fact body for explicit supersede or negation
+ * markers from the active locale pack (English by default). When a
+ * marker fires the stage emits a `supersede` outcome against the
+ * highest-scoring CONFLICT-CHECK candidate from Stage 2.
+ *
+ * @packageDocumentation
+ */
+
+import { evaluateMarkers } from '../locale-packs/index.js';
+import type { PipelineStage, StageOutcome } from '../types.js';
+
+export const stage3HeuristicRegex: PipelineStage = {
+  id: 'heuristic-regex',
+  async evaluate(ctx): Promise<StageOutcome> {
+    const top = ctx.existing[0];
+    if (top === undefined) {
+      return { kind: 'continue' };
+    }
+    const candidateText = ctx.candidate.text;
+    const supersede = evaluateMarkers(candidateText, ctx.localePack.supersedeMarkers);
+    if (supersede.matched) {
+      return {
+        kind: 'supersede',
+        existingId: top.record.id,
+        reason: formatReason('regex-supersede-marker', supersede),
+      };
+    }
+    const negation = evaluateMarkers(candidateText, ctx.localePack.negationMarkers);
+    if (negation.matched) {
+      return {
+        kind: 'supersede',
+        existingId: top.record.id,
+        reason: formatReason('regex-negation', negation),
+      };
+    }
+    return { kind: 'continue' };
+  },
+};
+
+function formatReason(
+  base: string,
+  match: { kind?: string | undefined; excerpt?: string | undefined },
+): string {
+  const parts = [base];
+  if (match.kind !== undefined) parts.push(`kind=${match.kind}`);
+  if (match.excerpt !== undefined) parts.push(`marker=${match.excerpt}`);
+  return parts.join('; ');
+}

package/src/conflict/stages/stage4-subject-predicate.ts

@@ -0,0 +1,46 @@
+/**
+ * Stage 4 - Subject / predicate match.
+ *
+ * Naively splits each fact body into `(subject, predicate, object)`
+ * triples using the locale pack's predicate normalisers. When the
+ * candidate and an existing CONFLICT-CHECK fact share the same
+ * subject + predicate but differ on the object, Stage 4 marks the
+ * existing fact as superseded.
+ *
+ * The split is intentionally simple: anything more sophisticated
+ * (full entity linking) lives in the post-MVP knowledge-graph layer.
+ *
+ * @packageDocumentation
+ */
+
+import type { PipelineStage, StageOutcome } from '../types.js';
+import { splitSubjectPredicateObject } from './helpers.js';
+
+export const stage4SubjectPredicate: PipelineStage = {
+  id: 'subject-predicate',
+  async evaluate(ctx): Promise<StageOutcome> {
+    const candidateTriple = splitSubjectPredicateObject(ctx.candidate.text, ctx.localePack);
+    if (candidateTriple === null) {
+      return { kind: 'continue' };
+    }
+    for (const hit of ctx.existing) {
+      const existingTriple = splitSubjectPredicateObject(hit.record.text, ctx.localePack);
+      if (existingTriple === null) continue;
+      if (candidateTriple.subject !== existingTriple.subject) continue;
+      if (candidateTriple.predicate !== existingTriple.predicate) continue;
+      if (candidateTriple.object === existingTriple.object) {
+        return {
+          kind: 'dedup',
+          existingId: hit.record.id,
+          reason: 'subject-predicate-match-same-object',
+        };
+      }
+      return {
+        kind: 'supersede',
+        existingId: hit.record.id,
+        reason: `subject-predicate-match; subject=${candidateTriple.subject}; predicate=${candidateTriple.predicate}`,
+      };
+    }
+    return { kind: 'continue' };
+  },
+};

package/src/conflict/stages/stage5-defer-to-deep.ts

@@ -0,0 +1,34 @@
+/**
+ * Stage 5 - Defer to deep LLM judge.
+ *
+ * Reached only when Stages 1-4 left the decision unresolved. The
+ * stage records every CONFLICT-CHECK candidate id surfaced by Stage 2
+ * so the orchestrator can hand the queue write to the storage layer
+ * (the actual `conflict_check_pending` insert is the orchestrator's
+ * responsibility - Stage 5 only computes the payload).
+ *
+ * The candidate fact itself is admitted with a `pending` flag so it
+ * remains searchable; the deep phase (Phase 10c) drains the queue and
+ * applies the resolution later.
+ *
+ * @packageDocumentation
+ */
+
+import type { PipelineStage, StageOutcome } from '../types.js';
+
+export const stage5DeferToDeep: PipelineStage = {
+  id: 'defer-to-deep',
+  async evaluate(ctx): Promise<StageOutcome> {
+    if (ctx.existing.length === 0) {
+      return { kind: 'admit', reason: 'stage5-no-candidates' };
+    }
+    const conflictingIds = ctx.existing.map((hit) => hit.record.id);
+    const top = ctx.existing[0];
+    return {
+      kind: 'pending',
+      conflictingIds,
+      ...(top !== undefined ? { similarity: top.score } : {}),
+      reason: 'awaiting-deep-llm-judge',
+    };
+  },
+};

package/src/conflict/types.ts

@@ -0,0 +1,263 @@
+/**
+ * Public types for the multi-stage conflict resolution pipeline shipped
+ * in `@graphorin/memory` Phase 10b (DEC-117 / ADR-018 ext / RB-02).
+ *
+ * @packageDocumentation
+ */
+
+import type { EmbedderProvider, Fact, MemoryHit, Tracer } from '@graphorin/core';
+import type { ConflictMemoryStoreExt, MemoryStoreAdapter } from '../internal/storage-adapter.js';
+import type { LocalePack } from './locale-packs/index.js';
+
+/**
+ * Stable lowercase identifier of the pipeline stage. Mirrored
+ * byte-for-byte by `@graphorin/store-sqlite`'s `ConflictPipelineStage`
+ * so `fact_conflicts` / `conflict_check_pending` rows can be grouped
+ * deterministically.
+ *
+ * @stable
+ */
+export type ConflictStage =
+  | 'exact-dedup'
+  | 'embedding-three-zone'
+  | 'heuristic-regex'
+  | 'subject-predicate'
+  | 'defer-to-deep';
+
+/**
+ * Configurable similarity thresholds for Stage 2 (embedding three-
+ * zone). Defaults pinned at `0.95 / 0.85 / 0.4` per RB-02 §8 - the
+ * production values for the default `Xenova/multilingual-e5-base`
+ * embedder (DEC-130).
+ *
+ * @stable
+ */
+export interface ConflictThresholds {
+  readonly hot: number;
+  readonly nearDup: number;
+  readonly cold: number;
+}
+
+/**
+ * Hard-coded defaults for the three-zone thresholds. Imported by the
+ * pipeline + the test suite so call sites stay aligned.
+ *
+ * @stable
+ */
+export const DEFAULT_CONFLICT_THRESHOLDS: ConflictThresholds = Object.freeze({
+  hot: 0.95,
+  nearDup: 0.85,
+  cold: 0.4,
+});
+
+/**
+ * Final pipeline outcome - discriminated union returned by
+ * {@link runConflictPipeline}. Mirrors RB-02 §8.1 / DEC-117 - every
+ * variant carries the originating `stage` so audit + replay tooling
+ * can pattern-match without inspecting the message.
+ *
+ * @stable
+ */
+export type ConflictDecision =
+  | {
+      readonly kind: 'admit';
+      readonly stage: ConflictStage;
+      readonly reason?: string;
+    }
+  | {
+      readonly kind: 'dedup';
+      readonly stage: ConflictStage;
+      readonly existingId: string;
+      readonly similarity?: number;
+      readonly reason?: string;
+    }
+  | {
+      readonly kind: 'supersede';
+      readonly stage: ConflictStage;
+      readonly existingId: string;
+      readonly reason: string;
+    }
+  | {
+      readonly kind: 'pending';
+      readonly stage: ConflictStage;
+      /** The candidate id that was admitted as `pending`. */
+      readonly candidateId: string;
+      /** Top-K conflicting existing fact ids from Stage 2's vector search. */
+      readonly conflictingIds: ReadonlyArray<string>;
+      readonly similarity?: number;
+      readonly reason?: string;
+    };
+
+/**
+ * Pipeline configuration accepted by `createMemory({ conflictPipeline:
+ * ... })` and surfaced through {@link createConflictPipeline}.
+ *
+ * The `mode` field is the master switch:
+ *
+ *  - `'on'` (default) - the multi-stage pipeline runs on every
+ *    `SemanticMemory.remember(...)` call.
+ *  - `'off'` - bypass the pipeline and fall back to 10a's straight-
+ *    through write. Emits a one-shot WARN (per process) so operators
+ *    notice the regression risk.
+ *
+ * @stable
+ */
+export interface ConflictPipelineOptions {
+  readonly mode?: 'on' | 'off';
+  readonly thresholds?: Partial<ConflictThresholds>;
+  readonly localePack?: LocalePack;
+  /** Per-list candidate count fed into Stage 2. Default `5` (RB-02 §8.1). */
+  readonly stage2TopK?: number;
+  /** Override the audit / pending sink. Defaults to `store.conflicts` when present. */
+  readonly conflictStore?: ConflictMemoryStoreExt;
+  /** Inject a deterministic clock. Defaults to `() => new Date().toISOString()`. */
+  readonly now?: () => string;
+}
+
+/**
+ * Inputs the orchestrator hands every stage. The `existing` array is
+ * populated during Stage 2 (vector search top-K); Stage 1 receives an
+ * empty array because the dedup hash is computed off the candidate
+ * alone.
+ *
+ * @stable
+ */
+export interface StageContext {
+  readonly candidate: Fact;
+  readonly existing: ReadonlyArray<MemoryHit<Fact>>;
+  readonly localePack: LocalePack;
+  readonly thresholds: ConflictThresholds;
+}
+
+/**
+ * Per-stage outcome surfaced to the orchestrator. `'admit'` means the
+ * stage decided not to short-circuit - the pipeline continues to the
+ * next stage. Every other variant terminates the pipeline.
+ *
+ * @stable
+ */
+export type StageOutcome =
+  | { readonly kind: 'continue' }
+  | { readonly kind: 'admit'; readonly reason?: string }
+  | {
+      readonly kind: 'dedup';
+      readonly existingId: string;
+      readonly similarity?: number;
+      readonly reason?: string;
+    }
+  | {
+      readonly kind: 'supersede';
+      readonly existingId: string;
+      readonly reason: string;
+    }
+  | {
+      readonly kind: 'pending';
+      readonly conflictingIds: ReadonlyArray<string>;
+      readonly similarity?: number;
+      readonly reason?: string;
+    };
+
+/**
+ * Stage interface every step implements. The orchestrator visits
+ * stages in declaration order and stops at the first non-`'continue'`
+ * outcome.
+ *
+ * @stable
+ */
+export interface PipelineStage {
+  readonly id: ConflictStage;
+  evaluate(ctx: StageContext): Promise<StageOutcome>;
+}
+
+/**
+ * Pre-built pipeline returned by {@link createConflictPipeline}. The
+ * `run` method is what `SemanticMemory.remember(...)` calls to make
+ * the conflict decision.
+ *
+ * @stable
+ */
+export interface ConflictPipeline {
+  readonly mode: 'on' | 'off';
+  readonly localePack: LocalePack;
+  readonly thresholds: ConflictThresholds;
+  run(deps: ConflictPipelineDeps, candidate: Fact): Promise<ConflictDecision>;
+}
+
+/**
+ * Per-call dependency bag handed to the pipeline by `SemanticMemory`.
+ *
+ * @stable
+ */
+export interface ConflictPipelineDeps {
+  readonly store: MemoryStoreAdapter;
+  readonly tracer: Tracer;
+  readonly embedder: EmbedderProvider | null;
+  readonly embedderId: string | null;
+  /** Optional cancellation signal forwarded to embedder + searchVector. */
+  readonly signal?: AbortSignal;
+}
+
+/**
+ * The four actions the neighbour-aware reconcile loop (P0-3) may choose
+ * for a candidate fact once the LLM has the most-similar existing
+ * memories in view. The de-facto-standard memory write loop
+ * (Mem0 / LangMem / Letta), with Graphorin's twist: `update` and
+ * `conflict` route through a **bi-temporal supersede** (close the old
+ * interval, insert the new) rather than a destructive delete.
+ *
+ * @stable
+ */
+export type ReconcileAction = 'add' | 'update' | 'noop' | 'conflict';
+
+/**
+ * Outcome of `reconcileCandidate`. `add` is independent / unsure
+ * (a fresh fact); `noop` is a duplicate that adds nothing; `update` is
+ * a newer version of `targetId`; `conflict` contradicts `targetId` and
+ * closes it. The three neighbour-referencing variants carry the
+ * `targetId` of the existing fact they resolve against.
+ *
+ * @stable
+ */
+export type ReconcileDecision =
+  | { readonly action: 'add'; readonly reason?: string }
+  | { readonly action: 'update'; readonly targetId: string; readonly reason: string }
+  | { readonly action: 'noop'; readonly targetId: string; readonly reason?: string }
+  | { readonly action: 'conflict'; readonly targetId: string; readonly reason: string };
+
+/**
+ * Map a {@link ReconcileDecision} onto the existing {@link ConflictDecision}
+ * so reconcile outcomes land in `fact_conflicts` through the same audit
+ * path as the inline pipeline (no new stage / schema): `add` → `admit`,
+ * `noop` → `dedup`, `update` / `conflict` → `supersede`. All reconcile
+ * decisions are stamped with the `defer-to-deep` stage - the reconcile
+ * loop is the consolidator's replacement for the deferred deep-judge
+ * step it supersedes.
+ *
+ * @stable
+ */
+export function reconcileToConflictDecision(decision: ReconcileDecision): ConflictDecision {
+  const stage: ConflictStage = 'defer-to-deep';
+  switch (decision.action) {
+    case 'add':
+      return {
+        kind: 'admit',
+        stage,
+        ...(decision.reason !== undefined ? { reason: decision.reason } : {}),
+      };
+    case 'noop':
+      return {
+        kind: 'dedup',
+        stage,
+        existingId: decision.targetId,
+        ...(decision.reason !== undefined ? { reason: decision.reason } : {}),
+      };
+    case 'update':
+    case 'conflict':
+      return {
+        kind: 'supersede',
+        stage,
+        existingId: decision.targetId,
+        reason: decision.reason,
+      };
+  }
+}