@illuma-ai/agents 1.1.28 → 1.3.1

package/src/memory/mmr.ts

@@ -0,0 +1,161 @@
+/**
+ * Maximal Marginal Relevance (MMR) re-ranking — Phase 2.
+ *
+ * Ported from upstream `extensions/memory-core/src/memory/mmr.ts` with
+ * minor adaptation for our `MemoryEntry` shape (content vs snippet, id vs
+ * path+startLine). Behavior is identical: normalize scores, iteratively
+ * pick the item that maximizes `λ * relevance - (1-λ) * max_similarity`
+ * using Jaccard on tokenized content.
+ *
+ * @see Carbonell & Goldstein, "The Use of MMR, Diversity-Based Reranking" (1998)
+ */
+
+export interface MMRConfig {
+  /** Opt-in. Upstream default is false. */
+  enabled: boolean;
+  /** 0 = max diversity, 1 = max relevance. Upstream default 0.7. */
+  lambda: number;
+}
+
+export const DEFAULT_MMR_CONFIG: MMRConfig = {
+  enabled: false,
+  lambda: 0.7,
+};
+
+/**
+ * CJK Unified Ideographs + Extension A, Hiragana/Katakana, Hangul.
+ * These lack whitespace boundaries so we must tokenize them differently.
+ */
+const CJK_RE =
+  /[\u3040-\u309f\u30a0-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\u1100-\u11ff]/;
+
+/**
+ * Tokenize content into a set for Jaccard similarity.
+ *
+ * ASCII: alphanumeric + underscore runs, lowercased.
+ * CJK: each char becomes a unigram; consecutive pairs become a bigram.
+ * Non-adjacent CJK chars (e.g. `我a好`) do NOT form a bigram.
+ */
+export function tokenize(text: string): Set<string> {
+  const lower = (text ?? '').toLowerCase();
+  const ascii = lower.match(/[a-z0-9_]+/g) ?? [];
+
+  const chars = Array.from(lower);
+  const cjkData: Array<{ char: string; index: number }> = [];
+  for (let i = 0; i < chars.length; i++) {
+    if (CJK_RE.test(chars[i])) cjkData.push({ char: chars[i], index: i });
+  }
+
+  const bigrams: string[] = [];
+  for (let i = 0; i < cjkData.length - 1; i++) {
+    if (cjkData[i + 1].index === cjkData[i].index + 1) {
+      bigrams.push(cjkData[i].char + cjkData[i + 1].char);
+    }
+  }
+
+  return new Set([...ascii, ...bigrams, ...cjkData.map((d) => d.char)]);
+}
+
+export function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
+  if (a.size === 0 && b.size === 0) return 1;
+  if (a.size === 0 || b.size === 0) return 0;
+
+  const [smaller, larger] = a.size <= b.size ? [a, b] : [b, a];
+  let intersection = 0;
+  for (const t of smaller) if (larger.has(t)) intersection++;
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+export function textSimilarity(a: string, b: string): number {
+  return jaccardSimilarity(tokenize(a), tokenize(b));
+}
+
+export function computeMMRScore(
+  relevance: number,
+  maxSimilarity: number,
+  lambda: number
+): number {
+  return lambda * relevance - (1 - lambda) * maxSimilarity;
+}
+
+export interface MMRItem {
+  id: string;
+  score: number;
+  content: string;
+}
+
+/**
+ * Re-rank items using MMR. Returns a new array in MMR order.
+ */
+export function mmrRerank<T extends MMRItem>(
+  items: T[],
+  config: Partial<MMRConfig> = {}
+): T[] {
+  const enabled = config.enabled ?? DEFAULT_MMR_CONFIG.enabled;
+  const rawLambda = config.lambda ?? DEFAULT_MMR_CONFIG.lambda;
+
+  if (!enabled || items.length <= 1) return [...items];
+
+  const lambda = Math.max(0, Math.min(1, rawLambda));
+  if (lambda === 1) return [...items].sort((a, b) => b.score - a.score);
+
+  const tokenCache = new Map<string, Set<string>>();
+  for (const item of items) tokenCache.set(item.id, tokenize(item.content));
+
+  const scores = items.map((i) => i.score);
+  const maxScore = Math.max(...scores);
+  const minScore = Math.min(...scores);
+  const range = maxScore - minScore;
+  const normalize = (s: number): number =>
+    range === 0 ? 1 : (s - minScore) / range;
+
+  const selected: T[] = [];
+  const remaining = new Set(items);
+
+  while (remaining.size > 0) {
+    let best: T | null = null;
+    let bestMMR = -Infinity;
+    for (const cand of remaining) {
+      const rel = normalize(cand.score);
+      const candTokens = tokenCache.get(cand.id)!;
+      let maxSim = 0;
+      for (const sel of selected) {
+        const sim = jaccardSimilarity(candTokens, tokenCache.get(sel.id)!);
+        if (sim > maxSim) maxSim = sim;
+      }
+      const mmr = computeMMRScore(rel, maxSim, lambda);
+      if (
+        mmr > bestMMR ||
+        (mmr === bestMMR && cand.score > (best?.score ?? -Infinity))
+      ) {
+        bestMMR = mmr;
+        best = cand;
+      }
+    }
+    if (!best) break;
+    selected.push(best);
+    remaining.delete(best);
+  }
+
+  return selected;
+}
+
+/**
+ * Adapter: apply MMR to an array of MemoryEntry-shaped hits.
+ *
+ * Uses (path|id|index) as the stable ID so two hits from the same file at
+ * different content still get distinct MMR identities.
+ */
+export function applyMMRToMemoryHits<
+  T extends { id: string; path: string; content: string; score: number },
+>(results: T[], config: Partial<MMRConfig> = {}): T[] {
+  if (results.length <= 1) return results;
+  const byId = new Map<string, T>();
+  const items: MMRItem[] = results.map((r, i) => {
+    const id = `${r.path}#${r.id}#${i}`;
+    byId.set(id, r);
+    return { id, score: r.score, content: r.content };
+  });
+  return mmrRerank(items, config).map((m) => byId.get(m.id)!);
+}

package/src/memory/paths.ts

@@ -0,0 +1,258 @@
+/**
+ * Autonomous memory — canonical path whitelist + tier utilities.
+ *
+ * Single source of truth for the 8 stable memory documents. Every write
+ * goes through {@link assertWritablePath}; every reader can ask
+ * {@link getTierForPath} what a row belongs to. Adding or removing a
+ * path means editing this file — and exactly this file.
+ *
+ * ## Why a whitelist?
+ *
+ * Earlier upstream-faithful designs used date-keyed files
+ * (`memory/YYYY-MM-DD.md`), which have three problems for a persistent
+ * multi-user agent:
+ *
+ * 1. **Unbounded growth** — one row per day per agent per user, forever.
+ * 2. **LLM routing ambiguity** — "which date file do I read?" has no
+ *    good answer, so the model reads (and ranks against) all of them.
+ * 3. **No natural deduplication** — the same fact written across three
+ *    days returns three near-identical hits.
+ *
+ * With 8 stable canonical documents the LLM knows *exactly* where to
+ * write ("is this a preference? → `memory/user/preferences.md`") and
+ * each row accumulates via UPSERT instead of piling up new rows.
+ *
+ * ## Two tiers
+ *
+ * - **Agent tier** (`memory/agent/*`) — operational knowledge that
+ *   every caller of the agent benefits from. Stored with `user_id = NULL`.
+ *   Shared across all users of a collaborative agent; the only tier that
+ *   exists for isolated/autonomous agents with no invoker.
+ *
+ * - **User tier** (`memory/user/*`) — per-invoker personalization.
+ *   Stored with `user_id = <caller>`. Read path filters so User A never
+ *   sees User B's rows, even for the same agent. Not writable unless the
+ *   flush scope carries a non-empty `userId`.
+ *
+ * The tier of a path is a function of its prefix (`memory/agent/` vs
+ * `memory/user/`), so adding a new document is one line here + the
+ * constant array entry; no store or route code has to change.
+ */
+
+import { MEMORY_PATH_PREFIX } from './constants';
+import type { MemoryScope } from './types';
+
+/** Tier discriminator — the two kinds of memory a row can belong to. */
+export type MemoryTier = 'agent' | 'user';
+
+/** Every canonical document the flush phase is allowed to write. */
+export interface MemoryPathDescriptor {
+  /** Full path including the `memory/` prefix. */
+  path: string;
+  /** Which tier the path lives under. */
+  tier: MemoryTier;
+  /** Short label shown in UI tier badges and the flush prompt rubric. */
+  label: string;
+  /** One-line description — fed to the LLM to make routing unambiguous. */
+  description: string;
+}
+
+/** Agent-tier documents — shared across all users of the agent. */
+export const MEMORY_AGENT_PATHS: readonly MemoryPathDescriptor[] =
+  Object.freeze([
+    Object.freeze({
+      path: 'memory/agent/playbook.md',
+      tier: 'agent' as const,
+      label: 'Playbook',
+      description:
+        'Successful task patterns and workflows that have worked for this agent — ' +
+        'reusable recipes, known-good tool sequences, approaches that consistently ' +
+        'produce the right result.',
+    }),
+    Object.freeze({
+      path: 'memory/agent/pitfalls.md',
+      tier: 'agent' as const,
+      label: 'Pitfalls',
+      description:
+        'Tool failures, error recoveries, schema mistakes, and invalid argument ' +
+        'shapes the agent has seen — so the same mistake is not repeated on future ' +
+        'turns. One note per distinct failure mode.',
+    }),
+    Object.freeze({
+      path: 'memory/agent/domain.md',
+      tier: 'agent' as const,
+      label: 'Domain',
+      description:
+        'Durable facts about the systems, APIs, data models, and business rules ' +
+        'this agent operates on — things that are true independent of any one user ' +
+        'and that the agent repeatedly needs to know.',
+    }),
+    Object.freeze({
+      path: 'memory/agent/style.md',
+      tier: 'agent' as const,
+      label: 'Style',
+      description:
+        'Tone, formatting, and response-shape conventions the agent has converged ' +
+        'on across the whole user base. Do NOT write user-specific preferences here ' +
+        '(those belong under the user tier).',
+    }),
+  ]);
+
+/** User-tier documents — per-invoker, never shared across users. */
+export const MEMORY_USER_PATHS: readonly MemoryPathDescriptor[] = Object.freeze(
+  [
+    Object.freeze({
+      path: 'memory/user/profile.md',
+      tier: 'user' as const,
+      label: 'Profile',
+      description:
+        "This specific user's identity — name, role, team, sign-off name, " +
+        'responsibilities, stable facts about who they are. Only facts ' +
+        'volunteered by the user themselves.',
+    }),
+    Object.freeze({
+      path: 'memory/user/preferences.md',
+      tier: 'user' as const,
+      label: 'Preferences',
+      description:
+        'How THIS user wants things done — preferred formats, verbosity, tone, ' +
+        'cadence, output length, language conventions. Explicit corrections they ' +
+        'have given count as preferences.',
+    }),
+    Object.freeze({
+      path: 'memory/user/projects.md',
+      tier: 'user' as const,
+      label: 'Projects',
+      description:
+        "This user's current initiatives, ongoing work, deadlines, and active " +
+        'project context. Include dates in the prose when relevant. Retire entries ' +
+        'by overwriting them on future flushes when the user indicates a project is done.',
+    }),
+    Object.freeze({
+      path: 'memory/user/references.md',
+      tier: 'user' as const,
+      label: 'References',
+      description:
+        'External systems, dashboards, accounts, links, channels, and resources ' +
+        "THIS user has pointed the agent at — the 'where to look' pointers that " +
+        'make later turns more efficient.',
+    }),
+  ]
+);
+
+/** All 8 canonical documents in display order: agent tier first, user tier second. */
+export const MEMORY_ALL_PATHS: readonly MemoryPathDescriptor[] = Object.freeze([
+  ...MEMORY_AGENT_PATHS,
+  ...MEMORY_USER_PATHS,
+]);
+
+/** Fast O(1) lookup map: path → descriptor. */
+const PATH_INDEX: ReadonlyMap<string, MemoryPathDescriptor> = new Map(
+  MEMORY_ALL_PATHS.map((p) => [p.path, p])
+);
+
+/** Set of all whitelisted paths — for `has()` lookups. */
+export const MEMORY_WRITABLE_PATHS: ReadonlySet<string> = new Set(
+  MEMORY_ALL_PATHS.map((p) => p.path)
+);
+
+/** Returns the descriptor for a canonical path, or `undefined` if not on the whitelist. */
+export function getPathDescriptor(
+  path: string
+): MemoryPathDescriptor | undefined {
+  return PATH_INDEX.get(path);
+}
+
+/**
+ * Returns the tier a given path belongs to.
+ *
+ * For unknown paths (e.g. legacy date-keyed rows that predate this schema),
+ * falls back to inspecting the `memory/agent/` or `memory/user/` prefix.
+ * Anything that matches neither is classified as `agent` — it will surface
+ * in the admin UI under the agent-tier header and stay read-only there.
+ */
+export function getTierForPath(path: string): MemoryTier {
+  const descriptor = PATH_INDEX.get(path);
+  if (descriptor) return descriptor.tier;
+  if (path.startsWith('memory/user/')) return 'user';
+  return 'agent';
+}
+
+/**
+ * Paths that are legal to write **for this caller's scope**.
+ *
+ * If `scope.userId` is absent (isolated/autonomous agent), the user-tier
+ * paths are dropped. This is the list the flush prompt renders into its
+ * rubric — the LLM never sees paths it cannot write to.
+ */
+export function getWritablePathsForScope(
+  scope: Pick<MemoryScope, 'userId'>
+): readonly MemoryPathDescriptor[] {
+  if (scope.userId == null || scope.userId === '') {
+    return MEMORY_AGENT_PATHS;
+  }
+  return MEMORY_ALL_PATHS;
+}
+
+/**
+ * Validates a write path against the whitelist AND the caller's scope.
+ *
+ * Throws with an actionable message when:
+ * - the path is missing the `memory/` prefix
+ * - the path is not on the 8-doc whitelist
+ * - the path is user-tier but `scope.userId` is missing
+ *
+ * The store's append() is the single call site; other callers should
+ * never hit this directly. Kept as a standalone function for testability.
+ */
+export function assertWritablePath(
+  path: string,
+  scope: Pick<MemoryScope, 'userId'>
+): MemoryPathDescriptor {
+  if (!path || !path.startsWith(MEMORY_PATH_PREFIX)) {
+    throw new Error(
+      `memory_append path must start with "${MEMORY_PATH_PREFIX}" (received "${path}")`
+    );
+  }
+  const descriptor = PATH_INDEX.get(path);
+  if (!descriptor) {
+    throw new Error(
+      `memory_append path "${path}" is not on the canonical whitelist. ` +
+        `Allowed: ${MEMORY_ALL_PATHS.map((p) => p.path).join(', ')}`
+    );
+  }
+  if (
+    descriptor.tier === 'user' &&
+    (scope.userId == null || scope.userId === '')
+  ) {
+    throw new Error(
+      `memory_append to user-tier path "${path}" requires a caller userId in scope, ` +
+        `but scope.userId is missing. This path is private to a specific user and ` +
+        `cannot be written from an isolated/autonomous context.`
+    );
+  }
+  return descriptor;
+}
+
+/**
+ * Renders the whitelist as a compact bullet list, used inside the flush
+ * prompt so the LLM sees the authoritative rubric at inference time.
+ *
+ * Output shape:
+ *   - memory/agent/playbook.md [Playbook, shared] — <description>
+ *   - memory/agent/pitfalls.md [Pitfalls, shared] — <description>
+ *   ...
+ *
+ * The `[Label, shared|private]` tag tells the model whether the doc is
+ * visible to all users (agent tier) or only to the caller (user tier).
+ * Filtered to only the paths writable for the given scope.
+ */
+export function renderPathsRubric(scope: Pick<MemoryScope, 'userId'>): string {
+  const writable = getWritablePathsForScope(scope);
+  return writable
+    .map((p) => {
+      const visibility = p.tier === 'agent' ? 'shared' : 'private';
+      return `- ${p.path} [${p.label}, ${visibility}] — ${p.description}`;
+    })
+    .join('\n');
+}