agent-working-memory 0.13.0 → 0.14.0

package/src/core/token-budget.ts

@@ -0,0 +1,160 @@
+// Copyright 2026 Robert Winter / Complete Ideas
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Token budgeting for memory_recall (0.13.3).
+ *
+ * WHY THIS EXISTS
+ * ---------------
+ * `limit: N` is a COUNT, which is token-blind: a 5-result recall can cost 400
+ * tokens or 4,000 depending on how long the memories happen to be. AWM's own
+ * benchmark shows the problem — in real sessions AWM retrieval averaged 4,514
+ * tokens per call against 2,106 for plain file retrieval. AWM wins 9.8:1 in
+ * AGGREGATE (it needs far fewer calls) but loses 2.1:1 PER CALL. Per-call cost
+ * is the exposed flank, and a caller had no way to bound it.
+ *
+ * `max_tokens` lets the caller say "I have 800 tokens of context to spare"
+ * instead of guessing at a result count.
+ *
+ * PACKING STRATEGY
+ * ----------------
+ * This is a 0/1 knapsack (maximise score within a token budget), so we use the
+ * standard greedy density heuristic — order by score-per-token — with one
+ * deliberate exception: the top-scored result is always admitted first if it
+ * fits at all. Density packing alone can drop the single most relevant memory
+ * purely for being long, which is exactly the answer the caller wanted.
+ *
+ * Selection is by density; OUTPUT is restored to score order, because a reader
+ * scanning results expects the best one first.
+ *
+ * ESTIMATOR
+ * ---------
+ * Deliberately dependency-free: no tokenizer, no model load, no async. It
+ * takes max(words x 1.3, chars / 4) — the two standard English/code
+ * approximations — and takes the larger so the estimate errs toward
+ * OVER-counting. Over-counting keeps us inside the budget; under-counting
+ * would silently blow it, which is the failure that matters.
+ */
+
+import type { ActivationResult } from '../types/engram.js';
+
+/** Rough token count. Over-estimates by design — see the note above. */
+export function estimateTokens(text: string): number {
+  if (!text) return 0;
+  const chars = text.length;
+  const words = text.split(/\s+/).filter(Boolean).length;
+  return Math.max(Math.ceil(words * 1.3), Math.ceil(chars / 4));
+}
+
+export interface PackedRecall {
+  /** Formatted lines that fit the budget, in score order. */
+  lines: string[];
+  /** How many results were kept. */
+  kept: number;
+  /** How many results were available before budgeting. */
+  total: number;
+  /** Estimated tokens of the kept lines. */
+  tokens: number;
+  /** Estimated tokens of everything dropped (0 when nothing was dropped). */
+  withheldTokens: number;
+}
+
+/**
+ * Select and format results to fit `maxTokens`.
+ *
+ * When `maxTokens` is undefined the behaviour is unchanged from pre-0.13.3 —
+ * everything is returned — but the accounting fields are still populated, so
+ * callers get per-call token visibility without opting into budgeting.
+ */
+export function packRecallByBudget(
+  results: ActivationResult[],
+  format: (r: ActivationResult, index: number) => string,
+  maxTokens?: number,
+  /**
+   * Tokens the caller will spend on the SAME reply outside the result lines —
+   * the accounting footer, and any peer-decisions suffix. Reserved up front so
+   * the budget bounds the whole reply rather than just the part this function
+   * happens to build.
+   *
+   * Found by the end-to-end eval, not by the unit tests: budgets of 600/250/80
+   * came back as 601/256/95 because the footer spent tokens it never counted.
+   */
+  reservedTokens = 0,
+): PackedRecall {
+  const total = results.length;
+  if (total === 0) return { lines: [], kept: 0, total: 0, tokens: 0, withheldTokens: 0 };
+  const effectiveBudget = maxTokens === undefined ? undefined
+    : Math.max(0, maxTokens - reservedTokens);
+
+  // Format once against the ORIGINAL index so displayed numbering matches the
+  // caller's mental model of "result 1 is the best match".
+  const measured = results.map((r, i) => {
+    const line = format(r, i);
+    return { line, tokens: estimateTokens(line), score: r.score, order: i };
+  });
+
+  const allTokens = measured.reduce((n, m) => n + m.tokens, 0);
+
+  // NOTE the asymmetry, which a unit test caught: `maxTokens` being absent or
+  // nonsense (<= 0) means "no budget — return everything", but an
+  // *effectiveBudget* of 0 means the reserve ate the entire budget, which must
+  // admit NOTHING. Collapsing those two cases returned the full result set on
+  // the tightest budgets — the exact opposite of what was asked for.
+  const unbudgeted = maxTokens === undefined || maxTokens <= 0;
+  if (unbudgeted || allTokens <= effectiveBudget!) {
+    return {
+      lines: measured.map(m => m.line),
+      kept: total,
+      total,
+      tokens: allTokens,
+      withheldTokens: 0,
+    };
+  }
+
+  const chosen: typeof measured = [];
+  let spent = 0;
+
+  // 1. Top-scored result gets first refusal, so a long best-match is never
+  //    dropped in favour of several short weak ones.
+  const top = measured[0];
+  if (top.tokens <= effectiveBudget!) {
+    chosen.push(top);
+    spent += top.tokens;
+  }
+
+  // 2. Everything else by value density (score per token), greedily.
+  const rest = measured
+    .filter(m => m !== top)
+    .sort((a, b) => (b.score / Math.max(b.tokens, 1)) - (a.score / Math.max(a.tokens, 1)));
+
+  for (const m of rest) {
+    if (spent + m.tokens > effectiveBudget!) continue;   // skip, don't stop: a later
+    chosen.push(m);                                // shorter result may still fit
+    spent += m.tokens;
+  }
+
+  // 3. Restore score order for display.
+  chosen.sort((a, b) => a.order - b.order);
+
+  return {
+    lines: chosen.map(m => m.line),
+    kept: chosen.length,
+    total,
+    tokens: spent,
+    withheldTokens: allTokens - spent,
+  };
+}
+
+/**
+ * One-line accounting footer. Costs ~15-25 tokens to report, which is a good
+ * trade against a recall that can run to thousands — and it is what makes the
+ * per-call cost visible to both the human and the harness. Without this,
+ * AWM's token behaviour is only observable in an offline benchmark.
+ */
+export function formatTokenFooter(p: PackedRecall, maxTokens?: number): string {
+  if (p.total === 0) return '';
+  const parts = [`~${p.tokens} tok`];
+  parts.push(p.kept === p.total ? `${p.total} results` : `${p.kept}/${p.total} results`);
+  if (maxTokens !== undefined && maxTokens > 0) parts.push(`budget ${maxTokens}`);
+  if (p.withheldTokens > 0) parts.push(`~${p.withheldTokens} tok withheld`);
+  return `\n\n[awm: ${parts.join(' · ')}]`;
+}

package/src/core/whoami.ts

@@ -1,92 +1,110 @@
-// Copyright 2026 Robert Winter / Complete Ideas
-// SPDX-License-Identifier: Apache-2.0
-
-/**
- * Instance identity (D3, 2026-07-30).
- *
- * Answers "which AWM am I talking to?" — the recurring confusion class where
- * sessions mix up the hosted multi-agent connector with the local instance,
- * or run blind to sibling agent spaces in the same store (an evaluation once
- * analyzed this project without seeing the `personal` space that held its
- * design decisions).
- *
- * Exposed as the `memory_whoami` MCP tool and the `GET /whoami` HTTP route.
- * Read-only; additive module.
- */
-
-import { dirname, join, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { VERSION } from '../version.js';
-import { getConfiguredBackend, getConfiguredPath } from '../storage/factory.js';
-
-export interface WhoamiInfo {
-  agentId: string;
-  workspace: string | null;
-  surface: 'mcp' | 'http';
-  mode: 'standalone' | 'hive';
-  backend: string;
-  storePath: string;
-  version: string;
-  codePath: string;
-  pid: number;
-  ports: { http: number | null; hookSidecar: number | null };
-  siblingAgents: string[];
-}
-
-/** Store subset whoami needs — every backend implements listAgentIds. */
-export interface WhoamiStore {
-  listAgentIds(): string[] | Promise<string[]>;
-}
-
-function packageRoot(): string {
-  // src/core/whoami.ts (dev) and dist/core/whoami.js (build) both sit two
-  // levels below the package root.
-  const here = dirname(fileURLToPath(import.meta.url));
-  return resolve(join(here, '..', '..'));
-}
-
-export async function buildWhoami(
-  store: WhoamiStore,
-  agentId: string,
-  surface: 'mcp' | 'http',
-): Promise<WhoamiInfo> {
-  const coordination = process.env.AWM_COORDINATION === 'true' || process.env.AWM_COORDINATION === '1';
-  let siblings: string[] = [];
-  try {
-    const all = await store.listAgentIds();
-    siblings = all.filter(a => a !== agentId).sort();
-  } catch { /* identity must never fail on a listing error */ }
-  const httpPort = Number(process.env.AWM_PORT ?? '');
-  const hookPort = Number(process.env.AWM_HOOK_PORT ?? '');
-  return {
-    agentId,
-    workspace: process.env.AWM_WORKSPACE ?? null,
-    surface,
-    mode: coordination ? 'hive' : 'standalone',
-    backend: getConfiguredBackend(),
-    storePath: getConfiguredPath(),
-    version: VERSION,
-    codePath: packageRoot(),
-    pid: process.pid,
-    ports: {
-      http: Number.isFinite(httpPort) && httpPort > 0 ? httpPort : null,
-      hookSidecar: Number.isFinite(hookPort) && hookPort > 0 ? hookPort : null,
-    },
-    siblingAgents: siblings,
-  };
-}
-
-/** Human-readable rendering for the MCP tool response. */
-export function formatWhoami(w: WhoamiInfo): string {
-  return [
-    `Agent: ${w.agentId}${w.workspace ? ` (workspace: ${w.workspace})` : ''}`,
-    `Mode: ${w.mode} · Surface: ${w.surface} · Version: ${w.version}`,
-    `Backend: ${w.backend}`,
-    `Store: ${w.storePath}`,
-    `Code: ${w.codePath} (pid ${w.pid})`,
-    `Ports: http=${w.ports.http ?? 'off'} hookSidecar=${w.ports.hookSidecar ?? 'off'}`,
-    w.siblingAgents.length
-      ? `Sibling agent spaces in this store: ${w.siblingAgents.join(', ')} — recall is scoped to '${w.agentId}'; other spaces need workspace recall, \`awm export --agent <id>\`, or a session configured for that agent.`
-      : 'Sibling agent spaces in this store: none',
-  ].join('\n');
-}
+// Copyright 2026 Robert Winter / Complete Ideas
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Instance identity (D3, 2026-07-30).
+ *
+ * Answers "which AWM am I talking to?" — the recurring confusion class where
+ * sessions mix up the hosted multi-agent connector with the local instance,
+ * or run blind to sibling agent spaces in the same store (an evaluation once
+ * analyzed this project without seeing the `personal` space that held its
+ * design decisions).
+ *
+ * Exposed as the `memory_whoami` MCP tool and the `GET /whoami` HTTP route.
+ * Read-only; additive module.
+ */
+
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { VERSION } from '../version.js';
+import { getConfiguredBackend, getConfiguredPath } from '../storage/factory.js';
+import { activeRecallConfig, recallConfigFingerprint } from './recall-config.js';
+import { embeddingHealth } from './embeddings.js';
+
+export interface WhoamiInfo {
+  agentId: string;
+  workspace: string | null;
+  surface: 'mcp' | 'http';
+  mode: 'standalone' | 'hive';
+  backend: string;
+  storePath: string;
+  version: string;
+  codePath: string;
+  pid: number;
+  ports: { http: number | null; hookSidecar: number | null };
+  /**
+   * Effective recall configuration. `whoami` is the "what am I actually
+   * running" tool, and the version alone does not answer that: a build can be
+   * current while the recall flags that change its behaviour are unset, or set
+   * to something unintended. GET /health already reports this, but HTTP is off
+   * by default, so an MCP-only session had no way to see it.
+   */
+  recall: { fingerprint: string; flags: Record<string, string> };
+  siblingAgents: string[];
+}
+
+/** Store subset whoami needs — every backend implements listAgentIds. */
+export interface WhoamiStore {
+  listAgentIds(): string[] | Promise<string[]>;
+}
+
+function packageRoot(): string {
+  // src/core/whoami.ts (dev) and dist/core/whoami.js (build) both sit two
+  // levels below the package root.
+  const here = dirname(fileURLToPath(import.meta.url));
+  return resolve(join(here, '..', '..'));
+}
+
+export async function buildWhoami(
+  store: WhoamiStore,
+  agentId: string,
+  surface: 'mcp' | 'http',
+): Promise<WhoamiInfo> {
+  const coordination = process.env.AWM_COORDINATION === 'true' || process.env.AWM_COORDINATION === '1';
+  let siblings: string[] = [];
+  try {
+    const all = await store.listAgentIds();
+    siblings = all.filter(a => a !== agentId).sort();
+  } catch { /* identity must never fail on a listing error */ }
+  const httpPort = Number(process.env.AWM_PORT ?? '');
+  const hookPort = Number(process.env.AWM_HOOK_PORT ?? '');
+  return {
+    agentId,
+    workspace: process.env.AWM_WORKSPACE ?? null,
+    surface,
+    mode: coordination ? 'hive' : 'standalone',
+    backend: getConfiguredBackend(),
+    storePath: getConfiguredPath(),
+    version: VERSION,
+    codePath: packageRoot(),
+    pid: process.pid,
+    ports: {
+      http: Number.isFinite(httpPort) && httpPort > 0 ? httpPort : null,
+      hookSidecar: Number.isFinite(hookPort) && hookPort > 0 ? hookPort : null,
+    },
+    recall: { fingerprint: recallConfigFingerprint(), flags: activeRecallConfig() },
+    siblingAgents: siblings,
+  };
+}
+
+/** Human-readable rendering for the MCP tool response. */
+export function formatWhoami(w: WhoamiInfo): string {
+  return [
+    `Agent: ${w.agentId}${w.workspace ? ` (workspace: ${w.workspace})` : ''}`,
+    `Mode: ${w.mode} · Surface: ${w.surface} · Version: ${w.version}`,
+    `Backend: ${w.backend}`,
+    `Store: ${w.storePath}`,
+    `Code: ${w.codePath} (pid ${w.pid})`,
+    `Ports: http=${w.ports.http ?? 'off'} hookSidecar=${w.ports.hookSidecar ?? 'off'}`,
+    `Recall config: ${w.recall.fingerprint}`,
+    ...(embeddingHealth().dimensionMismatches > 0
+      ? [`\u26a0 EMBEDDING INTEGRITY: ${embeddingHealth().dimensionMismatches} dimension mismatches ` +
+         `(expected ${embeddingHealth().expectedDimensions}d). Affected memories score 0 on the ` +
+         `vector channel — the corpus is probably half-migrated. Re-embed the whole store or ` +
+         `revert AWM_EMBED_MODEL/AWM_EMBED_DIMS.`]
+      : []),
+    w.siblingAgents.length
+      ? `Sibling agent spaces in this store: ${w.siblingAgents.join(', ')} — recall is scoped to '${w.agentId}'; other spaces need workspace recall, \`awm export --agent <id>\`, or a session configured for that agent.`
+      : 'Sibling agent spaces in this store: none',
+  ].join('\n');
+}

package/src/core/write-pipeline.ts

@@ -51,6 +51,7 @@ import { extractMetaTags } from './auto-tagger.js';
 import { extractEntitiesFromTags } from './entity-extract.js';
 import { reportWrite } from './write-telemetry.js';
 import { DEFAULT_AGENT_CONFIG } from '../types/agent.js';
+import { buildRetrievalText } from './retrieval-text.js';
 
 /** Confidence floor below which a matched engram is treated as "decaying out". */
 export const HEALTHY_CONFIDENCE_FLOOR = 0.3;
@@ -195,7 +196,7 @@ export async function performWrite(
   let prewriteEmbedding: number[] | null = null;
   if (process.env.AWM_NOVELTY_EMBED !== '0') {
     try {
-      prewriteEmbedding = await embed(`${input.concept} ${input.content}`);
+      prewriteEmbedding = await embed(buildRetrievalText(input.concept, input.content, input.tags));
     } catch { /* fall through — async embed will retry later */ }
   }
   tEmbed = performance.now() - tStartEmbed;
@@ -434,7 +435,9 @@ async function reinforceMatched(
         // beyond 512 tokens but the topic anchor (early content) drives
         // the vector for retrieval purposes.
         const conceptForEmbed = newConceptHint || matched.concept;
-        const newVec = await embed(`${conceptForEmbed} ${mergedContent}`);
+        // The reinforced engram's OWN tags — `input` is not in scope here, and
+        // the stored tags are the right vocabulary for the stored memory anyway.
+        const newVec = await embed(buildRetrievalText(conceptForEmbed, mergedContent, matched.tags));
         await store.updateEmbedding(matched.id, newVec);
       } catch { /* merge is best-effort — confidence bump already landed */ }
     }
@@ -574,7 +577,7 @@ async function createNewEngram(
   const alreadyEmbedded = prewriteEmbedding != null && prewriteEmbedding.length > 0;
   const shouldEmbed = (!isStructural || input.embed === true) && !alreadyEmbedded;
   if (shouldEmbed) {
-    embed(`${input.concept} ${input.content}`)
+    embed(buildRetrievalText(input.concept, input.content, input.tags))
       .then(async vec => {
         try { await store.updateEmbedding(engram.id, vec); } catch { /* engram may be evicted */ }
       })