@staix/agent-mem 0.6.0 → 0.7.0

package/src/memory/eval-set.json

@@ -0,0 +1,443 @@
+{
+  "schema": "amem.eval/v1",
+  "entries": [
+    {
+      "key": "shift-handover-old",
+      "kind": "decision",
+      "tier": "long",
+      "title": "Shift handover happens at the gate office",
+      "text": "The harbour shift handover is done at the gate office with a signed paper log.",
+      "about": [
+        "path:src/shift/handover.ts"
+      ]
+    },
+    {
+      "key": "harbor-retry",
+      "kind": "decision",
+      "title": "Harbor client retries idempotent calls three times",
+      "text": "The Harbor ferry client retries an idempotent call three times with exponential backoff, and never retries a booking write.",
+      "about": [
+        "path:src/ferry/client.ts"
+      ]
+    },
+    {
+      "key": "harbor-timeout",
+      "kind": "finding",
+      "title": "Dock sensor requests time out after eight seconds",
+      "text": "A dock sensor request that takes longer than eight seconds is abandoned and reported as stale, not as an error.",
+      "about": [
+        "path:src/dock/sensor.ts"
+      ],
+      "tier": "short"
+    },
+    {
+      "key": "tide-cache",
+      "kind": "decision",
+      "title": "Tide tables are cached for one day",
+      "text": "Tide tables come from the almanac service and are cached on disk for one day, because the almanac rate-limits per hour.",
+      "about": [
+        "path:src/tide/cache.ts"
+      ]
+    },
+    {
+      "key": "tide-rounding",
+      "kind": "finding",
+      "title": "Tide heights round half to even",
+      "text": "Tide heights are rounded half to even before display, so 2.5 metres shows as 2 and 3.5 as 4.",
+      "about": [
+        "path:src/tide/format.ts"
+      ]
+    },
+    {
+      "key": "crane-lock",
+      "kind": "contract",
+      "title": "Only one crane operator lock per berth",
+      "text": "A berth holds at most one crane operator lock. A second request waits, it does not steal the lock, and the lock expires after ninety seconds without a heartbeat.",
+      "about": [
+        "path:src/crane/lock.ts"
+      ]
+    },
+    {
+      "key": "crane-heartbeat",
+      "kind": "finding",
+      "title": "Crane heartbeat drift on the night shift",
+      "text": "The night shift crane heartbeat drifted by up to four seconds because the controller clock was not synced; the lock expiry still held.",
+      "about": [
+        "path:src/crane/heartbeat.ts"
+      ],
+      "tier": "short"
+    },
+    {
+      "key": "manifest-schema",
+      "kind": "contract",
+      "title": "Cargo manifest rows carry a container prefix",
+      "text": "Every cargo manifest row carries a four-letter container prefix and a seven digit serial. A row without both is rejected at import.",
+      "about": [
+        "path:src/cargo/manifest.ts"
+      ]
+    },
+    {
+      "key": "manifest-import",
+      "kind": "fail",
+      "title": "Importing manifests in one transaction blew the memory budget",
+      "text": "Importing a whole cargo manifest in one transaction used more than a gigabyte. Stream it in batches of five hundred rows instead.",
+      "about": [
+        "path:src/cargo/import.ts"
+      ]
+    },
+    {
+      "key": "pilot-roster",
+      "kind": "decision",
+      "title": "Pilot roster is published every Friday",
+      "text": "The pilot roster for the coming week is published on Friday at noon local time and is frozen afterwards except for sick cover.",
+      "about": [
+        "path:src/pilot/roster.ts"
+      ]
+    },
+    {
+      "key": "pilot-fatigue",
+      "kind": "contract",
+      "title": "A pilot may not exceed twelve hours on duty",
+      "text": "The scheduler refuses an assignment that takes a pilot past twelve hours on duty in any rolling twenty-four hours.",
+      "about": [
+        "path:src/pilot/fatigue.ts"
+      ]
+    },
+    {
+      "key": "fuel-ledger",
+      "kind": "decision",
+      "title": "Fuel is recorded in litres, converted once",
+      "text": "Bunker fuel is recorded in litres. Conversion from tonnes happens once at the edge, using the density on the delivery note.",
+      "about": [
+        "path:src/fuel/ledger.ts"
+      ]
+    },
+    {
+      "key": "fuel-density",
+      "kind": "finding",
+      "title": "Delivery note density is sometimes missing",
+      "text": "About one delivery note in forty has no fuel density. The ledger falls back to the supplier default and flags the row for review.",
+      "about": [
+        "path:src/fuel/density.ts"
+      ],
+      "tier": "short"
+    },
+    {
+      "key": "berth-clash",
+      "kind": "fail",
+      "title": "Two vessels were booked into the same berth window",
+      "text": "Two vessels were booked into one berth window because the overlap check compared local times across a daylight saving change. Compare in UTC.",
+      "about": [
+        "path:src/berth/overlap.ts"
+      ]
+    },
+    {
+      "key": "berth-priority",
+      "kind": "decision",
+      "title": "Passenger ferries outrank freight at the quay",
+      "text": "When a quay berth is contested the passenger ferry keeps it and the freight vessel moves to the outer berth.",
+      "about": [
+        "path:src/berth/priority.ts"
+      ]
+    },
+    {
+      "key": "alarm-quiet",
+      "kind": "decision",
+      "title": "Fog alarms are quiet between midnight and five",
+      "text": "The fog horn test alarm is silenced between midnight and five in the morning, the real fog alarm is never silenced.",
+      "about": [
+        "path:src/alarm/fog.ts"
+      ],
+      "tier": "short"
+    },
+    {
+      "key": "alarm-escalate",
+      "kind": "contract",
+      "title": "Unacknowledged alarms escalate after ten minutes",
+      "text": "An alarm nobody acknowledges within ten minutes escalates to the harbour master's phone, then to the duty manager after another ten.",
+      "about": [
+        "path:src/alarm/escalate.ts"
+      ]
+    },
+    {
+      "key": "radio-channel",
+      "kind": "finding",
+      "title": "Channel sixteen is monitored, not used for routine traffic",
+      "text": "Channel sixteen is monitored for distress calls only. Routine traffic moves to the working channel named in the berth assignment.",
+      "about": [
+        "path:src/radio/channels.ts"
+      ]
+    },
+    {
+      "key": "radio-log",
+      "kind": "decision",
+      "title": "Radio calls are logged with a vessel callsign",
+      "text": "Every radio call is logged with the vessel callsign and a UTC timestamp, never with the master's name.",
+      "about": [
+        "path:src/radio/log.ts"
+      ]
+    },
+    {
+      "key": "gate-badge",
+      "kind": "contract",
+      "title": "Gate badges expire at end of shift",
+      "text": "A gate badge is valid until the end of the holder's shift and a contractor badge never outlives the contract end date.",
+      "about": [
+        "path:src/gate/badge.ts"
+      ]
+    },
+    {
+      "key": "gate-camera",
+      "kind": "finding",
+      "title": "Gate camera plate reader fails in heavy rain",
+      "text": "The gate camera plate reader misreads plates in heavy rain; the guard confirms by hand when confidence is below ninety percent.",
+      "about": [
+        "path:src/gate/camera.ts"
+      ],
+      "tier": "short"
+    },
+    {
+      "key": "report-monthly",
+      "kind": "decision",
+      "title": "Throughput report closes on the third working day",
+      "text": "The monthly throughput report closes on the third working day, late manifests are counted in the next month.",
+      "about": [
+        "path:src/report/monthly.ts"
+      ]
+    },
+    {
+      "key": "report-units",
+      "kind": "finding",
+      "title": "Throughput is reported in twenty-foot equivalents",
+      "text": "Throughput is reported in twenty-foot equivalent units, a forty-foot container counts as two.",
+      "about": [
+        "path:src/report/units.ts"
+      ]
+    },
+    {
+      "key": "backup-nightly",
+      "kind": "decision",
+      "title": "Database backup runs at two in the morning",
+      "text": "The operations database is backed up at two in the morning and the copy is verified by a test restore every Sunday.",
+      "about": [
+        "path:ops/backup.sh"
+      ]
+    },
+    {
+      "key": "backup-restore",
+      "kind": "fail",
+      "title": "A backup restored without the audit table",
+      "text": "A restore test passed while the audit table was missing, because the verification only counted the cargo tables. Count every table.",
+      "about": [
+        "path:ops/restore-check.sh"
+      ]
+    },
+    {
+      "key": "shift-handover-new",
+      "kind": "finding",
+      "tier": "short",
+      "title": "Shift handover happens at the gate office",
+      "text": "The harbour shift handover is done at the gate office, and the signed paper log is being replaced by a tablet.",
+      "about": [
+        "path:src/shift/handover.ts"
+      ]
+    }
+  ],
+  "questions": [
+    {
+      "q": "how many times does the ferry client retry",
+      "expect": [
+        "harbor-retry"
+      ]
+    },
+    {
+      "q": "retries backoff idempotent",
+      "expect": [
+        "harbor-retry"
+      ]
+    },
+    {
+      "q": "dock sensor timeout",
+      "expect": [
+        "harbor-timeout"
+      ]
+    },
+    {
+      "q": "why are tide tables cached",
+      "expect": [
+        "tide-cache"
+      ]
+    },
+    {
+      "q": "almanac rate limit",
+      "expect": [
+        "tide-cache"
+      ]
+    },
+    {
+      "q": "round half to even tide height",
+      "expect": [
+        "tide-rounding"
+      ]
+    },
+    {
+      "q": "crane lock expiry without heartbeat",
+      "expect": [
+        "crane-lock",
+        "crane-heartbeat"
+      ]
+    },
+    {
+      "q": "night shift clock drift",
+      "expect": [
+        "crane-heartbeat"
+      ]
+    },
+    {
+      "q": "container prefix serial number manifest",
+      "expect": [
+        "manifest-schema"
+      ]
+    },
+    {
+      "q": "manifest import memory too large",
+      "expect": [
+        "manifest-import"
+      ]
+    },
+    {
+      "q": "when is the pilot roster published",
+      "expect": [
+        "pilot-roster"
+      ]
+    },
+    {
+      "q": "pilot twelve hours on duty",
+      "expect": [
+        "pilot-fatigue"
+      ]
+    },
+    {
+      "q": "fuel density litres tonnes",
+      "expect": [
+        "fuel-ledger",
+        "fuel-density"
+      ]
+    },
+    {
+      "q": "missing density on delivery note",
+      "expect": [
+        "fuel-density"
+      ]
+    },
+    {
+      "q": "berth double booking daylight saving",
+      "expect": [
+        "berth-clash"
+      ]
+    },
+    {
+      "q": "who wins a contested quay berth",
+      "expect": [
+        "berth-priority"
+      ]
+    },
+    {
+      "q": "fog horn alarm silenced at night",
+      "expect": [
+        "alarm-quiet"
+      ]
+    },
+    {
+      "q": "unacknowledged alarm escalates",
+      "expect": [
+        "alarm-escalate"
+      ]
+    },
+    {
+      "q": "distress channel sixteen",
+      "expect": [
+        "radio-channel"
+      ]
+    },
+    {
+      "q": "radio call log callsign",
+      "expect": [
+        "radio-log"
+      ]
+    },
+    {
+      "q": "contractor badge expiry",
+      "expect": [
+        "gate-badge"
+      ]
+    },
+    {
+      "q": "plate reader rain",
+      "expect": [
+        "gate-camera"
+      ]
+    },
+    {
+      "q": "monthly throughput report close date",
+      "expect": [
+        "report-monthly"
+      ]
+    },
+    {
+      "q": "forty foot container counts as two",
+      "expect": [
+        "report-units"
+      ]
+    },
+    {
+      "q": "database backup schedule test restore",
+      "expect": [
+        "backup-nightly",
+        "backup-restore"
+      ]
+    },
+    {
+      "q": "restore check missed the audit table",
+      "expect": [
+        "backup-restore"
+      ]
+    },
+    {
+      "q": "what stops two tugs sharing a mooring",
+      "expect": [
+        "berth-clash"
+      ]
+    },
+    {
+      "q": "staff badge validity period",
+      "expect": [
+        "gate-badge"
+      ]
+    },
+    {
+      "q": "vessel arrival prediction model",
+      "expect": [
+        "tide-cache"
+      ]
+    },
+    {
+      "q": "who approves overtime",
+      "expect": [
+        "pilot-fatigue"
+      ]
+    },
+    {
+      "q": "where is the shift handover done",
+      "expect": [
+        "shift-handover-new"
+      ]
+    },
+    {
+      "q": "paper log shift handover gate office",
+      "expect": [
+        "shift-handover-old"
+      ]
+    }
+  ]
+}

package/src/memory/eval.ts

@@ -0,0 +1,150 @@
+/**
+ * F13 (agent-mem#40, AC7): local recall evaluation. A question set is a list of (question, expected entry) pairs; the evaluation replays each question
+ * through a search function and reports recall@k, hit@k and MRR. No model judge, no network, no embedding: a result either is one of the expected
+ * entries or it is not. This leaf has no imports, so the parser and the arithmetic are testable without a store.
+ *
+ * Two shapes of set:
+ *  - self-contained: `entries` are synthetic notes seeded into a throwaway store, and `expect` names them by `key` (the committed fixture is one);
+ *  - the person's own: no `entries`, `expect` holds real entry ids (or `mem:<id>`, or a prefix of 8 or more hex characters) and the bound project's store is searched.
+ */
+export const EVAL_SCHEMA = "amem.eval/v1";
+export const MAX_QUESTIONS = 500, MAX_ENTRIES = 200, MAX_QUESTION_CHARS = 500, MAX_EXPECT = 20, MAX_ENTRY_CHARS = 4000, MAX_SET_BYTES = 1024 * 1024, MAX_K = 100;
+export const DEFAULT_KS: readonly number[] = [1, 5, 10];
+
+export const EVAL_KINDS = ["decision", "finding", "contract", "fail"] as const;
+export interface EvalEntry { key: string; title?: string; text: string; kind?: (typeof EVAL_KINDS)[number]; tier?: "short" | "long"; about?: string[] }
+export interface EvalQuestion { q: string; expect: string[] }
+export interface EvalSet { schema: typeof EVAL_SCHEMA; entries?: EvalEntry[]; questions: EvalQuestion[] }
+
+const fail = (message: string): never => { throw new Error(`invalid eval set: ${message}`); };
+const object = (value: unknown, what: string): Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : fail(`${what} must be an object`);
+function exact(row: Record<string, unknown>, allowed: readonly string[], what: string): void {
+  for (const key of Object.keys(row)) if (!allowed.includes(key)) fail(`${what} has an unknown key "${key.slice(0, 40)}"`);
+}
+const text = (value: unknown, what: string, max: number): string => typeof value === "string" && value.trim() !== "" && value.length <= max && !/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/.test(value) ? value : fail(`${what} must be a non-empty string of at most ${max} characters`);
+
+/** Strict, bounded parse of untrusted JSON. Unknown keys are errors, so a typo cannot silently change what is measured. */
+export function parseEvalSet(raw: unknown): EvalSet {
+  const root = object(raw, "the set");
+  exact(root, ["schema", "entries", "questions"], "the set");
+  if (root.schema !== EVAL_SCHEMA) fail(`schema must be "${EVAL_SCHEMA}"`);
+  const entries: EvalEntry[] = [], keys = new Set<string>();
+  if (root.entries !== undefined) {
+    if (!Array.isArray(root.entries) || !root.entries.length || root.entries.length > MAX_ENTRIES) fail(`entries must hold 1 to ${MAX_ENTRIES} items when present`);
+    for (const [index, item] of (root.entries as unknown[]).entries()) {
+      const row = object(item, `entries[${index}]`);
+      exact(row, ["key", "title", "text", "kind", "tier", "about"], `entries[${index}]`);
+      const key = text(row.key, `entries[${index}].key`, 80);
+      if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(key)) fail(`entries[${index}].key may use letters, digits, ".", "_" and "-"`);
+      if (keys.has(key)) fail(`entries[${index}].key "${key}" repeats`);
+      keys.add(key);
+      const kind = row.kind === undefined ? undefined : (EVAL_KINDS as readonly unknown[]).includes(row.kind) ? row.kind as EvalEntry["kind"] : fail(`entries[${index}].kind must be one of ${EVAL_KINDS.join(", ")}`);
+      const tier = row.tier === undefined ? undefined : row.tier === "short" || row.tier === "long" ? row.tier : fail(`entries[${index}].tier must be short or long`);
+      const about = row.about === undefined ? undefined : Array.isArray(row.about) && row.about.length <= 20 ? (row.about as unknown[]).map((value, at) => text(value, `entries[${index}].about[${at}]`, 200)) : fail(`entries[${index}].about must be a list of at most 20 strings`);
+      entries.push({ key, text: text(row.text, `entries[${index}].text`, MAX_ENTRY_CHARS), ...(row.title === undefined ? {} : { title: text(row.title, `entries[${index}].title`, 200) }), ...(kind ? { kind } : {}), ...(tier ? { tier } : {}), ...(about ? { about } : {}) });
+    }
+  }
+  if (!Array.isArray(root.questions) || !root.questions.length || root.questions.length > MAX_QUESTIONS) fail(`questions must hold 1 to ${MAX_QUESTIONS} items`);
+  const questions = (root.questions as unknown[]).map((item, index): EvalQuestion => {
+    const row = object(item, `questions[${index}]`);
+    exact(row, ["q", "expect"], `questions[${index}]`);
+    if (!Array.isArray(row.expect) || !row.expect.length || row.expect.length > MAX_EXPECT) fail(`questions[${index}].expect must hold 1 to ${MAX_EXPECT} entries`);
+    const expect = [...new Set((row.expect as unknown[]).map((value, at) => text(value, `questions[${index}].expect[${at}]`, 80)))];
+    for (const name of expect) {
+      if (keys.size && !keys.has(name)) fail(`questions[${index}].expect names "${name.slice(0, 40)}", which is not an entry key`);
+      if (!keys.size && !/^(?:mem:)?[0-9a-f]{8,32}$/.test(name)) fail(`questions[${index}].expect must be an entry id, optionally written mem:<id> as a brief prints it, or a prefix of 8 or more hex characters when the set has no entries`);
+    }
+    return { q: text(row.q, `questions[${index}].q`, MAX_QUESTION_CHARS), expect };
+  });
+  return { schema: EVAL_SCHEMA, ...(entries.length ? { entries } : {}), questions };
+}
+
+/** `--k 1,5,10`: ascending, unique, 1 to MAX_K. */
+export function parseKs(value: string | undefined): number[] {
+  if (value === undefined) return [...DEFAULT_KS];
+  const parts = value.split(",").map(part => part.trim());
+  const ks = parts.map(part => /^[0-9]{1,3}$/.test(part) ? Number(part) : NaN);
+  if (!ks.length || ks.some(k => !Number.isInteger(k) || k < 1 || k > MAX_K)) throw new Error(`--k must be a comma-separated list of integers from 1 to ${MAX_K}`);
+  return [...new Set(ks)].sort((a, b) => a - b);
+}
+
+export interface EvalQuestionResult {
+  q: string;
+  /** True when at least one expected entry exists in the searched store. Only those can be recalled. */
+  answerable: boolean;
+  /** 1-based rank of the first expected result, or null when none is in the examined window. */
+  rank: number | null;
+  /** How many of the question's present entries are within each k. */
+  found: Record<number, number>;
+  /** Names the question lists. */
+  named: number;
+  /** Distinct entries those names stand for in this store. A second name for the same entry adds nothing. */
+  present: number;
+  /** Distinct names the store holds no entry for. */
+  missing: number;
+}
+/** `recall` and `hit` are null when no question counts (no answerable question), never 0: an empty average is not a measured zero. */
+export interface EvalAggregate { questions: number; recall: Record<number, number | null>; hit: Record<number, number | null>; mrr: number | null }
+export interface EvalReport {
+  /** The search that was scored: the ranked, project-scoped `store.search` (BM25 with tier and recency weights) that MCP and hub recall use, NOT the `amem search` command's substring filter. */
+  engine: string;
+  ks: number[];
+  /** Every question. A question whose expected entry is not in the store scores 0, as a search that cannot find it would. */
+  all: EvalAggregate;
+  /** Only questions with at least one present entry, scored over their present entries only: what the ranking alone achieved. */
+  answerable: EvalAggregate;
+  perQuestion: EvalQuestionResult[];
+}
+
+const round = (value: number): number => Math.round(value * 10_000) / 10_000;
+function aggregate(rows: readonly EvalQuestionResult[], ks: readonly number[], every: boolean): EvalAggregate {
+  const mean = (pick: (row: EvalQuestionResult) => number): number | null => rows.length ? round(rows.reduce((sum, row) => sum + pick(row), 0) / rows.length) : null;
+  // In `all` the denominator is every distinct thing the question wanted (present entries plus names with no entry), so a question with nothing present scores 0; in `answerable`
+  // only the present entries count.
+  const denominator = (row: EvalQuestionResult) => every ? row.present + row.missing : row.present;
+  return {
+    questions: rows.length,
+    recall: Object.fromEntries(ks.map(k => [k, mean(row => denominator(row) ? row.found[k]! / denominator(row) : 0)])),
+    hit: Object.fromEntries(ks.map(k => [k, mean(row => row.rank !== null && row.rank <= k ? 1 : 0)])),
+    mrr: mean(row => row.rank === null ? 0 : 1 / row.rank),
+  };
+}
+
+/**
+ * Score a ranking. `rank(question)` returns the ranked entry ids, best first, at least `max(ks)` of them. `resolve(question)` returns, for each expected name of that question, the ONE
+ * entry id it stands for, or undefined when the store holds none (it is called once per question, never inside the ranking loop, and may throw to refuse an ambiguous name).
+ * Two names for the same entry count once. recall@k is the share of a question's present entries within the first k results, hit@k is whether any is, and MRR is the mean of
+ * 1/rank of the first expected result.
+ */
+export function runEval(rank: (question: string) => readonly string[], resolve: (question: EvalQuestion) => readonly (string | undefined)[], set: EvalSet, engine: string, ks: readonly number[] = DEFAULT_KS): EvalReport {
+  const window = Math.max(...ks);
+  const perQuestion = set.questions.map((question): EvalQuestionResult => {
+    const resolved = resolve(question);
+    const wanted = new Set(resolved.filter((id): id is string => id !== undefined));
+    const ranked = [...new Set(rank(question.q))].slice(0, window);
+    const found: Record<number, number> = Object.fromEntries(ks.map(k => [k, 0]));
+    let first: number | null = null;
+    ranked.forEach((id, index) => {
+      if (!wanted.has(id)) return;
+      first ??= index + 1;
+      for (const k of ks) if (index + 1 <= k) found[k]!++;
+    });
+    const missing = new Set(question.expect.filter((_, index) => resolved[index] === undefined)).size;
+    return { q: question.q, answerable: wanted.size > 0, rank: first, found, named: question.expect.length, present: wanted.size, missing };
+  });
+  return { engine, ks: [...ks], all: aggregate(perQuestion, ks, true), answerable: aggregate(perQuestion.filter(row => row.answerable), ks, false), perQuestion };
+}
+
+const pct = (value: number | null): string => value === null ? "  n/a" : value.toFixed(3);
+/** Human text for `amem eval`; the JSON report is the machine form. */
+export function evalText(report: EvalReport, label: string): string {
+  const head = (name: string, row: EvalAggregate) => `${name.padEnd(11)} ${String(row.questions).padStart(4)}  ${report.ks.map(k => `${pct(row.recall[k] ?? null)}/${pct(row.hit[k] ?? null)}`.padStart(13)).join(" ")}  ${pct(row.mrr).padStart(6)}`;
+  return [
+    `eval: ${label} (${report.engine === "substring" ? "substring fallback" : "ranked store.search, engine " + report.engine}; this is the ranking MCP and hub recall use, not the \`amem search\` filter)`,
+    `${"".padEnd(11)} ${"n".padStart(4)}  ${report.ks.map(k => `@${k} recall/hit`.padStart(13)).join(" ")}  ${"MRR".padStart(6)}`,
+    head("all", report.all),
+    head("answerable", report.answerable),
+    ...(report.answerable.questions < report.all.questions ? [`${report.all.questions - report.answerable.questions} question(s) name an entry this store does not hold (they score 0 in "all")`] : []),
+    ...report.perQuestion.filter(row => row.rank === null || row.rank > Math.min(...report.ks)).map(row => `  ${row.rank === null ? "miss" : `rank ${row.rank}`}${row.answerable ? "" : " (not in store)"}: ${row.q.slice(0, 100)}`),
+  ].join("\n");
+}