@staix/agent-mem 0.5.0 → 0.7.0

package/src/memory/eval-fixture.ts

@@ -0,0 +1,137 @@
+import { closeSync, constants, fstatSync, mkdirSync, mkdtempSync, openSync, readSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseEvalSet, runEval, type EvalQuestion, type EvalReport, type EvalSet, MAX_SET_BYTES } from "./eval.ts";
+import { MemoryStore, type View } from "./store.ts";
+import { realPath } from "./platform.ts";
+
+/**
+ * The committed synthetic question set: no person's memory, only invented entries about a made-up project. It lives under `src/` because the package ships
+ * `src/` and not `test/`, so a plain `amem eval` works from an installed copy. A file URL, converted with `fileURLToPath` wherever a path is needed: a plain
+ * `.pathname` stays percent-encoded and breaks under an install path with a space or non-ASCII characters.
+ */
+export const FIXTURE_PATH = new URL("./eval-set.json", import.meta.url);
+const PROJECT = "example.test/eval/fixture";
+const DAY = 86400_000;
+/** A fixed clock: entry ids hash their `at`, and recency weights the ranking, so a clock that only moves when the seeder moves it makes every run identical. */
+const EPOCH = Date.parse("2026-01-01T00:00:00.000Z");
+
+/**
+ * Read and parse a set file. One descriptor does everything: it must be a regular file, its size is bounded before and after the read, and the bytes read are bounded
+ * too, so a file that grows or is swapped between a size check and a read cannot slip a larger set through. A symlink is followed (the person named the path).
+ */
+export function readEvalSet(path: string | URL): EvalSet {
+  const file = typeof path === "string" ? path : fileURLToPath(path), label = typeof path === "string" ? path : "the built-in fixture";
+  const tooBig = () => new Error(`invalid eval set: ${label} is not a regular file within ${MAX_SET_BYTES} bytes`);
+  const fd = openSync(file, constants.O_RDONLY | constants.O_NONBLOCK);
+  let bytes: Buffer;
+  try {
+    const before = fstatSync(fd);
+    if (!before.isFile() || before.size > MAX_SET_BYTES) throw tooBig();
+    const buffer = Buffer.alloc(MAX_SET_BYTES + 1);
+    let length = 0;
+    while (length < buffer.length) {
+      const n = readSync(fd, buffer, length, buffer.length - length, length);
+      if (n === 0) break;
+      length += n;
+    }
+    const after = fstatSync(fd);
+    if (length > MAX_SET_BYTES || after.size !== before.size || length !== before.size) throw tooBig();
+    bytes = buffer.subarray(0, length);
+  } finally { closeSync(fd); }
+  let raw: unknown;
+  try { raw = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); } catch { throw new Error("invalid eval set: not valid JSON"); }
+  return parseEvalSet(raw);
+}
+
+/** The ranked ids of `store.search` for one question, curated tier only (a question set names notes, and the capture tier is machine-local recall, never an answer key). */
+function ranker(store: MemoryStore, project: string, window: number): { rank: (question: string) => string[]; engine: () => string } {
+  let engine = "unknown";
+  return {
+    rank: question => {
+      const result = store.search(question, { project, limit: window, includeCapture: false });
+      engine = result.engine;
+      return result.hits.flatMap(hit => hit.view ? [hit.view.id] : []);
+    },
+    engine: () => engine,
+  };
+}
+
+/** A ranking under test: the ids it returns for a question, best first, up to `window` of them. `amem eval` scores `store.search`; F14 candidates are scored by passing another one. */
+export type RankFn = (store: MemoryStore, question: string, window: number) => string[];
+
+/** Score a self-contained set: seed its synthetic entries into a throwaway store, search it, and delete it. Touches no memory home. `rank` replaces `store.search` (an experiment, never the command). */
+export function evalSeeded(set: EvalSet, ks?: readonly number[], rank?: RankFn): EvalReport {
+  if (!set.entries?.length) throw new Error("this set has no entries; it must be run against a store");
+  const dir = realPath(mkdtempSync(join(tmpdir(), "amem-eval-")));
+  // The cleanup is armed the moment the directory exists, so a failure while building the store cannot leak it.
+  let store: MemoryStore | undefined;
+  try {
+    const base = join(dir, "memory");
+    mkdirSync(base, { recursive: true });
+    // One entry per day, so the recency weight (a 30-day scale) is exercised, long-tier entries first and short-tier ones last: a short-term note leaves the index 14 days after it is
+    // written, and the clock stops a day after the last entry, so the short ones written in that window are live when the set is searched (a set with more short entries than that
+    // window holds loses its oldest, and a question naming one is then reported as not in the store, like any other missing entry).
+    let current = EPOCH;
+    store = new MemoryStore({ memoryDir: base, now: () => current, screen: () => "keep" });
+    const ids = new Map<string, string>();
+    const ordered = [...set.entries.filter(entry => entry.tier !== "short"), ...set.entries.filter(entry => entry.tier === "short")];
+    for (const [index, entry] of ordered.entries()) {
+      current = EPOCH + index * DAY;
+      ids.set(entry.key, store.note({ op: "note", project: PROJECT, scope: "project", tier: entry.tier ?? "long", kind: entry.kind ?? "finding", peer: "eval", source: "person",
+        ...(entry.title ? { title: entry.title } : {}), text: entry.text, about: entry.about ?? [] }, entry.tier === "short"));
+    }
+    current = EPOCH + ordered.length * DAY;
+    const window = Math.max(...(ks ?? [10]));
+    const search = rank ? { rank: (question: string) => rank(store!, question, window), engine: () => "custom" } : ranker(store, PROJECT, window);
+    // Resolved through `store.get`, so a note that has already expired is absent, not silently present: coverage stays separate from ranking.
+    const live = (name: string): string | undefined => { const id = ids.get(name); return id !== undefined && store!.get(id, PROJECT) ? id : undefined; };
+    const report = runEval(search.rank, question => question.expect.map(live), set, "pending", ks);
+    return { ...report, engine: search.engine() };
+  } finally { store?.close(); rmSync(dir, { recursive: true, force: true }); }
+}
+
+/**
+ * Score a set of real entry ids against the bound project's store. Each expected name is resolved through `store.get` (so a promoted short-term id, a `mem:` prefix and an id older
+ * than any list cap resolve like they do for `amem show`), then followed to the entry that is LIVE: a superseded entry points at its replacement, and `store.search` only ever returns the
+ * replacement, so an expected id that was superseded stands for the head of its chain (a broken or cyclic chain is "not in the store"). A prefix that `get` cannot place is checked against
+ * every visible id: more than one distinct live entry is refused with an error, because crediting whichever entry matched would let the ranking pick its own answer key.
+ *
+ * ponytail: `store.get` rebuilds the live view on every call, so the cost is names x store size. Names repeated across questions are resolved once (memoized for the run); the ceiling is
+ * 500 questions x 20 names on a very large store, tens of seconds. Upgrade: one snapshot of the visible views (`list` with `includeSuperseded`) indexed by id, `promotedFrom` and
+ * `supersededBy`, if a real set ever meets that.
+ */
+export function evalStore(store: MemoryStore, project: string, set: EvalSet, ks?: readonly number[]): EvalReport {
+  let visible: View[] | undefined;
+  const all = () => visible ??= store.list({ project, limit: Number.MAX_SAFE_INTEGER, includeSuperseded: true });
+  /** The id of the live entry `view` stands for: itself, or the head of its supersede chain. */
+  const live = (view: View): string | undefined => {
+    const seen = new Set<string>();
+    for (let current: View | undefined = view; current; current = store.get(current.supersededBy, project)) {
+      if (!current.supersededBy) return current.id;
+      if (seen.has(current.id)) return undefined;
+      seen.add(current.id);
+    }
+    return undefined;
+  };
+  const known = new Map<string, string | undefined>();
+  const resolve = (name: string): string | undefined => {
+    if (known.has(name)) return known.get(name);
+    let id: string | undefined;
+    const found = store.get(name, project);
+    if (found) id = live(found);
+    else {
+      const bare = name.replace(/^mem:/, "");
+      const heads = new Set<string>(), seenAny = all().filter(view => view.id.startsWith(bare));
+      for (const view of seenAny) { const head = live(view); if (head !== undefined) heads.add(head); }
+      if (heads.size > 1) throw new Error(`invalid eval set: "${name.slice(0, 40)}" is the start of ${heads.size} entries; use more characters`);
+      id = [...heads][0];
+    }
+    known.set(name, id);
+    return id;
+  };
+  const search = ranker(store, project, Math.max(...(ks ?? [10])));
+  const report = runEval(search.rank, (question: EvalQuestion) => question.expect.map(resolve), set, "pending", ks);
+  return { ...report, engine: search.engine() };
+}

package/src/memory/eval-set.json

@@ -0,0 +1,443 @@
+{
+  "schema": "amem.eval/v1",
+  "entries": [
+    {
+      "key": "shift-handover-old",
+      "kind": "decision",
+      "tier": "long",
+      "title": "Shift handover happens at the gate office",
+      "text": "The harbour shift handover is done at the gate office with a signed paper log.",
+      "about": [
+        "path:src/shift/handover.ts"
+      ]
+    },
+    {
+      "key": "harbor-retry",
+      "kind": "decision",
+      "title": "Harbor client retries idempotent calls three times",
+      "text": "The Harbor ferry client retries an idempotent call three times with exponential backoff, and never retries a booking write.",
+      "about": [
+        "path:src/ferry/client.ts"
+      ]
+    },
+    {
+      "key": "harbor-timeout",
+      "kind": "finding",
+      "title": "Dock sensor requests time out after eight seconds",
+      "text": "A dock sensor request that takes longer than eight seconds is abandoned and reported as stale, not as an error.",
+      "about": [
+        "path:src/dock/sensor.ts"
+      ],
+      "tier": "short"
+    },
+    {
+      "key": "tide-cache",
+      "kind": "decision",
+      "title": "Tide tables are cached for one day",
+      "text": "Tide tables come from the almanac service and are cached on disk for one day, because the almanac rate-limits per hour.",
+      "about": [
+        "path:src/tide/cache.ts"
+      ]
+    },
+    {
+      "key": "tide-rounding",
+      "kind": "finding",
+      "title": "Tide heights round half to even",
+      "text": "Tide heights are rounded half to even before display, so 2.5 metres shows as 2 and 3.5 as 4.",
+      "about": [
+        "path:src/tide/format.ts"
+      ]
+    },
+    {
+      "key": "crane-lock",
+      "kind": "contract",
+      "title": "Only one crane operator lock per berth",
+      "text": "A berth holds at most one crane operator lock. A second request waits, it does not steal the lock, and the lock expires after ninety seconds without a heartbeat.",
+      "about": [
+        "path:src/crane/lock.ts"
+      ]
+    },
+    {
+      "key": "crane-heartbeat",
+      "kind": "finding",
+      "title": "Crane heartbeat drift on the night shift",
+      "text": "The night shift crane heartbeat drifted by up to four seconds because the controller clock was not synced; the lock expiry still held.",
+      "about": [
+        "path:src/crane/heartbeat.ts"
+      ],
+      "tier": "short"
+    },
+    {
+      "key": "manifest-schema",
+      "kind": "contract",
+      "title": "Cargo manifest rows carry a container prefix",
+      "text": "Every cargo manifest row carries a four-letter container prefix and a seven digit serial. A row without both is rejected at import.",
+      "about": [
+        "path:src/cargo/manifest.ts"
+      ]
+    },
+    {
+      "key": "manifest-import",
+      "kind": "fail",
+      "title": "Importing manifests in one transaction blew the memory budget",
+      "text": "Importing a whole cargo manifest in one transaction used more than a gigabyte. Stream it in batches of five hundred rows instead.",
+      "about": [
+        "path:src/cargo/import.ts"
+      ]
+    },
+    {
+      "key": "pilot-roster",
+      "kind": "decision",
+      "title": "Pilot roster is published every Friday",
+      "text": "The pilot roster for the coming week is published on Friday at noon local time and is frozen afterwards except for sick cover.",
+      "about": [
+        "path:src/pilot/roster.ts"
+      ]
+    },
+    {
+      "key": "pilot-fatigue",
+      "kind": "contract",
+      "title": "A pilot may not exceed twelve hours on duty",
+      "text": "The scheduler refuses an assignment that takes a pilot past twelve hours on duty in any rolling twenty-four hours.",
+      "about": [
+        "path:src/pilot/fatigue.ts"
+      ]
+    },
+    {
+      "key": "fuel-ledger",
+      "kind": "decision",
+      "title": "Fuel is recorded in litres, converted once",
+      "text": "Bunker fuel is recorded in litres. Conversion from tonnes happens once at the edge, using the density on the delivery note.",
+      "about": [
+        "path:src/fuel/ledger.ts"
+      ]
+    },
+    {
+      "key": "fuel-density",
+      "kind": "finding",
+      "title": "Delivery note density is sometimes missing",
+      "text": "About one delivery note in forty has no fuel density. The ledger falls back to the supplier default and flags the row for review.",
+      "about": [
+        "path:src/fuel/density.ts"
+      ],
+      "tier": "short"
+    },
+    {
+      "key": "berth-clash",
+      "kind": "fail",
+      "title": "Two vessels were booked into the same berth window",
+      "text": "Two vessels were booked into one berth window because the overlap check compared local times across a daylight saving change. Compare in UTC.",
+      "about": [
+        "path:src/berth/overlap.ts"
+      ]
+    },
+    {
+      "key": "berth-priority",
+      "kind": "decision",
+      "title": "Passenger ferries outrank freight at the quay",
+      "text": "When a quay berth is contested the passenger ferry keeps it and the freight vessel moves to the outer berth.",
+      "about": [
+        "path:src/berth/priority.ts"
+      ]
+    },
+    {
+      "key": "alarm-quiet",
+      "kind": "decision",
+      "title": "Fog alarms are quiet between midnight and five",
+      "text": "The fog horn test alarm is silenced between midnight and five in the morning, the real fog alarm is never silenced.",
+      "about": [
+        "path:src/alarm/fog.ts"
+      ],
+      "tier": "short"
+    },
+    {
+      "key": "alarm-escalate",
+      "kind": "contract",
+      "title": "Unacknowledged alarms escalate after ten minutes",
+      "text": "An alarm nobody acknowledges within ten minutes escalates to the harbour master's phone, then to the duty manager after another ten.",
+      "about": [
+        "path:src/alarm/escalate.ts"
+      ]
+    },
+    {
+      "key": "radio-channel",
+      "kind": "finding",
+      "title": "Channel sixteen is monitored, not used for routine traffic",
+      "text": "Channel sixteen is monitored for distress calls only. Routine traffic moves to the working channel named in the berth assignment.",
+      "about": [
+        "path:src/radio/channels.ts"
+      ]
+    },
+    {
+      "key": "radio-log",
+      "kind": "decision",
+      "title": "Radio calls are logged with a vessel callsign",
+      "text": "Every radio call is logged with the vessel callsign and a UTC timestamp, never with the master's name.",
+      "about": [
+        "path:src/radio/log.ts"
+      ]
+    },
+    {
+      "key": "gate-badge",
+      "kind": "contract",
+      "title": "Gate badges expire at end of shift",
+      "text": "A gate badge is valid until the end of the holder's shift and a contractor badge never outlives the contract end date.",
+      "about": [
+        "path:src/gate/badge.ts"
+      ]
+    },
+    {
+      "key": "gate-camera",
+      "kind": "finding",
+      "title": "Gate camera plate reader fails in heavy rain",
+      "text": "The gate camera plate reader misreads plates in heavy rain; the guard confirms by hand when confidence is below ninety percent.",
+      "about": [
+        "path:src/gate/camera.ts"
+      ],
+      "tier": "short"
+    },
+    {
+      "key": "report-monthly",
+      "kind": "decision",
+      "title": "Throughput report closes on the third working day",
+      "text": "The monthly throughput report closes on the third working day, late manifests are counted in the next month.",
+      "about": [
+        "path:src/report/monthly.ts"
+      ]
+    },
+    {
+      "key": "report-units",
+      "kind": "finding",
+      "title": "Throughput is reported in twenty-foot equivalents",
+      "text": "Throughput is reported in twenty-foot equivalent units, a forty-foot container counts as two.",
+      "about": [
+        "path:src/report/units.ts"
+      ]
+    },
+    {
+      "key": "backup-nightly",
+      "kind": "decision",
+      "title": "Database backup runs at two in the morning",
+      "text": "The operations database is backed up at two in the morning and the copy is verified by a test restore every Sunday.",
+      "about": [
+        "path:ops/backup.sh"
+      ]
+    },
+    {
+      "key": "backup-restore",
+      "kind": "fail",
+      "title": "A backup restored without the audit table",
+      "text": "A restore test passed while the audit table was missing, because the verification only counted the cargo tables. Count every table.",
+      "about": [
+        "path:ops/restore-check.sh"
+      ]
+    },
+    {
+      "key": "shift-handover-new",
+      "kind": "finding",
+      "tier": "short",
+      "title": "Shift handover happens at the gate office",
+      "text": "The harbour shift handover is done at the gate office, and the signed paper log is being replaced by a tablet.",
+      "about": [
+        "path:src/shift/handover.ts"
+      ]
+    }
+  ],
+  "questions": [
+    {
+      "q": "how many times does the ferry client retry",
+      "expect": [
+        "harbor-retry"
+      ]
+    },
+    {
+      "q": "retries backoff idempotent",
+      "expect": [
+        "harbor-retry"
+      ]
+    },
+    {
+      "q": "dock sensor timeout",
+      "expect": [
+        "harbor-timeout"
+      ]
+    },
+    {
+      "q": "why are tide tables cached",
+      "expect": [
+        "tide-cache"
+      ]
+    },
+    {
+      "q": "almanac rate limit",
+      "expect": [
+        "tide-cache"
+      ]
+    },
+    {
+      "q": "round half to even tide height",
+      "expect": [
+        "tide-rounding"
+      ]
+    },
+    {
+      "q": "crane lock expiry without heartbeat",
+      "expect": [
+        "crane-lock",
+        "crane-heartbeat"
+      ]
+    },
+    {
+      "q": "night shift clock drift",
+      "expect": [
+        "crane-heartbeat"
+      ]
+    },
+    {
+      "q": "container prefix serial number manifest",
+      "expect": [
+        "manifest-schema"
+      ]
+    },
+    {
+      "q": "manifest import memory too large",
+      "expect": [
+        "manifest-import"
+      ]
+    },
+    {
+      "q": "when is the pilot roster published",
+      "expect": [
+        "pilot-roster"
+      ]
+    },
+    {
+      "q": "pilot twelve hours on duty",
+      "expect": [
+        "pilot-fatigue"
+      ]
+    },
+    {
+      "q": "fuel density litres tonnes",
+      "expect": [
+        "fuel-ledger",
+        "fuel-density"
+      ]
+    },
+    {
+      "q": "missing density on delivery note",
+      "expect": [
+        "fuel-density"
+      ]
+    },
+    {
+      "q": "berth double booking daylight saving",
+      "expect": [
+        "berth-clash"
+      ]
+    },
+    {
+      "q": "who wins a contested quay berth",
+      "expect": [
+        "berth-priority"
+      ]
+    },
+    {
+      "q": "fog horn alarm silenced at night",
+      "expect": [
+        "alarm-quiet"
+      ]
+    },
+    {
+      "q": "unacknowledged alarm escalates",
+      "expect": [
+        "alarm-escalate"
+      ]
+    },
+    {
+      "q": "distress channel sixteen",
+      "expect": [
+        "radio-channel"
+      ]
+    },
+    {
+      "q": "radio call log callsign",
+      "expect": [
+        "radio-log"
+      ]
+    },
+    {
+      "q": "contractor badge expiry",
+      "expect": [
+        "gate-badge"
+      ]
+    },
+    {
+      "q": "plate reader rain",
+      "expect": [
+        "gate-camera"
+      ]
+    },
+    {
+      "q": "monthly throughput report close date",
+      "expect": [
+        "report-monthly"
+      ]
+    },
+    {
+      "q": "forty foot container counts as two",
+      "expect": [
+        "report-units"
+      ]
+    },
+    {
+      "q": "database backup schedule test restore",
+      "expect": [
+        "backup-nightly",
+        "backup-restore"
+      ]
+    },
+    {
+      "q": "restore check missed the audit table",
+      "expect": [
+        "backup-restore"
+      ]
+    },
+    {
+      "q": "what stops two tugs sharing a mooring",
+      "expect": [
+        "berth-clash"
+      ]
+    },
+    {
+      "q": "staff badge validity period",
+      "expect": [
+        "gate-badge"
+      ]
+    },
+    {
+      "q": "vessel arrival prediction model",
+      "expect": [
+        "tide-cache"
+      ]
+    },
+    {
+      "q": "who approves overtime",
+      "expect": [
+        "pilot-fatigue"
+      ]
+    },
+    {
+      "q": "where is the shift handover done",
+      "expect": [
+        "shift-handover-new"
+      ]
+    },
+    {
+      "q": "paper log shift handover gate office",
+      "expect": [
+        "shift-handover-old"
+      ]
+    }
+  ]
+}