audrey 1.1.3 → 1.2.0

package/benchmarks/verify-submission-bundle.mjs

@@ -1,172 +0,0 @@
-import { existsSync, readdirSync, readFileSync } from 'node:fs';
-import { createHash } from 'node:crypto';
-import { join, relative, resolve } from 'node:path';
-import { validateGuardBenchArtifacts, validateSchema } from './validate-guardbench-artifacts.mjs';
-import { publicPath, scanFilesForLocalPaths } from './public-paths.mjs';
-
-const REQUIRED_FILES = [
-  'guardbench-conformance-card.json',
-  'guardbench-manifest.json',
-  'guardbench-raw.json',
-  'guardbench-summary.json',
-  'schemas/guardbench-adapter-registry.schema.json',
-  'schemas/guardbench-adapter-self-test.schema.json',
-  'schemas/guardbench-conformance-card.schema.json',
-  'schemas/guardbench-external-dry-run.schema.json',
-  'schemas/guardbench-external-evidence.schema.json',
-  'schemas/guardbench-external-run.schema.json',
-  'schemas/guardbench-leaderboard.schema.json',
-  'schemas/guardbench-manifest.schema.json',
-  'schemas/guardbench-publication-verification.schema.json',
-  'schemas/guardbench-raw.schema.json',
-  'schemas/guardbench-summary.schema.json',
-  'schemas/guardbench-submission-manifest.schema.json',
-  'validation-report.json',
-];
-
-function readJson(path) {
-  return JSON.parse(readFileSync(path, 'utf-8'));
-}
-
-function sha256File(path) {
-  return createHash('sha256').update(readFileSync(path)).digest('hex');
-}
-
-function walkFiles(dir, root = dir) {
-  return readdirSync(dir, { withFileTypes: true }).flatMap(entry => {
-    const path = join(dir, entry.name);
-    if (entry.isDirectory()) return walkFiles(path, root);
-    return relative(root, path).replaceAll('\\', '/');
-  });
-}
-
-export function verifyGuardBenchSubmissionBundle(options = {}) {
-  const dir = resolve(options.dir ?? 'benchmarks/output/submission-bundle');
-  const manifestPath = join(dir, 'submission-manifest.json');
-  const failures = [];
-  let manifest = null;
-
-  try {
-    manifest = readJson(manifestPath);
-  } catch (error) {
-    failures.push(`submission-manifest.json: ${error.message}`);
-  }
-
-  if (!manifest) {
-    return { ok: false, dir, failures };
-  }
-
-  if (manifest.suite !== 'GuardBench submission bundle') {
-    failures.push('submission-manifest.json: suite must be GuardBench submission bundle');
-  }
-  try {
-    const schema = readJson(join(dir, 'schemas', 'guardbench-submission-manifest.schema.json'));
-    for (const error of validateSchema(manifest, schema, 'submission-manifest')) {
-      failures.push(`submission-manifest.json: ${error}`);
-    }
-  } catch (error) {
-    failures.push(`schemas/guardbench-submission-manifest.schema.json: ${error.message}`);
-  }
-  const listed = new Map((manifest.files ?? []).map(file => [file.path, file]));
-  for (const file of REQUIRED_FILES) {
-    if (!listed.has(file))
-      failures.push(`submission-manifest.json: missing required file record ${file}`);
-  }
-  if (listed.has('submission-manifest.json')) {
-    failures.push('submission-manifest.json: must not include a self-hash file record');
-  }
-
-  for (const [file, record] of listed) {
-    const path = join(dir, file);
-    if (!existsSync(path)) {
-      failures.push(`${file}: listed in manifest but missing from bundle`);
-      continue;
-    }
-    const actualHash = sha256File(path);
-    if (record.sha256 !== actualHash) failures.push(`${file}: sha256 mismatch`);
-    const actualBytes = readFileSync(path).byteLength;
-    if (record.bytes !== actualBytes) failures.push(`${file}: byte length mismatch`);
-  }
-
-  const actualFiles = walkFiles(dir)
-    .filter(file => file !== 'submission-manifest.json')
-    .sort();
-  const listedFiles = [...listed.keys()].sort();
-  const actualSet = new Set(actualFiles);
-  const listedSet = new Set(listedFiles);
-  for (const file of actualFiles) {
-    if (!listedSet.has(file)) failures.push(`${file}: present in bundle but missing from manifest`);
-  }
-  for (const file of listedFiles) {
-    if (!actualSet.has(file))
-      failures.push(`${file}: listed in manifest but not present in bundle`);
-  }
-
-  const artifactValidation = validateGuardBenchArtifacts({
-    dir,
-    schemasDir: join(dir, 'schemas'),
-  });
-  if (!artifactValidation.ok) {
-    failures.push(...artifactValidation.failures.map(failure => `artifact validation: ${failure}`));
-  }
-  if (manifest.validation?.ok !== true) {
-    failures.push('submission-manifest.json: embedded validation status is not ok');
-  }
-  for (const file of scanFilesForLocalPaths(dir, actualFiles)) {
-    failures.push(`${file}: contains a local absolute path`);
-  }
-
-  return {
-    ok: failures.length === 0,
-    dir: publicPath(dir),
-    subject: manifest.subject ?? null,
-    files: listedFiles,
-    artifactValidation,
-    failures,
-  };
-}
-
-function parseArgs(argv = process.argv.slice(2)) {
-  const args = { dir: 'benchmarks/output/submission-bundle', json: false };
-  for (let i = 0; i < argv.length; i++) {
-    const token = argv[i];
-    if ((token === '--dir' || token === '--bundle-dir') && argv[i + 1]) args.dir = argv[++i];
-    else if (token === '--json') args.json = true;
-    else if (token === '--help' || token === '-h') args.help = true;
-    else throw new Error(`Unknown argument: ${token}`);
-  }
-  return args;
-}
-
-function usage() {
-  return [
-    'Usage: node benchmarks/verify-submission-bundle.mjs [--dir benchmarks/output/submission-bundle] [--json]',
-    '',
-    'Verifies a GuardBench submission bundle manifest, file hashes, bundled schemas,',
-    'and artifact validation report.',
-  ].join('\n');
-}
-
-async function main() {
-  const args = parseArgs();
-  if (args.help) {
-    console.log(usage());
-    return;
-  }
-  const report = verifyGuardBenchSubmissionBundle(args);
-  if (args.json) console.log(JSON.stringify(report, null, 2));
-  else if (report.ok)
-    console.log(`GuardBench submission bundle verification passed: ${report.dir}`);
-  else {
-    console.error('GuardBench submission bundle verification failed:');
-    for (const failure of report.failures) console.error(`- ${failure}`);
-  }
-  if (!report.ok) process.exit(1);
-}
-
-if (process.argv[1] && resolve(process.argv[1]).endsWith('verify-submission-bundle.mjs')) {
-  main().catch(error => {
-    console.error(error.stack ?? error.message);
-    process.exit(1);
-  });
-}

package/docs/AUDREY_PAPER_OUTLINE.md

@@ -1,178 +0,0 @@
-# Audrey Paper Outline
-
-## Working Title
-
-Audrey Guard: Local-First Pre-Action Memory Control for Tool-Using Agents
-
-## One-Sentence Thesis
-
-Long-term memory for agents should not stop at recall; it should run before tool use, connect prior outcomes to the next action, and return an auditable `allow`, `warn`, or `block` decision with evidence.
-
-## Abstract Draft
-
-Tool-using agents repeatedly fail in ways that are avoidable: they rerun broken commands, ignore project-specific procedures, lose the context behind prior failures, and trust degraded retrieval paths as if they were complete. Existing agent-memory systems focus mainly on storing and retrieving conversational facts. Audrey reframes memory as a local-first control loop for action: observe tool outcomes, encode durable lessons, build a memory capsule before the next action, generate reflexes, decide whether to allow, warn, or block, and validate whether the memory changed the result.
-
-This paper introduces Audrey Guard, a SQLite-backed memory controller for Model Context Protocol and CLI agents. Audrey Guard combines hybrid vector/FTS recall, memory capsules, preflight warnings, tool-trace learning, redaction-first audit logging, and evidence-linked impact measurement. The evaluation plan measures repeated-failure prevention, false-block rate, degraded-recall fail-closed behavior, redaction safety, and overhead. The result is a practical memory firewall for local agent work: not a replacement for general memory platforms, but an auditable layer that helps agents avoid repeating known mistakes before they touch tools.
-
-## Core Contributions
-
-1. Define pre-action memory control as a distinct problem from generic long-term memory retrieval.
-2. Present the Audrey Guard loop: `PostToolUse` observation -> memory encoding -> preflight/capsule/reflex generation -> `allow` / `warn` / `block` -> validation/impact.
-3. Show a local-first implementation over SQLite, vector search, FTS, MCP, CLI, REST, and Python clients.
-4. Introduce GuardBench, an evaluation suite focused on tool-use risk reduction rather than chat-memory accuracy alone.
-5. Measure safety properties that memory systems usually underreport: repeated-failure prevention, recall degradation handling, secret redaction, and audit lineage.
-
-## Paper Structure
-
-### 1. Introduction
-
-- Agents now operate tools, not just text conversations.
-- The failure mode is operational: the agent knows less than yesterday's run.
-- Generic memory recall is necessary but insufficient; the memory must participate before action.
-- Audrey's claim: a local memory controller can prevent repeated tool failures with low overhead and inspectable evidence.
-
-### 2. Background and Related Work
-
-- Agent-memory systems: Mem0, Letta/MemGPT, LangMem, Zep/Graphiti, Supermemory, OpenMemory, Cognee, LlamaIndex memory.
-- Memory-as-system-resource work: MemOS, procedural memory, evidence-driven retention, temporal graphs.
-- MCP tool safety: tool annotations, tool poisoning, descriptor drift, open-world tool risk.
-- Hook runtimes: Claude Code `PreToolUse`, `PostToolUse`, and `PostToolUseFailure` make pre-action memory control deployable.
-
-### 3. Problem Definition
-
-- Input: proposed agent action, tool name, command/action text, cwd, file scope, session id, and current memory store.
-- Output: decision, risk score, summary, evidence ids, recommended actions, reflexes, optional capsule, and preflight event id.
-- Desired behavior:
-  - Block exact repeated failures unless the action changed.
-  - Warn on relevant prior failures, must-follow procedures, contradictions, and degraded recall.
-  - Preserve evidence lineage and redact secrets before durable storage.
-  - Add low enough latency to run inside tool hooks.
-
-### 4. Audrey Guard Design
-
-- Memory substrate: episodic, semantic, procedural, event log, validation, decay, consolidation.
-- Recall: hybrid vector + FTS with tag/source/date filters and partial-failure diagnostics.
-- Capsules: budgeted evidence assembly for action context.
-- Preflight: warnings and risk scoring from capsule sections, status, and recent tool failures.
-- Reflexes: action-oriented responses generated from preflight evidence.
-- Controller: `beforeAction()` and `afterAction()` over existing Audrey primitives.
-- Audit safety: redaction-before-truncation, action hashing, file-scope hashing, event ids.
-
-### 5. Implementation
-
-- Runtime: Node.js 20+, TypeScript, SQLite, sqlite-vec, Hono REST, MCP stdio, Python client.
-- CLI:
-  - `audrey guard --tool Bash "npm run deploy"`
-  - `audrey demo --scenario repeated-failure`
-- MCP surfaces:
-  - Tools for recall, preflight, reflexes, observe-tool, impact, status.
-  - Resources for status, recent memories, and principles.
-  - Prompts for session briefing, recall, and reflection.
-- Docker behavior: fail-closed non-loopback REST sidecar with required API key.
-
-### 6. Evaluation: GuardBench
-
-Baselines:
-
-- No memory.
-- Recent-window memory.
-- Vector-only recall.
-- Keyword/FTS-only recall.
-- Audrey Guard with hybrid recall and exact-failure matching.
-
-Scenarios:
-
-- Repeated failed shell command.
-- Required preflight procedure missing.
-- Same command in a different file scope.
-- Same tool/action with changed command.
-- Prior failure plus successful fix.
-- Recall vector table missing.
-- FTS failure under hybrid recall.
-- Long secret near truncation boundary.
-- Conflicting project instructions.
-- High-volume irrelevant memory noise.
-
-Metrics:
-
-- Repeated-failure prevention rate.
-- False-block rate.
-- Useful-warning precision.
-- Evidence recall: whether the blocking evidence is surfaced.
-- Redaction safety: raw secret leakage count.
-- Recall-degradation detection rate.
-- Runtime overhead p50/p95.
-- Validation-linked impact count.
-
-### 7. Results Plan
-
-- Use the existing repeated-failure demo as the first qualitative figure.
-- Run `npm run bench:memory:check` as the memory-regression baseline.
-- Keep the `bench:guard` command wired into release evidence before paper submission.
-- Report machine provenance for all timings, matching the existing 0.22.2 benchmark snapshot style.
-- Include ablations:
-  - Without exact action hash.
-  - Without file scope.
-  - Without recall degradation warnings.
-  - Without redaction-aware truncation.
-
-### 8. Discussion
-
-- Why Audrey should not compete as "the best general memory store."
-- Why local-first matters for tool traces: secrets, filesystem paths, project rules, and private failures.
-- Why tool annotations are hints, not policy guarantees.
-- What Audrey borrows from graph memory without adding a graph database to the core.
-- Limitations:
-  - Audrey 1.1 applies lifecycle hooks for Claude Code and Codex, but it can
-    inspect only the events and tool paths emitted by each trusted host.
-  - Hooks start short-lived Audrey processes; a persistent local runtime and
-    measured host-version compatibility gates remain production work.
-  - Validation lineage is bound to exact preflight event evidence, but feedback
-    does not yet tune risk scoring.
-  - Local comparative GuardBench numbers exist; no external-system numbers yet.
-  - Temporal belief fields are still future work.
-
-### 9. Conclusion
-
-- Agent memory should be judged by whether it changes future actions, not just whether it retrieves relevant text.
-- Audrey Guard demonstrates a practical local loop for using memory as a pre-action control layer.
-- The next publishable milestone is live external-adapter GuardBench output plus measured real-host coverage and a persistent hook runtime.
-
-## Figures and Tables
-
-1. Guard loop diagram: observe -> encode -> capsule/preflight/reflex -> decision -> validate.
-2. Architecture diagram: SQLite store, event log, recall, controller, MCP/CLI/REST clients.
-3. Repeated-failure demo transcript with evidence ids.
-4. GuardBench table by baseline and scenario.
-5. Redaction/truncation safety table.
-6. Latency table: preflight p50/p95 by memory count.
-
-## Artifact Checklist Before Submission
-
-- `bench:guard` script and JSON output.
-- Public GuardBench scenario manifest, comparative adapter package, and external-run metadata bundle.
-- Reproducible benchmark snapshot with Node version, CPU, RAM, git SHA.
-- CLI smoke transcript for `audrey demo --scenario repeated-failure`.
-- MCP smoke transcript for `tools/list`, `resources/list`, `prompts/list`, and `memory_status`.
-- Python integration proof.
-- Docker fail-closed auth proof.
-- Paper appendix with exact commands.
-
-## Submission Strategy
-
-1. Publish an arXiv preprint after GuardBench exists.
-2. Submit to an agent-systems, AI engineering, or LLM applications workshop.
-3. Keep the first version implementation-centered, not theory-heavy.
-4. Release the evaluation artifact with the paper so the claim is falsifiable.
-
-## Source Map
-
-- MCP tool annotations and trust model: https://modelcontextprotocol.io/specification/2025-11-25/server/tools and https://modelcontextprotocol.io/specification/2025-11-25/schema
-- MCP annotation risk vocabulary: https://blog.modelcontextprotocol.io/posts/2026-03-16-tool-annotations/
-- Claude Code hooks: https://code.claude.com/docs/en/hooks
-- Codex hooks: https://developers.openai.com/codex/hooks
-- Mem0 token-efficient memory algorithm: https://mem0.ai/blog/mem0-the-token-efficient-memory-algorithm
-- MemOS: https://huggingface.co/papers/2507.03724
-- MCP Security Bench: https://huggingface.co/papers/2510.15994
-- Securing MCP against tool poisoning: https://papers.cool/arxiv/2512.06556
-- Zep/Graphiti temporal knowledge graph: https://help.getzep.com/graphiti/graphiti/overview

package/docs/MEMORY_BENCHMARKING.md

@@ -1,59 +0,0 @@
-# Audrey Memory Benchmarking Strategy
-
-Updated: 2026-05-05 for the 0.23.0 Audrey Guard release.
-
-Audrey should win trust before it tries to win leaderboards. The benchmark story is:
-separate what Audrey can prove locally, publish the exact harness and artifacts, and
-only compare against external systems on tasks that measure the same thing.
-
-## 0.23.0 Release Stance
-
-- Performance snapshots measure Audrey's local pipeline: SQLite, sqlite-vec,
-  hybrid ranking, and post-encode work. They intentionally exclude hosted
-  embedding latency and do not compare against unrelated systems.
-- The behavioral suite is split into retrieval, lifecycle operations, and guard
-  loop behavior. Guard cases stay out of the comparable aggregate because
-  "no controller" baselines are useful regressions, not fair leaderboard peers.
-- `npm run bench:memory:guard` is Audrey's new product benchmark: can memory
-  stop an agent before it repeats a known tool failure or violates a must-follow
-  release rule, and can the receipt boundary reject replayed or non-guard
-  outcomes?
-- The next public claim should be a reproducible report, not a slogan: commit
-  the command, raw JSON, machine provenance, model/provider configuration, and
-  any judge prompt or scoring rule used.
-
-## External Benchmark Map
-
-| Benchmark | What It Tests | Audrey Fit |
-|---|---|---|
-| LongMemEval | Information extraction, multi-session reasoning, temporal reasoning, knowledge updates, and abstention across scalable chat histories. | Good retrieval/lifecycle fit once Audrey has an adapter and evaluator. Source: https://arxiv.org/abs/2410.10813 |
-| LoCoMo | Very long-term conversations around 300 turns, 9K tokens on average, and up to 35 sessions, with QA, summarization, and multimodal dialogue tasks. | Useful external context, but Audrey should keep published scores separate from local synthetic cases. Source: https://arxiv.org/abs/2402.17753 |
-| MemoryAgentBench | Incremental multi-turn memory with accurate retrieval, test-time learning, long-range understanding, and selective forgetting. | Strong fit for Audrey's live agent posture because it evaluates online accumulation rather than static long-context reading. Source: https://arxiv.org/abs/2507.05257 |
-| StructMemEval | Whether agents organize long-term memory into useful structures such as ledgers, to-do lists, and trees rather than just recalling facts. | High-value 0.24 target for Audrey's memory-controller routing and future typed memory stores. Source: https://arxiv.org/abs/2602.11243 |
-| MemGUI-Bench | Cross-temporal and cross-spatial memory for mobile GUI agents, with memory-centric tasks and staged evaluation. | Not a direct coding-agent benchmark, but its failure taxonomy is relevant to tool-bound agents with UI state. Source: https://arxiv.org/abs/2602.06075 |
-
-## Release-Quality Rules
-
-1. Do not mix controller benchmarks with retrieval leaderboards unless all
-   compared systems receive equivalent controller affordances.
-2. Do not quote latency without the embedding provider, dimensions, corpus size,
-   recall mode, hardware, Node version, and whether warm caches were involved.
-3. Treat abstention, deletion, overwrite, conflict resolution, and selective
-   forgetting as first-class memory outcomes, not edge cases.
-4. Prefer task evidence over vibe: raw JSON, artifacts, evaluator code, and
-   reproduction commands should ship with every public benchmark claim.
-5. For coding-agent memory, measure prevented mistakes and time-to-recovery, not
-   only whether a stored fact was recalled.
-
-## 0.24 Benchmark Targets
-
-- Add a LongMemEval adapter that can run a small public shard with mock and real
-  embedding providers.
-- Add a MemoryAgentBench-style incremental harness with explicit selective
-  forgetting and test-time learning cases.
-- Add structured-memory cases that force Audrey to maintain a ledger, checklist,
-  or dependency tree across sessions.
-- Add an agent-tool benchmark where `beforeAction()` and `afterAction()` wrap a
-  scripted coding workflow and score prevented repeats, blocked violations, and
-  useful cautions.
-- Publish one reproducible external report before making any SOTA-style claim.