hippocamp 0.2.12 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +9 -1
- package/README.md +10 -2
- package/docs/development.md +21 -0
- package/docs/how-it-works.md +1 -1
- package/docs/results.md +56 -0
- package/package.json +2 -1
- package/scripts/hippocamp-memory.cjs +53 -8
- package/scripts/install-claude.cjs +8 -4
- package/scripts/eval-lagoon.cjs +0 -135
- package/scripts/eval-qa.cjs +0 -371
- package/scripts/eval-retrieval.cjs +0 -323
- package/test/dream-runner.test.cjs +0 -86
- package/test/dream.test.cjs +0 -190
- package/test/install-common.test.cjs +0 -224
- package/test/mcp-params.test.cjs +0 -80
- package/test/memory-efficiency.test.cjs +0 -718
- package/test/package-paths.test.cjs +0 -42
- package/test/sync-memory.test.cjs +0 -119
package/AGENTS.md
CHANGED
|
@@ -29,6 +29,13 @@ This repository is a local-first MCP package for Git-backed agent memory. Keep c
|
|
|
29
29
|
- `scripts/eval-lagoon.cjs`: read-only retrieval check against real Lagoon cases in `<Lagoon root>/evals/retrieval-cases.json`.
|
|
30
30
|
- `scripts/eval-qa.cjs`: LongMemEval-S answer-accuracy check (reader and grader models) for Hippocamp snippets against the labeled answer sessions.
|
|
31
31
|
- `scripts/eval-retrieval.cjs`: retrieval-only LongMemEval-S check for `searchMemory` against a BM25 reference.
|
|
32
|
+
- `scripts/eval-continuity.cjs`: work continuity benchmark runner and CLI: real headless Claude Code or Codex through successive sessions per arm (`cc-none`, `cc-native`, `cc-hippocamp`, `codex-native`, `codex-hippocamp`) in isolated sandboxes, with a frozen run manifest and `--report`.
|
|
33
|
+
- `scripts/continuity-adapters.cjs`: one adapter per benchmark arm, each installed the way a developer has it (Hippocamp through its real `install-claude` or `install-codex` installer).
|
|
34
|
+
- `scripts/continuity-judge.cjs`: blind task judge, miss attribution, and the memory-quality judge.
|
|
35
|
+
- `scripts/continuity-report.cjs`: outcome, efficiency and memory-quality tables, pooled over runs, with a probe-level bootstrap.
|
|
36
|
+
- `scripts/continuity-dataset.cjs`: loader and validator for the continuity dataset, including the retrieval-hint check on prompts.
|
|
37
|
+
- `scripts/eval-locomo.cjs`: LoCoMo answer-accuracy check with the MemoryTax protocol; its `complete` model client is shared with the continuity judge.
|
|
38
|
+
- `bench/continuity/`: synthetic continuity dataset: two seed repos, session tickets, planted facts and the run timeline.
|
|
32
39
|
- `scripts/install-common.cjs`: shared installer helpers: the MCP launch command and Lagoon repo preparation.
|
|
33
40
|
- `scripts/install-claude.cjs`: one-step installer for Claude Code MCP setup and user-level `CLAUDE.md` guidance.
|
|
34
41
|
- `scripts/install-codex.cjs`: one-step installer for the Hippocamp skill and MCP server in Codex.
|
|
@@ -38,7 +45,7 @@ This repository is a local-first MCP package for Git-backed agent memory. Keep c
|
|
|
38
45
|
- `assets/railway/run-dream.sh`: short-lived hosted Dream runner that clones Lagoon and opens one Dream PR per project.
|
|
39
46
|
- `Dockerfile.railway` and `railway.json`: optional Railway cron deployment for overnight Dream runs.
|
|
40
47
|
- `assets/brand/hippocamp-mascot.png`: README mascot asset.
|
|
41
|
-
- `assets/brand/
|
|
48
|
+
- `assets/brand/lagoon-graph.jpg`: README Obsidian graph view of a real Lagoon, loaded from GitHub and kept out of the npm package by `.npmignore` (keep `.npmignore` in sync with `.gitignore`).
|
|
42
49
|
- `docs/`: user docs (setup, how it works, Dream, results, development). The README stays a short overview that links to them.
|
|
43
50
|
|
|
44
51
|
## Project Invariants
|
|
@@ -79,6 +86,7 @@ When changing code, use the smallest verification that proves the change:
|
|
|
79
86
|
- Run `npm run dream -- --help` when changing Dream CLI arguments.
|
|
80
87
|
- Run `npm run eval:retrieval -- --limit 100` when changing search ranking or snippets; report Recall@5 against the BM25 row and snippet evidence. Also run `npm run eval:lagoon` when a Lagoon is available.
|
|
81
88
|
- Run `npm run eval:qa -- --limit 3 --dry-run` when changing `scripts/eval-qa.cjs`; a full run calls paid models, so run it only when asked.
|
|
89
|
+
- Run `npm test` and `npm run eval:continuity -- --dry-run` when changing the continuity benchmark or its dataset; a real run spawns paid Claude Code sessions, so run it only when asked.
|
|
82
90
|
|
|
83
91
|
## Notes For Future Agents
|
|
84
92
|
|
package/README.md
CHANGED
|
@@ -18,9 +18,11 @@ npx hippocamp@latest install
|
|
|
18
18
|
|
|
19
19
|
This installs Hippocamp into every supported agent it finds and creates your memory repo, `~/.lagoon`. Restart your agent and it starts each task by waking up from memory.
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
## Build and own your agents' memory
|
|
22
22
|
|
|
23
|
-
A
|
|
23
|
+

|
|
24
|
+
|
|
25
|
+
A real `~/.lagoon` opened in Obsidian. Every dot is a Markdown note your agents wrote, linked by keywords. It's plain files in your own Git repo, so you can open them in any editor.
|
|
24
26
|
|
|
25
27
|
## Why Hippocamp?
|
|
26
28
|
|
|
@@ -52,6 +54,12 @@ Reproducible with the scripts in this repo.
|
|
|
52
54
|
| Text returned per search | 5.9k chars | Whole sessions: 65.5k |
|
|
53
55
|
| Answer accuracy, Claude Sonnet 5, top 10 results | 77% | Correct sessions given directly: 90% |
|
|
54
56
|
| Unrelated questions that return nothing (our own test set) | 10 / 10 | |
|
|
57
|
+
| Work continuity, 21 real Claude Code sessions, checks passed | 100% (20 / 20) | Claude Code native memory: 96% (25 / 26) |
|
|
58
|
+
| Work continuity, same 21 sessions in Codex, checks passed | 86% (24 / 28) | Codex native memory: 86% (24 / 28) |
|
|
59
|
+
|
|
60
|
+
**Work continuity** tests memory on real engineering work, not chat questions. A real Claude Code or Codex agent works through 21 coding sessions on two small projects. Early sessions establish facts, such as a rejected library, a pinned Node version, a blocked vendor, a parked branch or a user preference. Later tickets only go well if the agent remembers them, and the prompts never mention memory. A blind judge grades 28 checks from the diff and the tool calls. Each arm is installed the way a developer has it.
|
|
61
|
+
|
|
62
|
+
The same Hippocamp server, plugged into either agent with no changes, keeps up with that agent's own memory. In Codex, whose built-in memory stored almost nothing, Hippocamp's memory covered 57% of the planted facts against 4%. Hippocamp uses about 25–45% more tokens per session. This is one run per arm, so the differences are within noise. Every grade, diff and judge prompt is published in [docs/results](docs/results/).
|
|
55
63
|
|
|
56
64
|
Full tables, method, and limits: [docs/results.md](docs/results.md).
|
|
57
65
|
|
package/docs/development.md
CHANGED
|
@@ -10,6 +10,7 @@ npm run mcp:help
|
|
|
10
10
|
npm run mcp:smoke
|
|
11
11
|
npm run dream
|
|
12
12
|
npm run dream:railway
|
|
13
|
+
npm run eval:continuity
|
|
13
14
|
npm run eval:lagoon
|
|
14
15
|
npm run eval:qa
|
|
15
16
|
npm run eval:retrieval
|
|
@@ -21,6 +22,26 @@ npm run upgrade:claude
|
|
|
21
22
|
npm run upgrade:grok
|
|
22
23
|
```
|
|
23
24
|
|
|
25
|
+
## Work continuity benchmark
|
|
26
|
+
|
|
27
|
+
`npm run eval:continuity` measures what Hippocamp is built for: a coding agent that keeps working on a project across independent sessions and acts on earlier decisions, constraints and state without being told to look them up.
|
|
28
|
+
|
|
29
|
+
- The dataset in `bench/continuity/` is synthetic: two small engineering projects (`relay`, a TypeScript webhook relay, and `ingest`, a Python CSV pipeline) on one 21-session timeline. Work sessions are ordinary tickets that plant facts; probe sessions are ordinary tickets with hidden checklists. No prompt hints at retrieval.
|
|
30
|
+
- Arms, each set up the way a developer has it: `cc-none` (auto-memory off), `cc-native` (Claude Code defaults) and `cc-hippocamp` (Hippocamp through its real installer, auto-memory off). Codex arms run `codex exec` (`gpt-6.1-sol`) on the ChatGPT sign-in copied from `~/.codex/auth.json`: `codex-native` (Codex `memories` on, with `min_rollout_idle_hours = 0` because sessions run back to back) and `codex-hippocamp` (Hippocamp through `install-codex`). Codex reports no dollar cost, so its cost column stays at $0.
|
|
31
|
+
- Each session is a fresh headless `claude -p --model claude-opus-5-5` process. The repo is committed after each session with the agent's own summary as the message. If the agent ends with an offer or a question, the simulated user answers once with "yes, go ahead". A retried session restores both the repo and the memory store.
|
|
32
|
+
- Sandboxes live in `HIPPOCAMP_BENCH_DIR` (default `~/.cache/hippocamp-bench`), each with its own `HOME` and `CLAUDE_CONFIG_DIR`, so your real `~/.claude` and `~/.lagoon` are never read or written, and runs survive reboots.
|
|
33
|
+
- Judging: a blind judge grades each check from the diff and a tool log with memory tools and paths anonymized. A second call explains each miss (never stored, stale memory, wrong scope, not recalled, ignored, asked instead of acting). At the end, a third grades every stored memory entry for coverage, precision, staleness, noise and duplicates. Prompts and replies are kept next to each transcript, and `validation.csv` lists every verdict for hand-checking.
|
|
34
|
+
- Each session records tokens, time, memory tool calls and memory bytes read or auto-loaded.
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
npm run eval:continuity -- --dry-run # free: validates and prints the cost range
|
|
38
|
+
npm run eval:continuity -- --systems cc-native,cc-hippocamp --limit 3 --run smoke
|
|
39
|
+
npm run eval:continuity -- --run final-1 # full run
|
|
40
|
+
npm run eval:continuity -- --report final-1,final-2 # pooled report, spawns nothing
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Sessions and the Claude judge run through Claude Code, so they use your Claude subscription when `CLAUDE_CODE_OAUTH_TOKEN` is set (create it with `claude setup-token` and put it in `.env`). Without it they bill `ANTHROPIC_API_KEY`. `--dry-run` prints the API-price range, and `--budget` caps each call. Results go to `.context/continuity/<run>/`. A run is frozen by `manifest.json` (dataset, harness, model and Claude Code version): reruns with the same `--run` skip finished sessions, new sessions appended to the timeline extend it, and any frozen change is refused unless `--force-manifest` records it. The report shows n next to every percentage, defines memory-required items as those `cc-none` fails in every run, and gives bootstrap intervals over probes.
|
|
44
|
+
|
|
24
45
|
## Releases
|
|
25
46
|
|
|
26
47
|
Pull requests and pushes to `main` run syntax and MCP smoke checks in GitHub Actions.
|
package/docs/how-it-works.md
CHANGED
|
@@ -70,7 +70,7 @@ Typical agent flow:
|
|
|
70
70
|
4. Use `append_event` for meaningful milestones, with a title and 1-8 concise keywords.
|
|
71
71
|
5. Update curated files like `current_state.md` and `open_threads.md` before finishing.
|
|
72
72
|
|
|
73
|
-
Writes sync by default. If sync fails or is skipped, call `sync_memory`.
|
|
73
|
+
Writes sync by default. If sync fails or is skipped, call `sync_memory`. `wake_up` and a `sync_memory` with nothing to send first pull what other machines pushed (fast-forward only, without credential prompts; when the remote cannot be reached, the pull gives up after 10 seconds and the agent continues with local memory), so a second machine starts from current memory. Sync only commits when the Lagoon is the top of its own Git repository; a Lagoon folder inside another repository, such as a code checkout, is never committed or pushed.
|
|
74
74
|
|
|
75
75
|
Event writes update a sibling `events/YYYY-MM-DD.index.json` file. The Markdown event remains the canonical memory; the sidecar is rebuildable. Search ranks indexed events and curated files with BM25 and typo tolerance ([MiniSearch](https://github.com/lucaong/minisearch)), weighting keywords above titles and titles above bodies. It returns small coherent evidence blocks from indexed events; an event found only by its keywords or title returns its first paragraphs. If no indexed result is strong enough, it returns no event result instead of scanning whole event logs.
|
|
76
76
|
|
package/docs/results.md
CHANGED
|
@@ -39,6 +39,60 @@ The same reader reaches 90% with the correct sessions, so the remaining gap is r
|
|
|
39
39
|
| Right event in top 5, 43 answerable questions | 42 / 43 |
|
|
40
40
|
| No results for 10 questions no memory is about | 10 / 10 |
|
|
41
41
|
|
|
42
|
+
## Work continuity
|
|
43
|
+
|
|
44
|
+
Does memory carry real engineering work from one coding session to the next? `npm run eval:continuity` runs real headless Claude Code (`claude-opus-5-5`, Claude Code 2.1.318 to 2.1.319) through 21 sessions over two small synthetic projects (a TypeScript webhook relay and a Python ingest job). Early sessions plant facts the way work does: a rejected library, a pinned Node version, a blocked vendor, a parked branch, a user preference. Later sessions give tickets that only go well if the agent remembers them. The prompts never say "remember" or point at memory.
|
|
45
|
+
|
|
46
|
+
In short: the same Hippocamp server, plugged unchanged into Claude Code or Codex, keeps up with each one's own memory (100% vs 96% in Claude Code, 86% vs 86% in Codex), with fewer than 5 points of difference and one run per arm. In Codex, whose built-in memory stored almost nothing, Hippocamp's memory covered 57% of the planted facts against 4%.
|
|
47
|
+
|
|
48
|
+
Each arm is installed the way a developer has it, in its own sandbox home: Claude Code defaults with native auto-memory, or Hippocamp through its real installer (`install-claude`) with auto-memory off. A blind judge (Opus, arm names and memory paths scrubbed) grades 28 checks against the diff and the tool calls. A check is n/a when the session wrote no code it applies to.
|
|
49
|
+
|
|
50
|
+
Run `full-1`, one run per arm:
|
|
51
|
+
|
|
52
|
+
| | Claude Code native memory | Hippocamp in Claude Code |
|
|
53
|
+
| --- | ---: | ---: |
|
|
54
|
+
| Checks passed | 96% (25 / 26) | 100% (20 / 20) |
|
|
55
|
+
| Checks marked as needing memory | 92% (11 / 12) | 100% (10 / 10) |
|
|
56
|
+
| Tokens per session, median | 179k | 224k |
|
|
57
|
+
| Time per session, median | 62 s | 73 s |
|
|
58
|
+
| Agent cost per run, API price | $8.84 | $9.19 |
|
|
59
|
+
| Memory entries at the end | 8 | 27 |
|
|
60
|
+
| Planted facts covered by memory | 22% | 57% |
|
|
61
|
+
| Memory entries that are useful (precision) | 88% | 81% |
|
|
62
|
+
|
|
63
|
+
What it shows: Hippocamp, plugged into Claude Code as a plain MCP server, does as well as Claude Code's own memory, at about the same cost (+4% dollars, +25% tokens per session). Native memory's one miss was a parked `fix/late-dedupe` branch it no longer knew about, so it rewrote the fix from scratch. Hippocamp's 8 n/a checks come mostly from one session where it correctly held a blocked vendor back and wrote no code; native also held the vendor back, but with gated code the checks could grade. A check is only counted when the session did work it applies to, so the two arms are scored on different numbers of checks (20 vs 26); this favors an arm that holds work back.
|
|
64
|
+
|
|
65
|
+
What it does not show: that Hippocamp is better. The difference is one check, from one run per arm (bootstrap 95% interval over probes: +4 points, 0 to +16). There is no no-memory run, so the "needing memory" label comes from the dataset, not from a measured baseline. An earlier development run (27 / 27 vs 24 / 27) is not used: its cases were added after a tie and its harness was unfair to native memory.
|
|
66
|
+
|
|
67
|
+
The diffs, judge prompts, grades with evidence and memory snapshots of this run are in [results/continuity-full-1](results/continuity-full-1/) (start with `report.md` and `validation.csv`). Full transcripts are left out for size.
|
|
68
|
+
|
|
69
|
+
### Same benchmark in Codex
|
|
70
|
+
|
|
71
|
+
Run `codex-1` repeats the 21 sessions in Codex (`codex exec`, codex-cli 0.159.2, `gpt-6.1-sol`, ChatGPT sign-in), same checks and same blind Opus judge. Codex native memory is its `memories` feature, turned on (it is off by default) with `min_rollout_idle_hours = 0`, because the benchmark runs sessions back to back instead of days apart. Hippocamp is installed with `install-codex`; nothing else changes.
|
|
72
|
+
|
|
73
|
+
| | Codex native memory | Hippocamp in Codex |
|
|
74
|
+
| --- | ---: | ---: |
|
|
75
|
+
| Checks passed | 86% (24 / 28) | 86% (24 / 28) |
|
|
76
|
+
| Checks marked as needing memory | 67% (8 / 12) | 75% (9 / 12) |
|
|
77
|
+
| Tokens per session, median | 166k | 238k |
|
|
78
|
+
| Time per session, median | 63 s | 82 s |
|
|
79
|
+
| Memory calls per session | 1.0 | 4.9 |
|
|
80
|
+
| Memory entries at the end | 4 | 31 |
|
|
81
|
+
| Planted facts covered by memory | 4% | 57% |
|
|
82
|
+
| Memory entries that are useful (precision) | 25% | 61% |
|
|
83
|
+
|
|
84
|
+
What it shows: the same Hippocamp server, with no Codex-specific code, works in Codex as it does in Claude Code. Codex called `wake_up`, `search_memory` and `append_event` on its own, and the memory it built covers as many planted facts as in Claude Code (57% in both). The two Codex arms tie on checks (bootstrap: 0 points, -14 to +14).
|
|
85
|
+
|
|
86
|
+
Codex's own memory stored almost nothing in this run: its background summaries never produced a memory from a past session, and the one real entry is a changelog preference the agent wrote as a note. The other three entries are Codex's empty templates, which the judge counts as noise. The harness turns Codex memory on but does not wait for or confirm its background step between sessions, so this arm measures Codex memory as it behaved here, not a verified working pipeline. Hippocamp's Codex memory also holds more duplicates (5) and stale entries (4) than in Claude Code, so its precision is lower (61%). Codex still passed most checks without it, from the repository itself (code, commit messages, notes it wrote in the repo); that is why the score does not drop more. Hippocamp's misses include one stale next step and one fact it never saved (a blocked vendor).
|
|
87
|
+
|
|
88
|
+
Codex reports no dollar cost, so this run has token and time numbers only.
|
|
89
|
+
|
|
90
|
+
### How memory quality is scored
|
|
91
|
+
|
|
92
|
+
At the end of each run, a judge labels every memory entry, across both projects, as a current fact, a stale fact, other useful context, noise or a duplicate. Coverage is the share of planted facts with at least one current entry. Each entry is credited with one fact at most, so a native memory file that lists several facts in one entry gets credit for one of them: coverage favors memories that store one fact per entry, like Hippocamp's events, and the native coverage figures are a lower bound. A first version of this report judged only one project of Hippocamp's memory and gave it 30% (Claude Code) and 35% (Codex); this was fixed and re-judged without re-running any session.
|
|
93
|
+
|
|
94
|
+
The run data is in [results/continuity-codex-1](results/continuity-codex-1/).
|
|
95
|
+
|
|
42
96
|
## Why less text matters
|
|
43
97
|
|
|
44
98
|
Memory results share the agent's context with code, diffs, and tool output, and agents search several times per task. About 6k characters per search is roughly 1.5k tokens; whole sessions would be about 16k. Fewer tokens make each turn cheaper and faster, and keep the relevant lines from being buried. Less text is only useful when the answer is still in it, which is why we report answer accuracy next to text size.
|
|
@@ -56,4 +110,6 @@ Memory results share the agent's context with code, diffs, and tool output, and
|
|
|
56
110
|
|
|
57
111
|
`npm run eval:lagoon` runs the same kind of check, read-only, against your real Lagoon. It reads cases from `<Lagoon root>/evals/retrieval-cases.json`: a project, a query, and the event ids that answer it. An empty list means no memory should be returned. Keep the cases in Lagoon, not in this repo, because they quote private work.
|
|
58
112
|
|
|
113
|
+
`npm run eval:continuity -- --systems cc-native,cc-hippocamp --run <name>` (or `codex-native,codex-hippocamp`, which needs a Codex ChatGPT sign-in) reruns the work continuity benchmark with your Claude subscription (`CLAUDE_CODE_OAUTH_TOKEN` in `.env`); `--report <name>` prints the tables, and `--dry-run` shows the cost estimate. A run stops cleanly at a usage limit and resumes where it stopped.
|
|
114
|
+
|
|
59
115
|
`npm run eval:qa` measures answer accuracy on the same LongMemEval-S data with the official LongMemEval reader and grader prompts. It compares Hippocamp top 5 (`hippocamp`), top 10 (`hippocamp-k10`), and the labeled answer sessions (`oracle`). It calls an OpenAI-compatible endpoint from `MANIFEST_BASE_URL` and `MANIFEST_API_KEY`; set `QA_READER_MODEL` and `QA_GRADER_MODEL` to change models. Results are cached in `.context/qa-results.jsonl`, and `--dry-run` shows the jobs and token estimates without any API call.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hippocamp",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.14",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Local Git-backed memory for AI agents.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
"scripts": {
|
|
31
31
|
"dream": "node scripts/hippocamp-dream.cjs",
|
|
32
32
|
"dream:railway": "bash assets/railway/run-dream.sh",
|
|
33
|
+
"eval:continuity": "node --env-file-if-exists=.env scripts/eval-continuity.cjs",
|
|
33
34
|
"eval:lagoon": "node scripts/eval-lagoon.cjs",
|
|
34
35
|
"eval:qa": "node scripts/eval-qa.cjs",
|
|
35
36
|
"eval:retrieval": "node scripts/eval-retrieval.cjs",
|
|
@@ -438,9 +438,9 @@ async function ensureParentDirectory(filePath) {
|
|
|
438
438
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
439
439
|
}
|
|
440
440
|
|
|
441
|
-
async function runGit(args, cwd) {
|
|
441
|
+
async function runGit(args, cwd, options = {}) {
|
|
442
442
|
try {
|
|
443
|
-
const result = await execFileAsync("git", args, { cwd });
|
|
443
|
+
const result = await execFileAsync("git", args, { cwd, ...options });
|
|
444
444
|
return {
|
|
445
445
|
ok: true,
|
|
446
446
|
stdout: result.stdout.trim(),
|
|
@@ -477,17 +477,54 @@ function realpathExisting(targetPath) {
|
|
|
477
477
|
}
|
|
478
478
|
}
|
|
479
479
|
|
|
480
|
-
|
|
480
|
+
// The Lagoon's own repository, or null: a repository that only encloses the Lagoon (a code
|
|
481
|
+
// checkout, a home-directory repo) must never receive memory commits or pushes.
|
|
482
|
+
async function getLagoonRepo() {
|
|
481
483
|
const repoCandidate = getGlobalRepoRoot();
|
|
482
484
|
const repoRoot = await getGitRepoRoot(repoCandidate);
|
|
483
|
-
const scopeRoot = getScopeRoot(scope, projectRoot);
|
|
484
485
|
|
|
485
486
|
if (!repoRoot) {
|
|
487
|
+
return { repoCandidate, reason: "not_git_repo" };
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
if (realpathExisting(repoRoot) !== realpathExisting(repoCandidate)) {
|
|
491
|
+
return { repoCandidate, repoRoot, reason: "lagoon_not_repo_root" };
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
return { repoCandidate, repoRoot };
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
// Brings in memory pushed from other machines. Fast-forward only: local commits that have not
|
|
498
|
+
// been pushed yet are reconciled by the push path, never here. Offline or slow remotes are skipped.
|
|
499
|
+
async function pullMemory() {
|
|
500
|
+
const { repoRoot, reason } = await getLagoonRepo();
|
|
501
|
+
|
|
502
|
+
if (!repoRoot) {
|
|
503
|
+
return { pulled: false, reason };
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
const upstream = await runGit(["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], repoRoot);
|
|
507
|
+
|
|
508
|
+
if (!upstream.ok) {
|
|
509
|
+
return { pulled: false, reason: "no_upstream" };
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
// Never prompt: saved credentials (credential helpers, SSH agent) still work; anything that would ask fails fast.
|
|
513
|
+
const env = { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "true", SSH_ASKPASS: "true", GIT_SSH_COMMAND: process.env.GIT_SSH_COMMAND || "ssh -o BatchMode=yes" };
|
|
514
|
+
const result = await runGit(["pull", "--ff-only", "--quiet"], repoRoot, { timeout: 10000, env });
|
|
515
|
+
return result.ok ? { pulled: true } : { pulled: false, reason: "pull_failed", error: result.stderr };
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
async function syncMemory({ scope, projectRoot, paths, message }) {
|
|
519
|
+
const { repoCandidate, repoRoot, reason } = await getLagoonRepo();
|
|
520
|
+
const scopeRoot = getScopeRoot(scope, projectRoot);
|
|
521
|
+
|
|
522
|
+
if (reason) {
|
|
486
523
|
return {
|
|
487
524
|
attempted: false,
|
|
488
525
|
skipped: true,
|
|
489
|
-
reason
|
|
490
|
-
repoRoot: repoCandidate,
|
|
526
|
+
reason,
|
|
527
|
+
repoRoot: repoRoot || repoCandidate,
|
|
491
528
|
};
|
|
492
529
|
}
|
|
493
530
|
|
|
@@ -530,11 +567,13 @@ async function syncMemory({ scope, projectRoot, paths, message }) {
|
|
|
530
567
|
const aheadResult = await runGit(["rev-list", "--count", "@{u}..HEAD"], repoRoot);
|
|
531
568
|
|
|
532
569
|
if (!aheadResult.ok || aheadResult.stdout === "0") {
|
|
570
|
+
// Nothing to send: an explicit sync still brings in what other machines pushed.
|
|
533
571
|
return {
|
|
534
572
|
attempted: true,
|
|
535
573
|
skipped: true,
|
|
536
574
|
reason: "no_changes_to_commit",
|
|
537
575
|
repoRoot,
|
|
576
|
+
pull: await pullMemory(),
|
|
538
577
|
};
|
|
539
578
|
}
|
|
540
579
|
|
|
@@ -1084,7 +1123,10 @@ async function readMemoryFile({ scope, path: relativePath, projectRoot }) {
|
|
|
1084
1123
|
async function writeMemoryFile({ scope, path: relativePath, content, projectRoot, sync = true }) {
|
|
1085
1124
|
const target = resolveScopedPath(scope, relativePath, projectRoot);
|
|
1086
1125
|
|
|
1087
|
-
|
|
1126
|
+
// Event logs are append-only in every scope, including a project's log reached from the global root.
|
|
1127
|
+
const fromGlobal = path.relative(getGlobalRoot(), target.absolutePath).split(path.sep);
|
|
1128
|
+
|
|
1129
|
+
if (target.path === "events" || target.path.startsWith("events/") || fromGlobal[0] === "events" || (fromGlobal[0] === "projects" && fromGlobal[2] === "events")) {
|
|
1088
1130
|
throw new Error("events/ is append-only. Use append_event with a title and keywords instead.");
|
|
1089
1131
|
}
|
|
1090
1132
|
|
|
@@ -1273,7 +1315,8 @@ async function collectEventDocuments({ scope, eventFiles }) {
|
|
|
1273
1315
|
index = null;
|
|
1274
1316
|
}
|
|
1275
1317
|
|
|
1276
|
-
|
|
1318
|
+
// The log always sits next to its index; a path field inside the index is never trusted.
|
|
1319
|
+
const markdownName = indexName.replace(/\.index\.json$/, ".md");
|
|
1277
1320
|
|
|
1278
1321
|
if (!index || !Array.isArray(index.events)) {
|
|
1279
1322
|
continue;
|
|
@@ -1622,6 +1665,7 @@ async function searchMemory({ query, scope = "both", projectRoot, maxResults = 5
|
|
|
1622
1665
|
}
|
|
1623
1666
|
|
|
1624
1667
|
async function wakeUp({ projectRoot } = {}) {
|
|
1668
|
+
const pull = await pullMemory();
|
|
1625
1669
|
const globalRoot = getScopeRoot("global", projectRoot);
|
|
1626
1670
|
const resolvedProjectRoot = getProjectRoot(projectRoot);
|
|
1627
1671
|
const projectSlug = getProjectSlug(projectRoot);
|
|
@@ -1695,6 +1739,7 @@ async function wakeUp({ projectRoot } = {}) {
|
|
|
1695
1739
|
missing,
|
|
1696
1740
|
globalFiles: globalFiles.map((file) => file.path),
|
|
1697
1741
|
projectFiles: projectFiles.map((file) => file.path),
|
|
1742
|
+
pull,
|
|
1698
1743
|
text: `${sections.join("\n").trim()}\n`,
|
|
1699
1744
|
};
|
|
1700
1745
|
}
|
|
@@ -273,7 +273,11 @@ async function main() {
|
|
|
273
273
|
printHints(lagoon);
|
|
274
274
|
}
|
|
275
275
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
276
|
+
if (require.main === module) {
|
|
277
|
+
main().catch((error) => {
|
|
278
|
+
console.error("Failed to install Hippocamp into Claude Code:", error.message);
|
|
279
|
+
process.exit(1);
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
module.exports = { buildClaudeManagedBlock, ensureClaudeInstructionsInstalled, ensureClaudeSkillInstalled };
|
package/scripts/eval-lagoon.cjs
DELETED
|
@@ -1,135 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
const fsSync = require("node:fs");
|
|
4
|
-
const fs = require("node:fs/promises");
|
|
5
|
-
const os = require("node:os");
|
|
6
|
-
const path = require("node:path");
|
|
7
|
-
const memory = require("./hippocamp-memory.cjs");
|
|
8
|
-
|
|
9
|
-
const MAX_TOP_K = 20;
|
|
10
|
-
const DEFAULT_CASES = "evals/retrieval-cases.json";
|
|
11
|
-
|
|
12
|
-
function printHelp() {
|
|
13
|
-
console.log(`Hippocamp Lagoon eval
|
|
14
|
-
|
|
15
|
-
Read-only retrieval check against your real Lagoon. No model or API key is used.
|
|
16
|
-
Each case names a project, a query, and the event ids that answer it.
|
|
17
|
-
An empty expect list means no memory should be returned.
|
|
18
|
-
|
|
19
|
-
Usage:
|
|
20
|
-
npm run eval:lagoon -- [--cases PATH] [--top-k K]
|
|
21
|
-
|
|
22
|
-
Options:
|
|
23
|
-
--cases PATH Cases JSON. Default: <Lagoon root>/${DEFAULT_CASES}
|
|
24
|
-
--top-k K Results retrieved per query, at most ${MAX_TOP_K}. Default: 5
|
|
25
|
-
--help Show this help
|
|
26
|
-
|
|
27
|
-
Cases file:
|
|
28
|
-
{ "cases": [ { "project": "<slug>", "query": "...", "expect": ["<event id>"] } ] }
|
|
29
|
-
`);
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
function parseArgs(argv) {
|
|
33
|
-
const options = { cases: path.join(memory.getGlobalRoot(), DEFAULT_CASES), topK: 5, help: false };
|
|
34
|
-
|
|
35
|
-
for (let index = 0; index < argv.length; index += 1) {
|
|
36
|
-
const token = argv[index];
|
|
37
|
-
|
|
38
|
-
if (token === "--help" || token === "-h") {
|
|
39
|
-
options.help = true;
|
|
40
|
-
} else if (token === "--cases") {
|
|
41
|
-
options.cases = argv[++index];
|
|
42
|
-
} else if (token === "--top-k") {
|
|
43
|
-
const number = Number(argv[++index]);
|
|
44
|
-
|
|
45
|
-
if (!Number.isInteger(number) || number < 1 || number > MAX_TOP_K) {
|
|
46
|
-
throw new Error(`--top-k must be an integer from 1 to ${MAX_TOP_K}.`);
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
options.topK = number;
|
|
50
|
-
} else {
|
|
51
|
-
throw new Error(`Unknown argument: ${token}`);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
return options;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
async function run(options) {
|
|
59
|
-
if (!fsSync.existsSync(options.cases)) {
|
|
60
|
-
throw new Error(`Missing ${options.cases}. Run with --help for the file format.`);
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
const { cases } = JSON.parse(await fs.readFile(options.cases, "utf8"));
|
|
64
|
-
// searchMemory derives the project slug from the folder name, so a temp path per
|
|
65
|
-
// slug reads that project's memory without touching any real repo. The folder is
|
|
66
|
-
// not created: an existing empty folder has no project and reads global memory.
|
|
67
|
-
const sandboxRoot = await fs.mkdtemp(path.join(os.tmpdir(), "hippocamp-eval-lagoon-"));
|
|
68
|
-
const answerable = { count: 0, hit: 0, mrr: 0, chars: 0, topScores: [] };
|
|
69
|
-
const empty = { count: 0, correct: 0, topScores: [] };
|
|
70
|
-
|
|
71
|
-
try {
|
|
72
|
-
for (const item of cases) {
|
|
73
|
-
if (!/^[a-z0-9][a-z0-9._-]*$/.test(String(item.project))) {
|
|
74
|
-
throw new Error(`Invalid project slug: ${item.project}`);
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
const projectRoot = path.join(sandboxRoot, item.project);
|
|
78
|
-
|
|
79
|
-
const search = await memory.searchMemory({
|
|
80
|
-
query: item.query,
|
|
81
|
-
scope: "project",
|
|
82
|
-
projectRoot,
|
|
83
|
-
maxResults: options.topK,
|
|
84
|
-
});
|
|
85
|
-
const topScore = search.results[0]?.score ?? 0;
|
|
86
|
-
|
|
87
|
-
if (!item.expect.length) {
|
|
88
|
-
empty.count += 1;
|
|
89
|
-
empty.correct += search.results.length ? 0 : 1;
|
|
90
|
-
empty.topScores.push(topScore);
|
|
91
|
-
console.log(`${search.results.length ? "MISS" : "ok "} empty top ${topScore} [${item.project}] ${item.query}`);
|
|
92
|
-
continue;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
const rank = search.results.findIndex((result) => item.expect.includes(result.id));
|
|
96
|
-
answerable.count += 1;
|
|
97
|
-
answerable.hit += rank === -1 ? 0 : 1;
|
|
98
|
-
answerable.mrr += rank === -1 ? 0 : 1 / (rank + 1);
|
|
99
|
-
answerable.chars += search.results.reduce((sum, result) => sum + result.snippet.length, 0);
|
|
100
|
-
answerable.topScores.push(topScore);
|
|
101
|
-
console.log(
|
|
102
|
-
`${rank === -1 ? "MISS" : "ok "} rank ${rank === -1 ? "-" : rank + 1} top ${topScore} [${item.project}] ${item.query}`,
|
|
103
|
-
);
|
|
104
|
-
}
|
|
105
|
-
} finally {
|
|
106
|
-
await fs.rm(sandboxRoot, { recursive: true, force: true });
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const percent = (part, whole) => `${((part / Math.max(whole, 1)) * 100).toFixed(1)}%`;
|
|
110
|
-
const range = (scores) => (scores.length ? `${Math.min(...scores)}-${Math.max(...scores)}` : "n/a");
|
|
111
|
-
|
|
112
|
-
console.log(`\nLagoon retrieval, K=${options.topK}, ${cases.length} cases`);
|
|
113
|
-
console.log(
|
|
114
|
-
`answerable (${answerable.count}) Hit@K ${percent(answerable.hit, answerable.count)} MRR ${(answerable.mrr / Math.max(answerable.count, 1)).toFixed(3)} context ${Math.round(answerable.chars / Math.max(answerable.count, 1))} chars top score ${range(answerable.topScores)}`,
|
|
115
|
-
);
|
|
116
|
-
console.log(
|
|
117
|
-
`no memory (${empty.count}) empty result ${percent(empty.correct, empty.count)} top score ${range(empty.topScores)}`,
|
|
118
|
-
);
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
async function main() {
|
|
122
|
-
const options = parseArgs(process.argv.slice(2));
|
|
123
|
-
|
|
124
|
-
if (options.help) {
|
|
125
|
-
printHelp();
|
|
126
|
-
return;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
await run(options);
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
main().catch((error) => {
|
|
133
|
-
console.error(error.message);
|
|
134
|
-
process.exit(1);
|
|
135
|
-
});
|