@justyork/repo-mind 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -0
- package/dist/ab-demo/arm-baseline.d.ts +8 -0
- package/dist/ab-demo/arm-baseline.js +40 -0
- package/dist/ab-demo/arm-repomind.d.ts +7 -0
- package/dist/ab-demo/arm-repomind.js +35 -0
- package/dist/ab-demo/estimate-tokens.d.ts +3 -0
- package/dist/ab-demo/estimate-tokens.js +10 -0
- package/dist/ab-demo/load-questions.d.ts +2 -0
- package/dist/ab-demo/load-questions.js +65 -0
- package/dist/ab-demo/paths.d.ts +5 -0
- package/dist/ab-demo/paths.js +31 -0
- package/dist/ab-demo/run-ab.d.ts +7 -0
- package/dist/ab-demo/run-ab.js +128 -0
- package/dist/ab-demo/run-arms.d.ts +3 -0
- package/dist/ab-demo/run-arms.js +67 -0
- package/dist/ab-demo/session-overhead.d.ts +3 -0
- package/dist/ab-demo/session-overhead.js +68 -0
- package/dist/ab-demo/types.d.ts +65 -0
- package/dist/ab-demo/types.js +1 -0
- package/dist/ab-demo/validate-corpus.d.ts +3 -0
- package/dist/ab-demo/validate-corpus.js +38 -0
- package/dist/check/collect-violations.d.ts +11 -0
- package/dist/check/collect-violations.js +127 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +147 -0
- package/dist/commands/check.d.ts +6 -0
- package/dist/commands/check.js +19 -0
- package/dist/commands/export.d.ts +8 -0
- package/dist/commands/export.js +80 -0
- package/dist/commands/init.d.ts +4 -0
- package/dist/commands/init.js +86 -0
- package/dist/commands/prepare.d.ts +7 -0
- package/dist/commands/prepare.js +61 -0
- package/dist/commands/setup.d.ts +11 -0
- package/dist/commands/setup.js +84 -0
- package/dist/commands/sync-links.d.ts +7 -0
- package/dist/commands/sync-links.js +41 -0
- package/dist/commands/ui.d.ts +5 -0
- package/dist/commands/ui.js +83 -0
- package/dist/index/asset-file.d.ts +4 -0
- package/dist/index/asset-file.js +26 -0
- package/dist/index/doc-index.d.ts +21 -0
- package/dist/index/doc-index.js +231 -0
- package/dist/index/knowledge-file.d.ts +4 -0
- package/dist/index/knowledge-file.js +17 -0
- package/dist/index/link-index.d.ts +41 -0
- package/dist/index/link-index.js +150 -0
- package/dist/index/path-inference.d.ts +9 -0
- package/dist/index/path-inference.js +33 -0
- package/dist/index/resolve-asset-href.d.ts +2 -0
- package/dist/index/resolve-asset-href.js +65 -0
- package/dist/index/resolve-md-href.d.ts +7 -0
- package/dist/index/resolve-md-href.js +116 -0
- package/dist/index/slug.d.ts +5 -0
- package/dist/index/slug.js +43 -0
- package/dist/index/types.d.ts +44 -0
- package/dist/index/types.js +71 -0
- package/dist/mcp/server.d.ts +1 -0
- package/dist/mcp/server.js +155 -0
- package/dist/package-version.d.ts +2 -0
- package/dist/package-version.js +15 -0
- package/dist/prepare/auto-links.d.ts +22 -0
- package/dist/prepare/auto-links.js +124 -0
- package/dist/prepare/prepare-docs.d.ts +36 -0
- package/dist/prepare/prepare-docs.js +106 -0
- package/dist/tools/explore-graph.d.ts +25 -0
- package/dist/tools/explore-graph.js +84 -0
- package/dist/tools/get-doc.d.ts +11 -0
- package/dist/tools/get-doc.js +21 -0
- package/dist/tools/get-glossary-term.d.ts +9 -0
- package/dist/tools/get-glossary-term.js +41 -0
- package/dist/tools/list-docs.d.ts +19 -0
- package/dist/tools/list-docs.js +35 -0
- package/dist/tools/search-docs.d.ts +14 -0
- package/dist/tools/search-docs.js +80 -0
- package/dist/ui/api-handlers.d.ts +12 -0
- package/dist/ui/api-handlers.js +223 -0
- package/dist/ui/catalog-meta.d.ts +4 -0
- package/dist/ui/catalog-meta.js +47 -0
- package/dist/ui/db/drafts-db.d.ts +49 -0
- package/dist/ui/db/drafts-db.js +179 -0
- package/dist/ui/diff.d.ts +8 -0
- package/dist/ui/diff.js +58 -0
- package/dist/ui/docs-watcher.d.ts +13 -0
- package/dist/ui/docs-watcher.js +59 -0
- package/dist/ui/draft-api.d.ts +7 -0
- package/dist/ui/draft-api.js +413 -0
- package/dist/ui/fs-operations.d.ts +52 -0
- package/dist/ui/fs-operations.js +304 -0
- package/dist/ui/fs-tree.d.ts +28 -0
- package/dist/ui/fs-tree.js +148 -0
- package/dist/ui/graph-all.d.ts +5 -0
- package/dist/ui/graph-all.js +50 -0
- package/dist/ui/link-cascade.d.ts +9 -0
- package/dist/ui/link-cascade.js +113 -0
- package/dist/ui/parse-multipart.d.ts +12 -0
- package/dist/ui/parse-multipart.js +64 -0
- package/dist/ui/publish.d.ts +14 -0
- package/dist/ui/publish.js +83 -0
- package/dist/ui/safe-path.d.ts +6 -0
- package/dist/ui/safe-path.js +42 -0
- package/dist/ui/serve-asset.d.ts +4 -0
- package/dist/ui/serve-asset.js +49 -0
- package/dist/ui/server.d.ts +17 -0
- package/dist/ui/server.js +237 -0
- package/dist/ui/stats.d.ts +9 -0
- package/dist/ui/stats.js +23 -0
- package/dist/ui/templates.d.ts +12 -0
- package/dist/ui/templates.js +39 -0
- package/dist/ui/upload-asset.d.ts +11 -0
- package/dist/ui/upload-asset.js +61 -0
- package/package.json +55 -0
- package/templates/adr-example.md +27 -0
- package/templates/agent-instruction-example.md +20 -0
- package/templates/combat-system-example.md +27 -0
- package/templates/feature-spec-example.md +27 -0
- package/templates/glossary-term-example.md +15 -0
- package/templates/open-question-example.md +26 -0
- package/ui/dist/assets/arc-DhC0JPue.js +1 -0
- package/ui/dist/assets/architectureDiagram-3BPJPVTR-Cun_Ijrv.js +36 -0
- package/ui/dist/assets/blockDiagram-GPEHLZMM-CgiNAArN.js +132 -0
- package/ui/dist/assets/c4Diagram-AAUBKEIU-BIwHcwcH.js +10 -0
- package/ui/dist/assets/channel-CNwAp9ic.js +1 -0
- package/ui/dist/assets/chunk-2J33WTMH-DXRgHPpp.js +1 -0
- package/ui/dist/assets/chunk-4BX2VUAB-BTb70kIb.js +1 -0
- package/ui/dist/assets/chunk-55IACEB6-BrAelyhX.js +1 -0
- package/ui/dist/assets/chunk-727SXJPM-BlYnlPdj.js +206 -0
- package/ui/dist/assets/chunk-AQP2D5EJ-DSPgdKZ8.js +231 -0
- package/ui/dist/assets/chunk-FMBD7UC4-BhH8ir2K.js +15 -0
- package/ui/dist/assets/chunk-ND2GUHAM-DCAuTSxB.js +1 -0
- package/ui/dist/assets/chunk-QZHKN3VN-DtYEkbYr.js +1 -0
- package/ui/dist/assets/classDiagram-4FO5ZUOK-DnHeGLmR.js +1 -0
- package/ui/dist/assets/classDiagram-v2-Q7XG4LA2-DnHeGLmR.js +1 -0
- package/ui/dist/assets/cose-bilkent-S5V4N54A-CAM4jLYo.js +1 -0
- package/ui/dist/assets/cytoscape.esm-DTSO7Bv0.js +331 -0
- package/ui/dist/assets/dagre-BM42HDAG-CISbgani.js +4 -0
- package/ui/dist/assets/defaultLocale-DX6XiGOO.js +1 -0
- package/ui/dist/assets/diagram-2AECGRRQ-BmXargwF.js +43 -0
- package/ui/dist/assets/diagram-5GNKFQAL-COlrLu0O.js +10 -0
- package/ui/dist/assets/diagram-KO2AKTUF-B-kUxuHX.js +3 -0
- package/ui/dist/assets/diagram-LMA3HP47-C3AVVxcm.js +24 -0
- package/ui/dist/assets/diagram-OG6HWLK6-JHeftSsO.js +24 -0
- package/ui/dist/assets/erDiagram-TEJ5UH35-BSWwMysi.js +85 -0
- package/ui/dist/assets/flowDiagram-I6XJVG4X-D-q1cK69.js +162 -0
- package/ui/dist/assets/ganttDiagram-6RSMTGT7-DrYn1H_t.js +292 -0
- package/ui/dist/assets/gitGraphDiagram-PVQCEYII-vJByl99X.js +106 -0
- package/ui/dist/assets/graph-CAnANduQ.js +1 -0
- package/ui/dist/assets/graph-DwoitsWW.js +2 -0
- package/ui/dist/assets/infoDiagram-5YYISTIA-D6zhGTMj.js +2 -0
- package/ui/dist/assets/init-Gi6I4Gst.js +1 -0
- package/ui/dist/assets/ishikawaDiagram-YF4QCWOH-CY-U_l7l.js +70 -0
- package/ui/dist/assets/journeyDiagram-JHISSGLW-jKj4lBEJ.js +139 -0
- package/ui/dist/assets/kanban-definition-UN3LZRKU-PZ-5AYw2.js +89 -0
- package/ui/dist/assets/katex-C5jXJg4s.js +257 -0
- package/ui/dist/assets/layout-DGIYPm2g.js +1 -0
- package/ui/dist/assets/linear-COY9pyF4.js +1 -0
- package/ui/dist/assets/main-BBzCq-49.js +308 -0
- package/ui/dist/assets/mermaid.core-Bddhr0ku.js +309 -0
- package/ui/dist/assets/mindmap-definition-RKZ34NQL-CPY2Fdu_.js +96 -0
- package/ui/dist/assets/ordinal-Cboi1Yqb.js +1 -0
- package/ui/dist/assets/pieDiagram-4H26LBE5-C7GJ49et.js +30 -0
- package/ui/dist/assets/quadrantDiagram-W4KKPZXB-DQyQN5K7.js +7 -0
- package/ui/dist/assets/requirementDiagram-4Y6WPE33-CDrkwz1t.js +84 -0
- package/ui/dist/assets/sankeyDiagram-5OEKKPKP-BrYb9Eql.js +40 -0
- package/ui/dist/assets/sequenceDiagram-3UESZ5HK-B8If_JZp.js +162 -0
- package/ui/dist/assets/stateDiagram-AJRCARHV-BbpTp9VX.js +1 -0
- package/ui/dist/assets/stateDiagram-v2-BHNVJYJU-BT4PvMFS.js +1 -0
- package/ui/dist/assets/theme-DV7vqTnV.js +1 -0
- package/ui/dist/assets/theme-SpsWsRN5.css +1 -0
- package/ui/dist/assets/timeline-definition-PNZ67QCA-DhUg6aIV.js +120 -0
- package/ui/dist/assets/transform-BwXaE9hv.js +1 -0
- package/ui/dist/assets/vennDiagram-CIIHVFJN-DpQVNNzF.js +34 -0
- package/ui/dist/assets/wardley-L42UT6IY-CyaxzHGP.js +173 -0
- package/ui/dist/assets/wardleyDiagram-YWT4CUSO-Bm0mA7wm.js +78 -0
- package/ui/dist/assets/xychartDiagram-2RQKCTM6-OJbmgDx6.js +7 -0
- package/ui/dist/graph.html +27 -0
- package/ui/dist/index.html +37 -0
package/README.md
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# RepoMind
|
|
2
|
+
|
|
3
|
+
**Unified project knowledge in `docs/`** — wiki, architecture, ADR, stack, and glossary live in your repository. Humans edit via a Confluence-style UI; AI agents query the same files through MCP. No Confluence or Notion required.
|
|
4
|
+
|
|
5
|
+
## Status
|
|
6
|
+
|
|
7
|
+
**v0.3.0** — keyboard nav, image upload, domain labels, yaml/json reader, `ab-demo` harness. Confluence-style UI, wikilinks, deep links (`?slug=`), prepare/sync-links.
|
|
8
|
+
|
|
9
|
+
| Artifact | Location |
|
|
10
|
+
|----------|----------|
|
|
11
|
+
| **Product roadmap** | [`docs/product/wiki/roadmap.md`](docs/product/wiki/roadmap.md) |
|
|
12
|
+
| Full product vision | [`idea.md`](idea.md) |
|
|
13
|
+
| docs/ pivot spec | [`.gstack/projects/repo-mind/specs/2026-06-21-repomind-docs-root-pivot.md`](.gstack/projects/repo-mind/specs/2026-06-21-repomind-docs-root-pivot.md) |
|
|
14
|
+
| Design system | [`DESIGN.md`](DESIGN.md) |
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
npm install -g @justyork/repo-mind
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
CLI command remains **`repo-mind`** (bin alias unchanged).
|
|
23
|
+
|
|
24
|
+
## Quick start
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
# From npm
|
|
28
|
+
npx -y @justyork/repo-mind init
|
|
29
|
+
npx -y @justyork/repo-mind setup
|
|
30
|
+
|
|
31
|
+
# From this repo (local dev)
|
|
32
|
+
npm install && npm run build
|
|
33
|
+
node dist/cli.js init
|
|
34
|
+
node dist/cli.js setup
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Then ask your agent a project question — it should call `search_docs` / `get_doc` via MCP against the same `docs/` you edit in the UI.
|
|
38
|
+
|
|
39
|
+
## CLI commands
|
|
40
|
+
|
|
41
|
+
| Command | Description |
|
|
42
|
+
|---------|-------------|
|
|
43
|
+
| `repo-mind init` | Scaffold `docs/` with example structured pages |
|
|
44
|
+
| `repo-mind setup` | Configure Cursor/Claude MCP + CLAUDE.md snippet |
|
|
45
|
+
| `repo-mind check` | Validate frontmatter schema and `related:` links |
|
|
46
|
+
| `repo-mind prepare` | Add frontmatter to legacy markdown (`--all` for batch) |
|
|
47
|
+
| `repo-mind sync-links` | Convert markdown links to wikilinks; sync `related:` |
|
|
48
|
+
| `repo-mind export` | Write `agents.md` to repo root |
|
|
49
|
+
| `repo-mind mcp` | Start the MCP stdio server |
|
|
50
|
+
| `repo-mind ui` | Confluence-style workspace over `docs/` (127.0.0.1:3847) |
|
|
51
|
+
| `npm run ab-demo` | Validate A/B demo fixture (repo checkout only) |
|
|
52
|
+
|
|
53
|
+
## Cursor skill
|
|
54
|
+
|
|
55
|
+
Agent skill for authoring `docs/` — **domains** (product, technical, game-design, analytics, …), frontmatter, structure, wikilinks: [`.cursor/skills/repomind-docs/`](.cursor/skills/repomind-docs/) ([structure.md](.cursor/skills/repomind-docs/structure.md)). Copy to `~/.cursor/skills/repomind-docs/` for other projects.
|
|
56
|
+
|
|
57
|
+
## Web UI
|
|
58
|
+
|
|
59
|
+
Confluence-style workspace over **`docs/`**:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
npm run build
|
|
63
|
+
repo-mind ui # http://127.0.0.1:3847
|
|
64
|
+
repo-mind ui --port 4000 --cwd /path/to/project
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
- **Catalog tree** — docs by domain (`Product`, `Technical`, …) and type
|
|
68
|
+
- **Prepare** — recursively find markdown without frontmatter; add schema in one click
|
|
69
|
+
- **Drafts + Publish** — SQLite drafts, publish to `docs/*.md`
|
|
70
|
+
- **Health** — schema check, publish queue, export `agents.md`
|
|
71
|
+
- **Graph** — full-page view at `/graph.html`
|
|
72
|
+
- **Theme** — light (default) / dark toggle
|
|
73
|
+
|
|
74
|
+
Binds **127.0.0.1** only. MCP reads published files in `docs/` only (not SQLite drafts).
|
|
75
|
+
|
|
76
|
+
## MCP tools
|
|
77
|
+
|
|
78
|
+
- `list_docs` — filter by type, status, tag
|
|
79
|
+
- `search_docs` — ranked full-text search
|
|
80
|
+
- `get_doc` — fetch one doc by slug
|
|
81
|
+
- `get_glossary_term` — resolve glossary entries
|
|
82
|
+
- `explore_graph` — BFS over `related:` links
|
|
83
|
+
|
|
84
|
+
## CI
|
|
85
|
+
|
|
86
|
+
Add to GitHub Actions or pre-commit:
|
|
87
|
+
|
|
88
|
+
```yaml
|
|
89
|
+
- run: npx @justyork/repo-mind check
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Development
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
npm install
|
|
96
|
+
npm run build
|
|
97
|
+
npm test
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## gstack workflow
|
|
101
|
+
|
|
102
|
+
This repo is developed with [gstack](https://github.com/garrytan/gstack). Project artifacts live in `.gstack/projects/repo-mind/` (versioned in git). Set `GSTACK_HOME=.gstack` when working in this repo (see `.env`).
|
|
103
|
+
|
|
104
|
+
## Roadmap
|
|
105
|
+
|
|
106
|
+
See [`docs/product/wiki/roadmap.md`](docs/product/wiki/roadmap.md) for v4.0–v4.2 phases. Next: v4.1 keyboard nav and image upload; v4.2 agent write (gated on kill-switch).
|
|
107
|
+
|
|
108
|
+
## License
|
|
109
|
+
|
|
110
|
+
MIT
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { DocIndex } from '../index/doc-index.js';
|
|
2
|
+
import type { ArmBaselineResult } from './types.js';
|
|
3
|
+
import type { AbQuestion } from './types.js';
|
|
4
|
+
/**
|
|
5
|
+
* Arm A: plain markdown + minimal CLAUDE.md.
|
|
6
|
+
* Simulates grep-then-read; falls back to reading the full corpus when nothing matches.
|
|
7
|
+
*/
|
|
8
|
+
export declare function runArmBaseline(index: DocIndex, question: AbQuestion, sessionOverheadPerQuestion: number): ArmBaselineResult;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { estimateTokens } from './estimate-tokens.js';
|
|
2
|
+
function queryTerms(prompt) {
|
|
3
|
+
return prompt
|
|
4
|
+
.toLowerCase()
|
|
5
|
+
.split(/[^a-z0-9]+/)
|
|
6
|
+
.filter((term) => term.length >= 3);
|
|
7
|
+
}
|
|
8
|
+
function docMatchesTerms(doc, terms) {
|
|
9
|
+
if (terms.length === 0) {
|
|
10
|
+
return true;
|
|
11
|
+
}
|
|
12
|
+
const haystack = `${doc.title}\n${doc.tags.join(' ')}\n${doc.body}`.toLowerCase();
|
|
13
|
+
return terms.some((term) => haystack.includes(term));
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Arm A: plain markdown + minimal CLAUDE.md.
|
|
17
|
+
* Simulates grep-then-read; falls back to reading the full corpus when nothing matches.
|
|
18
|
+
*/
|
|
19
|
+
export function runArmBaseline(index, question, sessionOverheadPerQuestion) {
|
|
20
|
+
const docs = index.refresh();
|
|
21
|
+
const terms = queryTerms(question.prompt);
|
|
22
|
+
let matched = docs.filter((doc) => docMatchesTerms(doc, terms));
|
|
23
|
+
let strategy = 'grep-then-read';
|
|
24
|
+
if (matched.length === 0) {
|
|
25
|
+
matched = docs;
|
|
26
|
+
strategy = 'read-all';
|
|
27
|
+
}
|
|
28
|
+
let tokens = sessionOverheadPerQuestion;
|
|
29
|
+
for (const doc of matched) {
|
|
30
|
+
tokens += estimateTokens(doc.body);
|
|
31
|
+
tokens += estimateTokens(doc.relativePath);
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
arm: 'baseline',
|
|
35
|
+
questionId: question.id,
|
|
36
|
+
tokens,
|
|
37
|
+
filesRead: matched.length,
|
|
38
|
+
strategy,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { DocIndex } from '../index/doc-index.js';
|
|
2
|
+
import type { ArmRepomindResult } from './types.js';
|
|
3
|
+
import type { AbQuestion } from './types.js';
|
|
4
|
+
/**
|
|
5
|
+
* Arm B: RepoMind MCP — search_docs then get_doc on top hits.
|
|
6
|
+
*/
|
|
7
|
+
export declare function runArmRepomind(index: DocIndex, question: AbQuestion, sessionOverheadPerQuestion: number): ArmRepomindResult;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { getDoc } from '../tools/get-doc.js';
|
|
2
|
+
import { searchDocs } from '../tools/search-docs.js';
|
|
3
|
+
import { estimateJsonTokens } from './estimate-tokens.js';
|
|
4
|
+
const GET_DOC_LIMIT = 3;
|
|
5
|
+
/**
|
|
6
|
+
* Arm B: RepoMind MCP — search_docs then get_doc on top hits.
|
|
7
|
+
*/
|
|
8
|
+
export function runArmRepomind(index, question, sessionOverheadPerQuestion) {
|
|
9
|
+
let tokens = sessionOverheadPerQuestion;
|
|
10
|
+
const searchInput = { query: question.prompt };
|
|
11
|
+
tokens += estimateJsonTokens({ tool: 'search_docs', input: searchInput });
|
|
12
|
+
const hits = searchDocs(index, searchInput);
|
|
13
|
+
tokens += estimateJsonTokens(hits);
|
|
14
|
+
const slugsToFetch = hits.slice(0, GET_DOC_LIMIT).map((hit) => hit.slug);
|
|
15
|
+
let docsFetched = 0;
|
|
16
|
+
for (const slug of slugsToFetch) {
|
|
17
|
+
tokens += estimateJsonTokens({ tool: 'get_doc', input: { slug } });
|
|
18
|
+
const doc = getDoc(index, slug);
|
|
19
|
+
if (doc.found && doc.body) {
|
|
20
|
+
tokens += estimateJsonTokens({
|
|
21
|
+
slug: doc.slug,
|
|
22
|
+
title: doc.frontmatter?.title,
|
|
23
|
+
body: doc.body,
|
|
24
|
+
});
|
|
25
|
+
docsFetched += 1;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return {
|
|
29
|
+
arm: 'repomind',
|
|
30
|
+
questionId: question.id,
|
|
31
|
+
tokens,
|
|
32
|
+
searchHits: hits.length,
|
|
33
|
+
docsFetched,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** Rough token estimate (~4 chars per token) for reproducible A/B comparison. */
|
|
2
|
+
export function estimateTokens(text) {
|
|
3
|
+
if (text.length === 0) {
|
|
4
|
+
return 0;
|
|
5
|
+
}
|
|
6
|
+
return Math.ceil(text.length / 4);
|
|
7
|
+
}
|
|
8
|
+
export function estimateJsonTokens(value) {
|
|
9
|
+
return estimateTokens(JSON.stringify(value));
|
|
10
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
function isNonEmptyString(value) {
|
|
3
|
+
return typeof value === 'string' && value.trim().length > 0;
|
|
4
|
+
}
|
|
5
|
+
function parseQuestion(raw, index) {
|
|
6
|
+
if (!raw || typeof raw !== 'object') {
|
|
7
|
+
throw new Error(`questions[${index}] must be an object`);
|
|
8
|
+
}
|
|
9
|
+
const record = raw;
|
|
10
|
+
if (!isNonEmptyString(record.id)) {
|
|
11
|
+
throw new Error(`questions[${index}].id must be a non-empty string`);
|
|
12
|
+
}
|
|
13
|
+
if (!isNonEmptyString(record.prompt)) {
|
|
14
|
+
throw new Error(`questions[${index}].prompt must be a non-empty string`);
|
|
15
|
+
}
|
|
16
|
+
if (!Array.isArray(record.anchorSlugs) || record.anchorSlugs.length === 0) {
|
|
17
|
+
throw new Error(`questions[${index}].anchorSlugs must be a non-empty array`);
|
|
18
|
+
}
|
|
19
|
+
for (const slug of record.anchorSlugs) {
|
|
20
|
+
if (!isNonEmptyString(slug)) {
|
|
21
|
+
throw new Error(`questions[${index}].anchorSlugs must contain strings`);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
const tags = Array.isArray(record.tags)
|
|
25
|
+
? record.tags.filter((tag) => isNonEmptyString(tag))
|
|
26
|
+
: undefined;
|
|
27
|
+
return {
|
|
28
|
+
id: record.id,
|
|
29
|
+
prompt: record.prompt,
|
|
30
|
+
anchorSlugs: record.anchorSlugs,
|
|
31
|
+
tags,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
export function loadQuestions(filePath) {
|
|
35
|
+
if (!fs.existsSync(filePath)) {
|
|
36
|
+
throw new Error(`questions file not found: ${filePath}`);
|
|
37
|
+
}
|
|
38
|
+
let parsed;
|
|
39
|
+
try {
|
|
40
|
+
parsed = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
44
|
+
throw new Error(`invalid JSON in ${filePath}: ${message}`);
|
|
45
|
+
}
|
|
46
|
+
if (!parsed || typeof parsed !== 'object') {
|
|
47
|
+
throw new Error(`questions file must be a JSON object: ${filePath}`);
|
|
48
|
+
}
|
|
49
|
+
const record = parsed;
|
|
50
|
+
if (record.version !== 1) {
|
|
51
|
+
throw new Error(`unsupported questions version (expected 1): ${filePath}`);
|
|
52
|
+
}
|
|
53
|
+
if (!Array.isArray(record.questions) || record.questions.length === 0) {
|
|
54
|
+
throw new Error(`questions array must be non-empty: ${filePath}`);
|
|
55
|
+
}
|
|
56
|
+
const questions = record.questions.map(parseQuestion);
|
|
57
|
+
const ids = new Set();
|
|
58
|
+
for (const question of questions) {
|
|
59
|
+
if (ids.has(question.id)) {
|
|
60
|
+
throw new Error(`duplicate question id: ${question.id}`);
|
|
61
|
+
}
|
|
62
|
+
ids.add(question.id);
|
|
63
|
+
}
|
|
64
|
+
return { version: 1, questions };
|
|
65
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export declare function resolveAbDemoRoot(startDir?: string): string;
|
|
2
|
+
export declare function corpusPath(abDemoRoot: string): string;
|
|
3
|
+
export declare function questionsPath(abDemoRoot: string): string;
|
|
4
|
+
export declare function resultsDir(abDemoRoot: string): string;
|
|
5
|
+
export declare function scoreRubricPath(abDemoRoot: string): string;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
|
|
5
|
+
export function resolveAbDemoRoot(startDir = process.cwd()) {
|
|
6
|
+
let current = path.resolve(startDir);
|
|
7
|
+
while (true) {
|
|
8
|
+
const candidate = path.join(current, 'ab-demo');
|
|
9
|
+
if (fs.existsSync(candidate) && fs.statSync(candidate).isDirectory()) {
|
|
10
|
+
return candidate;
|
|
11
|
+
}
|
|
12
|
+
const parent = path.dirname(current);
|
|
13
|
+
if (parent === current) {
|
|
14
|
+
break;
|
|
15
|
+
}
|
|
16
|
+
current = parent;
|
|
17
|
+
}
|
|
18
|
+
return path.join(PACKAGE_ROOT, 'ab-demo');
|
|
19
|
+
}
|
|
20
|
+
export function corpusPath(abDemoRoot) {
|
|
21
|
+
return path.join(abDemoRoot, 'corpus');
|
|
22
|
+
}
|
|
23
|
+
export function questionsPath(abDemoRoot) {
|
|
24
|
+
return path.join(abDemoRoot, 'questions.json');
|
|
25
|
+
}
|
|
26
|
+
export function resultsDir(abDemoRoot) {
|
|
27
|
+
return path.join(abDemoRoot, 'results');
|
|
28
|
+
}
|
|
29
|
+
export function scoreRubricPath(abDemoRoot) {
|
|
30
|
+
return path.join(abDemoRoot, 'score-hallucination.md');
|
|
31
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { fileURLToPath } from 'node:url';
|
|
5
|
+
import { DocIndex } from '../index/doc-index.js';
|
|
6
|
+
import { loadQuestions } from './load-questions.js';
|
|
7
|
+
import { corpusPath, questionsPath, resolveAbDemoRoot, resultsDir, scoreRubricPath, } from './paths.js';
|
|
8
|
+
import { runArmsComparison } from './run-arms.js';
|
|
9
|
+
import { materializeCorpusRepo, validateCorpusAgainstQuestions } from './validate-corpus.js';
|
|
10
|
+
function printUsage() {
|
|
11
|
+
console.log(`repo-mind A/B demo harness (kill-switch eval)
|
|
12
|
+
|
|
13
|
+
Usage:
|
|
14
|
+
npm run ab-demo -- [--dry-run] [--output <path>]
|
|
15
|
+
|
|
16
|
+
Options:
|
|
17
|
+
--dry-run Validate corpus + questions only (no agent runs)
|
|
18
|
+
--output Write results JSON (default: ab-demo/results/latest.json)
|
|
19
|
+
`);
|
|
20
|
+
}
|
|
21
|
+
function parseCliArgs(argv) {
|
|
22
|
+
const options = {};
|
|
23
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
24
|
+
const arg = argv[i];
|
|
25
|
+
if (arg === '--help' || arg === '-h') {
|
|
26
|
+
printUsage();
|
|
27
|
+
process.exit(0);
|
|
28
|
+
}
|
|
29
|
+
if (arg === '--dry-run') {
|
|
30
|
+
options.dryRun = true;
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
if (arg === '--output') {
|
|
34
|
+
const value = argv[i + 1];
|
|
35
|
+
if (!value) {
|
|
36
|
+
throw new Error('--output requires a path');
|
|
37
|
+
}
|
|
38
|
+
options.outputPath = value;
|
|
39
|
+
i += 1;
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
throw new Error(`unknown argument: ${arg}`);
|
|
43
|
+
}
|
|
44
|
+
return options;
|
|
45
|
+
}
|
|
46
|
+
export function runAbDemo(options = {}) {
|
|
47
|
+
const abRoot = resolveAbDemoRoot(options.cwd);
|
|
48
|
+
const corpus = corpusPath(abRoot);
|
|
49
|
+
const questionsFile = questionsPath(abRoot);
|
|
50
|
+
const rubric = scoreRubricPath(abRoot);
|
|
51
|
+
if (!fs.existsSync(rubric)) {
|
|
52
|
+
console.error(`missing rubric: ${rubric}`);
|
|
53
|
+
return 1;
|
|
54
|
+
}
|
|
55
|
+
let questions;
|
|
56
|
+
try {
|
|
57
|
+
questions = loadQuestions(questionsFile);
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
61
|
+
console.error(message);
|
|
62
|
+
return 1;
|
|
63
|
+
}
|
|
64
|
+
let report;
|
|
65
|
+
try {
|
|
66
|
+
report = validateCorpusAgainstQuestions(corpus, questions.questions);
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
70
|
+
console.error(message);
|
|
71
|
+
return 1;
|
|
72
|
+
}
|
|
73
|
+
if (report.missingAnchors.length > 0) {
|
|
74
|
+
console.error('corpus missing anchor slugs:');
|
|
75
|
+
for (const missing of report.missingAnchors) {
|
|
76
|
+
console.error(` ${missing.questionId}: ${missing.slug}`);
|
|
77
|
+
}
|
|
78
|
+
return 1;
|
|
79
|
+
}
|
|
80
|
+
console.log(`corpus ok: ${report.docCount} docs, ${report.questionCount} questions`);
|
|
81
|
+
if (options.dryRun) {
|
|
82
|
+
console.log('dry-run complete — agent arms not executed');
|
|
83
|
+
return 0;
|
|
84
|
+
}
|
|
85
|
+
const workDir = materializeCorpusRepo(corpus);
|
|
86
|
+
try {
|
|
87
|
+
const index = new DocIndex(workDir);
|
|
88
|
+
const arms = runArmsComparison(index, questions.questions);
|
|
89
|
+
const outDir = resultsDir(abRoot);
|
|
90
|
+
fs.mkdirSync(outDir, { recursive: true });
|
|
91
|
+
const outputPath = options.outputPath ?? path.join(outDir, 'latest.json');
|
|
92
|
+
const payload = {
|
|
93
|
+
runAt: new Date().toISOString(),
|
|
94
|
+
corpusPath: corpus,
|
|
95
|
+
questionsVersion: questions.version,
|
|
96
|
+
arms,
|
|
97
|
+
pass: arms.pass,
|
|
98
|
+
};
|
|
99
|
+
fs.writeFileSync(outputPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
|
|
100
|
+
console.log(`baseline median tokens: ${arms.baseline.medianTokens} | repomind: ${arms.repomind.medianTokens}`);
|
|
101
|
+
console.log(`token wins: repomind ${arms.repomindTokenWins}/${questions.questions.length} (need ${arms.passThreshold})`);
|
|
102
|
+
console.log(`token pass: ${arms.tokenPass ? 'yes' : 'no'}`);
|
|
103
|
+
console.log(`wrote ${outputPath}`);
|
|
104
|
+
if (!arms.tokenPass) {
|
|
105
|
+
return 1;
|
|
106
|
+
}
|
|
107
|
+
return 0;
|
|
108
|
+
}
|
|
109
|
+
finally {
|
|
110
|
+
fs.rmSync(workDir, { recursive: true, force: true });
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
function main() {
|
|
114
|
+
try {
|
|
115
|
+
const code = runAbDemo(parseCliArgs(process.argv.slice(2)));
|
|
116
|
+
process.exit(code);
|
|
117
|
+
}
|
|
118
|
+
catch (error) {
|
|
119
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
120
|
+
console.error(message);
|
|
121
|
+
process.exit(1);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
const isMain = process.argv[1] !== undefined &&
|
|
125
|
+
fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
|
|
126
|
+
if (isMain) {
|
|
127
|
+
main();
|
|
128
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { runArmBaseline } from './arm-baseline.js';
|
|
2
|
+
import { runArmRepomind } from './arm-repomind.js';
|
|
3
|
+
import { BASELINE_CLAUDE_SNIPPET, mcpToolSchemaTokenEstimate, } from './session-overhead.js';
|
|
4
|
+
import { estimateTokens } from './estimate-tokens.js';
|
|
5
|
+
function median(values) {
|
|
6
|
+
if (values.length === 0) {
|
|
7
|
+
return 0;
|
|
8
|
+
}
|
|
9
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
10
|
+
const mid = Math.floor(sorted.length / 2);
|
|
11
|
+
if (sorted.length % 2 === 0) {
|
|
12
|
+
return Math.round((sorted[mid - 1] + sorted[mid]) / 2);
|
|
13
|
+
}
|
|
14
|
+
return sorted[mid];
|
|
15
|
+
}
|
|
16
|
+
function summarizeArm(arm, perQuestion) {
|
|
17
|
+
const tokens = perQuestion.map((row) => arm === 'baseline' ? row.baseline.tokens : row.repomind.tokens);
|
|
18
|
+
return {
|
|
19
|
+
arm,
|
|
20
|
+
medianTokens: median(tokens),
|
|
21
|
+
perQuestionTokens: tokens,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export function runArmsComparison(index, questions) {
|
|
25
|
+
const n = questions.length;
|
|
26
|
+
const listing = index
|
|
27
|
+
.refresh()
|
|
28
|
+
.map((doc) => doc.relativePath)
|
|
29
|
+
.join('\n');
|
|
30
|
+
const baselineSession = estimateTokens(BASELINE_CLAUDE_SNIPPET) + estimateTokens(listing);
|
|
31
|
+
const repomindSession = mcpToolSchemaTokenEstimate();
|
|
32
|
+
const baselineOverhead = Math.ceil(baselineSession / n);
|
|
33
|
+
const repomindOverhead = Math.ceil(repomindSession / n);
|
|
34
|
+
const comparisons = questions.map((question) => {
|
|
35
|
+
const baseline = runArmBaseline(index, question, baselineOverhead);
|
|
36
|
+
const repomind = runArmRepomind(index, question, repomindOverhead);
|
|
37
|
+
return {
|
|
38
|
+
questionId: question.id,
|
|
39
|
+
prompt: question.prompt,
|
|
40
|
+
anchorSlugs: question.anchorSlugs,
|
|
41
|
+
baseline,
|
|
42
|
+
repomind,
|
|
43
|
+
tokenWinner: repomind.tokens < baseline.tokens
|
|
44
|
+
? 'repomind'
|
|
45
|
+
: repomind.tokens > baseline.tokens
|
|
46
|
+
? 'baseline'
|
|
47
|
+
: 'tie',
|
|
48
|
+
};
|
|
49
|
+
});
|
|
50
|
+
const baselineSummary = summarizeArm('baseline', comparisons);
|
|
51
|
+
const repomindSummary = summarizeArm('repomind', comparisons);
|
|
52
|
+
const repomindWins = comparisons.filter((row) => row.tokenWinner === 'repomind').length;
|
|
53
|
+
const passThreshold = Math.ceil((n * 2) / 3);
|
|
54
|
+
const tokenPass = repomindSummary.medianTokens < baselineSummary.medianTokens &&
|
|
55
|
+
repomindWins >= passThreshold;
|
|
56
|
+
return {
|
|
57
|
+
comparisons,
|
|
58
|
+
baseline: baselineSummary,
|
|
59
|
+
repomind: repomindSummary,
|
|
60
|
+
repomindTokenWins: repomindWins,
|
|
61
|
+
passThreshold,
|
|
62
|
+
tokenPass,
|
|
63
|
+
hallucinationPass: null,
|
|
64
|
+
pass: tokenPass ? null : false,
|
|
65
|
+
note: 'Automated token comparison only. Set humanScores in results JSON; pass becomes true when hallucination rubric also passes.',
|
|
66
|
+
};
|
|
67
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
/** Mirrors MCP ListTools payload shape for token budgeting. */
|
|
2
|
+
export declare function mcpToolSchemaTokenEstimate(): number;
|
|
3
|
+
export declare const BASELINE_CLAUDE_SNIPPET = "# Project knowledge\n\nRead markdown files under `docs/` to answer questions. Prefer grep or glob before reading entire files.\n";
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { DOC_DOMAINS, DOC_STATUSES, DOC_TYPES } from '../index/types.js';
|
|
2
|
+
import { estimateJsonTokens } from './estimate-tokens.js';
|
|
3
|
+
/** Mirrors MCP ListTools payload shape for token budgeting. */
|
|
4
|
+
export function mcpToolSchemaTokenEstimate() {
|
|
5
|
+
const tools = [
|
|
6
|
+
{
|
|
7
|
+
name: 'list_docs',
|
|
8
|
+
description: 'List project knowledge documents with optional filters.',
|
|
9
|
+
inputSchema: {
|
|
10
|
+
type: 'object',
|
|
11
|
+
properties: {
|
|
12
|
+
type: { type: 'string', enum: [...DOC_TYPES] },
|
|
13
|
+
status: { type: 'string', enum: [...DOC_STATUSES] },
|
|
14
|
+
tag: { type: 'string' },
|
|
15
|
+
domain: { type: 'string', enum: [...DOC_DOMAINS] },
|
|
16
|
+
},
|
|
17
|
+
},
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
name: 'search_docs',
|
|
21
|
+
description: 'Search project knowledge documents by query.',
|
|
22
|
+
inputSchema: {
|
|
23
|
+
type: 'object',
|
|
24
|
+
properties: {
|
|
25
|
+
query: { type: 'string' },
|
|
26
|
+
type: { type: 'string', enum: [...DOC_TYPES] },
|
|
27
|
+
domain: { type: 'string', enum: [...DOC_DOMAINS] },
|
|
28
|
+
},
|
|
29
|
+
required: ['query'],
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
name: 'get_doc',
|
|
34
|
+
description: 'Fetch a single document by slug.',
|
|
35
|
+
inputSchema: {
|
|
36
|
+
type: 'object',
|
|
37
|
+
properties: { slug: { type: 'string' } },
|
|
38
|
+
required: ['slug'],
|
|
39
|
+
},
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
name: 'get_glossary_term',
|
|
43
|
+
description: 'Resolve a glossary term by name.',
|
|
44
|
+
inputSchema: {
|
|
45
|
+
type: 'object',
|
|
46
|
+
properties: { name: { type: 'string' } },
|
|
47
|
+
required: ['name'],
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
name: 'explore_graph',
|
|
52
|
+
description: 'Explore related documents as a graph.',
|
|
53
|
+
inputSchema: {
|
|
54
|
+
type: 'object',
|
|
55
|
+
properties: {
|
|
56
|
+
slug: { type: 'string' },
|
|
57
|
+
depth: { type: 'number' },
|
|
58
|
+
},
|
|
59
|
+
required: ['slug'],
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
];
|
|
63
|
+
return estimateJsonTokens({ tools });
|
|
64
|
+
}
|
|
65
|
+
export const BASELINE_CLAUDE_SNIPPET = `# Project knowledge
|
|
66
|
+
|
|
67
|
+
Read markdown files under \`docs/\` to answer questions. Prefer grep or glob before reading entire files.
|
|
68
|
+
`;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
export interface AbQuestion {
|
|
2
|
+
id: string;
|
|
3
|
+
prompt: string;
|
|
4
|
+
/** Slugs that a grounded answer should cite or retrieve. */
|
|
5
|
+
anchorSlugs: string[];
|
|
6
|
+
tags?: string[];
|
|
7
|
+
}
|
|
8
|
+
export interface AbQuestionsFile {
|
|
9
|
+
version: number;
|
|
10
|
+
questions: AbQuestion[];
|
|
11
|
+
}
|
|
12
|
+
export interface AbDryRunReport {
|
|
13
|
+
corpusPath: string;
|
|
14
|
+
docCount: number;
|
|
15
|
+
questionCount: number;
|
|
16
|
+
missingAnchors: Array<{
|
|
17
|
+
questionId: string;
|
|
18
|
+
slug: string;
|
|
19
|
+
}>;
|
|
20
|
+
}
|
|
21
|
+
export interface ArmBaselineResult {
|
|
22
|
+
arm: 'baseline';
|
|
23
|
+
questionId: string;
|
|
24
|
+
tokens: number;
|
|
25
|
+
filesRead: number;
|
|
26
|
+
strategy: 'grep-then-read' | 'read-all';
|
|
27
|
+
}
|
|
28
|
+
export interface ArmRepomindResult {
|
|
29
|
+
arm: 'repomind';
|
|
30
|
+
questionId: string;
|
|
31
|
+
tokens: number;
|
|
32
|
+
searchHits: number;
|
|
33
|
+
docsFetched: number;
|
|
34
|
+
}
|
|
35
|
+
export interface AbQuestionComparison {
|
|
36
|
+
questionId: string;
|
|
37
|
+
prompt: string;
|
|
38
|
+
anchorSlugs: string[];
|
|
39
|
+
baseline: ArmBaselineResult;
|
|
40
|
+
repomind: ArmRepomindResult;
|
|
41
|
+
tokenWinner: 'baseline' | 'repomind' | 'tie';
|
|
42
|
+
}
|
|
43
|
+
export interface AbArmSummary {
|
|
44
|
+
arm: 'baseline' | 'repomind';
|
|
45
|
+
medianTokens: number;
|
|
46
|
+
perQuestionTokens: number[];
|
|
47
|
+
}
|
|
48
|
+
export interface AbArmsRunResult {
|
|
49
|
+
comparisons: AbQuestionComparison[];
|
|
50
|
+
baseline: AbArmSummary;
|
|
51
|
+
repomind: AbArmSummary;
|
|
52
|
+
repomindTokenWins: number;
|
|
53
|
+
passThreshold: number;
|
|
54
|
+
tokenPass: boolean;
|
|
55
|
+
hallucinationPass: boolean | null;
|
|
56
|
+
pass: boolean | null;
|
|
57
|
+
note: string;
|
|
58
|
+
}
|
|
59
|
+
export interface AbRunResult {
|
|
60
|
+
runAt: string;
|
|
61
|
+
corpusPath: string;
|
|
62
|
+
questionsVersion: number;
|
|
63
|
+
arms: AbArmsRunResult;
|
|
64
|
+
pass: boolean | null;
|
|
65
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|