@gamaze/hicortex 0.14.4 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -1
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +52 -0
- package/dist/cluster.d.ts +51 -0
- package/dist/cluster.js +118 -0
- package/dist/consolidate.d.ts +63 -1
- package/dist/consolidate.js +228 -2
- package/dist/db.js +26 -0
- package/dist/dedup.d.ts +157 -0
- package/dist/dedup.js +445 -0
- package/dist/eval/decay-eval.d.ts +110 -0
- package/dist/eval/decay-eval.js +252 -0
- package/dist/eval/dups.d.ts +100 -0
- package/dist/eval/dups.js +174 -0
- package/dist/eval/eval-db.d.ts +25 -0
- package/dist/eval/eval-db.js +67 -0
- package/dist/eval/graph-eval.d.ts +76 -0
- package/dist/eval/graph-eval.js +200 -0
- package/dist/eval/reflection-census.d.ts +19 -0
- package/dist/eval/reflection-census.js +25 -0
- package/dist/eval/run-eval.d.ts +17 -0
- package/dist/eval/run-eval.js +275 -0
- package/dist/init.js +10 -0
- package/dist/mcp-server.js +13 -11
- package/dist/nightly.js +19 -0
- package/dist/relink.d.ts +3 -2
- package/dist/relink.js +6 -11
- package/dist/seed-lesson.d.ts +1 -1
- package/dist/seed-lesson.js +1 -1
- package/dist/state.d.ts +10 -0
- package/dist/storage.d.ts +10 -0
- package/dist/storage.js +23 -0
- package/dist/telemetry-cli.d.ts +11 -0
- package/dist/telemetry-cli.js +82 -0
- package/dist/telemetry.d.ts +36 -2
- package/dist/telemetry.js +34 -4
- package/dist/types.d.ts +15 -0
- package/package.json +2 -1
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/**
|
|
4
|
+
* #191 mechanical audit baseline — orchestrates every section (D1 duplicate
|
|
5
|
+
* rate, D4 decay/prune/no-fit + #192 adoption, D6 link-graph health, the
|
|
6
|
+
* reflection census) into one markdown report.
|
|
7
|
+
*
|
|
8
|
+
* Read-only end to end: opens the snapshot via eval-db.ts (`openSnapshot`,
|
|
9
|
+
* never `initDb()`), never writes to the DB. Not wired into `cli.ts` — this
|
|
10
|
+
* is an internal measurement tool, run via the `eval` npm script:
|
|
11
|
+
*
|
|
12
|
+
* npm run eval -- <snapshot.db> [report.md]
|
|
13
|
+
*
|
|
14
|
+
* `state.json` is expected alongside the snapshot DB (same directory) for
|
|
15
|
+
* the `domainCursor`/`relinkCursor` watermarks. Report path defaults to
|
|
16
|
+
* `eval-report.md` next to the DB.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
const node_path_1 = require("node:path");
|
|
20
|
+
const node_fs_1 = require("node:fs");
|
|
21
|
+
const eval_db_js_1 = require("./eval-db.js");
|
|
22
|
+
const dups_js_1 = require("./dups.js");
|
|
23
|
+
const decay_eval_js_1 = require("./decay-eval.js");
|
|
24
|
+
const graph_eval_js_1 = require("./graph-eval.js");
|
|
25
|
+
const reflection_census_js_1 = require("./reflection-census.js");
|
|
26
|
+
const retrieval_js_1 = require("../retrieval.js");
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
// Formatting helpers
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
function pct(n) {
|
|
31
|
+
return `${(n * 100).toFixed(1)}%`;
|
|
32
|
+
}
|
|
33
|
+
function round(n, places = 4) {
|
|
34
|
+
const f = 10 ** places;
|
|
35
|
+
return Math.round(n * f) / f;
|
|
36
|
+
}
|
|
37
|
+
/** Extract the leading number from a bucket label (e.g. "[0.2, 0.3)" -> 0.2) so buckets render in ascending order. */
|
|
38
|
+
function bucketSortKey(label) {
|
|
39
|
+
const match = label.match(/-?\d+(\.\d+)?/);
|
|
40
|
+
return match ? Number(match[0]) : Number.POSITIVE_INFINITY;
|
|
41
|
+
}
|
|
42
|
+
function histTable(h) {
|
|
43
|
+
const entries = Object.entries(h).sort((a, b) => bucketSortKey(a[0]) - bucketSortKey(b[0]));
|
|
44
|
+
if (entries.length === 0)
|
|
45
|
+
return "_(no data)_\n";
|
|
46
|
+
return entries.map(([bucket, count]) => `| ${bucket} | ${count} |`).join("\n") + "\n";
|
|
47
|
+
}
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
// Report sections
|
|
50
|
+
// ---------------------------------------------------------------------------
|
|
51
|
+
function renderDups(d) {
|
|
52
|
+
const lines = [];
|
|
53
|
+
lines.push("## D1 — Duplicate rate\n");
|
|
54
|
+
lines.push(`Top-${d.knnK} vector KNN per memory (${d.totalMemories} memories), pairs kept at cosine >= 0.90 and clustered per threshold (union-find). "Excess" = rows that would disappear if every cluster merged to one.\n`);
|
|
55
|
+
lines.push("| Threshold | Clusters | Excess (rows) |");
|
|
56
|
+
lines.push("|---|---|---|");
|
|
57
|
+
for (const t of d.thresholds) {
|
|
58
|
+
lines.push(`| ${t.threshold} | ${t.clusterCount} | ${t.excess} |`);
|
|
59
|
+
}
|
|
60
|
+
lines.push("");
|
|
61
|
+
lines.push(`**Pair attribution** (over all ${d.pairAttribution.totalPairs} pairs at/above the lowest threshold, 0.90): ` +
|
|
62
|
+
`${d.pairAttribution.recoveryReingest} recovery/re-ingest-suspect (${pct(d.pairAttribution.totalPairs > 0 ? d.pairAttribution.recoveryReingest / d.pairAttribution.totalPairs : 0)}), ${d.pairAttribution.organic} organic (${pct(d.pairAttribution.totalPairs > 0 ? d.pairAttribution.organic / d.pairAttribution.totalPairs : 0)}).\n`);
|
|
63
|
+
lines.push(`### Top ${d.topClusters.length} clusters (0.90 threshold, largest first)\n`);
|
|
64
|
+
d.topClusters.forEach((cluster, i) => {
|
|
65
|
+
const mismatch = cluster.metadataMismatch;
|
|
66
|
+
const mismatchFlags = [
|
|
67
|
+
mismatch.projectMismatch ? "project" : null,
|
|
68
|
+
mismatch.privacyMismatch ? "privacy" : null,
|
|
69
|
+
mismatch.sourceAgentMismatch ? "source_agent" : null,
|
|
70
|
+
].filter(Boolean);
|
|
71
|
+
lines.push(`**Cluster ${i + 1}** — size ${cluster.size}, attribution: ${cluster.attribution.recoveryReingest} recovery-pair(s) / ${cluster.attribution.organic} organic-pair(s)` +
|
|
72
|
+
(mismatchFlags.length > 0 ? `, metadata mismatch: ${mismatchFlags.join(", ")}` : ", metadata consistent"));
|
|
73
|
+
for (const m of cluster.members) {
|
|
74
|
+
lines.push(` - \`${m.id.slice(0, 8)}\` ${m.created_at} — ${m.preview.replace(/\n/g, " ")}`);
|
|
75
|
+
}
|
|
76
|
+
lines.push("");
|
|
77
|
+
});
|
|
78
|
+
return lines.join("\n");
|
|
79
|
+
}
|
|
80
|
+
function renderDecay(backlog, prune, structural, adoption) {
|
|
81
|
+
const lines = [];
|
|
82
|
+
lines.push("## D4 — Decay / Prune / No-fit lifecycle + #192 adoption\n");
|
|
83
|
+
lines.push("### Domain classification backlog\n");
|
|
84
|
+
lines.push(`${backlog.nullDomainTotal} memories have NULL domain (of ${backlog.maxRowid} max rowid). Partitioned by ` +
|
|
85
|
+
`\`domainCursor\` (${backlog.domainCursor ?? "unset — classify-domains never run"}), documented in ` +
|
|
86
|
+
`classify-domains.ts as the last fully-committed rowid:\n`);
|
|
87
|
+
lines.push(`- **Never-classified backlog** (rowid > cursor): ${backlog.neverClassifiedBacklog}`);
|
|
88
|
+
lines.push(`- **Classified but no fitting domain** (rowid <= cursor, no-fit path): ${backlog.classifiedButEmpty}\n`);
|
|
89
|
+
lines.push("### Prune dry-run (real production predicate: `stageDecayPrune`, `dryRun: true`)\n");
|
|
90
|
+
lines.push(`Decay clock configured to \`decayHalfLifeDays: ${prune.decayHalfLifeDaysUsed}\` (bedrock has no override — this is the shipped default) before running.\n`);
|
|
91
|
+
lines.push(`- Candidates (would prune today): **${prune.candidates}**`);
|
|
92
|
+
lines.push(`- Pruned: ${prune.pruned} (0 expected — dry run)`);
|
|
93
|
+
lines.push(`- Failed: ${prune.failed}\n`);
|
|
94
|
+
lines.push("### Structural strength — who can EVER prune\n");
|
|
95
|
+
lines.push(`A memory can only ever cross the prune floor (effective strength < 0.01) if its asymptote is below that floor: ` +
|
|
96
|
+
`\`base_strength < sqrt(0.1) ≈ ${round(structural.everPrunableCeiling, 4)}\` (floor = base_strength × importance × 0.1, ` +
|
|
97
|
+
`importance defaults to base_strength — independent of access/link hardening, which only slows the approach).\n`);
|
|
98
|
+
lines.push(`**Ever-prunable: ${structural.everPrunableCount} / ${structural.totalMemories} (${pct(structural.totalMemories > 0 ? structural.everPrunableCount / structural.totalMemories : 0)})** — everyone else has a strength floor that never drops low enough to prune, no matter how long it goes unaccessed.\n`);
|
|
99
|
+
for (const [label, hist] of [
|
|
100
|
+
["Effective strength now", structural.effectiveStrengthNowHistogram],
|
|
101
|
+
["Simulated +180 days (no further access)", structural.effectiveStrengthAt180dHistogram],
|
|
102
|
+
["Simulated +365 days (no further access)", structural.effectiveStrengthAt365dHistogram],
|
|
103
|
+
["Asymptotic floor (never, base_strength² × 0.1)", structural.effectiveStrengthNeverHistogram],
|
|
104
|
+
]) {
|
|
105
|
+
lines.push(`**${label}**\n`);
|
|
106
|
+
lines.push("| Bucket | Count |");
|
|
107
|
+
lines.push("|---|---|");
|
|
108
|
+
lines.push(histTable(hist));
|
|
109
|
+
}
|
|
110
|
+
lines.push("### Adoption (#192 shown_count / access_count)\n");
|
|
111
|
+
lines.push("`shown_count` (migration v8) was deployed ~2 days before this snapshot — near-zero shown counts reflect the " +
|
|
112
|
+
"feature's youth as much as its effectiveness; read this section as a baseline to compare against, not a verdict.\n");
|
|
113
|
+
lines.push(`- Total shown (sum): ${adoption.totalShown}, total access (sum): ${adoption.totalAccess}`);
|
|
114
|
+
lines.push(`- Uses per showing (overall): ${adoption.usesPerShowingOverall !== null ? round(adoption.usesPerShowingOverall) : "n/a (nothing shown yet)"}`);
|
|
115
|
+
lines.push(`- Cold (shown=0 AND access=0): ${adoption.coldShare.coldCount} / ${adoption.coldShare.total} (${pct(adoption.coldShare.share)})\n`);
|
|
116
|
+
lines.push("**shown_count distribution**\n\n| Bucket | Count |\n|---|---|\n" + histTable(adoption.shownCountHistogram));
|
|
117
|
+
lines.push("**access_count distribution**\n\n| Bucket | Count |\n|---|---|\n" + histTable(adoption.accessCountHistogram));
|
|
118
|
+
lines.push("**Uses per showing by source_agent** (agents with at least one showing)\n");
|
|
119
|
+
lines.push("| source_agent | shown | access | ratio |");
|
|
120
|
+
lines.push("|---|---|---|---|");
|
|
121
|
+
const byAgent = Object.entries(adoption.usesPerShowingBySourceAgent)
|
|
122
|
+
.filter(([, v]) => v.shown > 0 || v.access > 0)
|
|
123
|
+
.sort((a, b) => b[1].shown - a[1].shown);
|
|
124
|
+
for (const [agent, v] of byAgent) {
|
|
125
|
+
lines.push(`| ${agent} | ${v.shown} | ${v.access} | ${v.ratio !== null ? round(v.ratio) : "n/a"} |`);
|
|
126
|
+
}
|
|
127
|
+
lines.push("");
|
|
128
|
+
lines.push("**Exposure age profile** — are new memories getting shown?\n");
|
|
129
|
+
lines.push("| Age bucket | Total | Ever shown | Share shown |");
|
|
130
|
+
lines.push("|---|---|---|---|");
|
|
131
|
+
for (const b of adoption.exposureAgeProfile) {
|
|
132
|
+
lines.push(`| ${b.bucket} | ${b.total} | ${b.everShown} | ${pct(b.shareShown)} |`);
|
|
133
|
+
}
|
|
134
|
+
lines.push("");
|
|
135
|
+
return lines.join("\n");
|
|
136
|
+
}
|
|
137
|
+
function renderGraph(g) {
|
|
138
|
+
const lines = [];
|
|
139
|
+
lines.push("## D6 — Link-graph health\n");
|
|
140
|
+
lines.push(`Total links: **${g.totalLinks}**\n`);
|
|
141
|
+
lines.push("| Relationship | Count |");
|
|
142
|
+
lines.push("|---|---|");
|
|
143
|
+
for (const [rel, count] of Object.entries(g.byRelationship)) {
|
|
144
|
+
lines.push(`| ${rel} | ${count} |`);
|
|
145
|
+
}
|
|
146
|
+
lines.push("");
|
|
147
|
+
lines.push(`### Degree distribution (${g.degree.memoriesWithLinks} / ${g.degree.totalMemories} memories have at least one link)\n`);
|
|
148
|
+
lines.push("| Degree bucket | Count |");
|
|
149
|
+
lines.push("|---|---|");
|
|
150
|
+
lines.push(histTable(g.degree.degreeHistogram));
|
|
151
|
+
lines.push(`### Top ${g.degree.topHubs.length} hubs\n`);
|
|
152
|
+
lines.push("| id | degree | project | domain | preview |");
|
|
153
|
+
lines.push("|---|---|---|---|---|");
|
|
154
|
+
for (const h of g.degree.topHubs) {
|
|
155
|
+
lines.push(`| \`${h.id.slice(0, 8)}\` | ${h.degree} | ${h.project ?? "-"} | ${h.domain ?? "-"} | ${h.preview.replace(/\|/g, "/")} |`);
|
|
156
|
+
}
|
|
157
|
+
lines.push("");
|
|
158
|
+
lines.push(`### Cosine drift (stored \`memory_links.strength\` vs recomputed, ${g.drift.sampleSize}-link random sample` +
|
|
159
|
+
(g.drift.skippedMissingEmbedding > 0 ? `, ${g.drift.skippedMissingEmbedding} skipped for missing embeddings` : "") +
|
|
160
|
+
")\n");
|
|
161
|
+
lines.push(`Mean |drift|: ${round(g.drift.meanAbsDrift, 8)}, max |drift|: ${round(g.drift.maxAbsDrift, 8)} ` +
|
|
162
|
+
"(float32 rounding-noise scale (~1e-7) counts as zero drift — strength was set from the same embeddings at link-creation time, so no staleness is expected).\n");
|
|
163
|
+
lines.push("| |drift| bucket | Count |");
|
|
164
|
+
lines.push("|---|---|");
|
|
165
|
+
lines.push(histTable(g.drift.driftHistogram));
|
|
166
|
+
lines.push(`### Dup-noise links (endpoints >= ${g.dupNoise.threshold} cosine, measured over ${g.dupNoise.measured} links with both embeddings)\n`);
|
|
167
|
+
lines.push(`${g.dupNoise.count} / ${g.dupNoise.measured} (${pct(g.dupNoise.share)}) of links connect near-duplicate memories.\n`);
|
|
168
|
+
lines.push(`### Relink migration partition (\`relinkCursor\`: ${g.relinkCursor ?? "unset — relink never run"})\n`);
|
|
169
|
+
lines.push("| Partition | Links | Dup-noise share |");
|
|
170
|
+
lines.push("|---|---|---|");
|
|
171
|
+
lines.push(`| Relinked (source rowid <= cursor) | ${g.relinkedPartition.linkCount} | ${pct(g.relinkedPartition.dupNoiseShare)} |`);
|
|
172
|
+
lines.push(`| Not yet relinked (source rowid > cursor) | ${g.notYetRelinkedPartition.linkCount} | ${pct(g.notYetRelinkedPartition.dupNoiseShare)} |`);
|
|
173
|
+
lines.push("");
|
|
174
|
+
return lines.join("\n");
|
|
175
|
+
}
|
|
176
|
+
function renderReflection(r) {
|
|
177
|
+
const lines = [];
|
|
178
|
+
lines.push("## Reflection census\n");
|
|
179
|
+
lines.push(`Lessons: ${r.totalLessons}, episodes: ${r.totalEpisodes}, yield ratio (lessons/episodes): ${r.yieldRatio !== null ? round(r.yieldRatio) : "n/a"}\n`);
|
|
180
|
+
lines.push("_Not a quality read — actionable-vs-noise sampling is deferred Phase-A grading work._\n");
|
|
181
|
+
if (r.lessonsByDate.length > 0) {
|
|
182
|
+
lines.push("| Date | Lessons |");
|
|
183
|
+
lines.push("|---|---|");
|
|
184
|
+
for (const row of r.lessonsByDate) {
|
|
185
|
+
lines.push(`| ${row.date} | ${row.count} |`);
|
|
186
|
+
}
|
|
187
|
+
lines.push("");
|
|
188
|
+
}
|
|
189
|
+
return lines.join("\n");
|
|
190
|
+
}
|
|
191
|
+
function renderPhaseB(dups, prune, structural, backlog, graph, adoption) {
|
|
192
|
+
const lines = [];
|
|
193
|
+
lines.push("## Phase-B decision inputs\n");
|
|
194
|
+
lines.push("**1. Merge-threshold curve (dup excess at each cosine)**\n");
|
|
195
|
+
for (const t of dups.thresholds) {
|
|
196
|
+
lines.push(`- ${t.threshold}: ${t.clusterCount} clusters, ${t.excess} excess rows`);
|
|
197
|
+
}
|
|
198
|
+
lines.push("");
|
|
199
|
+
lines.push("**2. Prune posture — inert by design or miscalibration?**\n");
|
|
200
|
+
lines.push(`- Dry-run candidates today: ${prune.candidates} (half-life ${prune.decayHalfLifeDaysUsed}d)`);
|
|
201
|
+
lines.push(`- Ever-prunable ceiling: ${structural.everPrunableCount} / ${structural.totalMemories} (${pct(structural.totalMemories > 0 ? structural.everPrunableCount / structural.totalMemories : 0)}) memories can EVER cross the floor regardless of age`);
|
|
202
|
+
lines.push(`- If the current trend holds, compare the now/+180d/+365d histograms above for how many additional memories drift into low-strength bands`);
|
|
203
|
+
lines.push("");
|
|
204
|
+
lines.push("**3. Classification backlog size**\n");
|
|
205
|
+
lines.push(`- ${backlog.neverClassifiedBacklog} never-classified (beyond domainCursor), ${backlog.classifiedButEmpty} classified-but-empty (no-fit), of ${backlog.nullDomainTotal} total NULL-domain`);
|
|
206
|
+
lines.push("");
|
|
207
|
+
lines.push("**4. Link-threshold pile-up evidence**\n");
|
|
208
|
+
lines.push(`- ${graph.dupNoise.count} / ${graph.dupNoise.measured} links (${pct(graph.dupNoise.share)}) connect near-duplicate memories (>= ${graph.dupNoise.threshold} cosine)`);
|
|
209
|
+
lines.push(`- Relinked partition dup-noise share: ${pct(graph.relinkedPartition.dupNoiseShare)} vs not-yet-relinked: ${pct(graph.notYetRelinkedPartition.dupNoiseShare)}`);
|
|
210
|
+
lines.push("");
|
|
211
|
+
lines.push("**5. Adoption snapshot**\n");
|
|
212
|
+
lines.push(`- Cold (never shown, never accessed): ${adoption.coldShare.coldCount} / ${adoption.coldShare.total} (${pct(adoption.coldShare.share)})`);
|
|
213
|
+
lines.push(`- Uses per showing overall: ${adoption.usesPerShowingOverall !== null ? round(adoption.usesPerShowingOverall) : "n/a"} — CAVEAT: shown_count is ~2 days old at snapshot time, read directionally only`);
|
|
214
|
+
lines.push("");
|
|
215
|
+
return lines.join("\n");
|
|
216
|
+
}
|
|
217
|
+
function renderReport(input) {
|
|
218
|
+
const parts = [];
|
|
219
|
+
parts.push("# Hicortex Mechanical Audit Baseline (#191)\n");
|
|
220
|
+
parts.push(`Snapshot: \`${input.dbPath}\` \nGenerated: ${input.generatedAt} \nRead-only, zero LLM, zero human grading — mechanical measurement only.\n`);
|
|
221
|
+
parts.push(renderDups(input.dups));
|
|
222
|
+
parts.push(renderDecay(input.domainBacklog, input.pruneDryRun, input.structural, input.adoption));
|
|
223
|
+
parts.push(renderGraph(input.graph));
|
|
224
|
+
parts.push(renderReflection(input.reflection));
|
|
225
|
+
parts.push(renderPhaseB(input.dups, input.pruneDryRun, input.structural, input.domainBacklog, input.graph, input.adoption));
|
|
226
|
+
return parts.join("\n");
|
|
227
|
+
}
|
|
228
|
+
// ---------------------------------------------------------------------------
|
|
229
|
+
// CLI entry
|
|
230
|
+
// ---------------------------------------------------------------------------
|
|
231
|
+
function main() {
|
|
232
|
+
const [, , dbPathArg, reportPathArg] = process.argv;
|
|
233
|
+
if (!dbPathArg) {
|
|
234
|
+
console.error("Usage: run-eval.js <snapshot.db> [report.md]");
|
|
235
|
+
process.exitCode = 1;
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
const dbPath = dbPathArg;
|
|
239
|
+
const stateDir = (0, node_path_1.dirname)(dbPath);
|
|
240
|
+
const reportPath = reportPathArg ?? (0, node_path_1.join)(stateDir, "eval-report.md");
|
|
241
|
+
const db = (0, eval_db_js_1.openSnapshot)(dbPath);
|
|
242
|
+
try {
|
|
243
|
+
console.log("[eval] D1 duplicate audit...");
|
|
244
|
+
const dups = (0, dups_js_1.runDupAudit)(db);
|
|
245
|
+
console.log("[eval] D4 domain backlog...");
|
|
246
|
+
const domainBacklog = (0, decay_eval_js_1.runDomainBacklogAudit)(db, stateDir);
|
|
247
|
+
console.log("[eval] D4 prune dry-run...");
|
|
248
|
+
const pruneDryRun = (0, decay_eval_js_1.runPruneDryRun)(db, retrieval_js_1.DEFAULT_DECAY_HALF_LIFE_DAYS);
|
|
249
|
+
console.log("[eval] D4 structural strength stats...");
|
|
250
|
+
const structural = (0, decay_eval_js_1.runStructuralStrengthStats)(db);
|
|
251
|
+
console.log("[eval] D4 adoption stats...");
|
|
252
|
+
const adoption = (0, decay_eval_js_1.runAdoptionStats)(db);
|
|
253
|
+
console.log("[eval] D6 link-graph audit...");
|
|
254
|
+
const graph = (0, graph_eval_js_1.runGraphAudit)(db, stateDir);
|
|
255
|
+
console.log("[eval] reflection census...");
|
|
256
|
+
const reflection = (0, reflection_census_js_1.runReflectionCensus)(db);
|
|
257
|
+
const report = renderReport({
|
|
258
|
+
dbPath,
|
|
259
|
+
generatedAt: new Date().toISOString(),
|
|
260
|
+
dups,
|
|
261
|
+
domainBacklog,
|
|
262
|
+
pruneDryRun,
|
|
263
|
+
structural,
|
|
264
|
+
adoption,
|
|
265
|
+
graph,
|
|
266
|
+
reflection,
|
|
267
|
+
});
|
|
268
|
+
(0, node_fs_1.writeFileSync)(reportPath, report, "utf-8");
|
|
269
|
+
console.log(`[eval] report written to ${reportPath}`);
|
|
270
|
+
}
|
|
271
|
+
finally {
|
|
272
|
+
db.close();
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
main();
|
package/dist/init.js
CHANGED
|
@@ -1296,6 +1296,11 @@ async function runInit(options = {}) {
|
|
|
1296
1296
|
console.log(` ✓ Removed old static lessons block from ${claudeMdPath} — lessons now injected at session start`);
|
|
1297
1297
|
}
|
|
1298
1298
|
console.log("\n✓ Hicortex setup complete!\n");
|
|
1299
|
+
// Telemetry disclosure at install time (informed consent, best practice):
|
|
1300
|
+
// opt-out telemetry is only acceptable if the user is TOLD about it.
|
|
1301
|
+
console.log("Anonymous usage telemetry (aggregate counts only, no content) is on by default —");
|
|
1302
|
+
console.log("see exactly what is sent with `hicortex telemetry`; to opt out, add");
|
|
1303
|
+
console.log('"telemetry": false to ~/.hicortex/config.json or set HICORTEX_TELEMETRY=off.\n');
|
|
1299
1304
|
console.log("Next steps:");
|
|
1300
1305
|
console.log(" 1. Restart Claude Code to pick up the new MCP server and SessionStart hook");
|
|
1301
1306
|
if (d.hermesFound) {
|
|
@@ -1461,6 +1466,11 @@ async function runClientInit(serverUrl, agentName) {
|
|
|
1461
1466
|
setupHermes(serverUrl, authToken);
|
|
1462
1467
|
}
|
|
1463
1468
|
console.log("\n✓ Hicortex client setup complete!\n");
|
|
1469
|
+
// Telemetry disclosure at install time (informed consent, best practice):
|
|
1470
|
+
// opt-out telemetry is only acceptable if the user is TOLD about it.
|
|
1471
|
+
console.log("Anonymous usage telemetry (aggregate counts only, no content) is on by default —");
|
|
1472
|
+
console.log("see exactly what is sent with `hicortex telemetry`; to opt out, add");
|
|
1473
|
+
console.log('"telemetry": false to ~/.hicortex/config.json or set HICORTEX_TELEMETRY=off.\n');
|
|
1464
1474
|
console.log("How it works:");
|
|
1465
1475
|
console.log(" • MCP tools (search, context, ingest) talk to the remote server");
|
|
1466
1476
|
console.log(" • Nightly pipeline denoises CC transcripts, POSTs to server for distillation");
|
package/dist/mcp-server.js
CHANGED
|
@@ -69,6 +69,7 @@ const memory_instructions_js_1 = require("./memory-instructions.js");
|
|
|
69
69
|
const recall_index_js_1 = require("./recall-index.js");
|
|
70
70
|
const seed_lesson_js_1 = require("./seed-lesson.js");
|
|
71
71
|
const distiller_js_1 = require("./distiller.js");
|
|
72
|
+
const dedup_js_1 = require("./dedup.js");
|
|
72
73
|
// ---------------------------------------------------------------------------
|
|
73
74
|
// Server state
|
|
74
75
|
// ---------------------------------------------------------------------------
|
|
@@ -790,30 +791,31 @@ async function startServer(options = {}) {
|
|
|
790
791
|
res.status(400).json({ error: "Provide either 'text' (string) or 'messages' (array)" });
|
|
791
792
|
return;
|
|
792
793
|
}
|
|
793
|
-
// Escape LIKE wildcards — Hermes ids contain "_" (e.g. 20260701_045744_...).
|
|
794
|
-
const escapeLike = (s) => s.replace(/[\\%_]/g, (m) => "\\" + m);
|
|
795
794
|
// Segment-exact dedup (#189): an incremental capture POST carries
|
|
796
795
|
// segment_id "<start>-<end>[.pN]". Skip iff THIS exact segment's chunks are
|
|
797
796
|
// already stored (keys "<sid>#<segment_id>#<i>"). This is what lets a failed
|
|
798
797
|
// segment be safely retried with the same id, and a legacy session-level row
|
|
799
798
|
// (key "<sid>#<i>", no segment) does NOT match — so the #189 recovery
|
|
800
799
|
// re-ingest is never blocked by night-1's whole-session rows.
|
|
800
|
+
// ALSO consults dedup_log (#100): a merged-away loser's source_session
|
|
801
|
+
// marker survives there after the `memories` row is deleted, so a
|
|
802
|
+
// `hicortex dedup --apply` merge can't be undone by a retried/recaptured
|
|
803
|
+
// segment silently re-ingesting the same content.
|
|
801
804
|
if (session_id && segment_id) {
|
|
802
|
-
const
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
res.status(200).json({ skipped: true, existing_count: existing.c });
|
|
805
|
+
const existingCount = (0, dedup_js_1.countExistingSegment)(db, session_id, segment_id);
|
|
806
|
+
if (existingCount > 0) {
|
|
807
|
+
res.status(200).json({ skipped: true, existing_count: existingCount });
|
|
806
808
|
return;
|
|
807
809
|
}
|
|
808
810
|
}
|
|
809
811
|
// Session-level dedup: when session_id is present and this is a whole-session
|
|
810
812
|
// POST (no segment_id — legacy ≤0.13.1 clients), skip if any chunk of this
|
|
811
|
-
// session is already stored
|
|
813
|
+
// session is already stored (memories OR dedup_log — see above).
|
|
814
|
+
// Unchanged: legacy clients keep exact behaviour.
|
|
812
815
|
if (session_id && !segment_id) {
|
|
813
|
-
const
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
res.status(200).json({ skipped: true, existing_count: existing.c });
|
|
816
|
+
const existingCount = (0, dedup_js_1.countExistingSession)(db, session_id);
|
|
817
|
+
if (existingCount > 0) {
|
|
818
|
+
res.status(200).json({ skipped: true, existing_count: existingCount });
|
|
817
819
|
return;
|
|
818
820
|
}
|
|
819
821
|
}
|
package/dist/nightly.js
CHANGED
|
@@ -390,6 +390,10 @@ async function runNightly(options = {}) {
|
|
|
390
390
|
domains: cfgDomains,
|
|
391
391
|
contentDomainsReady,
|
|
392
392
|
weakPrimaryFloor: (0, nofit_js_1.resolveWeakPrimaryFloor)(savedConfig),
|
|
393
|
+
}, {
|
|
394
|
+
minSimilarity: savedConfig?.supersessionMinSimilarity,
|
|
395
|
+
maxCalls: savedConfig?.supersessionMaxCalls,
|
|
396
|
+
penalty: savedConfig?.supersessionPenalty,
|
|
393
397
|
});
|
|
394
398
|
console.log(`[hicortex] Consolidation ${report.status} in ${report.elapsed_seconds}s` +
|
|
395
399
|
(report.stages.reflection ? ` (${report.stages.reflection.lessons_generated} lessons)` : ""));
|
|
@@ -419,15 +423,28 @@ async function runNightly(options = {}) {
|
|
|
419
423
|
ocBatches.length > 0 && "oc",
|
|
420
424
|
].filter(Boolean);
|
|
421
425
|
const agentType = kinds.length > 1 ? "mixed" : (kinds[0] ?? "cc");
|
|
426
|
+
// Adoption aggregates (0.15.1): corpus-wide exposure vs use. uses/shown
|
|
427
|
+
// is the recall-quality signal; cold is the never-touched share.
|
|
428
|
+
const adoption = db
|
|
429
|
+
.prepare(`SELECT COALESCE(SUM(shown_count), 0) AS shown,
|
|
430
|
+
COALESCE(SUM(access_count), 0) AS uses,
|
|
431
|
+
SUM(CASE WHEN COALESCE(shown_count, 0) = 0
|
|
432
|
+
AND COALESCE(access_count, 0) = 0 THEN 1 ELSE 0 END) AS cold
|
|
433
|
+
FROM memories`)
|
|
434
|
+
.get();
|
|
422
435
|
await (0, telemetry_js_1.sendTelemetry)({
|
|
423
436
|
id: (0, telemetry_js_1.getTelemetryId)(stateDir),
|
|
424
437
|
v: VERSION,
|
|
438
|
+
pv: telemetry_js_1.TELEMETRY_PAYLOAD_VERSION,
|
|
425
439
|
mode: "server",
|
|
426
440
|
agent: agentType,
|
|
427
441
|
mem: storage.countMemories(db),
|
|
428
442
|
lessons: storage.getLessons(db, 365).length,
|
|
429
443
|
sessions: batches.length,
|
|
430
444
|
ok: !hadTransientFailure,
|
|
445
|
+
shown: adoption.shown,
|
|
446
|
+
uses: adoption.uses,
|
|
447
|
+
cold: adoption.cold,
|
|
431
448
|
});
|
|
432
449
|
}
|
|
433
450
|
}
|
|
@@ -550,12 +567,14 @@ async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recapt
|
|
|
550
567
|
await (0, telemetry_js_1.sendTelemetry)({
|
|
551
568
|
id: (0, telemetry_js_1.getTelemetryId)(stateDir),
|
|
552
569
|
v: VERSION,
|
|
570
|
+
pv: telemetry_js_1.TELEMETRY_PAYLOAD_VERSION,
|
|
553
571
|
mode: "client",
|
|
554
572
|
agent: agentType,
|
|
555
573
|
mem: memoriesIngested,
|
|
556
574
|
lessons: 0, // client doesn't have direct DB access
|
|
557
575
|
sessions: batches.length,
|
|
558
576
|
ok: !hadTransientFailure,
|
|
577
|
+
// No adoption fields: a client install has no local DB to aggregate.
|
|
559
578
|
});
|
|
560
579
|
}
|
|
561
580
|
}
|
package/dist/relink.d.ts
CHANGED
|
@@ -37,7 +37,6 @@
|
|
|
37
37
|
* Server-mode only: relink needs the local DB. Client installs must run it
|
|
38
38
|
* on the server machine.
|
|
39
39
|
*/
|
|
40
|
-
import type Database from "better-sqlite3";
|
|
41
40
|
import type { EmbedFn } from "./retrieval.js";
|
|
42
41
|
export interface RelinkOptions {
|
|
43
42
|
/** Full discovery + would-be counts, zero writes, cursor untouched. */
|
|
@@ -87,8 +86,10 @@ export interface RelinkReport {
|
|
|
87
86
|
/**
|
|
88
87
|
* Read the stored embedding for a memory from memory_vectors.
|
|
89
88
|
* Returns null when the row is missing (caller falls back to re-embedding).
|
|
89
|
+
* @deprecated moved to storage.ts (shared with consolidate.ts's supersession
|
|
90
|
+
* stage); re-exported here so existing importers of relink.ts keep working.
|
|
90
91
|
*/
|
|
91
|
-
export
|
|
92
|
+
export { getStoredEmbedding } from "./storage.js";
|
|
92
93
|
/**
|
|
93
94
|
* Run the relink pass. Returns a structured report.
|
|
94
95
|
* Throws on unrecoverable errors (client mode, DB write failure) — the cursor
|
package/dist/relink.js
CHANGED
|
@@ -72,7 +72,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
72
72
|
};
|
|
73
73
|
})();
|
|
74
74
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
75
|
-
exports.getStoredEmbedding =
|
|
75
|
+
exports.getStoredEmbedding = void 0;
|
|
76
76
|
exports.runRelink = runRelink;
|
|
77
77
|
const paths_js_1 = require("./paths.js");
|
|
78
78
|
const node_fs_1 = require("node:fs");
|
|
@@ -107,16 +107,11 @@ function loadExistingPairs(db) {
|
|
|
107
107
|
/**
|
|
108
108
|
* Read the stored embedding for a memory from memory_vectors.
|
|
109
109
|
* Returns null when the row is missing (caller falls back to re-embedding).
|
|
110
|
+
* @deprecated moved to storage.ts (shared with consolidate.ts's supersession
|
|
111
|
+
* stage); re-exported here so existing importers of relink.ts keep working.
|
|
110
112
|
*/
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
.prepare("SELECT embedding FROM memory_vectors WHERE id = ?")
|
|
114
|
-
.get(memoryId);
|
|
115
|
-
if (!row?.embedding)
|
|
116
|
-
return null;
|
|
117
|
-
const buf = row.embedding;
|
|
118
|
-
return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
119
|
-
}
|
|
113
|
+
var storage_js_1 = require("./storage.js");
|
|
114
|
+
Object.defineProperty(exports, "getStoredEmbedding", { enumerable: true, get: function () { return storage_js_1.getStoredEmbedding; } });
|
|
120
115
|
/**
|
|
121
116
|
* Run the relink pass. Returns a structured report.
|
|
122
117
|
* Throws on unrecoverable errors (client mode, DB write failure) — the cursor
|
|
@@ -193,7 +188,7 @@ async function runRelink(options = {}) {
|
|
|
193
188
|
const { __rowid: _ignored, ...memRow } = row;
|
|
194
189
|
const mem = memRow;
|
|
195
190
|
try {
|
|
196
|
-
let embedding = getStoredEmbedding(db, mem.id);
|
|
191
|
+
let embedding = storage.getStoredEmbedding(db, mem.id);
|
|
197
192
|
if (!embedding) {
|
|
198
193
|
embedding = await (await getEmbedFn())(mem.content);
|
|
199
194
|
}
|
package/dist/seed-lesson.d.ts
CHANGED
|
@@ -3,5 +3,5 @@
|
|
|
3
3
|
* Shared between OC plugin and CC MCP server.
|
|
4
4
|
*/
|
|
5
5
|
import type Database from "better-sqlite3";
|
|
6
|
-
export declare const SEED_LESSON = "## Lesson: 1% Daily Self-Improvement\n\nSelf-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your instruction files (
|
|
6
|
+
export declare const SEED_LESSON = "## Lesson: 1% Daily Self-Improvement\n\nSelf-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your agent instruction files (e.g. SOUL.md / MEMORY.md for Hermes profiles, CLAUDE.md for Claude Code). Never overcorrect \u2014 feedback is guidance, not emergency. One fix per issue. Never rewrite entire systems. Before changing something, ask: is this a real pattern or am I reacting to noise?\n\nThe flywheel: Sessions \u2192 Hicortex captures \u2192 Nightly consolidation \u2192 Lessons generated \u2192 Lessons injected into your context \u2192 You apply one fix \u2192 Better sessions tomorrow.\n\nUse hicortex_lessons to check for new insights. If a lesson has high confidence, apply it. If medium, observe for one more day. If nothing needs fixing today, skip \u2014 that's fine.\n\n**Severity:** critical\n**Confidence:** high\n**Generated:** seed-lesson";
|
|
7
7
|
export declare function injectSeedLesson(database: Database.Database, log?: (msg: string) => void): Promise<void>;
|
package/dist/seed-lesson.js
CHANGED
|
@@ -43,7 +43,7 @@ const embedder_js_1 = require("./embedder.js");
|
|
|
43
43
|
const storage = __importStar(require("./storage.js"));
|
|
44
44
|
exports.SEED_LESSON = `## Lesson: 1% Daily Self-Improvement
|
|
45
45
|
|
|
46
|
-
Self-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your instruction files (
|
|
46
|
+
Self-improvement protocol: Review your sessions daily. Scan for mistakes, gaps, user corrections, and friction. Identify 1-2 issues maximum. Apply one targeted fix to your agent instruction files (e.g. SOUL.md / MEMORY.md for Hermes profiles, CLAUDE.md for Claude Code). Never overcorrect — feedback is guidance, not emergency. One fix per issue. Never rewrite entire systems. Before changing something, ask: is this a real pattern or am I reacting to noise?
|
|
47
47
|
|
|
48
48
|
The flywheel: Sessions → Hicortex captures → Nightly consolidation → Lessons generated → Lessons injected into your context → You apply one fix → Better sessions tomorrow.
|
|
49
49
|
|
package/dist/state.d.ts
CHANGED
|
@@ -50,6 +50,16 @@ export interface HicortexState {
|
|
|
50
50
|
* Same discipline as relinkCursor.
|
|
51
51
|
*/
|
|
52
52
|
domainCursor?: number;
|
|
53
|
+
/**
|
|
54
|
+
* Resume cursor for the nightly's supersession-detection stage (#191 Phase
|
|
55
|
+
* B) — highest memories.rowid whose decision/correction candidates have
|
|
56
|
+
* been evaluated (or infra-skipped) this run. Absent/0 = never run. Unlike
|
|
57
|
+
* relinkCursor/domainCursor (separate resumable CLI commands), this cursor
|
|
58
|
+
* advances a SMALL amount per night (config `supersessionMaxCalls`, default
|
|
59
|
+
* 30) as part of the regular nightly — the corpus is back-processed
|
|
60
|
+
* gradually over many nights.
|
|
61
|
+
*/
|
|
62
|
+
supersessionCursor?: number;
|
|
53
63
|
}
|
|
54
64
|
/**
|
|
55
65
|
* Load the state file. Returns an empty state if the file is missing
|
package/dist/storage.d.ts
CHANGED
|
@@ -89,6 +89,16 @@ export declare function getMemoryTagsWeighted(db: Database.Database, memoryId: s
|
|
|
89
89
|
tag: string;
|
|
90
90
|
weight: number | null;
|
|
91
91
|
}>;
|
|
92
|
+
/**
|
|
93
|
+
* Read the stored embedding for a memory from memory_vectors.
|
|
94
|
+
* Returns null when the row is missing (caller falls back to re-embedding).
|
|
95
|
+
*
|
|
96
|
+
* Shared by `hicortex relink` and the nightly's supersession stage
|
|
97
|
+
* (consolidate.ts) — lives here (not in relink.ts) so consolidate.ts can use
|
|
98
|
+
* it without importing from relink.ts, which itself imports from
|
|
99
|
+
* consolidate.ts (BudgetTracker, discoverLinkCandidates).
|
|
100
|
+
*/
|
|
101
|
+
export declare function getStoredEmbedding(db: Database.Database, memoryId: string): Float32Array | null;
|
|
92
102
|
/**
|
|
93
103
|
* Find similar memories by vector distance. Returns memories with distance field.
|
|
94
104
|
*/
|
package/dist/storage.js
CHANGED
|
@@ -15,6 +15,7 @@ exports.deleteMemory = deleteMemory;
|
|
|
15
15
|
exports.setMemoryTags = setMemoryTags;
|
|
16
16
|
exports.getMemoryTags = getMemoryTags;
|
|
17
17
|
exports.getMemoryTagsWeighted = getMemoryTagsWeighted;
|
|
18
|
+
exports.getStoredEmbedding = getStoredEmbedding;
|
|
18
19
|
exports.vectorSearch = vectorSearch;
|
|
19
20
|
exports.searchFts = searchFts;
|
|
20
21
|
exports.addLink = addLink;
|
|
@@ -115,6 +116,10 @@ const ALLOWED_UPDATE_FIELDS = new Set([
|
|
|
115
116
|
"base_strength",
|
|
116
117
|
"last_accessed",
|
|
117
118
|
"access_count",
|
|
119
|
+
// shown_count is normally maintained by touchMemoriesShown (bulk +1 per
|
|
120
|
+
// recall-index appearance); it's also allowed here so `hicortex dedup`
|
|
121
|
+
// (#100) can sum a merged cluster's counters onto the canonical row.
|
|
122
|
+
"shown_count",
|
|
118
123
|
"source_agent",
|
|
119
124
|
"source_session",
|
|
120
125
|
"project",
|
|
@@ -239,6 +244,24 @@ function getMemoryTagsWeighted(db, memoryId) {
|
|
|
239
244
|
.all(memoryId);
|
|
240
245
|
return rows;
|
|
241
246
|
}
|
|
247
|
+
/**
|
|
248
|
+
* Read the stored embedding for a memory from memory_vectors.
|
|
249
|
+
* Returns null when the row is missing (caller falls back to re-embedding).
|
|
250
|
+
*
|
|
251
|
+
* Shared by `hicortex relink` and the nightly's supersession stage
|
|
252
|
+
* (consolidate.ts) — lives here (not in relink.ts) so consolidate.ts can use
|
|
253
|
+
* it without importing from relink.ts, which itself imports from
|
|
254
|
+
* consolidate.ts (BudgetTracker, discoverLinkCandidates).
|
|
255
|
+
*/
|
|
256
|
+
function getStoredEmbedding(db, memoryId) {
|
|
257
|
+
const row = db
|
|
258
|
+
.prepare("SELECT embedding FROM memory_vectors WHERE id = ?")
|
|
259
|
+
.get(memoryId);
|
|
260
|
+
if (!row?.embedding)
|
|
261
|
+
return null;
|
|
262
|
+
const buf = row.embedding;
|
|
263
|
+
return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
264
|
+
}
|
|
242
265
|
// ---------------------------------------------------------------------------
|
|
243
266
|
// Vector search
|
|
244
267
|
// ---------------------------------------------------------------------------
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `hicortex telemetry` — transparency surface for anonymous usage telemetry.
|
|
3
|
+
*
|
|
4
|
+
* Read-only BY DESIGN (owner decision 30.07.2026). It shows the exact payload
|
|
5
|
+
* and both documented ways to switch telemetry off, but it does not flip the
|
|
6
|
+
* switch itself: the `telemetry` key is deliberately NOT scaffolded into
|
|
7
|
+
* config.json, and opting out is a deliberate edit the operator makes. Turning
|
|
8
|
+
* it off must stay completely possible and completely documented — just not a
|
|
9
|
+
* one-keystroke default-path action.
|
|
10
|
+
*/
|
|
11
|
+
export declare function runTelemetryCommand(args: string[]): void;
|