@gamaze/hicortex 0.22.1 → 0.22.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/dashboard.html +201 -74
- package/dist/calibration.d.ts +45 -0
- package/dist/calibration.js +52 -2
- package/dist/classify-domains.js +6 -1
- package/dist/consolidate.js +73 -14
- package/dist/dashboard.d.ts +11 -0
- package/dist/dashboard.js +9 -0
- package/dist/db.js +74 -0
- package/dist/mcp-server.js +35 -8
- package/dist/nightly.js +14 -0
- package/dist/recall-index.d.ts +46 -3
- package/dist/recall-index.js +83 -26
- package/dist/recall-precision.d.ts +212 -0
- package/dist/recall-precision.js +381 -0
- package/dist/types.d.ts +7 -0
- package/package.json +1 -1
- package/server.json +3 -3
package/dist/consolidate.js
CHANGED
|
@@ -467,8 +467,13 @@ async function scoreMemoriesImportance(db, memories, llm, opts = {}) {
|
|
|
467
467
|
}
|
|
468
468
|
return { scored, failed, skipped_budget: skippedBudget };
|
|
469
469
|
}
|
|
470
|
-
async function stageImportance(db, memories, llm, budget, dryRun, deadline
|
|
471
|
-
|
|
470
|
+
async function stageImportance(db, memories, llm, budget, dryRun, deadline,
|
|
471
|
+
/** #478: pool candidates the paid-gain guard dropped at the stage
|
|
472
|
+
* boundary — reported, not scored, so the run's evidence shows the
|
|
473
|
+
* promotion/enrichment gains that survived the nightly. */
|
|
474
|
+
guardSkipped = 0) {
|
|
475
|
+
const r = await scoreMemoriesImportance(db, memories, llm, { budget, deadline, dryRun });
|
|
476
|
+
return { ...r, guard_skipped: guardSkipped };
|
|
472
477
|
}
|
|
473
478
|
// ---------------------------------------------------------------------------
|
|
474
479
|
// Stage 2.5: Reflection
|
|
@@ -639,12 +644,18 @@ async function stageContentDomains(db, domains, llm, budget, embedFn, dryRun, st
|
|
|
639
644
|
// - domain NOT IN the current vocabulary (a rename/removal re-files), OR
|
|
640
645
|
// - no memory_tags rows yet (single-domain memories from feat/content-domains
|
|
641
646
|
// that have a primary but no tag set — backfill them to multi-tag).
|
|
647
|
+
// #477: absorbed dedup losers are excluded — the merge clears their tags
|
|
648
|
+
// and nulls domain before absorbing, so without the predicate every dead
|
|
649
|
+
// loser re-enters this scope nightly (one classify call + a halving on
|
|
650
|
+
// evidence no recall can ever see). Conjoined OUTSIDE the parenthesized OR
|
|
651
|
+
// group so SQL precedence cannot let an OR arm absorb it.
|
|
642
652
|
const placeholders = domains.map(() => "?").join(", ");
|
|
643
653
|
const rows = db
|
|
644
654
|
.prepare(`SELECT id, content, project FROM memories
|
|
645
|
-
WHERE
|
|
646
|
-
|
|
647
|
-
|
|
655
|
+
WHERE COALESCE(status, '') != 'absorbed'
|
|
656
|
+
AND (domain IS NULL
|
|
657
|
+
OR domain NOT IN (${placeholders})
|
|
658
|
+
OR id NOT IN (SELECT DISTINCT memory_id FROM memory_tags))`)
|
|
648
659
|
.all(...domains.map((d) => d.name));
|
|
649
660
|
if (dryRun) {
|
|
650
661
|
return { curated: false, domains: domains.length, classified: 0, reason: `dry_run (${rows.length} would classify)` };
|
|
@@ -1664,12 +1675,29 @@ deadline) {
|
|
|
1664
1675
|
};
|
|
1665
1676
|
// Stage 1: Pre-check
|
|
1666
1677
|
const precheck = stagePrecheck(db, stateDir);
|
|
1667
|
-
//
|
|
1678
|
+
// #478: the importance pool is ROW-AGE bound — young (ingested within
|
|
1679
|
+
// IMPORTANCE_SETTLE_WINDOW_DAYS) ∪ never-scored. The lastConsolidated
|
|
1680
|
+
// watermark no longer defines any part of it: a deferred run's stuck
|
|
1681
|
+
// watermark used to re-settle a growing cohort nightly, and every
|
|
1682
|
+
// re-settle overwrites base_strength outright (erasing gains
|
|
1683
|
+
// stagePromotion had already paid — 5/5 observed erasures). The watermark
|
|
1684
|
+
// cohort (precheck.newMemories) STILL feeds reflection + links below —
|
|
1685
|
+
// their work is batch retry by design; only importance's per-row settling
|
|
1686
|
+
// was mis-bound to the batch marker.
|
|
1687
|
+
const settleCutoff = new Date(Date.now() - CALIBRATION.IMPORTANCE_SETTLE_WINDOW_DAYS * 86_400_000).toISOString();
|
|
1688
|
+
// getMemoriesSince keys on ingested_at only — absorbed rows (invisible to
|
|
1689
|
+
// recall) are filtered here in the same vocabulary getUnscoredMemories
|
|
1690
|
+
// uses in SQL: no LLM call on dead evidence.
|
|
1691
|
+
const young = storage
|
|
1692
|
+
.getMemoriesSince(db, settleCutoff)
|
|
1693
|
+
.filter((m) => m.status !== "absorbed");
|
|
1694
|
+
const youngIds = new Set(young.map((m) => m.id));
|
|
1695
|
+
// Also check for unscored memories (#425 watermark pool) — first settle is
|
|
1696
|
+
// age-independent, so a row used before its first score is never stranded.
|
|
1668
1697
|
const unscored = storage.getUnscoredMemories(db);
|
|
1669
|
-
const newIds = new Set(precheck.newMemories.map((m) => m.id));
|
|
1670
1698
|
const scoreMemories = [
|
|
1671
|
-
...
|
|
1672
|
-
...unscored.filter((m) => !
|
|
1699
|
+
...young,
|
|
1700
|
+
...unscored.filter((m) => !youngIds.has(m.id)),
|
|
1673
1701
|
];
|
|
1674
1702
|
// #194 no-fit scope: untagged rows (domain IS NULL) stay in the
|
|
1675
1703
|
// re-evaluation scope — the decay/re-attempt contract ("re-halves once per
|
|
@@ -1678,15 +1706,26 @@ deadline) {
|
|
|
1678
1706
|
// never caught because stagePrecheck used to read the AMBIENT (always
|
|
1679
1707
|
// empty in the suite) state instead of the run's own watermark — threading
|
|
1680
1708
|
// stateDir (#357) exposed the divergence between test and production.
|
|
1681
|
-
|
|
1682
|
-
|
|
1709
|
+
// #477: absorbed dedup losers (tags cleared, domain NULLed by the merge)
|
|
1710
|
+
// are dead evidence — they must not keep the no-fit scope (and the run)
|
|
1711
|
+
// alive every night. Same predicate spelling as getUnscoredMemories.
|
|
1712
|
+
const nofitInScope = db
|
|
1713
|
+
.prepare("SELECT COUNT(*) AS n FROM memories WHERE domain IS NULL AND COALESCE(status, '') != 'absorbed'")
|
|
1714
|
+
.get().n;
|
|
1715
|
+
// #478: the quiet-night gate is the UNION — watermark cohort OR young OR
|
|
1716
|
+
// unscored OR no-fit scope. A stuck watermark alone must never silence
|
|
1717
|
+
// reflection/links (their pool is the watermark cohort, and their retry
|
|
1718
|
+
// semantics are the reason a deferred run holds the watermark at all).
|
|
1719
|
+
const skip = scoreMemories.length === 0 &&
|
|
1720
|
+
precheck.newMemories.length === 0 &&
|
|
1721
|
+
nofitInScope === 0;
|
|
1683
1722
|
report.stages.precheck = {
|
|
1684
1723
|
skip,
|
|
1685
1724
|
reason: skip
|
|
1686
1725
|
? precheck.reason
|
|
1687
|
-
: `${precheck.newMemories.length} new + ${scoreMemories.length -
|
|
1726
|
+
: `${precheck.newMemories.length} new (watermark) + ${young.length} young + ${scoreMemories.length - young.length} unscored memories`,
|
|
1688
1727
|
new_memory_count: precheck.newMemories.length,
|
|
1689
|
-
unscored_count:
|
|
1728
|
+
unscored_count: unscored.length,
|
|
1690
1729
|
};
|
|
1691
1730
|
// Strength promotion (#448) — runs in the pre-skip deterministic zone, in
|
|
1692
1731
|
// the same placement discipline as memory_cap BELOW it: accesses happen on
|
|
@@ -1724,7 +1763,27 @@ deadline) {
|
|
|
1724
1763
|
// deferred. Deferred stages drain next run (cursors hold below them).
|
|
1725
1764
|
// Stage 2: Importance Scoring
|
|
1726
1765
|
if (!deadline?.hit("importance")) {
|
|
1727
|
-
|
|
1766
|
+
// #478 paid-gain guard — evaluated HERE, at the stage boundary, not at
|
|
1767
|
+
// pool-build time: the pool above was built before this run's
|
|
1768
|
+
// stagePromotion payment, so a row whose FIRST use lands this run
|
|
1769
|
+
// passes a pool-build check and its just-paid gain is overwritten at
|
|
1770
|
+
// the first re-settle. This read sits after promotion by construction
|
|
1771
|
+
// and sees the advanced baseline. Already-scored rows carrying a paid
|
|
1772
|
+
// gain (promotion baseline advanced OR owner corroboration) keep it —
|
|
1773
|
+
// re-settling would stomp base_strength; `hicortex rescore-importance`
|
|
1774
|
+
// remains the wholesale operator re-judge. Never-scored rows
|
|
1775
|
+
// (importance_scored_at IS NULL) are NEVER skipped: first settle
|
|
1776
|
+
// always happens, whatever their use history.
|
|
1777
|
+
const paidGainIds = scoreMemories.length
|
|
1778
|
+
? new Set(db.prepare(`SELECT id FROM memories
|
|
1779
|
+
WHERE importance_scored_at IS NOT NULL
|
|
1780
|
+
AND (COALESCE(promotion_last_count, 0) > 0
|
|
1781
|
+
OR COALESCE(corroboration_count, 0) > 0)
|
|
1782
|
+
AND id IN (${scoreMemories.map(() => "?").join(", ")})`).all(...scoreMemories.map((m) => m.id))
|
|
1783
|
+
.map((r) => r.id))
|
|
1784
|
+
: new Set();
|
|
1785
|
+
const settleCandidates = scoreMemories.filter((m) => !paidGainIds.has(m.id));
|
|
1786
|
+
report.stages.importance = await stageImportance(db, settleCandidates, llm, budget, dryRun, deadline, scoreMemories.length - settleCandidates.length);
|
|
1728
1787
|
}
|
|
1729
1788
|
// Stage 2.5: Reflection
|
|
1730
1789
|
if (deadline?.hit("reflection")) {
|
package/dist/dashboard.d.ts
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
import type express from "express";
|
|
24
24
|
import type Database from "better-sqlite3";
|
|
25
25
|
import { type CaptureHealthRow } from "./capture-health.js";
|
|
26
|
+
import { type MemoryPrecision } from "./recall-precision.js";
|
|
26
27
|
import { type Stage } from "./stages.js";
|
|
27
28
|
/** Corpus-shape snapshot. `adoption` is null in backfilled rows (point-in-time,
|
|
28
29
|
* can't be reconstructed from created_at). */
|
|
@@ -257,6 +258,16 @@ export interface DashboardData {
|
|
|
257
258
|
last_outcome: string;
|
|
258
259
|
}>;
|
|
259
260
|
};
|
|
261
|
+
/**
|
|
262
|
+
* #476 — the Memory Precision card: Level 1 (pushed-index precision
|
|
263
|
+
* proxies over the recall_pushes/recall_events window) beside Level 2
|
|
264
|
+
* (recall depth = the uses-per-showing ratio over the SAME window, from
|
|
265
|
+
* snapshot deltas) + the divergence list. ALWAYS present; edges echoed
|
|
266
|
+
* like every threshold (the page never hardcodes), and the block measures
|
|
267
|
+
* ≤ 2 KB against the live 30-day payload. The page degrades to the
|
|
268
|
+
* pre-#476 recall-card rendering when the block is absent.
|
|
269
|
+
*/
|
|
270
|
+
memory_precision: MemoryPrecision;
|
|
260
271
|
digest: {
|
|
261
272
|
date: string | null;
|
|
262
273
|
run_at: string | null;
|
package/dist/dashboard.js
CHANGED
|
@@ -45,6 +45,7 @@ const recall_index_js_1 = require("./recall-index.js");
|
|
|
45
45
|
const config_read_js_1 = require("./config-read.js");
|
|
46
46
|
const consolidate_js_1 = require("./consolidate.js");
|
|
47
47
|
const capture_health_js_1 = require("./capture-health.js");
|
|
48
|
+
const recall_precision_js_1 = require("./recall-precision.js");
|
|
48
49
|
const capture_pause_js_1 = require("./capture-pause.js");
|
|
49
50
|
const state_js_1 = require("./state.js");
|
|
50
51
|
const retrieval_js_1 = require("./retrieval.js");
|
|
@@ -576,6 +577,14 @@ function handleDashboardData(db, query, config) {
|
|
|
576
577
|
by_source_machine: live.by_source_machine,
|
|
577
578
|
},
|
|
578
579
|
capture_health: { ...(0, capture_health_js_1.readCaptureHealth)(db), ...(0, capture_health_js_1.readCaptureHealthWindow)(db) },
|
|
580
|
+
// #476 Memory Precision: the window aggregation over the precision
|
|
581
|
+
// event tables + snapshot deltas. Level 2's "latest" is the SAME live
|
|
582
|
+
// adoption this handler already computed (one definition of corpus
|
|
583
|
+
// shape — computeDashboardMetrics); readMemoryPrecision owns the rest.
|
|
584
|
+
memory_precision: (0, recall_precision_js_1.readMemoryPrecision)(db, rangeParam, {
|
|
585
|
+
shown_sum: live.adoption?.shown_sum ?? 0,
|
|
586
|
+
used_sum: live.adoption?.used_sum ?? 0,
|
|
587
|
+
}),
|
|
579
588
|
// #423 phase 3: pauses + presence in one block — one source of truth
|
|
580
589
|
// for the rail's dots, toggles and the capture card's PAUSED badges.
|
|
581
590
|
fleet: {
|
package/dist/db.js
CHANGED
|
@@ -668,6 +668,80 @@ const MIGRATIONS = [
|
|
|
668
668
|
db.exec("UPDATE memories SET promotion_last_count = access_count WHERE promotion_last_count IS NULL");
|
|
669
669
|
},
|
|
670
670
|
},
|
|
671
|
+
{
|
|
672
|
+
version: 21,
|
|
673
|
+
name: "memory_layout",
|
|
674
|
+
up: (db) => {
|
|
675
|
+
// #464 — the console field's server-computed placement. One row per
|
|
676
|
+
// placed memory at one epoch; the CURRENT epoch is MAX(epoch) and the
|
|
677
|
+
// payload (dashboard.ts handleDashboardField) LEFT JOINs on it. A full
|
|
678
|
+
// re-projection (`hicortex layout --reproject`) writes a NEW epoch and
|
|
679
|
+
// drops older ones in the SAME transaction (atomic swap — a mid-write
|
|
680
|
+
// failure leaves the previous epoch fully intact). Epochs exist so the
|
|
681
|
+
// whole store moves at once (one announced visual shift), never as a
|
|
682
|
+
// mix of projections. Idempotent: IF NOT EXISTS (the v18 pattern —
|
|
683
|
+
// plain CREATE, no ALTER).
|
|
684
|
+
// Kept after the #464/#475 field revert (#474): nothing reads or writes
|
|
685
|
+
// this table now — a dormant orphan. Rc databases already applied v21
|
|
686
|
+
// (user_version 21), so removing it would fork schema versions; it
|
|
687
|
+
// stays per the append-only migration discipline.
|
|
688
|
+
db.exec(`
|
|
689
|
+
CREATE TABLE IF NOT EXISTS memory_layout (
|
|
690
|
+
memory_id TEXT PRIMARY KEY,
|
|
691
|
+
x REAL NOT NULL,
|
|
692
|
+
y REAL NOT NULL,
|
|
693
|
+
epoch INTEGER NOT NULL
|
|
694
|
+
)
|
|
695
|
+
`);
|
|
696
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_memory_layout_epoch ON memory_layout(epoch)");
|
|
697
|
+
},
|
|
698
|
+
},
|
|
699
|
+
{
|
|
700
|
+
version: 22,
|
|
701
|
+
name: "recall_precision_events",
|
|
702
|
+
up: (db) => {
|
|
703
|
+
// #476 — the Memory Precision card's event store (Option A, owner
|
|
704
|
+
// decision 2026-09-19: per-push event rows incl. a ≤256-char prompt
|
|
705
|
+
// excerpt — the exact-window Level-1 measures + the judge follow-up's
|
|
706
|
+
// sample frame). Two sidecar tables, the v16 distill_activity pattern:
|
|
707
|
+
// OPERATIONS telemetry, no memories FK — recall_events.memory_id is
|
|
708
|
+
// DATA (absorbed memories' history stays queryable; rows outliving
|
|
709
|
+
// their memory are valid). recall_pushes: one row per NON-skipped
|
|
710
|
+
// /recall-index call (short-prompt skips and resets record nothing; a
|
|
711
|
+
// silent turn records its push row with zero events — the silence rate
|
|
712
|
+
// is computable). recall_events: one kind='shown' row per pushed line
|
|
713
|
+
// (similarity = cosine vs the PURE prompt embedding, redundancy = max
|
|
714
|
+
// cosine vs the standing-context basis — both RAW, verdicts computed at
|
|
715
|
+
// render so recalibration never rewrites history) + one kind='fetch'
|
|
716
|
+
// row per handleMemoryGet (push_id NULL). Pruned nightly at the
|
|
717
|
+
// retention horizon (= the longest window, the capture-health
|
|
718
|
+
// single-constant law). Idempotent: IF NOT EXISTS everywhere.
|
|
719
|
+
db.exec(`
|
|
720
|
+
CREATE TABLE IF NOT EXISTS recall_pushes (
|
|
721
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
722
|
+
ts TEXT NOT NULL,
|
|
723
|
+
day TEXT NOT NULL,
|
|
724
|
+
session_id TEXT,
|
|
725
|
+
prompt_excerpt TEXT
|
|
726
|
+
)
|
|
727
|
+
`);
|
|
728
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_recall_pushes_day ON recall_pushes(day)");
|
|
729
|
+
db.exec(`
|
|
730
|
+
CREATE TABLE IF NOT EXISTS recall_events (
|
|
731
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
732
|
+
ts TEXT NOT NULL,
|
|
733
|
+
day TEXT NOT NULL,
|
|
734
|
+
push_id INTEGER,
|
|
735
|
+
memory_id TEXT NOT NULL,
|
|
736
|
+
similarity REAL,
|
|
737
|
+
redundancy REAL,
|
|
738
|
+
kind TEXT NOT NULL
|
|
739
|
+
)
|
|
740
|
+
`);
|
|
741
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_recall_events_day ON recall_events(day)");
|
|
742
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_recall_events_memory_kind ON recall_events(memory_id, kind)");
|
|
743
|
+
},
|
|
744
|
+
},
|
|
671
745
|
];
|
|
672
746
|
/**
|
|
673
747
|
* Run all pending migrations against the database.
|
package/dist/mcp-server.js
CHANGED
|
@@ -90,6 +90,7 @@ const dedup_js_1 = require("./dedup.js");
|
|
|
90
90
|
const reconsolidation_js_1 = require("./reconsolidation.js");
|
|
91
91
|
const redact_js_1 = require("./redact.js");
|
|
92
92
|
const capture_health_js_1 = require("./capture-health.js");
|
|
93
|
+
const recall_precision_js_1 = require("./recall-precision.js");
|
|
93
94
|
const capture_pause_js_1 = require("./capture-pause.js");
|
|
94
95
|
const init_js_1 = require("./init.js");
|
|
95
96
|
// ---------------------------------------------------------------------------
|
|
@@ -207,8 +208,10 @@ function createMcpServer() {
|
|
|
207
208
|
// (incl. the #204 FETCHED marker) is built in ONE place shared with the
|
|
208
209
|
// REST GET /memory path. CC reaches Hicortex through THIS MCP tool;
|
|
209
210
|
// before #207's fix it got a marker-less citation built inline here.
|
|
211
|
+
// #476: the fetch recorder forwards through the same funnel — exactly
|
|
212
|
+
// one precision event per fetch, whichever path served it.
|
|
210
213
|
try {
|
|
211
|
-
const r = (0, recall_index_js_1.formatMemoryGetText)(db, { id });
|
|
214
|
+
const r = (0, recall_index_js_1.formatMemoryGetText)(db, { id }, { recordFetch: recall_precision_js_1.recordRecallFetch });
|
|
212
215
|
return { content: [{ type: "text", text: r.text }], isError: r.status !== 200 };
|
|
213
216
|
}
|
|
214
217
|
catch (err) {
|
|
@@ -1053,6 +1056,22 @@ async function startServer(options = {}) {
|
|
|
1053
1056
|
// last_accessed, NOT access_count (that stays reserved for hicortex_get /
|
|
1054
1057
|
// GET /memory — real use). {reset: true} clears the session's dedup state
|
|
1055
1058
|
// (SessionStart/compaction).
|
|
1059
|
+
//
|
|
1060
|
+
// #476 Memory Precision: the route also wires the precision seams — the
|
|
1061
|
+
// factory's exposed embedPrompt (the per-request memo: the recorder
|
|
1062
|
+
// measures per-line similarity with ZERO extra embeds), the 24h-TTL
|
|
1063
|
+
// standing-context basis (lessons in /learnings order + identity only when
|
|
1064
|
+
// every known client is served — the redundancy reference), and the
|
|
1065
|
+
// fail-soft recorder. handleRecallIndex owns the fail-soft law: a
|
|
1066
|
+
// recording failure never touches the response.
|
|
1067
|
+
const standingContextBasis = (0, recall_precision_js_1.createStandingContextBasis)({
|
|
1068
|
+
// Lazy getters: identityClients is resolved at boot and stateDir is
|
|
1069
|
+
// assigned before the routes serve, but both land AFTER this provider is
|
|
1070
|
+
// created — read per cache rebuild (a config restart applies at the next
|
|
1071
|
+
// TTL), never captured once.
|
|
1072
|
+
clients: () => identityClients,
|
|
1073
|
+
identityDir: () => (0, node_path_1.join)(stateDir, "identity"),
|
|
1074
|
+
});
|
|
1056
1075
|
app.post("/recall-index", async (req, res) => {
|
|
1057
1076
|
if (!db) {
|
|
1058
1077
|
res.status(503).json({ error: "Server not initialized" });
|
|
@@ -1067,22 +1086,30 @@ async function startServer(options = {}) {
|
|
|
1067
1086
|
// that searches unblended and touches no centroid state. Extracted so the
|
|
1068
1087
|
// exact behavior is unit-testable without HTTP (blendQueryVector
|
|
1069
1088
|
// precedent); this adapter stays thin.
|
|
1089
|
+
const factory = (0, recall_index_js_1.createRecallRetrieveFn)({
|
|
1090
|
+
db,
|
|
1091
|
+
registry: recallRegistry,
|
|
1092
|
+
embedFn: embedder_js_1.embed,
|
|
1093
|
+
});
|
|
1070
1094
|
const r = await (0, recall_index_js_1.handleRecallIndex)({
|
|
1071
1095
|
db,
|
|
1072
1096
|
registry: recallRegistry,
|
|
1073
|
-
retrieveFn:
|
|
1074
|
-
db,
|
|
1075
|
-
registry: recallRegistry,
|
|
1076
|
-
embedFn: embedder_js_1.embed,
|
|
1077
|
-
}),
|
|
1097
|
+
retrieveFn: factory.retrieveFn,
|
|
1078
1098
|
options: recallIndexOptions,
|
|
1099
|
+
precision: {
|
|
1100
|
+
promptEmbed: factory.embedPrompt,
|
|
1101
|
+
basis: standingContextBasis,
|
|
1102
|
+
recorder: recall_precision_js_1.recordRecallPush,
|
|
1103
|
+
},
|
|
1079
1104
|
}, req.body);
|
|
1080
1105
|
res.status(r.status).json(r.body);
|
|
1081
1106
|
});
|
|
1082
1107
|
// REST /memory?id= — fetch one memory's full content (lazy-load counterpart
|
|
1083
1108
|
// of /recall-index for REST clients: Hermes/OC plugins). Marks it as used.
|
|
1084
1109
|
// Prefix ids resolve. 0.16.x: the `privacy` query param is accepted but
|
|
1085
|
-
// ignored (column is vestigial, never filtered). Logic in handleMemoryGet
|
|
1110
|
+
// ignored (column is vestigial, never filtered). Logic in handleMemoryGet;
|
|
1111
|
+
// the #476 fetch recorder rides the same funnel (exactly one event per
|
|
1112
|
+
// fetch, shared with the MCP hicortex_get path).
|
|
1086
1113
|
app.get("/memory", (req, res) => {
|
|
1087
1114
|
if (!db) {
|
|
1088
1115
|
res.status(503).json({ error: "Server not initialized" });
|
|
@@ -1090,7 +1117,7 @@ async function startServer(options = {}) {
|
|
|
1090
1117
|
}
|
|
1091
1118
|
warnDeprecatedPrivacyParamIfPresent(req.query, "memory");
|
|
1092
1119
|
try {
|
|
1093
|
-
const r = (0, recall_index_js_1.handleMemoryGet)(db, { id: req.query.id });
|
|
1120
|
+
const r = (0, recall_index_js_1.handleMemoryGet)(db, { id: req.query.id }, { recordFetch: recall_precision_js_1.recordRecallFetch });
|
|
1094
1121
|
res.status(r.status).json(r.body);
|
|
1095
1122
|
}
|
|
1096
1123
|
catch (err) {
|
package/dist/nightly.js
CHANGED
|
@@ -79,6 +79,7 @@ const capture_cursors_js_1 = require("./capture-cursors.js");
|
|
|
79
79
|
const capture_js_1 = require("./capture.js");
|
|
80
80
|
const run_deadline_js_1 = require("./run-deadline.js");
|
|
81
81
|
const dashboard_js_1 = require("./dashboard.js");
|
|
82
|
+
const recall_precision_js_1 = require("./recall-precision.js");
|
|
82
83
|
const telemetry_js_1 = require("./telemetry.js");
|
|
83
84
|
const init_js_1 = require("./init.js");
|
|
84
85
|
const backup_js_1 = require("./backup.js");
|
|
@@ -1071,6 +1072,19 @@ async function runNightly(options = {}) {
|
|
|
1071
1072
|
console.warn(`[hicortex] Dashboard snapshot write failed: ` +
|
|
1072
1073
|
`${snapErr instanceof Error ? snapErr.message : String(snapErr)}`);
|
|
1073
1074
|
}
|
|
1075
|
+
// #476 — recall-precision retention prune (zero-LLM, full nightly only):
|
|
1076
|
+
// both event tables drop rows outside the rolling window whose length IS
|
|
1077
|
+
// the retention constant (the capture-health single-constant law — the
|
|
1078
|
+
// Memory Precision card can never claim a window the store no longer
|
|
1079
|
+
// holds rows for). Own try/catch like the snapshot writer: telemetry
|
|
1080
|
+
// housekeeping must never fail the run.
|
|
1081
|
+
try {
|
|
1082
|
+
(0, recall_precision_js_1.pruneRecallPrecision)(db);
|
|
1083
|
+
}
|
|
1084
|
+
catch (pruneErr) {
|
|
1085
|
+
console.warn(`[hicortex] Recall-precision retention prune failed: ` +
|
|
1086
|
+
`${pruneErr instanceof Error ? pruneErr.message : String(pruneErr)}`);
|
|
1087
|
+
}
|
|
1074
1088
|
}
|
|
1075
1089
|
// Anonymous telemetry (fire-and-forget, full nightly only).
|
|
1076
1090
|
// Capture-only runs are excluded to avoid inflating install pings.
|
package/dist/recall-index.d.ts
CHANGED
|
@@ -51,6 +51,7 @@ import type Database from "better-sqlite3";
|
|
|
51
51
|
import type { MemorySearchResult } from "./types.js";
|
|
52
52
|
import * as storage from "./storage.js";
|
|
53
53
|
import { SessionRecallRegistry } from "./recall-registry.js";
|
|
54
|
+
import type { RecallPushEntry, RecallFetchRecorder } from "./recall-precision.js";
|
|
54
55
|
export interface RecallIndexOptions {
|
|
55
56
|
/** Minimum measured cosine for vector-only candidates (release-managed
|
|
56
57
|
* since #408 — calibration.ts RECALL_MIN_SIMILARITY; this field is the
|
|
@@ -161,6 +162,16 @@ export interface RecallFilters {
|
|
|
161
162
|
* RecallIndexDeps.retrieveFn). Named so the production factory
|
|
162
163
|
* (createRecallRetrieveFn) and test doubles share one type. */
|
|
163
164
|
export type RecallRetrieveFn = (query: string, limit: number, filters: RecallFilters | undefined, sessionId: string, purePrompt?: boolean) => Promise<MemorySearchResult[]>;
|
|
165
|
+
/** What createRecallRetrieveFn returns (#476): the search closure PLUS the
|
|
166
|
+
* per-request prompt-embed memo it already maintained — exposed so the
|
|
167
|
+
* precision recorder reuses the SAME embedding (zero extra embeds) instead
|
|
168
|
+
* of re-embedding the prompt to measure per-line similarity. */
|
|
169
|
+
export interface RecallRetrieveFactory {
|
|
170
|
+
retrieveFn: RecallRetrieveFn;
|
|
171
|
+
/** The single-entry embed memo (keyed on the query text; the factory is
|
|
172
|
+
* built per request, so the memo never outlives it). */
|
|
173
|
+
embedPrompt: (query: string) => Promise<Float32Array>;
|
|
174
|
+
}
|
|
164
175
|
export interface RecallIndexDeps {
|
|
165
176
|
db: Database.Database;
|
|
166
177
|
registry: SessionRecallRegistry;
|
|
@@ -176,6 +187,30 @@ export interface RecallIndexDeps {
|
|
|
176
187
|
* no breakage. */
|
|
177
188
|
retrieveFn: RecallRetrieveFn;
|
|
178
189
|
options?: RecallIndexOptions;
|
|
190
|
+
/** #476 Memory Precision seams — OPTIONAL so existing callers (tests,
|
|
191
|
+
* library use) keep their exact no-recording behavior. Absent → no event
|
|
192
|
+
* rows, the plain exposure write runs as before. mcp-server wires the
|
|
193
|
+
* production set (the real recorder, the request-memoized prompt embed,
|
|
194
|
+
* the 24h-TTL standing-context basis). Recording is FAIL-SOFT: any error
|
|
195
|
+
* is caught here and the recall response (200 + block) is never affected;
|
|
196
|
+
* on failure the exposure touch re-runs standalone so shown_count never
|
|
197
|
+
* depends on telemetry. */
|
|
198
|
+
precision?: RecallPrecisionDeps;
|
|
199
|
+
}
|
|
200
|
+
/** The #476 precision-recording seams (see RecallIndexDeps.precision). All
|
|
201
|
+
* three are injectable; tests force failures through them. */
|
|
202
|
+
export interface RecallPrecisionDeps {
|
|
203
|
+
/** The request's memoized pure-prompt embed (createRecallRetrieveFn's
|
|
204
|
+
* embedPrompt) — the recorder computes per-line cosines with ZERO extra
|
|
205
|
+
* embeds (the perf law). */
|
|
206
|
+
promptEmbed: (prompt: string) => Promise<Float32Array>;
|
|
207
|
+
/** Standing-context basis vectors (recall-precision.ts's 24h-TTL cached
|
|
208
|
+
* provider; empty array → redundancy NULL, unmeasured). */
|
|
209
|
+
basis: (db: Database.Database) => Promise<Float32Array[]>;
|
|
210
|
+
/** The recorder (recall-precision.ts recordRecallPush). Throws on failure —
|
|
211
|
+
* this handler catches (fail-soft) and falls back to the plain exposure
|
|
212
|
+
* write. */
|
|
213
|
+
recorder: (db: Database.Database, entry: RecallPushEntry, basis?: Float32Array[]) => void;
|
|
179
214
|
}
|
|
180
215
|
/**
|
|
181
216
|
* The PRODUCTION /recall-index retrieveFn (what mcp-server wires into
|
|
@@ -209,7 +244,7 @@ export declare function createRecallRetrieveFn(deps: {
|
|
|
209
244
|
embedFn: (text: string) => Promise<Float32Array>;
|
|
210
245
|
/** FTS resolution override (tests). Defaults to storage.searchFts. */
|
|
211
246
|
ftsFn?: typeof storage.searchFts;
|
|
212
|
-
}):
|
|
247
|
+
}): RecallRetrieveFactory;
|
|
213
248
|
/** Normalize a request-supplied string-list param: array of strings or a CSV
|
|
214
249
|
* string → string[] | undefined. Anything else (or an empty result) means
|
|
215
250
|
* "absent" — never a partial guess. Used by `mission_domains` (#203) so it
|
|
@@ -220,6 +255,14 @@ export declare function parseStringListParam(v: unknown): string[] | undefined;
|
|
|
220
255
|
* all behavior lives here so tests exercise it directly.
|
|
221
256
|
*/
|
|
222
257
|
export declare function handleRecallIndex(deps: RecallIndexDeps, body: unknown): Promise<RecallIndexResult>;
|
|
258
|
+
/** Optional #476 deps for handleMemoryGet: the fetch-event recorder. Absent
|
|
259
|
+
* (old callers, tests) → no recording, byte-identical behavior. mcp-server
|
|
260
|
+
* wires recordRecallFetch so BOTH fetch paths (REST GET /memory, MCP
|
|
261
|
+
* hicortex_get via formatMemoryGetText) record exactly once — this is the
|
|
262
|
+
* ONE funnel. Fail-soft: a recorder error is caught here, never surfaced. */
|
|
263
|
+
export interface MemoryGetDeps {
|
|
264
|
+
recordFetch: RecallFetchRecorder;
|
|
265
|
+
}
|
|
223
266
|
/**
|
|
224
267
|
* Handle a GET /memory request (lazy-load counterpart of the recall index for
|
|
225
268
|
* REST clients). Thin Express adapter in mcp-server.ts; behavior lives here so
|
|
@@ -235,7 +278,7 @@ export declare function handleRecallIndex(deps: RecallIndexDeps, body: unknown):
|
|
|
235
278
|
*/
|
|
236
279
|
export declare function handleMemoryGet(db: Database.Database, query: {
|
|
237
280
|
id?: unknown;
|
|
238
|
-
}): RecallIndexResult;
|
|
281
|
+
}, deps?: MemoryGetDeps): RecallIndexResult;
|
|
239
282
|
/**
|
|
240
283
|
* MCP `hicortex_get` presentation: handleMemoryGet's result framed as the
|
|
241
284
|
* text block the MCP tool returns (provenance header + the SHARED citation +
|
|
@@ -249,7 +292,7 @@ export declare function handleMemoryGet(db: Database.Database, query: {
|
|
|
249
292
|
*/
|
|
250
293
|
export declare function formatMemoryGetText(db: Database.Database, query: {
|
|
251
294
|
id?: unknown;
|
|
252
|
-
}): {
|
|
295
|
+
}, deps?: MemoryGetDeps): {
|
|
253
296
|
status: number;
|
|
254
297
|
text: string;
|
|
255
298
|
};
|
package/dist/recall-index.js
CHANGED
|
@@ -251,27 +251,34 @@ function createRecallRetrieveFn(deps) {
|
|
|
251
251
|
}
|
|
252
252
|
return ftsMemo.rows;
|
|
253
253
|
};
|
|
254
|
-
return
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
254
|
+
return {
|
|
255
|
+
retrieveFn: async (query, limit, filters, sessionId, purePrompt) => {
|
|
256
|
+
const { weight, alpha } = (0, retrieval_js_1.getSessionIntent)();
|
|
257
|
+
const promptEmb = await embedOnce(query);
|
|
258
|
+
const queryVec = (0, retrieval_js_1.recallQueryVector)(deps.registry, sessionId, promptEmb, {
|
|
259
|
+
weight,
|
|
260
|
+
alpha,
|
|
261
|
+
purePrompt,
|
|
262
|
+
});
|
|
263
|
+
return (0, retrieval_js_1.retrieve)(deps.db, deps.embedFn, query, {
|
|
264
|
+
limit,
|
|
265
|
+
noStrengthen: true,
|
|
266
|
+
// #203: project + mission_domains are SOFT affinity (zero-boost
|
|
267
|
+
// neutral), threaded into computeScore.
|
|
268
|
+
project: filters?.project,
|
|
269
|
+
missionDomains: filters?.mission_domains,
|
|
270
|
+
queryEmbedding: queryVec,
|
|
271
|
+
// #329: shared per-request FTS list. The recall path never passes
|
|
272
|
+
// sourceAgent, so the memo is keyed on (query, fetchLimit) only —
|
|
273
|
+
// exactly the two things retrieve() would pass to searchFts.
|
|
274
|
+
ftsCandidates: (fetchLimit) => ftsOnce(query, fetchLimit),
|
|
275
|
+
});
|
|
276
|
+
},
|
|
277
|
+
// #476: the memoized prompt embed, EXPOSED so the precision recorder
|
|
278
|
+
// computes per-line cosines with ZERO additional embeds (the perf law —
|
|
279
|
+
// the memo semantics are unchanged: one embed per distinct query string
|
|
280
|
+
// per factory instance).
|
|
281
|
+
embedPrompt: embedOnce,
|
|
275
282
|
};
|
|
276
283
|
}
|
|
277
284
|
/** Normalize a request-supplied string-list param: array of strings or a CSV
|
|
@@ -421,12 +428,48 @@ async function handleRecallIndex(deps, body) {
|
|
|
421
428
|
picked = [...picked, ...backfill];
|
|
422
429
|
}
|
|
423
430
|
if (picked.length === 0) {
|
|
431
|
+
// #476: a silent turn (searches ran, no line passed the gates) still
|
|
432
|
+
// records its PUSH row — the silence rate is computable later and the
|
|
433
|
+
// judge's population stays complete. Zero event rows, no exposure touch.
|
|
434
|
+
// Fail-soft like every recording site.
|
|
435
|
+
if (deps.precision) {
|
|
436
|
+
try {
|
|
437
|
+
deps.precision.recorder(deps.db, { sessionId, prompt, ids: [] });
|
|
438
|
+
}
|
|
439
|
+
catch (err) {
|
|
440
|
+
console.warn(`[hicortex] recall-precision push recording failed (fail-soft): ` +
|
|
441
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
424
444
|
return { status: 200, body: { block: null, shown: [], turn } };
|
|
425
445
|
}
|
|
426
446
|
const ids = picked.map((r) => r.id);
|
|
427
447
|
deps.registry.markShown(sessionId, ids);
|
|
428
|
-
|
|
429
|
-
|
|
448
|
+
const nowIso = new Date().toISOString();
|
|
449
|
+
// #476 Memory Precision: record the push + per-line events IN THE SAME
|
|
450
|
+
// TRANSACTION as the exposure write (the recorder owns the unit — it calls
|
|
451
|
+
// touchMemoriesShown inside its own transaction). FAIL-SOFT: on any error
|
|
452
|
+
// (or when the seams are absent — library callers, test doubles) the plain
|
|
453
|
+
// exposure write runs instead, so shown_count NEVER depends on telemetry.
|
|
454
|
+
let exposureWritten = false;
|
|
455
|
+
if (deps.precision) {
|
|
456
|
+
try {
|
|
457
|
+
const [promptEmb, basis] = await Promise.all([
|
|
458
|
+
deps.precision.promptEmbed(prompt),
|
|
459
|
+
deps.precision.basis(deps.db),
|
|
460
|
+
]);
|
|
461
|
+
deps.precision.recorder(deps.db, { ts: nowIso, sessionId, prompt, ids, promptEmbedding: promptEmb }, basis);
|
|
462
|
+
exposureWritten = true;
|
|
463
|
+
}
|
|
464
|
+
catch (err) {
|
|
465
|
+
console.warn(`[hicortex] recall-precision recording failed (fail-soft): ` +
|
|
466
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
if (!exposureWritten) {
|
|
470
|
+
// Exposure signal: shown_count + last_accessed refresh, NOT access_count.
|
|
471
|
+
storage.touchMemoriesShown(deps.db, ids, nowIso);
|
|
472
|
+
}
|
|
430
473
|
const lines = picked.map((r) => formatIndexLine(r, titleChars));
|
|
431
474
|
const block = [
|
|
432
475
|
"## Memory recall (auto)",
|
|
@@ -456,7 +499,7 @@ async function handleRecallIndex(deps, body) {
|
|
|
456
499
|
* never filtered. Callers may still send a `privacy` field (backward compat)
|
|
457
500
|
* but it is ignored.
|
|
458
501
|
*/
|
|
459
|
-
function handleMemoryGet(db, query) {
|
|
502
|
+
function handleMemoryGet(db, query, deps) {
|
|
460
503
|
const id = typeof query.id === "string" ? query.id : "";
|
|
461
504
|
if (!id)
|
|
462
505
|
return { status: 400, body: { error: "Missing 'id'" } };
|
|
@@ -471,6 +514,18 @@ function handleMemoryGet(db, query) {
|
|
|
471
514
|
if (!mem)
|
|
472
515
|
return notFound;
|
|
473
516
|
storage.strengthenMemory(db, fullId, new Date().toISOString());
|
|
517
|
+
// #476: one kind='fetch' precision event per successful fetch (the use
|
|
518
|
+
// signal for Level 2 / divergence). Fail-soft — the fetch result is never
|
|
519
|
+
// affected by a recording failure.
|
|
520
|
+
if (deps?.recordFetch) {
|
|
521
|
+
try {
|
|
522
|
+
deps.recordFetch(db, fullId, new Date().toISOString());
|
|
523
|
+
}
|
|
524
|
+
catch (err) {
|
|
525
|
+
console.warn(`[hicortex] recall-precision fetch recording failed (fail-soft): ` +
|
|
526
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
474
529
|
// `citation` is server-rendered so every plugin surfaces the same built-in
|
|
475
530
|
// provenance norm (owner directive 27.07) — see #193.
|
|
476
531
|
const date = (mem.created_at ?? "").slice(0, 10);
|
|
@@ -498,8 +553,10 @@ function handleMemoryGet(db, query) {
|
|
|
498
553
|
* built inline, while the REST path used handleMemoryGet — same contract, two
|
|
499
554
|
* implementations, one updated).
|
|
500
555
|
*/
|
|
501
|
-
function formatMemoryGetText(db, query) {
|
|
502
|
-
|
|
556
|
+
function formatMemoryGetText(db, query, deps) {
|
|
557
|
+
// deps (the #476 fetch recorder) forwards so the MCP path records exactly
|
|
558
|
+
// like the REST path — one funnel, one event per fetch.
|
|
559
|
+
const r = handleMemoryGet(db, query, deps);
|
|
503
560
|
if (r.status !== 200) {
|
|
504
561
|
return { status: r.status, text: String(r.body.error ?? `No memory with id ${query.id ?? ""}`) };
|
|
505
562
|
}
|