@gamaze/hicortex 0.23.0 → 0.23.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/assets/dashboard.html +455 -153
  2. package/dist/dashboard.d.ts +18 -7
  3. package/dist/dashboard.js +42 -10
  4. package/dist/dedup.js +2 -2
  5. package/dist/index.js +17 -2
  6. package/dist/learnings-identity.js +20 -1
  7. package/dist/llm.d.ts +12 -1
  8. package/dist/llm.js +14 -3
  9. package/dist/mcp-server.js +6 -2
  10. package/dist/mcp-stdio.d.ts +61 -3
  11. package/dist/mcp-stdio.js +272 -51
  12. package/dist/nightly.js +1 -1
  13. package/hermes-plugin/hicortex/provider.py +23 -0
  14. package/opencode-plugin/hicortex/index.ts +24 -1
  15. package/package.json +2 -1
  16. package/pi-extension/hicortex/index.ts +24 -1
  17. package/server.json +2 -2
  18. package/dist/eval/decay-eval.d.ts +0 -111
  19. package/dist/eval/decay-eval.js +0 -214
  20. package/dist/eval/dups.d.ts +0 -100
  21. package/dist/eval/dups.js +0 -174
  22. package/dist/eval/eval-clock.d.ts +0 -32
  23. package/dist/eval/eval-clock.js +0 -47
  24. package/dist/eval/eval-db.d.ts +0 -25
  25. package/dist/eval/eval-db.js +0 -67
  26. package/dist/eval/graph-eval.d.ts +0 -89
  27. package/dist/eval/graph-eval.js +0 -246
  28. package/dist/eval/importance-eval.d.ts +0 -85
  29. package/dist/eval/importance-eval.js +0 -286
  30. package/dist/eval/planted-eval.d.ts +0 -30
  31. package/dist/eval/planted-eval.js +0 -122
  32. package/dist/eval/planted-fixtures.d.ts +0 -107
  33. package/dist/eval/planted-fixtures.js +0 -283
  34. package/dist/eval/planted-harness.d.ts +0 -183
  35. package/dist/eval/planted-harness.js +0 -651
  36. package/dist/eval/ranking-battery.d.ts +0 -125
  37. package/dist/eval/ranking-battery.js +0 -289
  38. package/dist/eval/ranking-eval.d.ts +0 -61
  39. package/dist/eval/ranking-eval.js +0 -554
  40. package/dist/eval/ranking-fixtures.d.ts +0 -117
  41. package/dist/eval/ranking-fixtures.js +0 -485
  42. package/dist/eval/recall-sweep.d.ts +0 -87
  43. package/dist/eval/recall-sweep.js +0 -1030
  44. package/dist/eval/reflection-census.d.ts +0 -19
  45. package/dist/eval/reflection-census.js +0 -25
  46. package/dist/eval/relevance-eval.d.ts +0 -178
  47. package/dist/eval/relevance-eval.js +0 -2240
  48. package/dist/eval/run-eval.d.ts +0 -20
  49. package/dist/eval/run-eval.js +0 -299
@@ -1,111 +0,0 @@
1
- /**
2
- * D4 — decay / no-fit lifecycle + #192 adoption audit (#191 mechanical
3
- * baseline). Four sections, all read-only against a snapshot connection:
4
- *
5
- * (a) domain backlog — NULL-domain memories split into "never scanned
6
- * yet" vs "scanned, no fitting domain"
7
- * (b) prune dry-run — the REAL production predicate (stageDecayPrune,
8
- * dryRun=true), not a reimplementation
9
- * (c) structural strength — how many memories can EVER cross the prune
10
- * floor, now vs simulated further out
11
- * (d) adoption — #192 shown_count/access_count signal quality
12
- */
13
- import type Database from "better-sqlite3";
14
- export interface DomainBacklogReport {
15
- nullDomainTotal: number;
16
- /** rowid > domainCursor: the classify-domains scan has not reached these rows yet. */
17
- neverClassifiedBacklog: number;
18
- /** rowid <= domainCursor: scanned, but the LLM found no fitting domain (no-fit, below weakPrimaryFloor). */
19
- classifiedButEmpty: number;
20
- domainCursor: number | null;
21
- maxRowid: number;
22
- /** True when domainCursor was mapped 1:1 (classify-domains.ts documents it as "last committed rowid"). */
23
- cursorMappingConfident: boolean;
24
- }
25
- /**
26
- * Partition NULL-domain memories by the `domainCursor` rowid watermark.
27
- * `domainCursor` is documented in classify-domains.ts as "last fully
28
- * committed rowid" — a direct rowid position, not an opaque offset — so the
29
- * split is a plain rowid comparison. If a future version changes that
30
- * contract this function still degrades safely: with no cursor, everything
31
- * NULL is reported as backlog (worst case, never silently wrong).
32
- */
33
- export declare function runDomainBacklogAudit(db: Database.Database, stateDir?: string): DomainBacklogReport;
34
- export interface PruneDryRunReport {
35
- candidates: number;
36
- pruned: number;
37
- failed: number;
38
- decayHalfLifeDaysUsed: number;
39
- }
40
- /**
41
- * Run the actual `stageDecayPrune` (imported from consolidate.ts, `dryRun:
42
- * true`) against the snapshot. Configures the decay clock to the given
43
- * half-life first via the eval seam (#408: the half-life is a calibration
44
- * constant — the caller should pass the shipped default, see run-eval.ts) so
45
- * the eval and production score with the same clock.
46
- */
47
- export declare function runPruneDryRun(db: Database.Database, decayHalfLifeDays?: number): PruneDryRunReport;
48
- /**
49
- * effectiveStrength's asymptotic floor is `base_strength * importance * 0.1`
50
- * with `importance` defaulting to `base_strength` (retrieval.ts). A memory
51
- * can EVER cross the prune floor (0.01, stageDecayPrune) only if that
52
- * asymptote is itself below it: base_strength² * 0.1 < 0.01, i.e.
53
- * base_strength < sqrt(0.1).
54
- */
55
- export declare const EVER_PRUNABLE_BASE_STRENGTH_CEILING: number;
56
- export declare function histogram(values: number[], edges?: number[]): Record<string, number>;
57
- export interface StructuralStrengthStats {
58
- totalMemories: number;
59
- everPrunableCount: number;
60
- everPrunableCeiling: number;
61
- effectiveStrengthNowHistogram: Record<string, number>;
62
- effectiveStrengthAt180dHistogram: Record<string, number>;
63
- effectiveStrengthAt365dHistogram: Record<string, number>;
64
- /** The asymptotic floor per memory (base_strength² × 0.1) — what "at infinity" converges to. */
65
- effectiveStrengthNeverHistogram: Record<string, number>;
66
- }
67
- /**
68
- * Compute effective-strength distributions now, and simulated further out
69
- * (decay continuing with no further access or promotion — the
70
- * "if nothing changes" projection), plus the asymptotic floor per memory.
71
- * Requires `configureDecay` to already reflect the desired half-life (call
72
- * `runPruneDryRun` first, or `configureDecay` directly, in the same process).
73
- * `now` (#458): the instant "now" means — pinned by run-eval's --now for
74
- * wall-clock-independent before/after audits; default the live clock.
75
- */
76
- export declare function runStructuralStrengthStats(db: Database.Database, now?: Date): StructuralStrengthStats;
77
- export interface AdoptionStats {
78
- totalMemories: number;
79
- shownCountHistogram: Record<string, number>;
80
- accessCountHistogram: Record<string, number>;
81
- totalShown: number;
82
- totalAccess: number;
83
- /** sum(access_count) / sum(shown_count) across the corpus — null if nothing has ever been shown. */
84
- usesPerShowingOverall: number | null;
85
- usesPerShowingBySourceAgent: Record<string, {
86
- shown: number;
87
- access: number;
88
- ratio: number | null;
89
- }>;
90
- /** Never shown AND never accessed — completely inert memories. */
91
- coldShare: {
92
- coldCount: number;
93
- total: number;
94
- share: number;
95
- };
96
- /** By memory age bucket: total count, how many have ever been shown, and the share. */
97
- exposureAgeProfile: Array<{
98
- bucket: string;
99
- total: number;
100
- everShown: number;
101
- shareShown: number;
102
- }>;
103
- }
104
- /**
105
- * Adoption snapshot from the #192 exposure/use columns. `shown_count` was
106
- * added by migration v8 on 27.07.2026 — freshly deployed at snapshot time
107
- * (~2 days), so near-zero shown counts across the board reflect the
108
- * feature's youth as much as its effectiveness. Report callers should state
109
- * that caveat alongside the numbers, not just the numbers.
110
- */
111
- export declare function runAdoptionStats(db: Database.Database, now?: Date): AdoptionStats;
@@ -1,214 +0,0 @@
1
- "use strict";
2
- /**
3
- * D4 — decay / no-fit lifecycle + #192 adoption audit (#191 mechanical
4
- * baseline). Four sections, all read-only against a snapshot connection:
5
- *
6
- * (a) domain backlog — NULL-domain memories split into "never scanned
7
- * yet" vs "scanned, no fitting domain"
8
- * (b) prune dry-run — the REAL production predicate (stageDecayPrune,
9
- * dryRun=true), not a reimplementation
10
- * (c) structural strength — how many memories can EVER cross the prune
11
- * floor, now vs simulated further out
12
- * (d) adoption — #192 shown_count/access_count signal quality
13
- */
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- exports.EVER_PRUNABLE_BASE_STRENGTH_CEILING = void 0;
16
- exports.runDomainBacklogAudit = runDomainBacklogAudit;
17
- exports.runPruneDryRun = runPruneDryRun;
18
- exports.histogram = histogram;
19
- exports.runStructuralStrengthStats = runStructuralStrengthStats;
20
- exports.runAdoptionStats = runAdoptionStats;
21
- const retrieval_js_1 = require("../retrieval.js");
22
- const consolidate_js_1 = require("../consolidate.js");
23
- const state_js_1 = require("../state.js");
24
- /**
25
- * Partition NULL-domain memories by the `domainCursor` rowid watermark.
26
- * `domainCursor` is documented in classify-domains.ts as "last fully
27
- * committed rowid" — a direct rowid position, not an opaque offset — so the
28
- * split is a plain rowid comparison. If a future version changes that
29
- * contract this function still degrades safely: with no cursor, everything
30
- * NULL is reported as backlog (worst case, never silently wrong).
31
- */
32
- function runDomainBacklogAudit(db, stateDir) {
33
- const state = (0, state_js_1.loadState)(stateDir);
34
- const domainCursor = state.domainCursor ?? null;
35
- const maxRowid = db.prepare("SELECT MAX(rowid) AS r FROM memories").get().r ?? 0;
36
- const nullDomainTotal = db.prepare("SELECT COUNT(*) AS c FROM memories WHERE domain IS NULL").get().c;
37
- let neverClassifiedBacklog;
38
- if (domainCursor !== null) {
39
- neverClassifiedBacklog = db
40
- .prepare("SELECT COUNT(*) AS c FROM memories WHERE domain IS NULL AND rowid > ?")
41
- .get(domainCursor).c;
42
- }
43
- else {
44
- neverClassifiedBacklog = nullDomainTotal;
45
- }
46
- return {
47
- nullDomainTotal,
48
- neverClassifiedBacklog,
49
- classifiedButEmpty: nullDomainTotal - neverClassifiedBacklog,
50
- domainCursor,
51
- maxRowid,
52
- cursorMappingConfident: true,
53
- };
54
- }
55
- /**
56
- * Run the actual `stageDecayPrune` (imported from consolidate.ts, `dryRun:
57
- * true`) against the snapshot. Configures the decay clock to the given
58
- * half-life first via the eval seam (#408: the half-life is a calibration
59
- * constant — the caller should pass the shipped default, see run-eval.ts) so
60
- * the eval and production score with the same clock.
61
- */
62
- function runPruneDryRun(db, decayHalfLifeDays = retrieval_js_1.DEFAULT_DECAY_HALF_LIFE_DAYS) {
63
- (0, retrieval_js_1.configureDecay)(decayHalfLifeDays);
64
- const result = (0, consolidate_js_1.stageDecayPrune)(db, true);
65
- return { ...result, decayHalfLifeDaysUsed: decayHalfLifeDays };
66
- }
67
- // ---------------------------------------------------------------------------
68
- // (c) Structural strength stats
69
- // ---------------------------------------------------------------------------
70
- /**
71
- * effectiveStrength's asymptotic floor is `base_strength * importance * 0.1`
72
- * with `importance` defaulting to `base_strength` (retrieval.ts). A memory
73
- * can EVER cross the prune floor (0.01, stageDecayPrune) only if that
74
- * asymptote is itself below it: base_strength² * 0.1 < 0.01, i.e.
75
- * base_strength < sqrt(0.1).
76
- */
77
- exports.EVER_PRUNABLE_BASE_STRENGTH_CEILING = Math.sqrt(0.1);
78
- const STRENGTH_BUCKET_EDGES = [0, 0.01, 0.05, 0.1, 0.2, 0.3, 0.5, 0.7, 1.0];
79
- function bucketLabel(value, edges) {
80
- for (let i = 0; i < edges.length - 1; i++) {
81
- if (value >= edges[i] && value < edges[i + 1])
82
- return `[${edges[i]}, ${edges[i + 1]})`;
83
- }
84
- return `[${edges[edges.length - 1]}, +]`;
85
- }
86
- function histogram(values, edges = STRENGTH_BUCKET_EDGES) {
87
- const buckets = {};
88
- for (const v of values) {
89
- const label = bucketLabel(v, edges);
90
- buckets[label] = (buckets[label] ?? 0) + 1;
91
- }
92
- return buckets;
93
- }
94
- /**
95
- * Compute effective-strength distributions now, and simulated further out
96
- * (decay continuing with no further access or promotion — the
97
- * "if nothing changes" projection), plus the asymptotic floor per memory.
98
- * Requires `configureDecay` to already reflect the desired half-life (call
99
- * `runPruneDryRun` first, or `configureDecay` directly, in the same process).
100
- * `now` (#458): the instant "now" means — pinned by run-eval's --now for
101
- * wall-clock-independent before/after audits; default the live clock.
102
- */
103
- function runStructuralStrengthStats(db, now = new Date()) {
104
- const rows = db
105
- .prepare("SELECT id, base_strength, last_accessed FROM memories")
106
- .all();
107
- const nowVals = [];
108
- const at180 = [];
109
- const at365 = [];
110
- const neverVals = [];
111
- let everPrunable = 0;
112
- for (const r of rows) {
113
- const base = r.base_strength ?? 0.5;
114
- if (base < exports.EVER_PRUNABLE_BASE_STRENGTH_CEILING)
115
- everPrunable++;
116
- nowVals.push((0, retrieval_js_1.effectiveStrength)(base, r.last_accessed, now));
117
- at180.push((0, retrieval_js_1.effectiveStrength)(base, r.last_accessed, new Date(now.getTime() + 180 * 86_400_000)));
118
- at365.push((0, retrieval_js_1.effectiveStrength)(base, r.last_accessed, new Date(now.getTime() + 365 * 86_400_000)));
119
- neverVals.push(base * base * 0.1);
120
- }
121
- return {
122
- totalMemories: rows.length,
123
- everPrunableCount: everPrunable,
124
- everPrunableCeiling: exports.EVER_PRUNABLE_BASE_STRENGTH_CEILING,
125
- effectiveStrengthNowHistogram: histogram(nowVals),
126
- effectiveStrengthAt180dHistogram: histogram(at180),
127
- effectiveStrengthAt365dHistogram: histogram(at365),
128
- effectiveStrengthNeverHistogram: histogram(neverVals),
129
- };
130
- }
131
- // ---------------------------------------------------------------------------
132
- // (d) Adoption stats (#192 columns: shown_count, access_count)
133
- // ---------------------------------------------------------------------------
134
- const COUNT_BUCKET_EDGES = [0, 1, 2, 5, 10, 25, 50, 100, 1000];
135
- const AGE_BUCKET_EDGES_DAYS = [0, 7, 30, 90, 180, 365];
136
- function ageBucketLabel(ageDays) {
137
- for (let i = 0; i < AGE_BUCKET_EDGES_DAYS.length - 1; i++) {
138
- if (ageDays >= AGE_BUCKET_EDGES_DAYS[i] && ageDays < AGE_BUCKET_EDGES_DAYS[i + 1]) {
139
- return `${AGE_BUCKET_EDGES_DAYS[i]}-${AGE_BUCKET_EDGES_DAYS[i + 1]}d`;
140
- }
141
- }
142
- return `${AGE_BUCKET_EDGES_DAYS[AGE_BUCKET_EDGES_DAYS.length - 1]}d+`;
143
- }
144
- /**
145
- * Adoption snapshot from the #192 exposure/use columns. `shown_count` was
146
- * added by migration v8 on 27.07.2026 — freshly deployed at snapshot time
147
- * (~2 days), so near-zero shown counts across the board reflect the
148
- * feature's youth as much as its effectiveness. Report callers should state
149
- * that caveat alongside the numbers, not just the numbers.
150
- */
151
- function runAdoptionStats(db, now = new Date()) {
152
- const rows = db
153
- .prepare("SELECT id, shown_count, access_count, source_agent, created_at FROM memories")
154
- .all();
155
- const shownVals = [];
156
- const accessVals = [];
157
- let totalShown = 0;
158
- let totalAccess = 0;
159
- let coldCount = 0;
160
- const byAgent = new Map();
161
- const ageBuckets = new Map();
162
- for (const r of rows) {
163
- const shown = r.shown_count ?? 0;
164
- const access = r.access_count ?? 0;
165
- shownVals.push(shown);
166
- accessVals.push(access);
167
- totalShown += shown;
168
- totalAccess += access;
169
- if (shown === 0 && access === 0)
170
- coldCount++;
171
- const agentAgg = byAgent.get(r.source_agent) ?? { shown: 0, access: 0 };
172
- agentAgg.shown += shown;
173
- agentAgg.access += access;
174
- byAgent.set(r.source_agent, agentAgg);
175
- const ageDays = (now.getTime() - Date.parse(r.created_at)) / 86_400_000;
176
- const label = ageBucketLabel(Number.isFinite(ageDays) ? Math.max(ageDays, 0) : 0);
177
- const bucketAgg = ageBuckets.get(label) ?? { total: 0, everShown: 0 };
178
- bucketAgg.total++;
179
- if (shown > 0)
180
- bucketAgg.everShown++;
181
- ageBuckets.set(label, bucketAgg);
182
- }
183
- const usesPerShowingBySourceAgent = {};
184
- for (const [agent, agg] of byAgent) {
185
- usesPerShowingBySourceAgent[agent] = {
186
- shown: agg.shown,
187
- access: agg.access,
188
- ratio: agg.shown > 0 ? agg.access / agg.shown : null,
189
- };
190
- }
191
- const exposureAgeProfile = AGE_BUCKET_EDGES_DAYS.map((edge, i) => {
192
- const label = i < AGE_BUCKET_EDGES_DAYS.length - 1
193
- ? `${edge}-${AGE_BUCKET_EDGES_DAYS[i + 1]}d`
194
- : `${edge}d+`;
195
- const agg = ageBuckets.get(label) ?? { total: 0, everShown: 0 };
196
- return {
197
- bucket: label,
198
- total: agg.total,
199
- everShown: agg.everShown,
200
- shareShown: agg.total > 0 ? agg.everShown / agg.total : 0,
201
- };
202
- }).filter((b) => b.total > 0);
203
- return {
204
- totalMemories: rows.length,
205
- shownCountHistogram: histogram(shownVals, COUNT_BUCKET_EDGES),
206
- accessCountHistogram: histogram(accessVals, COUNT_BUCKET_EDGES),
207
- totalShown,
208
- totalAccess,
209
- usesPerShowingOverall: totalShown > 0 ? totalAccess / totalShown : null,
210
- usesPerShowingBySourceAgent,
211
- coldShare: { coldCount, total: rows.length, share: rows.length > 0 ? coldCount / rows.length : 0 },
212
- exposureAgeProfile,
213
- };
214
- }
@@ -1,100 +0,0 @@
1
- /**
2
- * D1 — duplicate-rate audit (#191 mechanical baseline).
3
- *
4
- * For each memory, finds its top-10 nearest neighbors by vector distance and
5
- * keeps every pair whose cosine similarity clears the lowest of three report
6
- * thresholds (0.90 / 0.92 / 0.95). Clusters are built per threshold with
7
- * union-find; the headline number at each threshold is "excess" — how many
8
- * rows would disappear if every cluster were merged down to one memory.
9
- *
10
- * Duplicate pairs are additionally attributed to either the #189 recovery
11
- * re-ingest (a retried capture segment produced a second row for content
12
- * already stored) or organic near-duplication (independent sessions that
13
- * happened to cover the same ground), using two signals: a shared base
14
- * `source_session` (the `#<segment>` suffix stripped), or ingestion runs that
15
- * differ while `created_at` is near-identical (a re-ingest preserves the
16
- * original session date but lands in a later ingestion run).
17
- *
18
- * The clustering primitives (union-find, KNN edge building, metadata-mismatch
19
- * check) live in `../cluster.js` — extracted (#100/#191) so `hicortex dedup`
20
- * reuses the exact same math instead of re-implementing it. Re-exported below
21
- * for backward compatibility with existing importers of this module.
22
- */
23
- import type Database from "better-sqlite3";
24
- import { UnionFind, clusterEdges, clusterExcess, clusterMetadataMismatch, type Edge, type ClusterMetadataMismatch } from "../cluster.js";
25
- export { UnionFind, clusterEdges, clusterExcess, clusterMetadataMismatch, type ClusterMetadataMismatch };
26
- /** @deprecated import `Edge` from `../cluster.js` instead. */
27
- export type DupEdge = Edge;
28
- /** Cosine thresholds the report evaluates, low to high. */
29
- export declare const DUP_THRESHOLDS: readonly [0.9, 0.92, 0.95];
30
- export interface DupMemoryRow {
31
- id: string;
32
- content: string;
33
- created_at: string;
34
- ingested_at: string;
35
- source_session: string | null;
36
- project: string | null;
37
- privacy: string;
38
- source_agent: string;
39
- }
40
- /**
41
- * Partition memories into ingestion "runs" by `ingested_at`: sorted
42
- * ascending, a gap greater than `gapHours` starts a new run. Returns a map
43
- * of memory id -> run id (0-based, monotonically increasing).
44
- */
45
- export declare function sessionizeByIngestedAt(rows: Array<{
46
- id: string;
47
- ingested_at: string;
48
- }>, gapHours?: number): Map<string, number>;
49
- /** Strip the `#<segment>` suffix from a `source_session` value. Null-safe. */
50
- export declare function baseSessionId(sourceSession: string | null): string | null;
51
- /** Whether two ISO timestamps fall within `toleranceDays` of each other. */
52
- export declare function nearIdenticalDate(a: string, b: string, toleranceDays?: number): boolean;
53
- export type PairAttribution = "recovery_reingest" | "organic";
54
- /**
55
- * Classify a duplicate pair as a #189 recovery re-ingest or organic overlap.
56
- * Recovery re-ingest when either: both sides share the same base
57
- * `source_session`, or they landed in different ingestion runs but
58
- * `created_at` is near-identical (the re-ingest preserves the original
59
- * session date while landing in a later run).
60
- */
61
- export declare function attributePair(a: DupMemoryRow, b: DupMemoryRow, runOf: Map<string, number>): PairAttribution;
62
- export interface DupThresholdResult {
63
- threshold: number;
64
- clusterCount: number;
65
- excess: number;
66
- /** Cluster sizes, largest first — a quick histogram without dumping content. */
67
- clusterSizes: number[];
68
- }
69
- export interface DupClusterDump {
70
- size: number;
71
- members: Array<{
72
- id: string;
73
- created_at: string;
74
- preview: string;
75
- }>;
76
- attribution: {
77
- recoveryReingest: number;
78
- organic: number;
79
- };
80
- metadataMismatch: ClusterMetadataMismatch;
81
- }
82
- export interface DupReport {
83
- totalMemories: number;
84
- knnK: number;
85
- thresholds: DupThresholdResult[];
86
- /** Attribution over every pair at/above the lowest threshold (broadest view). */
87
- pairAttribution: {
88
- recoveryReingest: number;
89
- organic: number;
90
- totalPairs: number;
91
- };
92
- /** Top clusters at the lowest threshold, largest first, for manual review. */
93
- topClusters: DupClusterDump[];
94
- }
95
- /**
96
- * Run the full D1 duplicate audit against an open (readonly) snapshot
97
- * connection. Pure aside from the DB reads — safe against a readonly
98
- * connection, no writes attempted.
99
- */
100
- export declare function runDupAudit(db: Database.Database, topN?: number): DupReport;
package/dist/eval/dups.js DELETED
@@ -1,174 +0,0 @@
1
- "use strict";
2
- /**
3
- * D1 — duplicate-rate audit (#191 mechanical baseline).
4
- *
5
- * For each memory, finds its top-10 nearest neighbors by vector distance and
6
- * keeps every pair whose cosine similarity clears the lowest of three report
7
- * thresholds (0.90 / 0.92 / 0.95). Clusters are built per threshold with
8
- * union-find; the headline number at each threshold is "excess" — how many
9
- * rows would disappear if every cluster were merged down to one memory.
10
- *
11
- * Duplicate pairs are additionally attributed to either the #189 recovery
12
- * re-ingest (a retried capture segment produced a second row for content
13
- * already stored) or organic near-duplication (independent sessions that
14
- * happened to cover the same ground), using two signals: a shared base
15
- * `source_session` (the `#<segment>` suffix stripped), or ingestion runs that
16
- * differ while `created_at` is near-identical (a re-ingest preserves the
17
- * original session date but lands in a later ingestion run).
18
- *
19
- * The clustering primitives (union-find, KNN edge building, metadata-mismatch
20
- * check) live in `../cluster.js` — extracted (#100/#191) so `hicortex dedup`
21
- * reuses the exact same math instead of re-implementing it. Re-exported below
22
- * for backward compatibility with existing importers of this module.
23
- */
24
- Object.defineProperty(exports, "__esModule", { value: true });
25
- exports.DUP_THRESHOLDS = exports.clusterMetadataMismatch = exports.clusterExcess = exports.clusterEdges = exports.UnionFind = void 0;
26
- exports.sessionizeByIngestedAt = sessionizeByIngestedAt;
27
- exports.baseSessionId = baseSessionId;
28
- exports.nearIdenticalDate = nearIdenticalDate;
29
- exports.attributePair = attributePair;
30
- exports.runDupAudit = runDupAudit;
31
- const cluster_js_1 = require("../cluster.js");
32
- Object.defineProperty(exports, "UnionFind", { enumerable: true, get: function () { return cluster_js_1.UnionFind; } });
33
- Object.defineProperty(exports, "clusterEdges", { enumerable: true, get: function () { return cluster_js_1.clusterEdges; } });
34
- Object.defineProperty(exports, "clusterExcess", { enumerable: true, get: function () { return cluster_js_1.clusterExcess; } });
35
- Object.defineProperty(exports, "clusterMetadataMismatch", { enumerable: true, get: function () { return cluster_js_1.clusterMetadataMismatch; } });
36
- /** Cosine thresholds the report evaluates, low to high. */
37
- exports.DUP_THRESHOLDS = [0.9, 0.92, 0.95];
38
- /** Neighbors requested per memory (excluding the memory itself). */
39
- const KNN_K = 10;
40
- /** Ingestion-run gap: a pause longer than this starts a new "run". */
41
- const DEFAULT_RUN_GAP_HOURS = 1;
42
- /** created_at proximity treated as "near-identical" for cross-run pairs. */
43
- const DEFAULT_SAME_DATE_TOLERANCE_DAYS = 1;
44
- // ---------------------------------------------------------------------------
45
- // Sessionization + attribution (pure — unit tested)
46
- // ---------------------------------------------------------------------------
47
- /**
48
- * Partition memories into ingestion "runs" by `ingested_at`: sorted
49
- * ascending, a gap greater than `gapHours` starts a new run. Returns a map
50
- * of memory id -> run id (0-based, monotonically increasing).
51
- */
52
- function sessionizeByIngestedAt(rows, gapHours = DEFAULT_RUN_GAP_HOURS) {
53
- const sorted = [...rows].sort((a, b) => a.ingested_at.localeCompare(b.ingested_at));
54
- const gapMs = gapHours * 3_600_000;
55
- const runOf = new Map();
56
- let runId = -1;
57
- let prevTime = null;
58
- for (const r of sorted) {
59
- const t = Date.parse(r.ingested_at);
60
- const valid = Number.isFinite(t);
61
- if (prevTime === null || !valid || t - prevTime > gapMs) {
62
- runId++;
63
- }
64
- runOf.set(r.id, runId);
65
- if (valid)
66
- prevTime = t;
67
- }
68
- return runOf;
69
- }
70
- /** Strip the `#<segment>` suffix from a `source_session` value. Null-safe. */
71
- function baseSessionId(sourceSession) {
72
- if (!sourceSession)
73
- return null;
74
- const idx = sourceSession.indexOf("#");
75
- return idx === -1 ? sourceSession : sourceSession.slice(0, idx);
76
- }
77
- /** Whether two ISO timestamps fall within `toleranceDays` of each other. */
78
- function nearIdenticalDate(a, b, toleranceDays = DEFAULT_SAME_DATE_TOLERANCE_DAYS) {
79
- const ta = Date.parse(a);
80
- const tb = Date.parse(b);
81
- if (!Number.isFinite(ta) || !Number.isFinite(tb))
82
- return false;
83
- return Math.abs(ta - tb) / 86_400_000 <= toleranceDays;
84
- }
85
- /**
86
- * Classify a duplicate pair as a #189 recovery re-ingest or organic overlap.
87
- * Recovery re-ingest when either: both sides share the same base
88
- * `source_session`, or they landed in different ingestion runs but
89
- * `created_at` is near-identical (the re-ingest preserves the original
90
- * session date while landing in a later run).
91
- */
92
- function attributePair(a, b, runOf) {
93
- const baseA = baseSessionId(a.source_session);
94
- const baseB = baseSessionId(b.source_session);
95
- if (baseA !== null && baseA === baseB)
96
- return "recovery_reingest";
97
- const runA = runOf.get(a.id);
98
- const runB = runOf.get(b.id);
99
- if (runA !== undefined &&
100
- runB !== undefined &&
101
- runA !== runB &&
102
- nearIdenticalDate(a.created_at, b.created_at)) {
103
- return "recovery_reingest";
104
- }
105
- return "organic";
106
- }
107
- /**
108
- * Run the full D1 duplicate audit against an open (readonly) snapshot
109
- * connection. Pure aside from the DB reads — safe against a readonly
110
- * connection, no writes attempted.
111
- */
112
- function runDupAudit(db, topN = 15) {
113
- const rows = db
114
- .prepare(`SELECT id, content, created_at, ingested_at, source_session, project, privacy, source_agent
115
- FROM memories`)
116
- .all();
117
- const byId = new Map(rows.map((r) => [r.id, r]));
118
- const runOf = sessionizeByIngestedAt(rows.map((r) => ({ id: r.id, ingested_at: r.ingested_at })));
119
- const lowestThreshold = Math.min(...exports.DUP_THRESHOLDS);
120
- const edges = (0, cluster_js_1.buildKnnEdges)(db, { k: KNN_K, minCosine: lowestThreshold });
121
- const thresholds = exports.DUP_THRESHOLDS.map((threshold) => {
122
- const clusters = (0, cluster_js_1.clusterEdges)(edges, threshold);
123
- return {
124
- threshold,
125
- clusterCount: clusters.length,
126
- excess: (0, cluster_js_1.clusterExcess)(clusters),
127
- clusterSizes: clusters.map((c) => c.length).sort((a, b) => b - a),
128
- };
129
- });
130
- let recoveryReingest = 0;
131
- let organic = 0;
132
- for (const e of edges) {
133
- const a = byId.get(e.a);
134
- const b = byId.get(e.b);
135
- if (!a || !b)
136
- continue;
137
- if (attributePair(a, b, runOf) === "recovery_reingest")
138
- recoveryReingest++;
139
- else
140
- organic++;
141
- }
142
- const broadestClusters = (0, cluster_js_1.clusterEdges)(edges, lowestThreshold).sort((a, b) => b.length - a.length);
143
- const topClusters = broadestClusters.slice(0, topN).map((memberIds) => {
144
- const members = memberIds.map((id) => byId.get(id)).filter((m) => !!m);
145
- let recovery = 0;
146
- let organicCount = 0;
147
- for (let i = 0; i < members.length; i++) {
148
- for (let j = i + 1; j < members.length; j++) {
149
- if (attributePair(members[i], members[j], runOf) === "recovery_reingest")
150
- recovery++;
151
- else
152
- organicCount++;
153
- }
154
- }
155
- const sortedMembers = [...members].sort((a, b) => a.created_at.localeCompare(b.created_at));
156
- return {
157
- size: members.length,
158
- members: sortedMembers.map((m) => ({
159
- id: m.id,
160
- created_at: m.created_at,
161
- preview: m.content.slice(0, 80),
162
- })),
163
- attribution: { recoveryReingest: recovery, organic: organicCount },
164
- metadataMismatch: (0, cluster_js_1.clusterMetadataMismatch)(members),
165
- };
166
- });
167
- return {
168
- totalMemories: rows.length,
169
- knnK: KNN_K,
170
- thresholds,
171
- pairAttribution: { recoveryReingest, organic, totalPairs: edges.length },
172
- topClusters,
173
- };
174
- }
@@ -1,32 +0,0 @@
1
- /**
2
- * #458 — the shared pinned-clock helper for eval harnesses.
3
- *
4
- * The recall evals score against wall-clock time (recency + decay terms in
5
- * computeScore/effectiveStrength), so a pre-photo and a post-run hours apart
6
- * drift even on identical code — PR D evidenced mode-OFF units moving with no
7
- * changed code in the path. The fix is a harness-visible pin: every eval that
8
- * calls retrieve() (or the decay/adoption stats) accepts a `--now <ISO>` flag
9
- * and threads the SAME instant into the retrieve() `now` seam (retrieval.ts —
10
- * the injectable-clock idiom of run-deadline.ts), making before/after runs
11
- * wall-clock-independent. Default stays the live clock — pinning is opt-in,
12
- * for comparability.
13
- *
14
- * Fail-explicit by design (the repo convention): an invalid pin THROWS with a
15
- * clear message rather than silently falling back to the live clock — a run
16
- * that believed it was pinned but wasn't would silently reintroduce the drift
17
- * class this helper exists to remove.
18
- */
19
- /**
20
- * Parse a `--now` flag value into the pinned instant.
21
- *
22
- * @param raw the raw flag value. undefined or empty/whitespace → null (the
23
- * live clock — the flag was not passed). Anything else must parse as a
24
- * Date; garbage or an invalid instant (e.g. month 13) throws.
25
- * @returns the pinned Date, or null for the live clock.
26
- */
27
- export declare function parsePinnedNow(raw: string | undefined): Date | null;
28
- /**
29
- * Human-readable clock mode for report headers / photo JSON: "pinned <ISO>"
30
- * or "live" — so artifacts are self-describing about which clock produced them.
31
- */
32
- export declare function clockLabel(now: Date | null): string;
@@ -1,47 +0,0 @@
1
- "use strict";
2
- /**
3
- * #458 — the shared pinned-clock helper for eval harnesses.
4
- *
5
- * The recall evals score against wall-clock time (recency + decay terms in
6
- * computeScore/effectiveStrength), so a pre-photo and a post-run hours apart
7
- * drift even on identical code — PR D evidenced mode-OFF units moving with no
8
- * changed code in the path. The fix is a harness-visible pin: every eval that
9
- * calls retrieve() (or the decay/adoption stats) accepts a `--now <ISO>` flag
10
- * and threads the SAME instant into the retrieve() `now` seam (retrieval.ts —
11
- * the injectable-clock idiom of run-deadline.ts), making before/after runs
12
- * wall-clock-independent. Default stays the live clock — pinning is opt-in,
13
- * for comparability.
14
- *
15
- * Fail-explicit by design (the repo convention): an invalid pin THROWS with a
16
- * clear message rather than silently falling back to the live clock — a run
17
- * that believed it was pinned but wasn't would silently reintroduce the drift
18
- * class this helper exists to remove.
19
- */
20
- Object.defineProperty(exports, "__esModule", { value: true });
21
- exports.parsePinnedNow = parsePinnedNow;
22
- exports.clockLabel = clockLabel;
23
- /**
24
- * Parse a `--now` flag value into the pinned instant.
25
- *
26
- * @param raw the raw flag value. undefined or empty/whitespace → null (the
27
- * live clock — the flag was not passed). Anything else must parse as a
28
- * Date; garbage or an invalid instant (e.g. month 13) throws.
29
- * @returns the pinned Date, or null for the live clock.
30
- */
31
- function parsePinnedNow(raw) {
32
- if (raw === undefined || raw.trim() === "")
33
- return null;
34
- const parsed = new Date(raw);
35
- if (Number.isNaN(parsed.getTime())) {
36
- throw new Error(`eval clock: invalid --now value "${raw}" — pass a full ISO 8601 instant ` +
37
- `(e.g. 2026-09-17T09:00:00.000Z) or omit the flag to run on the live clock`);
38
- }
39
- return parsed;
40
- }
41
- /**
42
- * Human-readable clock mode for report headers / photo JSON: "pinned <ISO>"
43
- * or "live" — so artifacts are self-describing about which clock produced them.
44
- */
45
- function clockLabel(now) {
46
- return now === null ? "live" : `pinned ${now.toISOString()}`;
47
- }