@gamaze/hicortex 0.18.1 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/dashboard.html +8 -3
- package/assets/viz.html +9 -3
- package/dist/consolidate.js +7 -7
- package/dist/dashboard.js +3 -3
- package/dist/db.js +24 -1
- package/dist/distiller.d.ts +9 -7
- package/dist/distiller.js +37 -19
- package/dist/eval/recall-sweep.js +2 -2
- package/dist/eval/reflection-census.js +3 -3
- package/dist/hosted-boot.d.ts +61 -0
- package/dist/hosted-boot.js +72 -0
- package/dist/index.js +13 -13
- package/dist/init.d.ts +15 -0
- package/dist/init.js +99 -8
- package/dist/learnings-identity.js +4 -4
- package/dist/localhost-bypass.d.ts +27 -0
- package/dist/localhost-bypass.js +71 -0
- package/dist/mcp-server.js +115 -19
- package/dist/prompts.js +12 -12
- package/dist/recall-index.js +7 -2
- package/dist/retrieval.js +2 -1
- package/dist/seed-lesson.js +1 -1
- package/dist/status.d.ts +8 -0
- package/dist/status.js +15 -1
- package/dist/storage.js +4 -4
- package/dist/token-budget.d.ts +34 -0
- package/dist/token-budget.js +131 -0
- package/dist/type-classify.d.ts +29 -26
- package/dist/type-classify.js +52 -45
- package/dist/type-labels.d.ts +48 -17
- package/dist/type-labels.js +89 -18
- package/dist/types.d.ts +10 -1
- package/dist/viz.d.ts +9 -1
- package/dist/viz.js +11 -2
- package/hermes-plugin/hicortex/client.py +1 -1
- package/hermes-plugin/hicortex/provider.py +13 -13
- package/package.json +1 -1
package/dist/recall-index.js
CHANGED
|
@@ -255,10 +255,15 @@ function handleMemoryGet(db, query) {
|
|
|
255
255
|
// `citation` is server-rendered so every plugin surfaces the same built-in
|
|
256
256
|
// provenance norm (owner directive 27.07) — see #193.
|
|
257
257
|
const date = (mem.created_at ?? "").slice(0, 10);
|
|
258
|
+
// Shallow-copy and apply the human-term label to memory_type so the REST
|
|
259
|
+
// response surfaces the user-facing vocabulary, not the raw DB enum. The
|
|
260
|
+
// underlying DB row (`mem`) is NOT mutated — the DB IS the raw-enum source
|
|
261
|
+
// of truth; the label is a presentation concern applied at the boundary.
|
|
262
|
+
const memory = { ...mem, memory_type: (0, type_labels_js_1.labelForType)(mem.memory_type) };
|
|
258
263
|
return {
|
|
259
264
|
status: 200,
|
|
260
265
|
body: {
|
|
261
|
-
memory
|
|
266
|
+
memory,
|
|
262
267
|
citation: `(memory ${String(mem.id).slice(0, 8)}, ${date}, from ${mem.source_agent ?? "unknown"}, FETCHED)`,
|
|
263
268
|
},
|
|
264
269
|
};
|
|
@@ -284,7 +289,7 @@ function formatMemoryGetText(db, query) {
|
|
|
284
289
|
const date = (mem.created_at ?? "").slice(0, 10);
|
|
285
290
|
// #264 WS2: render the human-term label (Knowledge/Experience/...), not the
|
|
286
291
|
// internal enum, in the citation header shown to the agent/user.
|
|
287
|
-
const header = `[memory ${mem.id} | ${(0, type_labels_js_1.labelForType)(mem.memory_type ?? "
|
|
292
|
+
const header = `[memory ${mem.id} | ${(0, type_labels_js_1.labelForType)(mem.memory_type ?? "experience")} | ${mem.project ?? "-"} | from ${mem.source_agent ?? "unknown"} | ${date}]\n` +
|
|
288
293
|
`Cite as ${citation} where this shapes your answer; it may be stale — newer memories supersede older.`;
|
|
289
294
|
return { status: 200, text: `${header}\n\n${mem.content ?? ""}` };
|
|
290
295
|
}
|
package/dist/retrieval.js
CHANGED
|
@@ -69,6 +69,7 @@ exports.retrieve = retrieve;
|
|
|
69
69
|
exports.searchRecent = searchRecent;
|
|
70
70
|
const storage = __importStar(require("./storage.js"));
|
|
71
71
|
const schema_prototypes_js_1 = require("./schema-prototypes.js");
|
|
72
|
+
const type_labels_js_1 = require("./type-labels.js");
|
|
72
73
|
/** Default decay half-life (days) at importance 0.5. #192: was 0.0005/h
|
|
73
74
|
* (~115-day half-life at base 0.5) — aggressive enough to bury the long tail
|
|
74
75
|
* in ranking. Long-term remembering is the product; time preference stays,
|
|
@@ -450,7 +451,7 @@ function formatResult(memory, score, effStr, connections, provenance) {
|
|
|
450
451
|
score: Math.round(score * 1e6) / 1e6,
|
|
451
452
|
effective_strength: Math.round(effStr * 1e6) / 1e6,
|
|
452
453
|
access_count: memory.access_count ?? 0,
|
|
453
|
-
memory_type: memory.memory_type ?? "
|
|
454
|
+
memory_type: (0, type_labels_js_1.labelForType)(memory.memory_type ?? "experience"),
|
|
454
455
|
project: memory.project ?? null,
|
|
455
456
|
source_agent: memory.source_agent ?? null,
|
|
456
457
|
created_at: memory.created_at ?? "",
|
package/dist/seed-lesson.js
CHANGED
|
@@ -63,7 +63,7 @@ async function injectSeedLesson(database, log = console.log) {
|
|
|
63
63
|
storage.insertMemory(database, exports.SEED_LESSON, embedding, {
|
|
64
64
|
sourceAgent: "hicortex/seed",
|
|
65
65
|
project: "global",
|
|
66
|
-
memoryType: "
|
|
66
|
+
memoryType: "learnings",
|
|
67
67
|
baseStrength: 0.95,
|
|
68
68
|
});
|
|
69
69
|
log("[hicortex] Seed lesson injected: Daily Self-Improvement Protocol");
|
package/dist/status.d.ts
CHANGED
|
@@ -10,4 +10,12 @@
|
|
|
10
10
|
* out as invalid (the hook sends none) rather than silently accepted.
|
|
11
11
|
*/
|
|
12
12
|
export declare function statusAgentLine(config: Record<string, unknown>): string;
|
|
13
|
+
/**
|
|
14
|
+
* Format the memory-type breakdown for `hicortex status`. Each raw DB enum key
|
|
15
|
+
* is rendered through {@link labelForType} so the printed line uses the human
|
|
16
|
+
* vocabulary (Knowledge/Experience/Decisions/Learnings), not the raw enum.
|
|
17
|
+
* Extracted from `runStatus` so the labeling is unit-testable without booting
|
|
18
|
+
* the full status printer. Unknown keys pass through verbatim (forward-compat).
|
|
19
|
+
*/
|
|
20
|
+
export declare function formatTypeBreakdown(byType: Record<string, number>): string;
|
|
13
21
|
export declare function runStatus(): Promise<void>;
|
package/dist/status.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.statusAgentLine = statusAgentLine;
|
|
7
|
+
exports.formatTypeBreakdown = formatTypeBreakdown;
|
|
7
8
|
exports.runStatus = runStatus;
|
|
8
9
|
const paths_js_1 = require("./paths.js");
|
|
9
10
|
const node_fs_1 = require("node:fs");
|
|
@@ -14,6 +15,7 @@ const db_js_1 = require("./db.js");
|
|
|
14
15
|
const features_js_1 = require("./features.js");
|
|
15
16
|
const state_js_1 = require("./state.js");
|
|
16
17
|
const identity_store_js_1 = require("./identity-store.js");
|
|
18
|
+
const type_labels_js_1 = require("./type-labels.js");
|
|
17
19
|
const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
|
|
18
20
|
const CC_SETTINGS = (0, node_path_1.join)((0, node_os_1.homedir)(), ".claude", "settings.json");
|
|
19
21
|
const OC_CONFIG = (0, node_path_1.join)((0, node_os_1.homedir)(), ".openclaw", "openclaw.json");
|
|
@@ -36,6 +38,18 @@ function statusAgentLine(config) {
|
|
|
36
38
|
return "(not set — global identity)";
|
|
37
39
|
}
|
|
38
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* Format the memory-type breakdown for `hicortex status`. Each raw DB enum key
|
|
43
|
+
* is rendered through {@link labelForType} so the printed line uses the human
|
|
44
|
+
* vocabulary (Knowledge/Experience/Decisions/Learnings), not the raw enum.
|
|
45
|
+
* Extracted from `runStatus` so the labeling is unit-testable without booting
|
|
46
|
+
* the full status printer. Unknown keys pass through verbatim (forward-compat).
|
|
47
|
+
*/
|
|
48
|
+
function formatTypeBreakdown(byType) {
|
|
49
|
+
return Object.entries(byType)
|
|
50
|
+
.map(([k, v]) => `${(0, type_labels_js_1.labelForType)(k)}=${v}`)
|
|
51
|
+
.join(", ");
|
|
52
|
+
}
|
|
39
53
|
async function runStatus() {
|
|
40
54
|
console.log("Hicortex Status");
|
|
41
55
|
console.log("─".repeat(40));
|
|
@@ -48,7 +62,7 @@ async function runStatus() {
|
|
|
48
62
|
const { initDb, getStats } = await import("./db.js");
|
|
49
63
|
const db = initDb(dbPath);
|
|
50
64
|
const stats = getStats(db, dbPath);
|
|
51
|
-
const typeStr =
|
|
65
|
+
const typeStr = formatTypeBreakdown(stats.by_type);
|
|
52
66
|
console.log(`Memories: ${stats.memories} (${typeStr || "none"})`);
|
|
53
67
|
console.log(`Links: ${stats.links}`);
|
|
54
68
|
console.log(`DB size: ${(stats.db_size_bytes / 1024).toFixed(1)} KB`);
|
package/dist/storage.js
CHANGED
|
@@ -72,7 +72,7 @@ function insertMemory(db, content, embedding, opts = {}) {
|
|
|
72
72
|
created_at, ingested_at, source_agent, source_agent_id, source_session,
|
|
73
73
|
source_domain, project, privacy, memory_type)
|
|
74
74
|
VALUES (?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
75
|
-
.run(id, content, opts.baseStrength ?? 0.5, ts, ts, ingestedTs, opts.sourceAgent ?? "default", opts.sourceAgentId ?? null, sourceSession, opts.sourceDomain ?? null, opts.project ?? null, opts.privacy ?? null, opts.memoryType ?? "
|
|
75
|
+
.run(id, content, opts.baseStrength ?? 0.5, ts, ts, ingestedTs, opts.sourceAgent ?? "default", opts.sourceAgentId ?? null, sourceSession, opts.sourceDomain ?? null, opts.project ?? null, opts.privacy ?? null, opts.memoryType ?? "experience");
|
|
76
76
|
if (result.changes > 0) {
|
|
77
77
|
// New row — store its vector.
|
|
78
78
|
db.prepare("INSERT INTO memory_vectors (id, embedding) VALUES (?, ?)").run(id, embedToBlob(embedding));
|
|
@@ -469,7 +469,7 @@ function insertMemoriesBatch(db, memories) {
|
|
|
469
469
|
for (const mem of memories) {
|
|
470
470
|
const id = (0, node_crypto_1.randomUUID)();
|
|
471
471
|
const ts = nowIso();
|
|
472
|
-
insertMem.run(id, mem.content, mem.baseStrength ?? 0.5, ts, ts, ts, mem.sourceAgent ?? "default", mem.sourceAgentId ?? null, mem.sourceSession ?? null, mem.sourceDomain ?? null, mem.project ?? null, mem.privacy ?? null, mem.memoryType ?? "
|
|
472
|
+
insertMem.run(id, mem.content, mem.baseStrength ?? 0.5, ts, ts, ts, mem.sourceAgent ?? "default", mem.sourceAgentId ?? null, mem.sourceSession ?? null, mem.sourceDomain ?? null, mem.project ?? null, mem.privacy ?? null, mem.memoryType ?? "experience");
|
|
473
473
|
insertVec.run(id, embedToBlob(mem.embedding));
|
|
474
474
|
count++;
|
|
475
475
|
}
|
|
@@ -514,14 +514,14 @@ function getLessons(db, days = 7, project) {
|
|
|
514
514
|
if (project) {
|
|
515
515
|
const rows = db
|
|
516
516
|
.prepare(`SELECT * FROM memories
|
|
517
|
-
WHERE memory_type = '
|
|
517
|
+
WHERE memory_type = 'learnings' AND created_at > ? AND project = ?
|
|
518
518
|
ORDER BY created_at DESC`)
|
|
519
519
|
.all(cutoff, project);
|
|
520
520
|
return rows.map(rowToMemory);
|
|
521
521
|
}
|
|
522
522
|
const rows = db
|
|
523
523
|
.prepare(`SELECT * FROM memories
|
|
524
|
-
WHERE memory_type = '
|
|
524
|
+
WHERE memory_type = 'learnings' AND created_at > ?
|
|
525
525
|
ORDER BY created_at DESC`)
|
|
526
526
|
.all(cutoff);
|
|
527
527
|
return rows.map(rowToMemory);
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve the effective cap: env (HICORTEX_TOKEN_CAP) takes precedence over the
|
|
3
|
+
* config key. A positive, finite env wins; otherwise the config value (0/absent
|
|
4
|
+
* = unlimited). Pure — exported for tests.
|
|
5
|
+
*/
|
|
6
|
+
export declare function resolveTokenCap(configCap: unknown): number;
|
|
7
|
+
/**
|
|
8
|
+
* Initialise at server boot (after stateDir is known). Resolves + caches the cap
|
|
9
|
+
* and seeds the 80%-warn dedup so a restart mid-period doesn't re-warn.
|
|
10
|
+
*/
|
|
11
|
+
export declare function initTokenBudget(stateDir: string, configCap: unknown): void;
|
|
12
|
+
/** The resolved monthly cap (0 = unlimited / enforcement off). */
|
|
13
|
+
export declare function getTokenCap(): number;
|
|
14
|
+
/**
|
|
15
|
+
* Pre-call check for /distill: refuse (429) when the tenant is already at/over
|
|
16
|
+
* the monthly cap. Reuses `shouldThrottleTokens(cap, period, 0)` — lastRunTokens
|
|
17
|
+
* is 0 because we cannot predict a call's cost before making it, so this refuses
|
|
18
|
+
* only when already over (a tenant exactly at the cap is refused on the next
|
|
19
|
+
* call). Reads state.json fresh so the nightly process's writes are reflected.
|
|
20
|
+
*/
|
|
21
|
+
export declare function isTokenBudgetExceeded(stateDir: string): boolean;
|
|
22
|
+
/**
|
|
23
|
+
* After a successful distill, add the consumed tokens to the monthly counter and
|
|
24
|
+
* emit the 80% warning once per period. Synchronous read-modify-write via
|
|
25
|
+
* `updateState` (serializes concurrent in-process /distill; picks up the nightly
|
|
26
|
+
* process's writes via the fresh read). Accumulates the full breakdown
|
|
27
|
+
* (prompt/completion/total) so the dashboard's prompt+completion stays
|
|
28
|
+
* consistent with total (distill + consolidation).
|
|
29
|
+
*/
|
|
30
|
+
export declare function recordDistillUsage(stateDir: string, usage: {
|
|
31
|
+
prompt: number;
|
|
32
|
+
completion: number;
|
|
33
|
+
total: number;
|
|
34
|
+
}): void;
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.resolveTokenCap = resolveTokenCap;
|
|
4
|
+
exports.initTokenBudget = initTokenBudget;
|
|
5
|
+
exports.getTokenCap = getTokenCap;
|
|
6
|
+
exports.isTokenBudgetExceeded = isTokenBudgetExceeded;
|
|
7
|
+
exports.recordDistillUsage = recordDistillUsage;
|
|
8
|
+
/**
|
|
9
|
+
* Per-tenant monthly token-budget enforcement (#110 Phase 0B item #5).
|
|
10
|
+
*
|
|
11
|
+
* Limits LLM token consumption over /distill (the cost-generating path the
|
|
12
|
+
* nightly consolidation throttle did NOT cover). Mode-agnostic — gates on
|
|
13
|
+
* `cap > 0`, never on `hostedMode`:
|
|
14
|
+
* - Self-hosted: the cap is the operator's own `llmTokensPerMonth` config
|
|
15
|
+
* (default 0 = unlimited → never throttles). Protects the operator's wallet
|
|
16
|
+
* from a runaway nightly on an expensive model.
|
|
17
|
+
* - Hosted: the cap is provider-set via the `HICORTEX_TOKEN_CAP` env, which
|
|
18
|
+
* takes PRECEDENCE over config. The tenant process cannot mutate boot-time
|
|
19
|
+
* env, so a hosted tenant cannot raise its own cap (the config.json
|
|
20
|
+
* self-edit loophole is closed). Protects the provider's wallet.
|
|
21
|
+
*
|
|
22
|
+
* Reuses the existing machinery: `shouldThrottleTokens` (consolidate.ts) for the
|
|
23
|
+
* decision (incl. monthly reset), and `llmTokensThisPeriod` + `updateState`
|
|
24
|
+
* (state.ts) for the counter + atomic persistence.
|
|
25
|
+
*
|
|
26
|
+
* Concurrency: state.json is read fresh for each check and written via
|
|
27
|
+
* `updateState` (a synchronous read-modify-write; Node's single thread
|
|
28
|
+
* serializes concurrent /distill calls within the server process, so no
|
|
29
|
+
* in-process tally is needed). The nightly consolidation is a SEPARATE process
|
|
30
|
+
* that also writes state.json; a rare cross-process write collision can lose a
|
|
31
|
+
* small increment — negligible on a multi-million-token monthly budget. A
|
|
32
|
+
* DB-backed counter (WAL transactions serialize across processes) is the future
|
|
33
|
+
* hardening if it ever matters.
|
|
34
|
+
*/
|
|
35
|
+
const state_js_1 = require("./state.js");
|
|
36
|
+
const consolidate_js_1 = require("./consolidate.js");
|
|
37
|
+
/** Env override (hosted: provider-set, tenant-immutable at runtime). */
|
|
38
|
+
const TOKEN_CAP_ENV = "HICORTEX_TOKEN_CAP";
|
|
39
|
+
let cap = 0;
|
|
40
|
+
/** periodStart we last emitted the 80% warning at, to dedup within a period. */
|
|
41
|
+
let warnedPeriod = null;
|
|
42
|
+
/**
|
|
43
|
+
* Resolve the effective cap: env (HICORTEX_TOKEN_CAP) takes precedence over the
|
|
44
|
+
* config key. A positive, finite env wins; otherwise the config value (0/absent
|
|
45
|
+
* = unlimited). Pure — exported for tests.
|
|
46
|
+
*/
|
|
47
|
+
function resolveTokenCap(configCap) {
|
|
48
|
+
const envCap = Number(process.env[TOKEN_CAP_ENV]);
|
|
49
|
+
if (Number.isFinite(envCap) && envCap > 0)
|
|
50
|
+
return envCap;
|
|
51
|
+
const cfg = Number(configCap);
|
|
52
|
+
return Number.isFinite(cfg) && cfg > 0 ? cfg : 0;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Initialise at server boot (after stateDir is known). Resolves + caches the cap
|
|
56
|
+
* and seeds the 80%-warn dedup so a restart mid-period doesn't re-warn.
|
|
57
|
+
*/
|
|
58
|
+
function initTokenBudget(stateDir, configCap) {
|
|
59
|
+
cap = resolveTokenCap(configCap);
|
|
60
|
+
if (cap > 0) {
|
|
61
|
+
const p = (0, state_js_1.loadState)(stateDir).llmTokensThisPeriod;
|
|
62
|
+
warnedPeriod = p && p.total >= cap * 0.8 ? (p.periodStart ?? null) : null;
|
|
63
|
+
// Label the source truthfully: only claim env if the env value was actually
|
|
64
|
+
// used (a malformed env falls back to config, so the label must not lie).
|
|
65
|
+
const fromEnv = Number(process.env[TOKEN_CAP_ENV]) === cap;
|
|
66
|
+
console.log(`[hicortex] Token budget: ${cap.toLocaleString()}/month${fromEnv ? " (HICORTEX_TOKEN_CAP)" : ""}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** The resolved monthly cap (0 = unlimited / enforcement off). */
|
|
70
|
+
function getTokenCap() {
|
|
71
|
+
return cap;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Pre-call check for /distill: refuse (429) when the tenant is already at/over
|
|
75
|
+
* the monthly cap. Reuses `shouldThrottleTokens(cap, period, 0)` — lastRunTokens
|
|
76
|
+
* is 0 because we cannot predict a call's cost before making it, so this refuses
|
|
77
|
+
* only when already over (a tenant exactly at the cap is refused on the next
|
|
78
|
+
* call). Reads state.json fresh so the nightly process's writes are reflected.
|
|
79
|
+
*/
|
|
80
|
+
function isTokenBudgetExceeded(stateDir) {
|
|
81
|
+
if (cap <= 0)
|
|
82
|
+
return false;
|
|
83
|
+
const period = (0, state_js_1.loadState)(stateDir).llmTokensThisPeriod;
|
|
84
|
+
return (0, consolidate_js_1.shouldThrottleTokens)(cap, period, 0).throttle;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* After a successful distill, add the consumed tokens to the monthly counter and
|
|
88
|
+
* emit the 80% warning once per period. Synchronous read-modify-write via
|
|
89
|
+
* `updateState` (serializes concurrent in-process /distill; picks up the nightly
|
|
90
|
+
* process's writes via the fresh read). Accumulates the full breakdown
|
|
91
|
+
* (prompt/completion/total) so the dashboard's prompt+completion stays
|
|
92
|
+
* consistent with total (distill + consolidation).
|
|
93
|
+
*/
|
|
94
|
+
function recordDistillUsage(stateDir, usage) {
|
|
95
|
+
if (cap <= 0 || usage.total <= 0)
|
|
96
|
+
return;
|
|
97
|
+
let newTotal = 0;
|
|
98
|
+
let periodStart = "";
|
|
99
|
+
(0, state_js_1.updateState)((s) => {
|
|
100
|
+
const prev = s.llmTokensThisPeriod;
|
|
101
|
+
// Monthly reset (year+month) — matches shouldThrottleTokens's staleness check.
|
|
102
|
+
const stale = !prev?.periodStart ||
|
|
103
|
+
new Date(prev.periodStart).getUTCFullYear() !== new Date().getUTCFullYear() ||
|
|
104
|
+
new Date(prev.periodStart).getUTCMonth() !== new Date().getUTCMonth();
|
|
105
|
+
if (stale) {
|
|
106
|
+
s.llmTokensThisPeriod = {
|
|
107
|
+
prompt: usage.prompt,
|
|
108
|
+
completion: usage.completion,
|
|
109
|
+
total: usage.total,
|
|
110
|
+
periodStart: new Date().toISOString(),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
const base = prev;
|
|
115
|
+
s.llmTokensThisPeriod = {
|
|
116
|
+
prompt: (base.prompt ?? 0) + usage.prompt,
|
|
117
|
+
completion: (base.completion ?? 0) + usage.completion,
|
|
118
|
+
total: (base.total ?? 0) + usage.total,
|
|
119
|
+
periodStart: base.periodStart,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
newTotal = s.llmTokensThisPeriod.total;
|
|
123
|
+
periodStart = s.llmTokensThisPeriod.periodStart;
|
|
124
|
+
}, stateDir);
|
|
125
|
+
// 80% warning — dedup per period (once per month per threshold crossing).
|
|
126
|
+
if (periodStart && warnedPeriod !== periodStart && newTotal >= cap * 0.8) {
|
|
127
|
+
warnedPeriod = periodStart;
|
|
128
|
+
const pct = Math.round((newTotal / cap) * 100);
|
|
129
|
+
console.warn(`[hicortex] Token usage at ${pct}% of monthly cap (${newTotal.toLocaleString()}/${cap.toLocaleString()}).`);
|
|
130
|
+
}
|
|
131
|
+
}
|
package/dist/type-classify.d.ts
CHANGED
|
@@ -1,41 +1,41 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `hicortex classify-types` — deliberate, resumable
|
|
3
|
-
* reclassification pass over the memories corpus (#216).
|
|
2
|
+
* `hicortex classify-types` — deliberate, resumable experience→knowledge/
|
|
3
|
+
* decisions reclassification pass over the memories corpus (#216).
|
|
4
4
|
*
|
|
5
5
|
* WHY THIS EXISTS
|
|
6
6
|
* ---------------
|
|
7
7
|
* Before #216 the distiller NEVER set memory_type — every distilled memory
|
|
8
|
-
* defaulted to "
|
|
9
|
-
* corpus was ~98%
|
|
10
|
-
* time via the [E]/[
|
|
11
|
-
* one-shot backfill. This command is that backfill — modelled on
|
|
8
|
+
* defaulted to "experience" (storage.ts insertMemory `?? "experience"`), so
|
|
9
|
+
* the corpus was ~98% experiences. The distiller now classifies each entry at
|
|
10
|
+
* extract time via the [E]/[K]/[D] tag (distiller.ts), but the EXISTING corpus
|
|
11
|
+
* needs a one-shot backfill. This command is that backfill — modelled on
|
|
12
12
|
* `classify-domains` (resumable cursor, batched, infra-error-safe).
|
|
13
13
|
*
|
|
14
14
|
* WHAT IT DOES
|
|
15
15
|
* ------------
|
|
16
16
|
* Walks memories ordered by rowid in batches (default 200). Default scope =
|
|
17
|
-
*
|
|
18
|
-
* regardless of current type **except
|
|
19
|
-
* ONE constrained LLM call asks the model to classify the content as
|
|
20
|
-
*
|
|
17
|
+
* experiences only (`memory_type = 'experience'`); `--all` reclassifies every
|
|
18
|
+
* memory regardless of current type **except learnings** (see below). For each
|
|
19
|
+
* memory, ONE constrained LLM call asks the model to classify the content as
|
|
20
|
+
* experience / knowledge / decisions. The reply is parsed + validated, and
|
|
21
21
|
* `UPDATE memories SET memory_type = ? WHERE id = ?` runs inside a per-batch
|
|
22
22
|
* transaction. The cursor (`typeCursor` in state.json) advances to the last
|
|
23
23
|
* committed rowid after each batch — crash-safe and infra-abort-safe (same
|
|
24
24
|
* discipline as classify-domains).
|
|
25
25
|
*
|
|
26
|
-
*
|
|
27
|
-
* scope explicitly excludes `memory_type = '
|
|
28
|
-
* defence. (The prompt asks only for
|
|
29
|
-
* DID enter scope would be overwritten
|
|
30
|
-
* "
|
|
31
|
-
* the main guard.)
|
|
26
|
+
* Learnings are NEVER touched here: the reflection stage owns them. The
|
|
27
|
+
* `--all` scope explicitly excludes `memory_type = 'learnings'` — this is the
|
|
28
|
+
* primary defence. (The prompt asks only for experience/knowledge/decisions,
|
|
29
|
+
* so a learning that DID enter scope would be overwritten — the model never
|
|
30
|
+
* replies "learnings". `parseTypeReply`'s rejection of a "learnings" reply is
|
|
31
|
+
* a backstop, not the main guard.)
|
|
32
32
|
*
|
|
33
33
|
* This command does NOT use the consolidation budget — it is a standalone CLI,
|
|
34
34
|
* not a nightly stage.
|
|
35
35
|
*/
|
|
36
36
|
import { LlmClient } from "./llm.js";
|
|
37
37
|
export interface ClassifyTypesOptions {
|
|
38
|
-
/** Reclassify EVERY memory, not just
|
|
38
|
+
/** Reclassify EVERY memory, not just experiences. */
|
|
39
39
|
all?: boolean;
|
|
40
40
|
/** Memories per batch (default 200). Cursor advances per committed batch. */
|
|
41
41
|
batchSize?: number;
|
|
@@ -55,7 +55,7 @@ export interface ClassifyTypesReport {
|
|
|
55
55
|
scanned: number;
|
|
56
56
|
/** Memories whose memory_type was changed. */
|
|
57
57
|
reclassified: number;
|
|
58
|
-
/**
|
|
58
|
+
/** Experiences confirmed as experience (no change). */
|
|
59
59
|
unchanged: number;
|
|
60
60
|
/** Memories skipped due to an infra error (LLM threw twice). */
|
|
61
61
|
failed: number;
|
|
@@ -70,23 +70,26 @@ export interface ClassifyTypesReport {
|
|
|
70
70
|
}
|
|
71
71
|
/**
|
|
72
72
|
* Build the constrained type-classification prompt for one memory. The model
|
|
73
|
-
* must reply with ONLY the type word (
|
|
74
|
-
* distinction mirrors the distiller's [E]/[
|
|
75
|
-
* so distill-time and backfill-time classification stay
|
|
73
|
+
* must reply with ONLY the type word (experience/knowledge/decisions) — no
|
|
74
|
+
* prose. The distinction mirrors the distiller's [E]/[K]/[D] tag definitions
|
|
75
|
+
* (prompts.ts), so distill-time and backfill-time classification stay
|
|
76
|
+
* consistent. (The stored enum was renamed in #264: episode→experience,
|
|
77
|
+
* fact→knowledge, decision→decisions; the conceptual definitions are
|
|
78
|
+
* unchanged.)
|
|
76
79
|
*/
|
|
77
80
|
export declare function buildTypeClassifyPrompt(content: string): string;
|
|
78
81
|
/**
|
|
79
82
|
* Parse the model's reply into a validated type. Accepts the bare word
|
|
80
83
|
* (case-insensitive), tolerating surrounding whitespace, a trailing period, a
|
|
81
|
-
* leading "Type:" label, and markdown emphasis. "
|
|
82
|
-
* (the reflection stage owns
|
|
83
|
-
* null so the caller retries.
|
|
84
|
+
* leading "Type:" label, and markdown emphasis. "learnings" (and the legacy
|
|
85
|
+
* "lesson") are NEVER accepted (the reflection stage owns learnings; a model
|
|
86
|
+
* that emits either is wrong) — returns null so the caller retries.
|
|
84
87
|
*
|
|
85
88
|
* Returns null on anything unparseable or out-of-vocabulary so the caller can
|
|
86
89
|
* retry once (matching classify-domains' two-attempt discipline).
|
|
87
90
|
*/
|
|
88
91
|
export declare function parseTypeReply(reply: string): {
|
|
89
|
-
type: "
|
|
92
|
+
type: "experience" | "knowledge" | "decisions";
|
|
90
93
|
score: number;
|
|
91
94
|
} | null;
|
|
92
95
|
/**
|
|
@@ -95,7 +98,7 @@ export declare function parseTypeReply(reply: string): {
|
|
|
95
98
|
* (caller leaves the memory untouched and retries via the cursor next run).
|
|
96
99
|
*/
|
|
97
100
|
export declare function classifyMemoryType(content: string, llm: LlmClient): Promise<{
|
|
98
|
-
type: "
|
|
101
|
+
type: "experience" | "knowledge" | "decisions";
|
|
99
102
|
score: number;
|
|
100
103
|
} | null>;
|
|
101
104
|
/**
|
package/dist/type-classify.js
CHANGED
|
@@ -1,35 +1,35 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* `hicortex classify-types` — deliberate, resumable
|
|
4
|
-
* reclassification pass over the memories corpus (#216).
|
|
3
|
+
* `hicortex classify-types` — deliberate, resumable experience→knowledge/
|
|
4
|
+
* decisions reclassification pass over the memories corpus (#216).
|
|
5
5
|
*
|
|
6
6
|
* WHY THIS EXISTS
|
|
7
7
|
* ---------------
|
|
8
8
|
* Before #216 the distiller NEVER set memory_type — every distilled memory
|
|
9
|
-
* defaulted to "
|
|
10
|
-
* corpus was ~98%
|
|
11
|
-
* time via the [E]/[
|
|
12
|
-
* one-shot backfill. This command is that backfill — modelled on
|
|
9
|
+
* defaulted to "experience" (storage.ts insertMemory `?? "experience"`), so
|
|
10
|
+
* the corpus was ~98% experiences. The distiller now classifies each entry at
|
|
11
|
+
* extract time via the [E]/[K]/[D] tag (distiller.ts), but the EXISTING corpus
|
|
12
|
+
* needs a one-shot backfill. This command is that backfill — modelled on
|
|
13
13
|
* `classify-domains` (resumable cursor, batched, infra-error-safe).
|
|
14
14
|
*
|
|
15
15
|
* WHAT IT DOES
|
|
16
16
|
* ------------
|
|
17
17
|
* Walks memories ordered by rowid in batches (default 200). Default scope =
|
|
18
|
-
*
|
|
19
|
-
* regardless of current type **except
|
|
20
|
-
* ONE constrained LLM call asks the model to classify the content as
|
|
21
|
-
*
|
|
18
|
+
* experiences only (`memory_type = 'experience'`); `--all` reclassifies every
|
|
19
|
+
* memory regardless of current type **except learnings** (see below). For each
|
|
20
|
+
* memory, ONE constrained LLM call asks the model to classify the content as
|
|
21
|
+
* experience / knowledge / decisions. The reply is parsed + validated, and
|
|
22
22
|
* `UPDATE memories SET memory_type = ? WHERE id = ?` runs inside a per-batch
|
|
23
23
|
* transaction. The cursor (`typeCursor` in state.json) advances to the last
|
|
24
24
|
* committed rowid after each batch — crash-safe and infra-abort-safe (same
|
|
25
25
|
* discipline as classify-domains).
|
|
26
26
|
*
|
|
27
|
-
*
|
|
28
|
-
* scope explicitly excludes `memory_type = '
|
|
29
|
-
* defence. (The prompt asks only for
|
|
30
|
-
* DID enter scope would be overwritten
|
|
31
|
-
* "
|
|
32
|
-
* the main guard.)
|
|
27
|
+
* Learnings are NEVER touched here: the reflection stage owns them. The
|
|
28
|
+
* `--all` scope explicitly excludes `memory_type = 'learnings'` — this is the
|
|
29
|
+
* primary defence. (The prompt asks only for experience/knowledge/decisions,
|
|
30
|
+
* so a learning that DID enter scope would be overwritten — the model never
|
|
31
|
+
* replies "learnings". `parseTypeReply`'s rejection of a "learnings" reply is
|
|
32
|
+
* a backstop, not the main guard.)
|
|
33
33
|
*
|
|
34
34
|
* This command does NOT use the consolidation budget — it is a standalone CLI,
|
|
35
35
|
* not a nightly stage.
|
|
@@ -49,8 +49,8 @@ const type_labels_js_1 = require("./type-labels.js");
|
|
|
49
49
|
const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
|
|
50
50
|
/** Max chars of memory content fed to the classify prompt. */
|
|
51
51
|
const CLASSIFY_CONTENT_MAX_CHARS = 1500;
|
|
52
|
-
/** Valid distillation-time memory types (NO
|
|
53
|
-
const VALID_TYPES = new Set(["
|
|
52
|
+
/** Valid distillation-time memory types (NO learnings — reflection owns that). */
|
|
53
|
+
const VALID_TYPES = new Set(["experience", "knowledge", "decisions"]);
|
|
54
54
|
function readConfig(stateDir) {
|
|
55
55
|
try {
|
|
56
56
|
return JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(stateDir, "config.json"), "utf-8"));
|
|
@@ -61,9 +61,12 @@ function readConfig(stateDir) {
|
|
|
61
61
|
}
|
|
62
62
|
/**
|
|
63
63
|
* Build the constrained type-classification prompt for one memory. The model
|
|
64
|
-
* must reply with ONLY the type word (
|
|
65
|
-
* distinction mirrors the distiller's [E]/[
|
|
66
|
-
* so distill-time and backfill-time classification stay
|
|
64
|
+
* must reply with ONLY the type word (experience/knowledge/decisions) — no
|
|
65
|
+
* prose. The distinction mirrors the distiller's [E]/[K]/[D] tag definitions
|
|
66
|
+
* (prompts.ts), so distill-time and backfill-time classification stay
|
|
67
|
+
* consistent. (The stored enum was renamed in #264: episode→experience,
|
|
68
|
+
* fact→knowledge, decision→decisions; the conceptual definitions are
|
|
69
|
+
* unchanged.)
|
|
67
70
|
*/
|
|
68
71
|
function buildTypeClassifyPrompt(content) {
|
|
69
72
|
const truncated = content.length > CLASSIFY_CONTENT_MAX_CHARS
|
|
@@ -71,28 +74,31 @@ function buildTypeClassifyPrompt(content) {
|
|
|
71
74
|
: content;
|
|
72
75
|
return (`You are classifying a single memory by its TYPE and IMPORTANCE.\n\n` +
|
|
73
76
|
`TYPES:\n` +
|
|
74
|
-
`-
|
|
77
|
+
`- experience: a specific event, interaction, or narrative — a one-time ` +
|
|
75
78
|
`occurrence ("tried X, failed because Y", a correction, a debugging session).\n` +
|
|
76
|
-
`-
|
|
77
|
-
`moment ("the API is at :8787", "uv is used for packages").\n` +
|
|
78
|
-
`-
|
|
79
|
-
`can supersede ("switched from gemma4 to qwen3.5", "adopted the
|
|
80
|
-
`tag model"). Not
|
|
79
|
+
`- knowledge: a durable truth that holds across sessions, not tied to a ` +
|
|
80
|
+
`single moment ("the API is at :8787", "uv is used for packages").\n` +
|
|
81
|
+
`- decisions: a choice made that future work builds on and a later ` +
|
|
82
|
+
`decision can supersede ("switched from gemma4 to qwen3.5", "adopted the ` +
|
|
83
|
+
`graded-schema tag model"). Not knowledge (it can change) and not an ` +
|
|
84
|
+
`experience (it persists).\n\n` +
|
|
81
85
|
`IMPORTANCE (0.0–1.0):\n` +
|
|
82
|
-
`- 0.8–1.0: load-bearing — a core
|
|
86
|
+
`- 0.8–1.0: load-bearing — a core piece of knowledge or a decision the ` +
|
|
87
|
+
`agent must know.\n` +
|
|
83
88
|
`- 0.5–0.8: useful context — relevant to current and future work.\n` +
|
|
84
89
|
`- 0.2–0.5: marginal — situational, likely to fade.\n` +
|
|
85
90
|
`- 0.0–0.2: noise — low value, safe to forget.\n` +
|
|
86
|
-
`
|
|
91
|
+
`Knowledge and decisions tend to score higher than experiences (they ` +
|
|
92
|
+
`persist).\n\n` +
|
|
87
93
|
`MEMORY:\n${truncated}\n\n` +
|
|
88
|
-
`Reply with ONLY: type importance (e.g. "
|
|
94
|
+
`Reply with ONLY: type importance (e.g. "knowledge 0.8"). No prose, no explanation.`);
|
|
89
95
|
}
|
|
90
96
|
/**
|
|
91
97
|
* Parse the model's reply into a validated type. Accepts the bare word
|
|
92
98
|
* (case-insensitive), tolerating surrounding whitespace, a trailing period, a
|
|
93
|
-
* leading "Type:" label, and markdown emphasis. "
|
|
94
|
-
* (the reflection stage owns
|
|
95
|
-
* null so the caller retries.
|
|
99
|
+
* leading "Type:" label, and markdown emphasis. "learnings" (and the legacy
|
|
100
|
+
* "lesson") are NEVER accepted (the reflection stage owns learnings; a model
|
|
101
|
+
* that emits either is wrong) — returns null so the caller retries.
|
|
96
102
|
*
|
|
97
103
|
* Returns null on anything unparseable or out-of-vocabulary so the caller can
|
|
98
104
|
* retry once (matching classify-domains' two-attempt discipline).
|
|
@@ -112,8 +118,8 @@ function parseTypeReply(reply) {
|
|
|
112
118
|
.replace(/^[*_`"'\s]+/, "")
|
|
113
119
|
.replace(/[*_`"']+$/, "")
|
|
114
120
|
.trim();
|
|
115
|
-
// Expected format: "type score" (e.g. "
|
|
116
|
-
const match = cleaned.toLowerCase().match(/^(
|
|
121
|
+
// Expected format: "type score" (e.g. "knowledge 0.8"). Parse both.
|
|
122
|
+
const match = cleaned.toLowerCase().match(/^(experience|knowledge|decisions)\s+([0-9]*\.?[0-9]+)/);
|
|
117
123
|
if (match) {
|
|
118
124
|
const type = match[1];
|
|
119
125
|
let score = parseFloat(match[2]);
|
|
@@ -158,7 +164,7 @@ async function classifyMemoryType(content, llm) {
|
|
|
158
164
|
}
|
|
159
165
|
}
|
|
160
166
|
// Two successful calls, neither parseable → leave the memory's type unchanged.
|
|
161
|
-
// We do NOT default to
|
|
167
|
+
// We do NOT default to experience here: a model that can't decide should not
|
|
162
168
|
// silently overwrite an existing type. Return null so the caller records a
|
|
163
169
|
// failed classification and the cursor still advances past this row.
|
|
164
170
|
return null;
|
|
@@ -208,17 +214,18 @@ async function runClassifyTypes(options = {}) {
|
|
|
208
214
|
try {
|
|
209
215
|
let cursor = options.reset ? 0 : ((0, state_js_1.loadState)(stateDir).typeCursor ?? 0);
|
|
210
216
|
report.cursor = cursor;
|
|
211
|
-
console.log(`[hicortex] classify-types starting: scope ${all ? "ALL" : "
|
|
217
|
+
console.log(`[hicortex] classify-types starting: scope ${all ? "ALL" : "experiences only"}, ` +
|
|
212
218
|
`batch ${batchSize}, cursor ${cursor}${options.reset ? " (reset)" : ""}`);
|
|
213
|
-
// Scope filter: default =
|
|
214
|
-
//
|
|
215
|
-
// reclassified here. (The prompt asks only for
|
|
216
|
-
//
|
|
217
|
-
// The scope exclusion is therefore the real guard;
|
|
218
|
-
// "
|
|
219
|
+
// Scope filter: default = experiences only; --all = everything EXCEPT
|
|
220
|
+
// learnings. Learnings are owned by the reflection stage — they must never
|
|
221
|
+
// be reclassified here. (The prompt asks only for experience/knowledge/
|
|
222
|
+
// decisions, so a learning row in scope gets overwritten: the model never
|
|
223
|
+
// replies "learnings". The scope exclusion is therefore the real guard;
|
|
224
|
+
// parseTypeReply's "learnings" rejection is a backstop, not the primary
|
|
225
|
+
// defence.)
|
|
219
226
|
const scopeSql = all
|
|
220
|
-
? "rowid > ? AND (memory_type IS NULL OR memory_type != '
|
|
221
|
-
: "rowid > ? AND memory_type = '
|
|
227
|
+
? "rowid > ? AND (memory_type IS NULL OR memory_type != 'learnings')"
|
|
228
|
+
: "rowid > ? AND memory_type = 'experience'";
|
|
222
229
|
const batchStmt = db.prepare(`SELECT rowid AS __rowid, id, content, memory_type FROM memories
|
|
223
230
|
WHERE ${scopeSql} ORDER BY rowid ASC LIMIT ?`);
|
|
224
231
|
// Set true when the classifier returns null (infra error): finish the
|