@gamaze/hicortex 0.23.2 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/dashboard.html +39 -0
- package/dist/calibration.d.ts +31 -0
- package/dist/calibration.js +38 -1
- package/dist/capture.d.ts +7 -0
- package/dist/capture.js +10 -1
- package/dist/dashboard.d.ts +14 -0
- package/dist/dashboard.js +8 -0
- package/dist/db.js +45 -0
- package/dist/distill-queue.d.ts +203 -0
- package/dist/distill-queue.js +440 -0
- package/dist/health.d.ts +13 -1
- package/dist/health.js +6 -1
- package/dist/hosted-boot.d.ts +1 -1
- package/dist/localhost-bypass.js +1 -1
- package/dist/mcp-server.js +86 -5
- package/dist/nightly.js +87 -0
- package/dist/nofit.d.ts +1 -1
- package/dist/nofit.js +1 -1
- package/dist/schema-prototypes.d.ts +1 -1
- package/dist/schema-prototypes.js +1 -1
- package/dist/status.d.ts +11 -0
- package/dist/status.js +25 -0
- package/dist/types.d.ts +20 -0
- package/hermes-plugin/hicortex/README.md +13 -6
- package/hermes-plugin/hicortex/__init__.py +7 -0
- package/hermes-plugin/hicortex/client.py +64 -2
- package/hermes-plugin/hicortex/plugin.yaml +1 -1
- package/hermes-plugin/hicortex/provider.py +1 -1
- package/package.json +1 -1
- package/server.json +2 -2
package/assets/dashboard.html
CHANGED
|
@@ -865,6 +865,15 @@
|
|
|
865
865
|
/* #423 phase 3: the capture-pause badge — the same amber family as the
|
|
866
866
|
held badge (an operator state, not an error) */
|
|
867
867
|
.cap-paused{ color:var(--amber); margin-left:5px; font-size:9px; letter-spacing:.08em; }
|
|
868
|
+
/* #530 follow-up: the distill-inbox row — stale badge + bar fill in the
|
|
869
|
+
amber family once the oldest queued item passes the payload's ECHOED warn
|
|
870
|
+
age. [hidden] must win over .cap-row2's flex (author display beats the
|
|
871
|
+
UA's hidden — the #billing-banner idiom) so a pre-#529 server shows
|
|
872
|
+
nothing, theme tokens only */
|
|
873
|
+
.cap-qrow[hidden]{ display:none; }
|
|
874
|
+
.cap-stale{ color:var(--amber); margin-left:5px; }
|
|
875
|
+
.cap-qrow .fill.warn{ background:var(--amber); }
|
|
876
|
+
.cap-qrow.stale .cap-dot{ background:var(--amber); }
|
|
868
877
|
|
|
869
878
|
/* ================= the memory dropdown — opens UNDER the left search ================= */
|
|
870
879
|
#panel{
|
|
@@ -1277,6 +1286,11 @@
|
|
|
1277
1286
|
<div class="brow">
|
|
1278
1287
|
<div class="bseg" id="g-capture">
|
|
1279
1288
|
<div class="gh"><span class="t">Capture health</span><span class="sub" id="cap-sub">memory counts</span></div>
|
|
1289
|
+
<!-- #530 follow-up: the distill inbox — the drain-side counterpart of
|
|
1290
|
+
this card's delivery-side health. Pinned above the scrollable
|
|
1291
|
+
#cap-rows; hidden until buildCapture's payload guard unhides it
|
|
1292
|
+
(pre-#529 servers omit the block). -->
|
|
1293
|
+
<div class="cap-row2 cap-qrow" id="cap-queue" hidden></div>
|
|
1280
1294
|
<div id="cap-rows"></div>
|
|
1281
1295
|
</div>
|
|
1282
1296
|
<div class="bseg" id="g-roll">
|
|
@@ -3766,6 +3780,31 @@ function agoLabel(iso){
|
|
|
3766
3780
|
function buildCapture(){
|
|
3767
3781
|
const host=el('cap-rows');
|
|
3768
3782
|
host.innerHTML='';
|
|
3783
|
+
/* #530 follow-up: the distill-inbox row — rendered FIRST because every
|
|
3784
|
+
render branch below returns early (a call after them would only run in
|
|
3785
|
+
the degrade). The absent-block guard keeps it hidden on a pre-#529
|
|
3786
|
+
server; the warn threshold is always the payload's echo (#476
|
|
3787
|
+
degrade-literal precedent), never a page literal. */
|
|
3788
|
+
const qrow=el('cap-queue');
|
|
3789
|
+
qrow.innerHTML='';
|
|
3790
|
+
qrow.className='cap-row2 cap-qrow';
|
|
3791
|
+
const q=D&&D.distill_queue?D.distill_queue:null;
|
|
3792
|
+
qrow.hidden=!q;
|
|
3793
|
+
if(q){
|
|
3794
|
+
const thr=q.warn_age_hours||24;
|
|
3795
|
+
const oldest=(typeof q.oldest_age_hours==='number')?q.oldest_age_hours:null;
|
|
3796
|
+
const stale=oldest!==null&&oldest>thr;
|
|
3797
|
+
if(stale) qrow.classList.add('stale');
|
|
3798
|
+
qrow.title='Distill inbox — deliveries are stored durably at POST time and distilled by the scheduled nightly run before consolidation'
|
|
3799
|
+
+(stale?' — the oldest item is '+oldest+' h old (warn age '+thr+' h): the scheduled drain is behind':'');
|
|
3800
|
+
qrow.innerHTML='<span class="cap-dot"></span>'
|
|
3801
|
+
+'<span class="hn">Distill inbox</span>'
|
|
3802
|
+
+'<span class="cap-bar"><span class="fill'+(stale?' warn':'')+'" style="width:'+(oldest===null?0:Math.min(1,oldest/thr)*100)+'%"></span></span>'
|
|
3803
|
+
+'<span class="st"><b>'+fmt(q.depth||0)+'</b> queued'
|
|
3804
|
+
+(oldest===null?' · inbox empty':' · oldest '+(oldest>=48?(oldest/24).toFixed(1)+'d':oldest.toFixed(1)+'h'))
|
|
3805
|
+
+(stale?'<span class="cap-stale" title="oldest queued item past the '+thr+' h warn age — the scheduled drain is behind">⚠ stale</span>':'')
|
|
3806
|
+
+'</span>';
|
|
3807
|
+
}
|
|
3769
3808
|
const sub=el('cap-sub');
|
|
3770
3809
|
const ch=D&&D.capture_health?D.capture_health:null;
|
|
3771
3810
|
/* fix round 7: the rolling-window view — the card shows the normal 30 day
|
package/dist/calibration.d.ts
CHANGED
|
@@ -416,3 +416,34 @@ export declare function resolveOllamaFlushEvery(): number;
|
|
|
416
416
|
* HICORTEX_OLLAMA_FLUSH_WAIT_MS wins; invalid warns and keeps
|
|
417
417
|
* OLLAMA_FLUSH_WAIT_MS. */
|
|
418
418
|
export declare function resolveOllamaFlushWaitMs(): number;
|
|
419
|
+
/**
|
|
420
|
+
* Re-check interval (ms) while the drain's yield signal reports the LLM
|
|
421
|
+
* endpoint busy (#529 owner ruling 05.10.2026: interactive use always
|
|
422
|
+
* yields). The drain waits one interval between probe re-checks instead of
|
|
423
|
+
* polling in a tight loop; 30 s is short enough that the run resumes within
|
|
424
|
+
* half a minute of the owner going idle, long enough not to hammer the
|
|
425
|
+
* signal endpoint while a long interactive session runs.
|
|
426
|
+
*/
|
|
427
|
+
export declare const DRAIN_YIELD_POLL_MS = 30000;
|
|
428
|
+
/**
|
|
429
|
+
* Per-item cap (ms) on yield waits (#529). The run deadline is the primary
|
|
430
|
+
* bound; this cap is the fail-open backstop so a misconfigured signal that
|
|
431
|
+
* always answers "busy" cannot stall one item past half an hour — the item
|
|
432
|
+
* proceeds and the next between-items checks decide again. 30 min matches
|
|
433
|
+
* the order of a long local-model generation, the longest legitimate busy.
|
|
434
|
+
*/
|
|
435
|
+
export declare const DRAIN_YIELD_WAIT_CAP_MS: number;
|
|
436
|
+
/**
|
|
437
|
+
* Timeout (ms) for ONE yield-probe GET (#529). Deliberately short: the probe
|
|
438
|
+
* sits between drain items, and an endpoint that answers busy-signals at all
|
|
439
|
+
* answers them fast — anything slower reads as unreachable (fail-open).
|
|
440
|
+
*/
|
|
441
|
+
export declare const DRAIN_YIELD_PROBE_TIMEOUT_MS = 3000;
|
|
442
|
+
/**
|
|
443
|
+
* Oldest-item age (hours) above which the distill inbox warns in
|
|
444
|
+
* `hicortex status` and on the dashboard (#529). 24 h ≈ two missed scheduled
|
|
445
|
+
* runs on the default 2×/day consolidation grid — items older than that are
|
|
446
|
+
* not "waiting for tonight" anymore, they are stuck, and the operator should
|
|
447
|
+
* look at the drain's last outcome.
|
|
448
|
+
*/
|
|
449
|
+
export declare const DRAIN_QUEUE_WARN_AGE_HOURS = 24;
|
package/dist/calibration.js
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
*/
|
|
26
26
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
27
27
|
exports.STAGE_BELIEF_STRENGTH = exports.STAGE_FADING_STRENGTH = exports.STAGE_FADING_DAYS = exports.WEAK_PRIMARY_FLOOR = exports.CORRECTION_REWRITE_MIN_CONFIDENCE = exports.CORRECTION_MIN_SIMILARITY = exports.DEDUP_AUTO_MERGE_THRESHOLD = exports.IMPORTANCE_SETTLE_WINDOW_DAYS = exports.IMPORTANCE_CEILING = exports.BM25_WEIGHT_DOMAIN = exports.BM25_WEIGHT_PROJECT = exports.BM25_WEIGHT_BODY = exports.BOTH_CHANNEL_BOOST = exports.RRF_VECTOR_WEIGHT = exports.RRF_FTS_WEIGHT = exports.RRF_COMPOSITE_WEIGHT = exports.RRF_K = exports.SCOPE_AFFINITY_WEIGHT = exports.SUPERSEDED_DEMOTION = exports.RECENCY_HEAD_DAYS = exports.RECENCY_HEAD_WEIGHT = exports.RECENCY_HOURLY_DECAY = exports.SCORE_RECENCY_WEIGHT = exports.CONNECTIONS_SATURATION_DEGREE = exports.SCORE_CONNECTIONS_WEIGHT = exports.SCORE_STRENGTH_WEIGHT = exports.SCORE_SIMILARITY_WEIGHT = exports.VOLATILE_GATE_MAX_CHARS = exports.VOLATILE_STATUS_FILTER = exports.MEMORY_PRECISION_PROMPT_EXCERPT_CHARS = exports.MEMORY_PRECISION_DIVERGENCE_MIN_SHOWN = exports.MEMORY_PRECISION_REDUNDANT_ABOVE = exports.MEMORY_PRECISION_WINDOW_DAYS = exports.RECALL_USES_AXIS_MAX = exports.RECALL_USES_NORMAL_MAX = exports.RECALL_USES_LOW_MAX = exports.RECALL_RESHOW_TURNS = exports.NOVELTY_FLOOR_SLOTS = exports.RECALL_TITLE_CHARS = exports.RECALL_MIN_PROMPT_CHARS = exports.RECALL_MAX_ITEMS = exports.RECALL_MIN_SIMILARITY = exports.SESSION_INTENT_WEIGHT = exports.COLD_EXPOSURE_SLOTS = exports.RECENT_WINDOW_DAYS = exports.RECENT_LIMIT = exports.SEARCH_LIMIT = exports.PROMOTION_STRENGTH_FLOOR = exports.PROMOTION_RATE = exports.DECAY_HALF_LIFE_DAYS = void 0;
|
|
28
|
-
exports.DIAGNOSTIC_ENV_TIER = exports.OLLAMA_FLUSH_WAIT_MS = exports.OLLAMA_FLUSH_EVERY = exports.NUM_CTX = exports.DEFAULT_FIRST_RUN_LOOKBACK_DAYS = exports.ENRICH_STRENGTH_DELTA = exports.STAGE_TRUTH_STRENGTH = void 0;
|
|
28
|
+
exports.DRAIN_QUEUE_WARN_AGE_HOURS = exports.DRAIN_YIELD_PROBE_TIMEOUT_MS = exports.DRAIN_YIELD_WAIT_CAP_MS = exports.DRAIN_YIELD_POLL_MS = exports.DIAGNOSTIC_ENV_TIER = exports.OLLAMA_FLUSH_WAIT_MS = exports.OLLAMA_FLUSH_EVERY = exports.NUM_CTX = exports.DEFAULT_FIRST_RUN_LOOKBACK_DAYS = exports.ENRICH_STRENGTH_DELTA = exports.STAGE_TRUTH_STRENGTH = void 0;
|
|
29
29
|
exports.resolveNumCtx = resolveNumCtx;
|
|
30
30
|
exports.resolveOllamaFlushEvery = resolveOllamaFlushEvery;
|
|
31
31
|
exports.resolveOllamaFlushWaitMs = resolveOllamaFlushWaitMs;
|
|
@@ -514,3 +514,40 @@ function resolveOllamaFlushWaitMs() {
|
|
|
514
514
|
console.warn(`[hicortex] env HICORTEX_OLLAMA_FLUSH_WAIT_MS=${JSON.stringify(raw)} is not a positive finite number — using default ${exports.OLLAMA_FLUSH_WAIT_MS}.`);
|
|
515
515
|
return exports.OLLAMA_FLUSH_WAIT_MS;
|
|
516
516
|
}
|
|
517
|
+
// ---------------------------------------------------------------------------
|
|
518
|
+
// Distill inbox / drain family (#529) — delivery-compute separation. Delivery
|
|
519
|
+
// (cheap, must never lose data) stores segments durably; compute (expensive,
|
|
520
|
+
// scheduled) happens only inside the nightly's drain stage. These constants
|
|
521
|
+
// shape the drain's interactive-yield seam and the inbox's visibility signal.
|
|
522
|
+
// ---------------------------------------------------------------------------
|
|
523
|
+
/**
|
|
524
|
+
* Re-check interval (ms) while the drain's yield signal reports the LLM
|
|
525
|
+
* endpoint busy (#529 owner ruling 05.10.2026: interactive use always
|
|
526
|
+
* yields). The drain waits one interval between probe re-checks instead of
|
|
527
|
+
* polling in a tight loop; 30 s is short enough that the run resumes within
|
|
528
|
+
* half a minute of the owner going idle, long enough not to hammer the
|
|
529
|
+
* signal endpoint while a long interactive session runs.
|
|
530
|
+
*/
|
|
531
|
+
exports.DRAIN_YIELD_POLL_MS = 30_000;
|
|
532
|
+
/**
|
|
533
|
+
* Per-item cap (ms) on yield waits (#529). The run deadline is the primary
|
|
534
|
+
* bound; this cap is the fail-open backstop so a misconfigured signal that
|
|
535
|
+
* always answers "busy" cannot stall one item past half an hour — the item
|
|
536
|
+
* proceeds and the next between-items checks decide again. 30 min matches
|
|
537
|
+
* the order of a long local-model generation, the longest legitimate busy.
|
|
538
|
+
*/
|
|
539
|
+
exports.DRAIN_YIELD_WAIT_CAP_MS = 30 * 60_000;
|
|
540
|
+
/**
|
|
541
|
+
* Timeout (ms) for ONE yield-probe GET (#529). Deliberately short: the probe
|
|
542
|
+
* sits between drain items, and an endpoint that answers busy-signals at all
|
|
543
|
+
* answers them fast — anything slower reads as unreachable (fail-open).
|
|
544
|
+
*/
|
|
545
|
+
exports.DRAIN_YIELD_PROBE_TIMEOUT_MS = 3_000;
|
|
546
|
+
/**
|
|
547
|
+
* Oldest-item age (hours) above which the distill inbox warns in
|
|
548
|
+
* `hicortex status` and on the dashboard (#529). 24 h ≈ two missed scheduled
|
|
549
|
+
* runs on the default 2×/day consolidation grid — items older than that are
|
|
550
|
+
* not "waiting for tonight" anymore, they are stuck, and the operator should
|
|
551
|
+
* look at the drain's last outcome.
|
|
552
|
+
*/
|
|
553
|
+
exports.DRAIN_QUEUE_WARN_AGE_HOURS = 24;
|
package/dist/capture.d.ts
CHANGED
|
@@ -90,6 +90,13 @@ export interface PostResult {
|
|
|
90
90
|
completion: number;
|
|
91
91
|
total: number;
|
|
92
92
|
};
|
|
93
|
+
/**
|
|
94
|
+
* #529: the 201 confirmed durable QUEUEING, not distillation — a queue-mode
|
|
95
|
+
* server stores the segment and the nightly's drain distills it in the
|
|
96
|
+
* scheduled run. Contract-wise identical to any other 201 (the cursor
|
|
97
|
+
* advances); the capture loop only swaps its log line on it.
|
|
98
|
+
*/
|
|
99
|
+
queued?: boolean;
|
|
93
100
|
}
|
|
94
101
|
export type PostFn = (body: DistillBody) => Promise<PostResult>;
|
|
95
102
|
export interface CaptureOptions {
|
package/dist/capture.js
CHANGED
|
@@ -301,7 +301,16 @@ async function captureBatches(batches, opts) {
|
|
|
301
301
|
}
|
|
302
302
|
if (advancesBoundary)
|
|
303
303
|
lastConfirmedEnd = seg.segEnd;
|
|
304
|
-
|
|
304
|
+
// #529: a queue-mode 201 means "durably stored, distilled on the
|
|
305
|
+
// server's next scheduled run" — same confirmation, different log
|
|
306
|
+
// line, so a nightly log never reads "0 memories" as a failure when
|
|
307
|
+
// the server simply queued the segment.
|
|
308
|
+
if (result.queued) {
|
|
309
|
+
console.log(`[hicortex] Queued on server (segment ${body.segment_id}) — distilled on the next scheduled run`);
|
|
310
|
+
}
|
|
311
|
+
else {
|
|
312
|
+
console.log(`[hicortex] → ${result.distilled ?? 0} memories (segment ${body.segment_id})`);
|
|
313
|
+
}
|
|
305
314
|
for (const d of result.dropped ?? []) {
|
|
306
315
|
// #489: the response array carries BOTH gate kinds (substance +
|
|
307
316
|
// volatility) — the server's own log names the exact gate; this
|
package/dist/dashboard.d.ts
CHANGED
|
@@ -268,6 +268,20 @@ export interface DashboardData {
|
|
|
268
268
|
* pre-#476 recall-card rendering when the block is absent.
|
|
269
269
|
*/
|
|
270
270
|
memory_precision: MemoryPrecision;
|
|
271
|
+
/**
|
|
272
|
+
* #529 — the distill inbox: live queue depth + oldest-item age (the
|
|
273
|
+
* drain-side health signal; delivery-side health is capture_health
|
|
274
|
+
* above). ALWAYS present; `oldest_age_hours` null when empty. The warn
|
|
275
|
+
* threshold is ECHOED like every threshold (the page renders its own
|
|
276
|
+
* stale badge from it, never hardcodes 24). Live view only — no snapshot
|
|
277
|
+
* history (depth is ~0 at snapshot time by construction). A pre-#529
|
|
278
|
+
* server omits the block; the page guards.
|
|
279
|
+
*/
|
|
280
|
+
distill_queue: {
|
|
281
|
+
depth: number;
|
|
282
|
+
oldest_age_hours: number | null;
|
|
283
|
+
warn_age_hours: number;
|
|
284
|
+
};
|
|
271
285
|
digest: {
|
|
272
286
|
date: string | null;
|
|
273
287
|
run_at: string | null;
|
package/dist/dashboard.js
CHANGED
|
@@ -46,6 +46,7 @@ const config_read_js_1 = require("./config-read.js");
|
|
|
46
46
|
const consolidate_js_1 = require("./consolidate.js");
|
|
47
47
|
const capture_health_js_1 = require("./capture-health.js");
|
|
48
48
|
const recall_precision_js_1 = require("./recall-precision.js");
|
|
49
|
+
const distill_queue_js_1 = require("./distill-queue.js");
|
|
49
50
|
const capture_pause_js_1 = require("./capture-pause.js");
|
|
50
51
|
const state_js_1 = require("./state.js");
|
|
51
52
|
const retrieval_js_1 = require("./retrieval.js");
|
|
@@ -585,6 +586,13 @@ function handleDashboardData(db, query, config) {
|
|
|
585
586
|
shown_sum: live.adoption?.shown_sum ?? 0,
|
|
586
587
|
used_sum: live.adoption?.used_sum ?? 0,
|
|
587
588
|
}),
|
|
589
|
+
// #529: live inbox state — depth + oldest age + the echoed warn
|
|
590
|
+
// threshold. Live only (no series): depth at snapshot time is ~0 by
|
|
591
|
+
// construction, so history would be a flat line.
|
|
592
|
+
distill_queue: {
|
|
593
|
+
...(0, distill_queue_js_1.readQueueStats)(db),
|
|
594
|
+
warn_age_hours: calibration_js_1.DRAIN_QUEUE_WARN_AGE_HOURS,
|
|
595
|
+
},
|
|
588
596
|
// #423 phase 3: pauses + presence in one block — one source of truth
|
|
589
597
|
// for the rail's dots, toggles and the capture card's PAUSED badges.
|
|
590
598
|
fleet: {
|
package/dist/db.js
CHANGED
|
@@ -763,6 +763,51 @@ const MIGRATIONS = [
|
|
|
763
763
|
`);
|
|
764
764
|
},
|
|
765
765
|
},
|
|
766
|
+
{
|
|
767
|
+
version: 24,
|
|
768
|
+
name: "distill_queue",
|
|
769
|
+
up: (db) => {
|
|
770
|
+
// #529 — the durable distill inbox. POST /distill in queue mode stores
|
|
771
|
+
// the REDACTED segment here and answers 201 immediately (no LLM call);
|
|
772
|
+
// the nightly drain distills rows oldest-first and deletes each row in
|
|
773
|
+
// the same transaction as its memory inserts. One table in the shared
|
|
774
|
+
// SQLite DB — one store, same backup and tooling as the corpus (owner
|
|
775
|
+
// decision; not files). The row carries every /distill wire field needed
|
|
776
|
+
// to re-distill later (attribution, project, session_date, privacy) so
|
|
777
|
+
// the drain reproduces the sync path's insert exactly.
|
|
778
|
+
//
|
|
779
|
+
// UNIQUE(session_id, segment_id) is the delivery-time idempotency key: a
|
|
780
|
+
// queued-but-undistilled segment re-POST answers 200 skipped (the
|
|
781
|
+
// handler's prechecks consult this table), and a failed drain attempt
|
|
782
|
+
// never duplicates on retry. Legacy no-segment posts normalize
|
|
783
|
+
// segment_id ''. session_id stays NULL for posts that carry no session
|
|
784
|
+
// key — SQLite unique indexes treat NULLs as distinct, so those rows
|
|
785
|
+
// (which have no dedup key by construction, same as the sync path's
|
|
786
|
+
// sourceSession-undefined inserts) each enqueue separately.
|
|
787
|
+
// Idempotent: IF NOT EXISTS (the v18/v21 pattern — plain CREATE).
|
|
788
|
+
db.exec(`
|
|
789
|
+
CREATE TABLE IF NOT EXISTS distill_queue (
|
|
790
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
791
|
+
session_id TEXT,
|
|
792
|
+
segment_id TEXT NOT NULL DEFAULT '',
|
|
793
|
+
source_agent TEXT,
|
|
794
|
+
source_agent_id TEXT,
|
|
795
|
+
source_domain TEXT,
|
|
796
|
+
source_machine TEXT,
|
|
797
|
+
project TEXT,
|
|
798
|
+
session_date TEXT,
|
|
799
|
+
privacy TEXT,
|
|
800
|
+
text TEXT NOT NULL,
|
|
801
|
+
arrived_at TEXT NOT NULL
|
|
802
|
+
)
|
|
803
|
+
`);
|
|
804
|
+
db.exec(`
|
|
805
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_distill_queue_session_segment
|
|
806
|
+
ON distill_queue(session_id, segment_id)
|
|
807
|
+
`);
|
|
808
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_distill_queue_arrived ON distill_queue(arrived_at)");
|
|
809
|
+
},
|
|
810
|
+
},
|
|
766
811
|
];
|
|
767
812
|
/**
|
|
768
813
|
* Run all pending migrations against the database.
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The durable distill inbox + its drain (#529).
|
|
3
|
+
*
|
|
4
|
+
* Delivery and compute are separated: `POST /distill` in queue mode stores the
|
|
5
|
+
* REDACTED segment in the `distill_queue` table (migration v24) and answers
|
|
6
|
+
* 201 immediately — no LLM call, no probe, no chunk-size detection, no token
|
|
7
|
+
* gate (all of those move to the drain). The nightly's drain stage then
|
|
8
|
+
* distills queued items oldest-first, round-robin per client, before
|
|
9
|
+
* consolidation, so all distill LLM traffic happens inside the owner's
|
|
10
|
+
* scheduled runs instead of on the clients' drifting capture grids.
|
|
11
|
+
*
|
|
12
|
+
* This module holds BOTH halves as pure units:
|
|
13
|
+
* - the inbox STORE (enqueue, dedup mirrors, depth/age stats, delete) — real
|
|
14
|
+
* functions the /distill handler calls directly;
|
|
15
|
+
* - the DRAIN — dependency-injected (LlmClient-like object, embed fn,
|
|
16
|
+
* yield-probe, token-budget gate, run deadline, sleep) so the whole stage
|
|
17
|
+
* is unit-testable with no HTTP and no real LLM. Every LLM call goes
|
|
18
|
+
* through `distillSession` (distiller.ts) on the injected client — never a
|
|
19
|
+
* bespoke fetch — so the drain inherits the #337 undici dispatcher, #355
|
|
20
|
+
* single-flight, the retry ladder, and the circuit breaker by construction.
|
|
21
|
+
*
|
|
22
|
+
* Ordering contract (spec #529): oldest-first (rowid = arrival order),
|
|
23
|
+
* round-robin per client key (`source_machine` + `source_agent`) so one
|
|
24
|
+
* client's giant backlog cannot starve the others, and a session's segments
|
|
25
|
+
* in arrival order (natural — a client POSTs its segments sequentially, so
|
|
26
|
+
* ascending rowid within a client is per-session ascending).
|
|
27
|
+
*
|
|
28
|
+
* Checkpointing contract: per item, ALL memory inserts and the inbox-row
|
|
29
|
+
* delete commit in ONE transaction. An interrupted run (crash, deadline,
|
|
30
|
+
* endpoint outage) leaves processed items done and the rest queued — the
|
|
31
|
+
* segment-exact dedup keys (`<sid>#<segment_id>#<i>`) make the retry
|
|
32
|
+
* idempotent, dup-over-loss.
|
|
33
|
+
*/
|
|
34
|
+
import type Database from "better-sqlite3";
|
|
35
|
+
import { detectChunkSize } from "./distiller.js";
|
|
36
|
+
import type { LlmUsage, LlmConfig } from "./llm.js";
|
|
37
|
+
import type { RunDeadline } from "./run-deadline.js";
|
|
38
|
+
/** One queued delivery, as stored in `distill_queue`. */
|
|
39
|
+
export interface DistillQueueRow {
|
|
40
|
+
id: number;
|
|
41
|
+
session_id: string | null;
|
|
42
|
+
segment_id: string;
|
|
43
|
+
source_agent: string | null;
|
|
44
|
+
source_agent_id: string | null;
|
|
45
|
+
source_domain: string | null;
|
|
46
|
+
source_machine: string | null;
|
|
47
|
+
project: string | null;
|
|
48
|
+
session_date: string | null;
|
|
49
|
+
privacy: string | null;
|
|
50
|
+
text: string;
|
|
51
|
+
arrived_at: string;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* The wire fields of one /distill POST, as the queue-mode handler resolved
|
|
55
|
+
* them (post-redaction). `session_date` defaults to today AT ENQUEUE TIME —
|
|
56
|
+
* the sync path stamps `new Date()` at delivery, and the queue must not let a
|
|
57
|
+
* segment that waited ~12 h in the inbox drift to the drain-day's date.
|
|
58
|
+
*/
|
|
59
|
+
export interface EnqueueDistillInput {
|
|
60
|
+
/** REDACTED conversation text (the handler redacts before enqueueing). */
|
|
61
|
+
text: string;
|
|
62
|
+
source_agent?: unknown;
|
|
63
|
+
source_agent_id?: unknown;
|
|
64
|
+
source_domain?: unknown;
|
|
65
|
+
source_machine?: unknown;
|
|
66
|
+
project?: unknown;
|
|
67
|
+
session_id?: unknown;
|
|
68
|
+
segment_id?: unknown;
|
|
69
|
+
session_date?: unknown;
|
|
70
|
+
privacy?: unknown;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Insert one delivery into the inbox. Idempotent on the UNIQUE(session_id,
|
|
74
|
+
* segment_id) index — a re-POST of the SAME key (only reachable through a
|
|
75
|
+
* race: the handler's prechecks already answered the sequential re-POST with
|
|
76
|
+
* a 200 skip) reports `{ duplicate: true }` and inserts nothing. Posts with
|
|
77
|
+
* no session_id have no dedup key by construction and always insert (SQLite
|
|
78
|
+
* unique indexes treat NULLs as distinct).
|
|
79
|
+
*/
|
|
80
|
+
export declare function enqueueDistill(db: Database.Database, input: EnqueueDistillInput): {
|
|
81
|
+
duplicate: boolean;
|
|
82
|
+
};
|
|
83
|
+
/**
|
|
84
|
+
* Inbox mirror of the segment-exact delivery precheck: how many QUEUED rows
|
|
85
|
+
* match this exact `session_id` + `segment_id`. The handler adds this to
|
|
86
|
+
* `countExistingSegment` so a queued-but-undistilled segment's re-POST
|
|
87
|
+
* answers 200 skipped — the client's cursor advances and the segment is
|
|
88
|
+
* never queued twice.
|
|
89
|
+
*/
|
|
90
|
+
export declare function countQueuedSegment(db: Database.Database, sessionId: string, segmentId: string): number;
|
|
91
|
+
/**
|
|
92
|
+
* Inbox mirror of the legacy session-level precheck: how many QUEUED rows
|
|
93
|
+
* belong to this session (whole-session OR any segment) — a legacy
|
|
94
|
+
* whole-session re-POST while any part of the session sits queued skips.
|
|
95
|
+
*/
|
|
96
|
+
export declare function countQueuedSession(db: Database.Database, sessionId: string): number;
|
|
97
|
+
/** Queue depth (rows pending distillation). */
|
|
98
|
+
export declare function queueDepth(db: Database.Database): number;
|
|
99
|
+
/** Age (ms) of the oldest queued item; null when the inbox is empty. */
|
|
100
|
+
export declare function oldestQueueAgeMs(db: Database.Database, now?: number): number | null;
|
|
101
|
+
/** The visibility shape shared by `hicortex status`, /health/detail, and the
|
|
102
|
+
* dashboard payload: depth + oldest-item age in hours (null = empty). */
|
|
103
|
+
export interface QueueStats {
|
|
104
|
+
depth: number;
|
|
105
|
+
oldest_age_hours: number | null;
|
|
106
|
+
}
|
|
107
|
+
export declare function readQueueStats(db: Database.Database, now?: number): QueueStats;
|
|
108
|
+
/**
|
|
109
|
+
* #529 bootstrap carve-out: a FRESH brain (empty memories table AND empty
|
|
110
|
+
* inbox) distills synchronously, exactly as pre-#529 — the first-ever
|
|
111
|
+
* delivery must not wait ~12 h for the next scheduled run. Anything else
|
|
112
|
+
* (memories exist OR rows are already queued) is ordinary queue mode.
|
|
113
|
+
*/
|
|
114
|
+
export declare function isFreshBrain(db: Database.Database): boolean;
|
|
115
|
+
/**
|
|
116
|
+
* The ONE branch condition into the sync flow (owner ruling #529: kill
|
|
117
|
+
* switch and bootstrap carve-out share it — no third path):
|
|
118
|
+
*
|
|
119
|
+
* queueMode = distillQueue enabled && NOT a fresh brain
|
|
120
|
+
*
|
|
121
|
+
* `distillQueueEnabled` is the already-strict-boolean-resolved kill switch
|
|
122
|
+
* (`readStrictBoolean(config, "distillQueue") !== false`, default ON); false
|
|
123
|
+
* lands here always → today's synchronous distill byte-for-byte.
|
|
124
|
+
*/
|
|
125
|
+
export declare function resolveQueueMode(db: Database.Database, distillQueueEnabled: boolean): boolean;
|
|
126
|
+
/**
|
|
127
|
+
* The structural minimum the drain needs from an LLM client: `complete()` is
|
|
128
|
+
* the ONE surface (#405). Real `LlmClient` satisfies it; tests inject a stub.
|
|
129
|
+
* (LlmClient has private fields, so it cannot be the parameter type itself —
|
|
130
|
+
* distillSession only ever calls `complete()` on the value we pass.)
|
|
131
|
+
*/
|
|
132
|
+
export interface DrainLlm {
|
|
133
|
+
complete(prompt: string): Promise<{
|
|
134
|
+
text: string;
|
|
135
|
+
usage?: LlmUsage;
|
|
136
|
+
}>;
|
|
137
|
+
}
|
|
138
|
+
/** One yield-probe outcome (the HTTP shape is the caller's concern — nightly). */
|
|
139
|
+
export type YieldStatus = "busy" | "idle" | "unreachable";
|
|
140
|
+
export interface DrainOptions {
|
|
141
|
+
/** The LLM client all distill calls go through (null = no LLM configured). */
|
|
142
|
+
llm: DrainLlm | null;
|
|
143
|
+
/** Its config — drives the cached chunk-size detection, like the handler. */
|
|
144
|
+
llmConfig: LlmConfig | null;
|
|
145
|
+
/** Embed fn for the distilled entries (the nightly's embedder). */
|
|
146
|
+
embed: (text: string) => Promise<Float32Array>;
|
|
147
|
+
/**
|
|
148
|
+
* #5 token-budget gate, checked BETWEEN items (and once before the first
|
|
149
|
+
* item — zero LLM calls when already over the monthly cap). Injected so the
|
|
150
|
+
* drain unit never touches state.json; the nightly passes
|
|
151
|
+
* `() => isTokenBudgetExceeded(stateDir)`.
|
|
152
|
+
*/
|
|
153
|
+
budgetExceeded?: () => boolean;
|
|
154
|
+
/** The ONE run deadline (#405) — `hit("drain")` between items. */
|
|
155
|
+
deadline?: RunDeadline;
|
|
156
|
+
/**
|
|
157
|
+
* #529 yield signal: the configured `drainYieldUrl`. Absent/unset ⇒ never
|
|
158
|
+
* wait. Provided with `probeBusy`, the drain waits out "busy" answers
|
|
159
|
+
* between items (bounded by the deadline + the wait cap).
|
|
160
|
+
*/
|
|
161
|
+
yieldUrl?: string;
|
|
162
|
+
/** The probe itself (HTTP lives in nightly.ts; tests inject a fake). */
|
|
163
|
+
probeBusy?: (url: string) => Promise<YieldStatus>;
|
|
164
|
+
/** Injectable clock-wait so tests exercise busy-loops instantly. */
|
|
165
|
+
sleep?: (ms: number) => Promise<void>;
|
|
166
|
+
/**
|
|
167
|
+
* Injectable chunk-size detector (tests pin it; production defaults to the
|
|
168
|
+
* real detectChunkSize, cached per endpoint like the /distill handler).
|
|
169
|
+
*/
|
|
170
|
+
detectChunkSize?: typeof detectChunkSize;
|
|
171
|
+
}
|
|
172
|
+
export interface DrainReport {
|
|
173
|
+
/**
|
|
174
|
+
* "empty" (nothing queued — the common case right after a clean run),
|
|
175
|
+
* "completed" (every item done), "deferred" (run deadline fired; the rest
|
|
176
|
+
* stays queued for the next scheduled run), "budget" (monthly token cap;
|
|
177
|
+
* zero LLM calls past the refusal), "no_llm" (no LLM configured — items
|
|
178
|
+
* stay queued, consolidation reports its own no_llm), or "endpoint_down"
|
|
179
|
+
* (the first item's distillation threw after the ladder/breaker — never a
|
|
180
|
+
* tight retry loop; processed items stay done).
|
|
181
|
+
*/
|
|
182
|
+
outcome: "empty" | "completed" | "deferred" | "budget" | "no_llm" | "endpoint_down";
|
|
183
|
+
/** Items fully processed (distilled or duplicate-skipped) this run. */
|
|
184
|
+
processed: number;
|
|
185
|
+
/** Of those, items that were already stored and only had their row deleted. */
|
|
186
|
+
duplicates: number;
|
|
187
|
+
/** Memories inserted this run. */
|
|
188
|
+
memories: number;
|
|
189
|
+
/** Items still queued after this run. */
|
|
190
|
+
remaining: number;
|
|
191
|
+
/** Tokens metered this drain (incl. a failed item's partial usage). */
|
|
192
|
+
usage: {
|
|
193
|
+
prompt: number;
|
|
194
|
+
completion: number;
|
|
195
|
+
total: number;
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Drain the distill inbox. See the module doc for the ordering + checkpoint
|
|
200
|
+
* contracts. Pure w.r.t. its injected dependencies — no HTTP, no state.json
|
|
201
|
+
* reads (the nightly records the reported usage against the monthly meter).
|
|
202
|
+
*/
|
|
203
|
+
export declare function drainDistillQueue(db: Database.Database, opts: DrainOptions): Promise<DrainReport>;
|