@gamaze/hicortex 0.23.1 → 0.24.0

package/dist/eval/reflection-census.d.ts

@@ -1,19 +0,0 @@
-/**
- * Reflection census (#191 mechanical baseline) — lesson output volume over
- * time and the lesson/episode yield ratio. This is NOT a quality read
- * (sampling lessons for actionable-vs-noise is Phase-A grading work,
- * deferred per the issue's owner comment until recall is active) — just the
- * mechanical count.
- */
-import type Database from "better-sqlite3";
-export interface ReflectionCensus {
-    lessonsByDate: Array<{
-        date: string;
-        count: number;
-    }>;
-    totalLessons: number;
-    totalEpisodes: number;
-    /** lessons / episodes — null when there are no episodes to divide by. */
-    yieldRatio: number | null;
-}
-export declare function runReflectionCensus(db: Database.Database): ReflectionCensus;

package/dist/eval/reflection-census.js

@@ -1,25 +0,0 @@
-"use strict";
-/**
- * Reflection census (#191 mechanical baseline) — lesson output volume over
- * time and the lesson/episode yield ratio. This is NOT a quality read
- * (sampling lessons for actionable-vs-noise is Phase-A grading work,
- * deferred per the issue's owner comment until recall is active) — just the
- * mechanical count.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runReflectionCensus = runReflectionCensus;
-function runReflectionCensus(db) {
-    const lessonsByDate = db
-        .prepare(`SELECT date(created_at) AS date, COUNT(*) AS count
-       FROM memories WHERE memory_type = 'learnings'
-       GROUP BY date ORDER BY date`)
-        .all();
-    const totalLessons = db.prepare("SELECT COUNT(*) AS c FROM memories WHERE memory_type = 'learnings'").get().c;
-    const totalEpisodes = db.prepare("SELECT COUNT(*) AS c FROM memories WHERE memory_type = 'experience'").get().c;
-    return {
-        lessonsByDate,
-        totalLessons,
-        totalEpisodes,
-        yieldRatio: totalEpisodes > 0 ? totalLessons / totalEpisodes : null,
-    };
-}

package/dist/eval/relevance-eval.d.ts

@@ -1,178 +0,0 @@
-#!/usr/bin/env node
-/**
- * Real-query relevance + SNIPPET eval — recall QUALITY on real agent prompts.
- *
- * v2 (spec `specs/2026-08-02-relevance-eval.md`) extends the v1 selection-only
- * eval with the SNIPPET layer: v1 asked "did retrieve() surface the right
- * memories?" (judge sees up to 2000 chars). Production shows the agent only a
- * ~100-char one-liner (`recall-index.ts#memoryTitle`), so a memory can be
- * genuinely relevant while its rendered line is useless — v1 scored that as a
- * win. v2 grades BOTH: `full_verdict` (selection quality, judge sees full
- * content) and `line_verdict` (snippet quality, judge sees ONLY the rendered
- * production one-liner — imported from `recall-index.ts`, never reimplemented).
- *
- * v2 additions (spec §4, §5, §6, §6b) layered onto the v1 base (prompt
- * sampling, readonly snapshot handling, embed-once + neverCalledEmbed,
- * lenient JSON parse, distribution/CI reporting):
- *   1. Dual verdict per surfaced memory — TWO separate, blind judge calls.
- *   2. Snippet-length sweep (100/200/300/title+1st-sentence) on a fixed
- *      40-prompt subset (8 per source).
- *   3. Similarity-floor + retrieval-source analysis (near-free — logged, not
- *      re-judged).
- *   4. Token-cost estimate (char/4) per K and per snippet-length variant.
- *   5. ~20 rendered ACTUAL production blocks dumped into the report.
- *   6. Redundancy — one set-level judge call per (prompt × mode) over the
- *      production 6.
- *   7. Rate limiting + resumability (§6b, MANDATORY): serial calls,
- *      `--judge-delay-ms` (default 2000), exponential backoff with jitter on
- *      429/5xx/timeout (5→10→20→40→80s, max 5 retries, respects
- *      `Retry-After`), checkpoint-per-call to a `.jsonl` sidecar, `--resume`,
- *      progress logging, 10%-error-rate abort, `--max-calls` budget guard
- *      (default 900).
- *
- * Prompt corpus (spec §2, owner decision §11.1): EVEN split, 20 prompts per
- * source × 5 sources — Hermes (lenny, raider, nano) + CC (the DevOps
- * `infrastructure` project, the `aironic-marine` project). Saved to
- * `data/prompts.json`, stable/reused verbatim once a valid v2 set exists.
- *
- * Judge: GLM-5.2 via z.ai — the INSTRUMENT only. It never picks candidates;
- * retrieve() (LLM-free) does. A dedicated raw HTTP caller (NOT `LlmClient`) is
- * used here on purpose: `LlmClient.completeReflect` bakes in a
- * nightly-tolerant retry policy (30s/60s/120s, unlimited rate-limit patience)
- * that conflicts with §6b's specific real-time batch policy (5/10/20/40/80s +
- * jitter, 5 retries, a hard call budget). Implemented directly here rather
- * than adding a second retry mode to `llm.ts` (out of scope for this eval,
- * and another agent is concurrently working elsewhere in this repo).
- *
- * Honesty invariants (non-negotiable — mirror recall-sweep.ts + spec §7):
- *   - Snapshot opened READONLY via openSnapshot — never initDb.
- *   - noStrengthen: true on every retrieve() call.
- *   - Real bge-small-en-v1.5 embedder, embed-once + queryEmbedding reuse.
- *   - neverCalledEmbed self-check ABORTS the run if retrieve() ignores
- *     queryEmbedding (would invalidate every measured number).
- *   - GLM-5.2 is the JUDGE only, real prompts, no synthetic queries.
- *   - Production renderer (`formatIndexLine`/`memoryTitle`) imported from
- *     `recall-index.ts`, never reimplemented.
- *   - judge_error batches/units excluded from every denominator, reported
- *     separately.
- *
- * Run:
- *   npm run eval:relevance -- <snapshot.db> [prompts.json] [report.md] \
- *       [--judge-delay-ms=2000] [--max-calls=900] [--resume] \
- *       [--verdicts-json=path] [--verdicts-jsonl=path] \
- *       [--now=<ISO>] [--runs=N]
- *
- * #458 clock pin + judge variance protocol:
- *   - `--now=<ISO>` pins the clock the retrieve sweep scores against (the
- *     retrieve() `now` seam) — before/after runs become wall-clock-independent
- *     (default: live clock, the pre-#458 behavior).
- *   - `--runs=N` (default 1) runs judge phases 1+2 N times over the SAME
- *     selections (one retrieve sweep) and reports the run-to-run noise floor
- *     (median + spread + selection-identical flip counts, §15b). Phases 3+4
- *     stay single-shot; the shared --max-calls budget spans all runs and the
- *     default scales ×N unless --max-calls was passed explicitly.
- */
-type Verdict = "relevant" | "noise" | "stale";
-interface JudgeVerdict {
-    /** Q1..Q8 label — mapped back to the real memory id via the caller's
-     *  position-ordered topIds list. */
-    id: string;
-    verdict: Verdict;
-    reason: string;
-}
-type VerdictsOrError = JudgeVerdict[] | {
-    judgeError: string;
-};
-interface CliArgs {
-    positionals: string[];
-    judgeDelayMs: number;
-    maxCalls: number;
-    /** True when --max-calls was passed explicitly (#458) — an explicit budget
-     *  was sized by the caller for the whole invocation and must NOT be scaled
-     *  by --runs (see scaleMaxCalls). */
-    maxCallsExplicit: boolean;
-    resume: boolean;
-    /** Override the raw-verdicts JSON path (default VERDICTS_JSON_PATH) — added
-     *  2026-08-02 so a re-run against a NEW snapshot can never silently
-     *  overwrite a baseline's raw verdicts. */
-    verdictsJsonPath: string;
-    /** Override the checkpoint sidecar / raw-verdicts JSONL path (default
-     *  VERDICTS_JSONL_PATH) — same reason. */
-    verdictsJsonlPath: string;
-    /** #458: raw --now=<ISO> value — undefined = live clock. Validated in main
-     *  via parsePinnedNow (invalid → usage-style error, exit 1). */
-    nowIso: string | undefined;
-    /** #458: judge runs over the fixed selections (phases 1+2 only). Default 1
-     *  = the pre-#458 single-run behavior. */
-    runs: number;
-}
-/** Parsed and exported for tests (#458). Throws on an invalid --runs value
- *  (NaN or <1) — fail-explicit, never a silent default. */
-export declare function parseArgs(argv: string[]): CliArgs;
-/**
- * #458 — resolve the effective --max-calls budget. The ONE shared budget
- * spans ALL judge runs, so when --runs>1 and the caller did NOT pass
- * --max-calls explicitly, the default scales ×N (each run re-judges every
- * unit). An explicit budget always wins — the caller sized it for the whole
- * invocation. Pure, exported for tests.
- */
-export declare function scaleMaxCalls(runs: number, explicitMaxCalls: number | undefined, defaultMaxCalls: number): number;
-/** Median of a numeric sample (even count → mean of the two middle values).
- *  Empty input → NaN, the mean() convention above. Does not mutate input. */
-export declare function median(xs: number[]): number;
-/** One judged unit's verdicts for variance accounting (#458): `key` is the
- *  stable (promptIdx|mode) unit identity, identical across runs because the
- *  retrieve sweep runs ONCE. */
-export interface VarianceUnitVerdicts {
-    key: string;
-    verdicts: VerdictsOrError;
-}
-/** A headline metric measured once per run (e.g. precision@5 OFF). Values are
- *  in raw fractions; null = not computable that run (excluded from
- *  median/spread). */
-export interface VarianceMetricSeries {
-    label: string;
-    values: Array<number | null>;
-}
-export interface VarianceMetricStats extends VarianceMetricSeries {
-    median: number | null;
-    /** (max − min) × 100, in points — the measured run-to-run noise floor. */
-    spreadPts: number | null;
-}
-/** Pairwise flip counts between runs i and j (1-based run numbers). */
-export interface PairwiseFlips {
-    runA: number;
-    runB: number;
-    /** Verdict ROWS compared: (unit, Q-label) pairs present as parsed verdict
-     *  arrays in BOTH runs. judge_error units and labels the judge omitted in
-     *  one run are not comparable — excluded, never counted as flips. */
-    rowsCompared: number;
-    rowsFlipped: number;
-    unitsCompared: number;
-    /** A UNIT flips if any of its comparable rows differ. */
-    unitsFlipped: number;
-}
-export interface JudgeVariance {
-    runs: number;
-    metrics: VarianceMetricStats[];
-    flips: PairwiseFlips[];
-    maxRowsFlipped: number;
-    maxUnitsFlipped: number;
-    meanRowsFlipped: number;
-    meanUnitsFlipped: number;
-    /** Comparable rows of the first pair — selections are identical across
-     *  runs, so only judge errors shrink this (reported per run below). */
-    totalComparableRows: number;
-    /** judge_error UNITS per run (a failed unit yields no parsed rows; it is
-     *  excluded from every flip denominator). */
-    judgeErrorUnitsPerRun: number[];
-}
-/**
- * Compute the #458 judge-variance account over N runs of the same selections:
- * per-metric median + spread (the noise floor), and pairwise verdict flip
- * counts (rows = one Q-label judgment on one (prompt,mode) unit; units flip
- * if any row does). `perRunVerdicts[r]` is run r+1's units in a stable
- * order/identity; `metricSeries[s].values[r]` is run r+1's headline value.
- */
-export declare function computeJudgeVariance(perRunVerdicts: VarianceUnitVerdicts[][], metricSeries: VarianceMetricSeries[]): JudgeVariance;
-export {};