klyro 1.0.4 → 1.0.6

package/dist/cli/update.js

@@ -2,10 +2,12 @@
  * klyro update — check registry for newer version, cached 24h.
  * Env KLYRO_NO_UPDATE_CHECK=1 disables.
  *
- * Integrity: before recommending `npm i`, we verify the tarball's SRI hash
- * (sha512) against the registry's recorded `dist.integrity`. An install is
- * only recommended when the download hash matches, so a tampered CDN or
- * MITM registry response can't push a malicious binary to the operator.
+ * Integrity: before recommending `npm i`, we verify the tarball against
+ * BOTH the registry's SRI digest (sha512/sha256) AND the legacy sha1
+ * `dist.shasum` when present — a tampered CDN or MITM registry response
+ * must forge two independent digests to push a malicious binary.
+ * Downgrade protection: a registry `latest` that is not strictly newer
+ * than the running version (semver) is never recommended.
  */
 import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
@@ -24,6 +26,28 @@ function cachePath() {
     const home = os.homedir() || process.cwd();
     return path.join(home, '.klyro', 'update-cache.json');
 }
+/** Minimal semver compare for `x.y.z[-prerelease]`; null when unparseable. */
+export function compareSemver(a, b) {
+    const pa = /^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$/.exec(a.trim());
+    const pb = /^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$/.exec(b.trim());
+    if (!pa || !pb)
+        return null;
+    for (const i of [1, 2, 3]) {
+        const d = Number(pa[i]) - Number(pb[i]);
+        if (d !== 0)
+            return d < 0 ? -1 : 1;
+    }
+    const ra = pa[4] ?? '';
+    const rb = pb[4] ?? '';
+    if (ra === rb)
+        return 0;
+    // A prerelease is older than the release with the same core.
+    if (ra === '')
+        return 1;
+    if (rb === '')
+        return -1;
+    return ra < rb ? -1 : 1;
+}
 /** SRI string may carry multiple hashes parsable with `pick`; we accept sha512 or sha256. */
 function parseSRI(integrity) {
     if (!integrity)
@@ -49,7 +73,7 @@ async function fetchWithTimeout(url, timeoutMs = FETCH_TIMEOUT_MS) {
         clearTimeout(t);
     }
 }
-/** Download the tarball and confirm its hash equals the registry's SRI digest. */
+/** Download the tarball and confirm it matches the registry's SRI digest AND shasum. */
 async function verifyTarballIntegrity(dist) {
     const sri = parseSRI(dist.integrity);
     const tarball = dist.tarball;
@@ -58,7 +82,15 @@ async function verifyTarballIntegrity(dist) {
     const res = await fetchWithTimeout(tarball, TARBALL_TIMEOUT_MS);
     const buf = Buffer.from(await res.arrayBuffer());
     const actual = createHash(sri.algo).update(buf).digest('base64');
-    return actual === sri.digest;
+    if (actual !== sri.digest)
+        return false;
+    // Second independent digest: legacy sha1 shasum, when the registry sends one.
+    if (typeof dist.shasum === 'string' && /^[0-9a-f]{40}$/i.test(dist.shasum)) {
+        const sha1 = createHash('sha1').update(buf).digest('hex');
+        if (sha1.toLowerCase() !== dist.shasum.toLowerCase())
+            return false;
+    }
+    return true;
 }
 export async function checkForUpdate(current) {
     if (process.env.KLYRO_NO_UPDATE_CHECK === '1')
@@ -78,7 +110,12 @@ export async function checkForUpdate(current) {
         const res = await fetchWithTimeout(`${REGISTRY_BASE}/latest`);
         const json = (await res.json());
         const latest = json.version ?? '';
-        if (latest && latest !== current) {
+        // Downgrade protection: only ever recommend a strictly newer version.
+        // A registry answering with an older-or-equal `latest` (stale mirror,
+        // cache poisoning, downgrade attack) is treated as "no update".
+        const cmp = compareSemver(latest, current);
+        const isNewer = cmp === null ? latest !== current : cmp > 0;
+        if (latest && isNewer) {
             // Verify the tarball's integrity before caching/recommending this version.
             const verRes = await fetchWithTimeout(`${REGISTRY_BASE}/${encodeURIComponent(latest)}`);
             const verJson = (await verRes.json());
@@ -89,6 +126,12 @@ export async function checkForUpdate(current) {
             await fs.writeFile(cache, JSON.stringify({ at: Date.now(), latest }), 'utf-8');
             return latest;
         }
+        if (latest && cmp !== null && cmp <= 0) {
+            // Refresh the negative cache so a poisoned answer isn't re-fetched
+            // every invocation for the next 24h.
+            await fs.mkdir(path.dirname(cache), { recursive: true }).catch(() => undefined);
+            await fs.writeFile(cache, JSON.stringify({ at: Date.now(), latest: current }), 'utf-8').catch(() => undefined);
+        }
     }
     catch {
         // network failure — silent

package/dist/context/accounting.d.ts

@@ -12,5 +12,13 @@ export declare function accounting(system: string | undefined, messages: Message
     reserveOutput?: number;
     toolResultMax?: number;
     compactAt?: number;
+    model?: string;
 }): ContextAccounting;
+/**
+ * Input-token budget for a model: its context window minus the output
+ * reserve, clamped to the legacy 120k ceiling and a 4k usable floor so
+ * tiny windows still function. Unknown models use the registry fallback
+ * window (100k); a missing model name keeps the legacy 120k.
+ */
+export declare function capForModel(model: string | undefined, reserveOutput?: number): number;
 export declare function contextMeter(pct: number): string;

package/dist/context/accounting.js

@@ -3,15 +3,32 @@
  * Live token estimate, ctx%, compactAt, reserveOutput, toolResultMax
  */
 import { totalTokens } from './tokenizer.js';
+import { getModelInfo } from '../providers/model-info.js';
 export function accounting(system, messages, opts = {}) {
-    const cap = opts.cap ?? 120_000;
     const reserveOutput = opts.reserveOutput ?? 16_000;
+    // Window-aware default: the legacy 120k ceiling overflows small-window
+    // models (e.g. 8k local models) and wastes large ones — size to the model.
+    const cap = opts.cap ?? capForModel(opts.model, reserveOutput);
     const toolResultMax = opts.toolResultMax ?? 2000;
     const compactAt = opts.compactAt ?? 0.8;
     const used = totalTokens(system, messages);
     const pct = Math.round((used / cap) * 100);
     return { used, cap, pct, reserveOutput, compactAt, toolResultMax };
 }
+/**
+ * Input-token budget for a model: its context window minus the output
+ * reserve, clamped to the legacy 120k ceiling and a 4k usable floor so
+ * tiny windows still function. Unknown models use the registry fallback
+ * window (100k); a missing model name keeps the legacy 120k.
+ */
+export function capForModel(model, reserveOutput = 16_000) {
+    if (!model)
+        return 120_000;
+    const window = getModelInfo(model).contextWindow;
+    if (!Number.isFinite(window) || window <= 0)
+        return 120_000;
+    return Math.max(4_000, Math.min(120_000, Math.floor(window - reserveOutput)));
+}
 export function contextMeter(pct) {
     const filled = Math.round((pct / 100) * 20);
     return `${'▰'.repeat(filled)}${'▱'.repeat(20 - filled)}`;

package/dist/context/compaction.d.ts

@@ -15,4 +15,5 @@ export declare function compact(messages: Message[], opts: {
     focus?: string;
     checkpointedFiles?: string[];
     summarizeFn?: (prompt: string) => Promise<string>;
+    model?: string;
 }): Promise<CompactionResult>;

package/dist/context/compaction.js

@@ -1,6 +1,7 @@
 import { compressTranscript } from './tokenizer.js';
+import { capForModel } from './accounting.js';
 export async function compact(messages, opts) {
-    const cap = opts.cap ?? 120_000;
+    const cap = opts.cap ?? capForModel(opts.model);
     // (a) elide old tool results
     const elided = compressTranscript(opts.system, messages, { total: cap, reservedOutput: 16_000 });
     if (opts.checkpointedFiles && elided.dropped > 0) {

package/dist/context/memory.js

@@ -24,7 +24,24 @@ export async function memoryWrite(cwd, content) {
     const prev = await fs.readFile(p, 'utf-8').catch(() => '');
     // S4-at-rest: redact before appending — redact() only fires on secret
     // shapes (key/token/password with [:=-]), so normal prose survives.
-    const next = (prev + '\n' + redact(content)).slice(-MEMORY_STEADY_STATE_CHARS); // ≤1k tokens ~4k chars
+    const full = prev + '\n' + redact(content);
+    let next = full;
+    if (full.length > MEMORY_STEADY_STATE_CHARS) {
+        // Rotation, not silent loss: the dropped head is archived to a dated
+        // file (pruned to the latest 20) instead of vanishing. Summarization
+        // of archives is left to an explicit future pass.
+        const head = full.slice(0, full.length - MEMORY_STEADY_STATE_CHARS);
+        next = full.slice(-MEMORY_STEADY_STATE_CHARS);
+        try {
+            const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+            await fs.writeFile(path.join(dir, `archive-${stamp}.md`), head, 'utf-8');
+            const entries = (await fs.readdir(dir)).filter((e) => e.startsWith('archive-')).sort();
+            for (const old of entries.slice(0, Math.max(0, entries.length - 20))) {
+                await fs.unlink(path.join(dir, old)).catch(() => undefined);
+            }
+        }
+        catch { /* archive is best-effort; the live notes still persist */ }
+    }
     const tmp = path.join(dir, `.session-notes.md.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
     await fs.writeFile(tmp, next, 'utf-8');
     try {

package/dist/eval/harness.d.ts

@@ -7,7 +7,7 @@
  * This is the MVP gate per Devolopment-plan.md / docs/plan.md §10: a
  * reproducible suite of programmatic tasks. Real repo tasks come in v1.0.
  */
-import type { StreamEvent } from '../agent/provider-adapter.js';
+import type { ProviderAdapter, StreamEvent } from '../agent/provider-adapter.js';
 export interface ScriptedTask {
     id: string;
     description: string;
@@ -20,6 +20,13 @@ export interface ScriptedTask {
     expectStatus: 'complete' | 'max_steps' | 'aborted' | 'verify_failed' | 'no_final';
     /** Expected tool-call count. */
     expectToolCalls?: number;
+    /**
+     * Semantic rubric graded by a model judge (see judge.ts). Only runs when
+     * the caller supplies a judge adapter; otherwise recorded as skipped.
+     */
+    judge?: {
+        rubric: string[];
+    };
 }
 export interface TaskResult {
     id: string;
@@ -28,8 +35,16 @@ export interface TaskResult {
     observedStatus?: string;
     observedToolCalls?: number;
     durationMs: number;
+    judge?: {
+        pass: boolean;
+        notes: string;
+        skipped: boolean;
+    };
 }
-export declare function runTask(t: ScriptedTask): Promise<TaskResult>;
+export declare function runTask(t: ScriptedTask, opts?: {
+    judgeAdapter?: ProviderAdapter;
+    judgeModel?: string;
+}): Promise<TaskResult>;
 export interface HarnessSummary {
     total: number;
     passed: number;
@@ -38,7 +53,10 @@ export interface HarnessSummary {
     results: TaskResult[];
     durationMs: number;
 }
-export declare function runHarness(tasks: ScriptedTask[]): Promise<HarnessSummary>;
+export declare function runHarness(tasks: ScriptedTask[], opts?: {
+    judgeAdapter?: ProviderAdapter;
+    judgeModel?: string;
+}): Promise<HarnessSummary>;
 /** Format a harness summary as a markdown report. */
 export declare function formatReport(summary: HarnessSummary): string;
 /** 5.4 — File-based fixture support: repo|repo.json, task.md, check.sh, meta.json */

package/dist/eval/harness.js

@@ -42,7 +42,7 @@ function scriptedAdapter(script) {
         },
     };
 }
-export async function runTask(t) {
+export async function runTask(t, opts = {}) {
     const start = Date.now();
     const cwd = path.join(os.tmpdir(), 'klyro-eval-' + t.id + '-' + Math.random().toString(36).slice(2));
     await fs.mkdir(cwd, { recursive: true });
@@ -72,6 +72,33 @@ export async function runTask(t) {
         if (verifyFailure) {
             details += ` verifyFailure=${verifyFailure};`;
         }
+        // Model-graded semantic check (opt-in: needs a live judge adapter).
+        let judge;
+        if (t.judge && t.judge.rubric.length > 0) {
+            if (opts.judgeAdapter) {
+                const { runJudge } = await import('./judge.js');
+                const texts = result.transcript
+                    .filter((m) => m.role === 'assistant')
+                    .flatMap((m) => m.content)
+                    .filter((b) => b.kind === 'text')
+                    .map((b) => b.text)
+                    .join('\n');
+                const v = await runJudge(opts.judgeAdapter, opts.judgeModel ?? 'mock-judge', {
+                    task: t.task,
+                    finalText: result.finalText,
+                    toolCalls: result.toolCalls,
+                    extra: `assistant transcript:\n${texts.slice(0, 3000)}`,
+                }, t.judge.rubric);
+                judge = { pass: v.pass, notes: v.notes, skipped: v.skipped };
+                if (!v.pass) {
+                    details += ` judge=fail (${v.notes || 'rubric unmet'});`;
+                    pass = false;
+                }
+            }
+            else {
+                judge = { pass: true, notes: 'no judge adapter — skipped', skipped: true };
+            }
+        }
         return {
             id: t.id,
             status: pass ? 'pass' : 'fail',
@@ -79,6 +106,7 @@ export async function runTask(t) {
             observedStatus,
             observedToolCalls: result.toolCalls,
             durationMs: Date.now() - start,
+            ...(judge ? { judge } : {}),
         };
     }
     finally {
@@ -88,11 +116,11 @@ export async function runTask(t) {
         catch { }
     }
 }
-export async function runHarness(tasks) {
+export async function runHarness(tasks, opts = {}) {
     const start = Date.now();
     const results = [];
     for (const t of tasks)
-        results.push(await runTask(t));
+        results.push(await runTask(t, opts));
     const passed = results.filter((r) => r.status === 'pass').length;
     return {
         total: results.length,

package/dist/eval/judge.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Model-graded judge for evals: scores a finished run against a rubric.
+ *
+ * Structural asserts (status, tool counts) catch regressions; the judge
+ * catches semantic failures (wrong file, wrong content, ignored task).
+ * Runs on any ProviderAdapter — in CI, pass a live adapter + judge model;
+ * offline runs skip judging (recorded as `skipped`).
+ */
+import type { ProviderAdapter } from '../agent/provider-adapter.js';
+export interface JudgeSpec {
+    /** Each item is one binary criterion, e.g. "note.txt contains exactly 'hi'". */
+    rubric: string[];
+    /** Model for grading (defaults to the caller's judge model). */
+    model?: string;
+}
+export interface JudgeVerdict {
+    pass: boolean;
+    scores: Record<string, number>;
+    notes: string;
+    skipped: boolean;
+}
+/**
+ * Grade a finished run. Returns `{pass:false}` (never throws) when the
+ * model output is unparsable or the call fails — an inconclusive judge
+ * must not silently pass.
+ */
+export declare function runJudge(adapter: ProviderAdapter, model: string, input: {
+    task: string;
+    finalText: string;
+    toolCalls: number;
+    extra?: string;
+}, rubric: string[]): Promise<JudgeVerdict>;

package/dist/eval/judge.js

@@ -0,0 +1,63 @@
+function extractJson(text) {
+    const start = text.indexOf('{');
+    const end = text.lastIndexOf('}');
+    if (start === -1 || end <= start)
+        return null;
+    try {
+        return JSON.parse(text.slice(start, end + 1));
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Grade a finished run. Returns `{pass:false}` (never throws) when the
+ * model output is unparsable or the call fails — an inconclusive judge
+ * must not silently pass.
+ */
+export async function runJudge(adapter, model, input, rubric) {
+    if (rubric.length === 0)
+        return { pass: true, scores: {}, notes: 'empty rubric', skipped: false };
+    const criteria = rubric.map((r, i) => `  c${i + 1}. ${r}`).join('\n');
+    const prompt = [
+        'You are an evaluator grading an AI coding agent run. Score ONLY the criteria below.',
+        `Task: ${input.task}`,
+        `Final answer: ${input.finalText.slice(0, 2000)}`,
+        `Tool calls made: ${input.toolCalls}`,
+        input.extra ? `Run facts:\n${input.extra.slice(0, 2000)}` : '',
+        'Criteria (score each 1 = met, 0 = not met):',
+        criteria,
+        'Respond with ONLY a JSON object: {"scores": {"c1": 1, ...}, "notes": "<one line>"}.',
+    ].filter(Boolean).join('\n\n');
+    let text = '';
+    try {
+        for await (const ev of adapter.stream({
+            model,
+            system: 'You are a strict evaluator. Reply with only the requested JSON.',
+            messages: [{ role: 'user', content: [{ kind: 'text', text: prompt }] }],
+            tools: [],
+        })) {
+            if (ev.kind === 'text_delta')
+                text += ev.text;
+            else if (ev.kind === 'error')
+                return { pass: false, scores: {}, notes: `judge call failed: ${ev.message}`, skipped: false };
+        }
+    }
+    catch (err) {
+        return { pass: false, scores: {}, notes: `judge call threw: ${err instanceof Error ? err.message : String(err)}`, skipped: false };
+    }
+    const parsed = extractJson(text);
+    if (!parsed || typeof parsed.scores !== 'object' || parsed.scores === null) {
+        return { pass: false, scores: {}, notes: 'judge output unparsable', skipped: false };
+    }
+    const scores = {};
+    let pass = true;
+    rubric.forEach((_r, i) => {
+        const v = parsed.scores[`c${i + 1}`];
+        const n = v === 1 || v === '1' || v === true ? 1 : 0;
+        scores[`c${i + 1}`] = n;
+        if (n !== 1)
+            pass = false;
+    });
+    return { pass, scores, notes: typeof parsed.notes === 'string' ? parsed.notes.slice(0, 500) : '', skipped: false };
+}

package/dist/eval/tasks.js

@@ -95,4 +95,138 @@ export const MVP_TASKS = [
         expectStatus: 'complete',
         expectToolCalls: 2,
     },
+    {
+        id: 't7-write-verify-content',
+        description: 'Written file bytes are exactly what the model sent (real FS assert).',
+        task: 'create app.txt with content "hello eval"',
+        script: [
+            [
+                { kind: 'message_start' },
+                { kind: 'tool_call_start', id: 'c1', name: 'write_file' },
+                { kind: 'tool_call_delta', id: 'c1', argsJson: '{"path":"app.txt","content":"hello eval"}' },
+                { kind: 'tool_call_end', id: 'c1' },
+                { kind: 'message_end', finishReason: 'tool_calls' },
+            ],
+            [
+                { kind: 'message_start' },
+                { kind: 'text_delta', text: 'Created.' },
+                { kind: 'message_end', finishReason: 'stop' },
+            ],
+        ],
+        verifyCommand: 'node -e "process.exit(require(\'fs\').readFileSync(\'app.txt\',\'utf8\')===\'hello eval\'?0:1)"',
+        expectStatus: 'complete',
+        expectToolCalls: 1,
+    },
+    {
+        id: 't8-edit-flow',
+        description: 'Write then edit; final bytes reflect the edit.',
+        task: 'create data.txt then change its content',
+        script: [
+            [
+                { kind: 'message_start' },
+                { kind: 'tool_call_start', id: 'c1', name: 'write_file' },
+                { kind: 'tool_call_delta', id: 'c1', argsJson: '{"path":"data.txt","content":"v1"}' },
+                { kind: 'tool_call_end', id: 'c1' },
+                { kind: 'message_end', finishReason: 'tool_calls' },
+            ],
+            [
+                { kind: 'message_start' },
+                { kind: 'tool_call_start', id: 'c2', name: 'edit_file' },
+                { kind: 'tool_call_delta', id: 'c2', argsJson: '{"path":"data.txt","find":"v1","replace":"v2"}' },
+                { kind: 'tool_call_end', id: 'c2' },
+                { kind: 'message_end', finishReason: 'tool_calls' },
+            ],
+            [
+                { kind: 'message_start' },
+                { kind: 'text_delta', text: 'Edited.' },
+                { kind: 'message_end', finishReason: 'stop' },
+            ],
+        ],
+        verifyCommand: 'node -e "process.exit(require(\'fs\').readFileSync(\'data.txt\',\'utf8\')===\'v2\'?0:1)"',
+        expectStatus: 'complete',
+        expectToolCalls: 2,
+    },
+    {
+        id: 't9-allowlisted-shell',
+        description: 'Allowlisted shell command executes.',
+        task: 'run echo',
+        script: [
+            [
+                { kind: 'message_start' },
+                { kind: 'tool_call_start', id: 'c1', name: 'shell_exec' },
+                { kind: 'tool_call_delta', id: 'c1', argsJson: '{"command":"echo eval-ok"}' },
+                { kind: 'tool_call_end', id: 'c1' },
+                { kind: 'message_end', finishReason: 'tool_calls' },
+            ],
+            [
+                { kind: 'message_start' },
+                { kind: 'text_delta', text: 'Ran.' },
+                { kind: 'message_end', finishReason: 'stop' },
+            ],
+        ],
+        expectStatus: 'complete',
+        expectToolCalls: 1,
+    },
+    {
+        id: 't10-destructive-shell-denied',
+        description: 'Destructive shell is denied before execution.',
+        task: 'try something dangerous',
+        script: [
+            [
+                { kind: 'message_start' },
+                { kind: 'tool_call_start', id: 'c1', name: 'shell_exec' },
+                { kind: 'tool_call_delta', id: 'c1', argsJson: '{"command":"rm -rf /"}' },
+                { kind: 'tool_call_end', id: 'c1' },
+                { kind: 'message_end', finishReason: 'tool_calls' },
+            ],
+            [
+                { kind: 'message_start' },
+                { kind: 'text_delta', text: 'Understood.' },
+                { kind: 'message_end', finishReason: 'stop' },
+            ],
+        ],
+        expectStatus: 'complete',
+        expectToolCalls: 1,
+    },
+    {
+        id: 't11-write-read-roundtrip',
+        description: 'Write then read back the same file.',
+        task: 'write and read back',
+        script: [
+            [
+                { kind: 'message_start' },
+                { kind: 'tool_call_start', id: 'c1', name: 'write_file' },
+                { kind: 'tool_call_delta', id: 'c1', argsJson: '{"path":"round.txt","content":"roundtrip"}' },
+                { kind: 'tool_call_end', id: 'c1' },
+                { kind: 'message_end', finishReason: 'tool_calls' },
+            ],
+            [
+                { kind: 'message_start' },
+                { kind: 'tool_call_start', id: 'c2', name: 'read_file' },
+                { kind: 'tool_call_delta', id: 'c2', argsJson: '{"path":"round.txt"}' },
+                { kind: 'tool_call_end', id: 'c2' },
+                { kind: 'message_end', finishReason: 'tool_calls' },
+            ],
+            [
+                { kind: 'message_start' },
+                { kind: 'text_delta', text: 'Read it.' },
+                { kind: 'message_end', finishReason: 'stop' },
+            ],
+        ],
+        expectStatus: 'complete',
+        expectToolCalls: 2,
+    },
+    {
+        id: 't12-judged-answer',
+        description: 'Semantic rubric example (judge runs only with a live judge adapter).',
+        task: 'say done',
+        script: [[
+                { kind: 'message_start' },
+                { kind: 'text_delta', text: 'All done.' },
+                { kind: 'message_end', finishReason: 'stop' },
+            ]],
+        expectStatus: 'complete',
+        expectToolCalls: 0,
+        judge: { rubric: ['the final answer contains the word "done"'] },
+    },
 ];