klyro 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +50 -0
- package/dist/agent/anthropic-adapter.d.ts +75 -0
- package/dist/agent/anthropic-adapter.js +297 -0
- package/dist/agent/message.d.ts +40 -0
- package/dist/agent/message.js +21 -0
- package/dist/agent/observation.d.ts +47 -0
- package/dist/agent/observation.js +53 -0
- package/dist/agent/provider-adapter.d.ts +101 -0
- package/dist/agent/provider-adapter.js +254 -0
- package/dist/agent/registry.d.ts +42 -0
- package/dist/agent/registry.js +86 -0
- package/dist/agent/retry.d.ts +34 -0
- package/dist/agent/retry.js +91 -0
- package/dist/agent/runtime.d.ts +130 -0
- package/dist/agent/runtime.js +221 -0
- package/dist/agent/worker-spawner.d.ts +41 -0
- package/dist/agent/worker-spawner.js +76 -0
- package/dist/chat.d.ts +39 -0
- package/dist/chat.js +235 -0
- package/dist/chat.js.map +1 -0
- package/dist/cli/eval.d.ts +75 -0
- package/dist/cli/eval.js +190 -0
- package/dist/cli/repl.d.ts +15 -0
- package/dist/cli/repl.js +194 -0
- package/dist/cli/run.d.ts +67 -0
- package/dist/cli/run.js +198 -0
- package/dist/cli/slash/parser.d.ts +39 -0
- package/dist/cli/slash/parser.js +47 -0
- package/dist/context/level6.d.ts +45 -0
- package/dist/context/level6.js +231 -0
- package/dist/context/level7.d.ts +84 -0
- package/dist/context/level7.js +156 -0
- package/dist/context/project-map.d.ts +48 -0
- package/dist/context/project-map.js +438 -0
- package/dist/context/repo-map.d.ts +30 -0
- package/dist/context/repo-map.js +168 -0
- package/dist/context/selector.d.ts +22 -0
- package/dist/context/selector.js +37 -0
- package/dist/context/snippets.d.ts +17 -0
- package/dist/context/snippets.js +15 -0
- package/dist/context/tokenizer.d.ts +49 -0
- package/dist/context/tokenizer.js +97 -0
- package/dist/eval/harness.d.ts +48 -0
- package/dist/eval/harness.js +149 -0
- package/dist/eval/tasks.d.ts +6 -0
- package/dist/eval/tasks.js +98 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.js +128 -0
- package/dist/index.js.map +1 -0
- package/dist/persistence/audit.d.ts +75 -0
- package/dist/persistence/audit.js +15 -0
- package/dist/persistence/store.d.ts +67 -0
- package/dist/persistence/store.js +106 -0
- package/dist/policy/approval.d.ts +35 -0
- package/dist/policy/approval.js +54 -0
- package/dist/policy/engine.d.ts +76 -0
- package/dist/policy/engine.js +163 -0
- package/dist/policy/path-guard.d.ts +30 -0
- package/dist/policy/path-guard.js +100 -0
- package/dist/policy/secret-redactor.d.ts +19 -0
- package/dist/policy/secret-redactor.js +52 -0
- package/dist/providers.d.ts +23 -0
- package/dist/providers.js +83 -0
- package/dist/repl.d.ts +9 -0
- package/dist/repl.js +170 -0
- package/dist/repl.js.map +1 -0
- package/dist/tools/fs/edit-file.d.ts +48 -0
- package/dist/tools/fs/edit-file.js +84 -0
- package/dist/tools/fs/list-dir.d.ts +40 -0
- package/dist/tools/fs/list-dir.js +72 -0
- package/dist/tools/fs/read-file.d.ts +66 -0
- package/dist/tools/fs/read-file.js +75 -0
- package/dist/tools/fs/write-file.d.ts +24 -0
- package/dist/tools/fs/write-file.js +48 -0
- package/dist/tools/git/git-diff.d.ts +35 -0
- package/dist/tools/git/git-diff.js +68 -0
- package/dist/tools/git/git-status.d.ts +29 -0
- package/dist/tools/git/git-status.js +57 -0
- package/dist/tools/normalize.d.ts +31 -0
- package/dist/tools/normalize.js +88 -0
- package/dist/tools/registry.d.ts +23 -0
- package/dist/tools/registry.js +87 -0
- package/dist/tools/schema.d.ts +29 -0
- package/dist/tools/schema.js +183 -0
- package/dist/tools/search/dependencies.d.ts +44 -0
- package/dist/tools/search/dependencies.js +188 -0
- package/dist/tools/search/glob.d.ts +26 -0
- package/dist/tools/search/glob.js +88 -0
- package/dist/tools/search/grep.d.ts +52 -0
- package/dist/tools/search/grep.js +125 -0
- package/dist/tools/search/recent-files.d.ts +33 -0
- package/dist/tools/search/recent-files.js +100 -0
- package/dist/tools/search/search-files.d.ts +39 -0
- package/dist/tools/search/search-files.js +110 -0
- package/dist/tools/shell/shell-exec.d.ts +60 -0
- package/dist/tools/shell/shell-exec.js +133 -0
- package/dist/tools/types.d.ts +54 -0
- package/dist/tools/types.js +15 -0
- package/dist/tools/verify/run-verify.d.ts +60 -0
- package/dist/tools/verify/run-verify.js +115 -0
- package/dist/tui/app.d.ts +27 -0
- package/dist/tui/app.js +79 -0
- package/dist/tui/app.test.d.ts +1 -0
- package/dist/tui/app.test.js +77 -0
- package/dist/tui/status.d.ts +21 -0
- package/dist/tui/status.js +13 -0
- package/dist/tui/status.test.d.ts +1 -0
- package/dist/tui/status.test.js +39 -0
- package/dist/tui/transcript.d.ts +40 -0
- package/dist/tui/transcript.js +28 -0
- package/dist/tui/transcript.test.d.ts +1 -0
- package/dist/tui/transcript.test.js +71 -0
- package/dist/verification/detect.d.ts +29 -0
- package/dist/verification/detect.js +85 -0
- package/dist/verification/engine.d.ts +25 -0
- package/dist/verification/engine.js +57 -0
- package/package.json +58 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Snippet reader — read a slice of a file (line window). Used by the
|
|
3
|
+
* runtime when the model needs a specific region, not the whole file.
|
|
4
|
+
*/
|
|
5
|
+
export interface Snippet {
|
|
6
|
+
path: string;
|
|
7
|
+
startLine: number;
|
|
8
|
+
endLine: number;
|
|
9
|
+
content: string;
|
|
10
|
+
}
|
|
11
|
+
export interface SnippetOptions {
|
|
12
|
+
cwd: string;
|
|
13
|
+
path: string;
|
|
14
|
+
startLine?: number;
|
|
15
|
+
maxLines?: number;
|
|
16
|
+
}
|
|
17
|
+
export declare function readSnippet(opts: SnippetOptions): Promise<Snippet>;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Snippet reader — read a slice of a file (line window). Used by the
|
|
3
|
+
* runtime when the model needs a specific region, not the whole file.
|
|
4
|
+
*/
|
|
5
|
+
import * as fs from 'node:fs/promises';
|
|
6
|
+
import { resolveWithinCwd } from '../policy/path-guard.js';
|
|
7
|
+
export async function readSnippet(opts) {
|
|
8
|
+
const { resolved } = resolveWithinCwd(opts.cwd, opts.path);
|
|
9
|
+
const content = await fs.readFile(resolved, 'utf-8');
|
|
10
|
+
const lines = content.split(/\r?\n/);
|
|
11
|
+
const start = Math.max(1, opts.startLine ?? 1);
|
|
12
|
+
const max = Math.min(opts.maxLines ?? 200, lines.length - start + 1);
|
|
13
|
+
const end = start + max - 1;
|
|
14
|
+
return { path: opts.path, startLine: start, endLine: end, content: lines.slice(start - 1, end).join('\n') };
|
|
15
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token budget — keeps each model call under a hard cap by trimming the
|
|
3
|
+
* transcript before sending it to the provider.
|
|
4
|
+
*
|
|
5
|
+
* For MVP we count tokens with a simple heuristic (chars/4) so we don't
|
|
6
|
+
* pull in a heavy BPE dependency. The cap is conservative; the runtime
|
|
7
|
+
* never overflows the model's true window because the heuristic
|
|
8
|
+
* overestimates mixed text.
|
|
9
|
+
*
|
|
10
|
+
* Strategy:
|
|
11
|
+
* 1. Always preserve the system prompt, the latest user task, and the
|
|
12
|
+
* latest assistant message.
|
|
13
|
+
* 2. If still over budget, drop the oldest tool_result observations
|
|
14
|
+
* that have been "consumed" (a later assistant message referenced
|
|
15
|
+
* their tool_call_id).
|
|
16
|
+
* 3. If still over budget, summarize the surviving tail into a single
|
|
17
|
+
* user message ("Earlier in this session: …").
|
|
18
|
+
*/
|
|
19
|
+
import type { Message } from '../agent/message.js';
|
|
20
|
+
export interface TokenBudget {
|
|
21
|
+
/** Total input cap (system + messages). */
|
|
22
|
+
total: number;
|
|
23
|
+
/** Reserve for the response (we don't trim these; the cap is on input). */
|
|
24
|
+
reservedOutput: number;
|
|
25
|
+
}
|
|
26
|
+
export interface BudgetCheck {
|
|
27
|
+
ok: boolean;
|
|
28
|
+
used: number;
|
|
29
|
+
cap: number;
|
|
30
|
+
}
|
|
31
|
+
export declare function estimateTokens(s: string): number;
|
|
32
|
+
export declare function estimateMessage(m: Message): number;
|
|
33
|
+
/** Total input tokens for a transcript + optional system prompt. */
|
|
34
|
+
export declare function totalTokens(system: string | undefined, messages: Message[]): number;
|
|
35
|
+
/** True if the input fits under the budget cap. */
|
|
36
|
+
export declare function withinBudget(system: string | undefined, messages: Message[], budget: TokenBudget): BudgetCheck;
|
|
37
|
+
/**
|
|
38
|
+
* Trim a transcript to fit under the budget. Strategy:
|
|
39
|
+
* - Keep first user task (so the model never forgets the goal).
|
|
40
|
+
* - Keep the last 2 messages verbatim.
|
|
41
|
+
* - Compress intermediate messages: tool_result blocks shorter, text
|
|
42
|
+
* blocks truncated to 500 chars each.
|
|
43
|
+
* Returns the new transcript plus a count of dropped observations.
|
|
44
|
+
*/
|
|
45
|
+
export declare function compressTranscript(system: string | undefined, messages: Message[], budget: TokenBudget): {
|
|
46
|
+
system: string | undefined;
|
|
47
|
+
messages: Message[];
|
|
48
|
+
dropped: number;
|
|
49
|
+
};
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token budget — keeps each model call under a hard cap by trimming the
|
|
3
|
+
* transcript before sending it to the provider.
|
|
4
|
+
*
|
|
5
|
+
* For MVP we count tokens with a simple heuristic (chars/4) so we don't
|
|
6
|
+
* pull in a heavy BPE dependency. The cap is conservative; the runtime
|
|
7
|
+
* never overflows the model's true window because the heuristic
|
|
8
|
+
* overestimates mixed text.
|
|
9
|
+
*
|
|
10
|
+
* Strategy:
|
|
11
|
+
* 1. Always preserve the system prompt, the latest user task, and the
|
|
12
|
+
* latest assistant message.
|
|
13
|
+
* 2. If still over budget, drop the oldest tool_result observations
|
|
14
|
+
* that have been "consumed" (a later assistant message referenced
|
|
15
|
+
* their tool_call_id).
|
|
16
|
+
* 3. If still over budget, summarize the surviving tail into a single
|
|
17
|
+
* user message ("Earlier in this session: …").
|
|
18
|
+
*/
|
|
19
|
+
export function estimateTokens(s) {
|
|
20
|
+
return Math.ceil(s.length / 4);
|
|
21
|
+
}
|
|
22
|
+
export function estimateMessage(m) {
|
|
23
|
+
let n = 4; // role + structural overhead
|
|
24
|
+
for (const b of m.content)
|
|
25
|
+
n += estimateBlock(b);
|
|
26
|
+
return n;
|
|
27
|
+
}
|
|
28
|
+
function estimateBlock(b) {
|
|
29
|
+
if (b.kind === 'text')
|
|
30
|
+
return estimateTokens(b.text) + 4;
|
|
31
|
+
if (b.kind === 'tool_use')
|
|
32
|
+
return estimateTokens(b.name) + estimateTokens(JSON.stringify(b.input)) + 16;
|
|
33
|
+
if (b.kind === 'tool_result') {
|
|
34
|
+
const out = typeof b.output === 'string' ? b.output : JSON.stringify(b.output ?? '');
|
|
35
|
+
return estimateTokens(out) + estimateTokens(b.name) + 16;
|
|
36
|
+
}
|
|
37
|
+
return 0;
|
|
38
|
+
}
|
|
39
|
+
/** Total input tokens for a transcript + optional system prompt. */
|
|
40
|
+
export function totalTokens(system, messages) {
|
|
41
|
+
let n = system ? estimateTokens(system) : 0;
|
|
42
|
+
for (const m of messages)
|
|
43
|
+
n += estimateMessage(m);
|
|
44
|
+
return n;
|
|
45
|
+
}
|
|
46
|
+
/** True if the input fits under the budget cap. */
|
|
47
|
+
export function withinBudget(system, messages, budget) {
|
|
48
|
+
const used = totalTokens(system, messages);
|
|
49
|
+
return { ok: used <= budget.total, used, cap: budget.total };
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Trim a transcript to fit under the budget. Strategy:
|
|
53
|
+
* - Keep first user task (so the model never forgets the goal).
|
|
54
|
+
* - Keep the last 2 messages verbatim.
|
|
55
|
+
* - Compress intermediate messages: tool_result blocks shorter, text
|
|
56
|
+
* blocks truncated to 500 chars each.
|
|
57
|
+
* Returns the new transcript plus a count of dropped observations.
|
|
58
|
+
*/
|
|
59
|
+
export function compressTranscript(system, messages, budget) {
|
|
60
|
+
const result = messages.slice();
|
|
61
|
+
let dropped = 0;
|
|
62
|
+
// Phase 1: find tool_call_ids referenced in any later assistant message.
|
|
63
|
+
const consumed = new Set();
|
|
64
|
+
for (const m of result) {
|
|
65
|
+
if (m.role !== 'assistant')
|
|
66
|
+
continue;
|
|
67
|
+
for (const b of m.content)
|
|
68
|
+
if (b.kind === 'tool_use')
|
|
69
|
+
consumed.add(b.id);
|
|
70
|
+
}
|
|
71
|
+
// Phase 2: shrink tool_result blocks older than the last 4 messages.
|
|
72
|
+
const tailStart = Math.max(0, result.length - 4);
|
|
73
|
+
for (let i = 0; i < tailStart; i++) {
|
|
74
|
+
const m = result[i];
|
|
75
|
+
if (!m || m.role !== 'tool')
|
|
76
|
+
continue;
|
|
77
|
+
for (const b of m.content) {
|
|
78
|
+
if (b.kind !== 'tool_result')
|
|
79
|
+
continue;
|
|
80
|
+
if (!consumed.has(b.toolCallId)) {
|
|
81
|
+
// Drop the observation entirely.
|
|
82
|
+
b.output = '[earlier observation removed to fit context]';
|
|
83
|
+
dropped++;
|
|
84
|
+
}
|
|
85
|
+
else if (typeof b.output === 'string' && b.output.length > 400) {
|
|
86
|
+
b.output = b.output.slice(0, 400) + '... [truncated]';
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
// Phase 3: hard cap — drop oldest messages until we fit, but always
|
|
91
|
+
// preserve the first message (the user's task).
|
|
92
|
+
while (totalTokens(system, result) > budget.total && result.length > 2) {
|
|
93
|
+
result.splice(1, 1);
|
|
94
|
+
dropped++;
|
|
95
|
+
}
|
|
96
|
+
return { system, messages: result, dropped };
|
|
97
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval harness — runs a sequence of scripted tasks against the runtime
|
|
3
|
+
* using a MockProviderAdapter. Verifies (a) the runtime completes the
|
|
4
|
+
* expected tool-call sequence, (b) the verification engine detects the
|
|
5
|
+
* right failure type, and (c) the compressor preserves the goal.
|
|
6
|
+
*
|
|
7
|
+
* This is the MVP gate per Devolopment-plan.md / docs/plan.md §10: a
|
|
8
|
+
* reproducible suite of programmatic tasks. Real repo tasks come in v1.0.
|
|
9
|
+
*/
|
|
10
|
+
import type { StreamEvent } from '../agent/provider-adapter.js';
|
|
11
|
+
export interface ScriptedTask {
|
|
12
|
+
id: string;
|
|
13
|
+
description: string;
|
|
14
|
+
task: string;
|
|
15
|
+
/** A list of event scripts — each inner list is one assistant turn. */
|
|
16
|
+
script: StreamEvent[][];
|
|
17
|
+
/** Optional verify command to run after the runtime finishes. */
|
|
18
|
+
verifyCommand?: string;
|
|
19
|
+
/** Expected final status. */
|
|
20
|
+
expectStatus: 'complete' | 'max_steps' | 'aborted' | 'verify_failed' | 'no_final';
|
|
21
|
+
/** Expected tool-call count. */
|
|
22
|
+
expectToolCalls?: number;
|
|
23
|
+
}
|
|
24
|
+
export interface TaskResult {
|
|
25
|
+
id: string;
|
|
26
|
+
status: 'pass' | 'fail';
|
|
27
|
+
details: string;
|
|
28
|
+
observedStatus?: string;
|
|
29
|
+
observedToolCalls?: number;
|
|
30
|
+
durationMs: number;
|
|
31
|
+
}
|
|
32
|
+
export declare function runTask(t: ScriptedTask): Promise<TaskResult>;
|
|
33
|
+
export interface HarnessSummary {
|
|
34
|
+
total: number;
|
|
35
|
+
passed: number;
|
|
36
|
+
failed: number;
|
|
37
|
+
passRate: number;
|
|
38
|
+
results: TaskResult[];
|
|
39
|
+
durationMs: number;
|
|
40
|
+
}
|
|
41
|
+
export declare function runHarness(tasks: ScriptedTask[]): Promise<HarnessSummary>;
|
|
42
|
+
/** Format a harness summary as a markdown report. */
|
|
43
|
+
export declare function formatReport(summary: HarnessSummary): string;
|
|
44
|
+
/** Hook the harness up to a session + audit log so task runs are durable. */
|
|
45
|
+
export declare function runHarnessWithPersistence(tasks: ScriptedTask[], opts: {
|
|
46
|
+
storeDir: string;
|
|
47
|
+
auditPath: string;
|
|
48
|
+
}): Promise<HarnessSummary>;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval harness — runs a sequence of scripted tasks against the runtime
|
|
3
|
+
* using a MockProviderAdapter. Verifies (a) the runtime completes the
|
|
4
|
+
* expected tool-call sequence, (b) the verification engine detects the
|
|
5
|
+
* right failure type, and (c) the compressor preserves the goal.
|
|
6
|
+
*
|
|
7
|
+
* This is the MVP gate per Devolopment-plan.md / docs/plan.md §10: a
|
|
8
|
+
* reproducible suite of programmatic tasks. Real repo tasks come in v1.0.
|
|
9
|
+
*/
|
|
10
|
+
import { run } from '../agent/runtime.js';
|
|
11
|
+
import { ToolRegistry } from '../tools/registry.js';
|
|
12
|
+
import { readFileTool } from '../tools/fs/read-file.js';
|
|
13
|
+
import { writeFileTool } from '../tools/fs/write-file.js';
|
|
14
|
+
import { editFileTool } from '../tools/fs/edit-file.js';
|
|
15
|
+
import { shellExecTool } from '../tools/shell/shell-exec.js';
|
|
16
|
+
import { runVerifyTool } from '../tools/verify/run-verify.js';
|
|
17
|
+
import { listDirTool } from '../tools/fs/list-dir.js';
|
|
18
|
+
import { globTool } from '../tools/search/glob.js';
|
|
19
|
+
import { grepTool } from '../tools/search/grep.js';
|
|
20
|
+
import { gitStatusTool } from '../tools/git/git-status.js';
|
|
21
|
+
import { gitDiffTool } from '../tools/git/git-diff.js';
|
|
22
|
+
import { PolicyEngine, builtinRules, DEFAULT_POLICY_CONFIG } from '../policy/engine.js';
|
|
23
|
+
import { DenyAllApprovalPrompt } from '../policy/approval.js';
|
|
24
|
+
import { defaultSystemPrompt } from '../agent/runtime.js';
|
|
25
|
+
import { verify } from '../verification/engine.js';
|
|
26
|
+
import { SessionStore } from '../persistence/store.js';
|
|
27
|
+
import { AuditLog } from '../persistence/audit.js';
|
|
28
|
+
import * as fs from 'node:fs/promises';
|
|
29
|
+
import * as os from 'node:os';
|
|
30
|
+
import * as path from 'node:path';
|
|
31
|
+
function scriptedAdapter(script) {
|
|
32
|
+
let i = 0;
|
|
33
|
+
return {
|
|
34
|
+
id: 'mock',
|
|
35
|
+
async *stream() {
|
|
36
|
+
if (i < script.length) {
|
|
37
|
+
const turn = script[i++];
|
|
38
|
+
if (turn)
|
|
39
|
+
for (const ev of turn)
|
|
40
|
+
yield ev;
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export async function runTask(t) {
|
|
46
|
+
const start = Date.now();
|
|
47
|
+
const cwd = path.join(os.tmpdir(), 'klyro-eval-' + t.id + '-' + Math.random().toString(36).slice(2));
|
|
48
|
+
await fs.mkdir(cwd, { recursive: true });
|
|
49
|
+
const reg = new ToolRegistry()
|
|
50
|
+
.register(readFileTool).register(writeFileTool).register(editFileTool)
|
|
51
|
+
.register(listDirTool).register(globTool).register(grepTool)
|
|
52
|
+
.register(shellExecTool).register(runVerifyTool)
|
|
53
|
+
.register(gitStatusTool).register(gitDiffTool);
|
|
54
|
+
const policy = new PolicyEngine(builtinRules(), DEFAULT_POLICY_CONFIG);
|
|
55
|
+
try {
|
|
56
|
+
const result = await run({ task: t.task, cwd, model: 'mock', maxSteps: 12, nonInteractive: true }, { adapter: scriptedAdapter(t.script), registry: reg, policy, approval: new DenyAllApprovalPrompt(), systemPrompt: defaultSystemPrompt });
|
|
57
|
+
let observedStatus = result.status;
|
|
58
|
+
let verifyFailure;
|
|
59
|
+
if (t.verifyCommand) {
|
|
60
|
+
const v = await verify({ cwd, command: t.verifyCommand });
|
|
61
|
+
if (!v.ok) {
|
|
62
|
+
observedStatus = 'verify_failed';
|
|
63
|
+
verifyFailure = v.failure?.type ?? 'unknown';
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
let details = '';
|
|
67
|
+
let pass = result.status === t.expectStatus;
|
|
68
|
+
if (t.expectToolCalls !== undefined && result.toolCalls !== t.expectToolCalls) {
|
|
69
|
+
details += ` toolCalls=${result.toolCalls} expected=${t.expectToolCalls};`;
|
|
70
|
+
pass = false;
|
|
71
|
+
}
|
|
72
|
+
if (verifyFailure) {
|
|
73
|
+
details += ` verifyFailure=${verifyFailure};`;
|
|
74
|
+
}
|
|
75
|
+
return {
|
|
76
|
+
id: t.id,
|
|
77
|
+
status: pass ? 'pass' : 'fail',
|
|
78
|
+
details: details.trim() || `status=${observedStatus}`,
|
|
79
|
+
observedStatus,
|
|
80
|
+
observedToolCalls: result.toolCalls,
|
|
81
|
+
durationMs: Date.now() - start,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
finally {
|
|
85
|
+
try {
|
|
86
|
+
await fs.rm(cwd, { recursive: true, force: true });
|
|
87
|
+
}
|
|
88
|
+
catch { }
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
export async function runHarness(tasks) {
|
|
92
|
+
const start = Date.now();
|
|
93
|
+
const results = [];
|
|
94
|
+
for (const t of tasks)
|
|
95
|
+
results.push(await runTask(t));
|
|
96
|
+
const passed = results.filter((r) => r.status === 'pass').length;
|
|
97
|
+
return {
|
|
98
|
+
total: results.length,
|
|
99
|
+
passed,
|
|
100
|
+
failed: results.length - passed,
|
|
101
|
+
passRate: results.length === 0 ? 0 : passed / results.length,
|
|
102
|
+
results,
|
|
103
|
+
durationMs: Date.now() - start,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
/** Format a harness summary as a markdown report. */
|
|
107
|
+
export function formatReport(summary) {
|
|
108
|
+
const lines = [
|
|
109
|
+
`# Klyro Harness Report`,
|
|
110
|
+
``,
|
|
111
|
+
`**Total:** ${summary.total} **Passed:** ${summary.passed} **Failed:** ${summary.failed} **Pass rate:** ${(summary.passRate * 100).toFixed(1)}%`,
|
|
112
|
+
`**Wall time:** ${(summary.durationMs / 1000).toFixed(1)}s`,
|
|
113
|
+
``,
|
|
114
|
+
`| Task | Status | Details | Tool calls | Duration |`,
|
|
115
|
+
`|------|--------|---------|------------|----------|`,
|
|
116
|
+
...summary.results.map((r) => `| ${r.id} | ${r.status} | ${r.details} | ${r.observedToolCalls ?? '-'} | ${r.durationMs}ms |`),
|
|
117
|
+
];
|
|
118
|
+
return lines.join('\n');
|
|
119
|
+
}
|
|
120
|
+
/** Hook the harness up to a session + audit log so task runs are durable. */
|
|
121
|
+
export async function runHarnessWithPersistence(tasks, opts) {
|
|
122
|
+
const store = new SessionStore(opts.storeDir);
|
|
123
|
+
const audit = new AuditLog(opts.auditPath);
|
|
124
|
+
const start = Date.now();
|
|
125
|
+
const results = [];
|
|
126
|
+
for (const t of tasks) {
|
|
127
|
+
await audit.write({ kind: 'verification_attempted', sessionId: t.id, command: t.task, ts: Date.now() });
|
|
128
|
+
const r = await runTask(t);
|
|
129
|
+
results.push(r);
|
|
130
|
+
await audit.write({
|
|
131
|
+
kind: r.status === 'pass' ? 'verification_succeeded' : 'verification_failed',
|
|
132
|
+
sessionId: t.id,
|
|
133
|
+
exitCode: 0,
|
|
134
|
+
type: r.details,
|
|
135
|
+
ts: Date.now(),
|
|
136
|
+
});
|
|
137
|
+
// Suppress unused warning.
|
|
138
|
+
void store;
|
|
139
|
+
}
|
|
140
|
+
const passed = results.filter((r) => r.status === 'pass').length;
|
|
141
|
+
return {
|
|
142
|
+
total: results.length,
|
|
143
|
+
passed,
|
|
144
|
+
failed: results.length - passed,
|
|
145
|
+
passRate: results.length === 0 ? 0 : passed / results.length,
|
|
146
|
+
results,
|
|
147
|
+
durationMs: Date.now() - start,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The default MVP task suite. All tasks are programmatic — they use a
|
|
3
|
+
* scripted mock provider, so the suite is hermetic and CI-friendly.
|
|
4
|
+
*/
|
|
5
|
+
export const MVP_TASKS = [
|
|
6
|
+
{
|
|
7
|
+
id: 't1-direct-answer',
|
|
8
|
+
description: 'Model says hello without calling any tool.',
|
|
9
|
+
task: 'say hi',
|
|
10
|
+
script: [[
|
|
11
|
+
{ kind: 'message_start' },
|
|
12
|
+
{ kind: 'text_delta', text: 'Hello there.' },
|
|
13
|
+
{ kind: 'message_end', finishReason: 'stop' },
|
|
14
|
+
]],
|
|
15
|
+
expectStatus: 'complete',
|
|
16
|
+
expectToolCalls: 0,
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
id: 't2-write-then-answer',
|
|
20
|
+
description: 'Model writes a file then confirms.',
|
|
21
|
+
task: 'create note.txt with content "hi"',
|
|
22
|
+
script: [
|
|
23
|
+
[
|
|
24
|
+
{ kind: 'message_start' },
|
|
25
|
+
{ kind: 'tool_call_start', id: 'c1', name: 'write_file' },
|
|
26
|
+
{ kind: 'tool_call_delta', id: 'c1', argsJson: '{"path":"note.txt","content":"hi"}' },
|
|
27
|
+
{ kind: 'tool_call_end', id: 'c1' },
|
|
28
|
+
{ kind: 'message_end', finishReason: 'tool_calls' },
|
|
29
|
+
],
|
|
30
|
+
[
|
|
31
|
+
{ kind: 'message_start' },
|
|
32
|
+
{ kind: 'text_delta', text: 'Created.' },
|
|
33
|
+
{ kind: 'message_end', finishReason: 'stop' },
|
|
34
|
+
],
|
|
35
|
+
],
|
|
36
|
+
expectStatus: 'complete',
|
|
37
|
+
expectToolCalls: 1,
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
id: 't3-policy-deny',
|
|
41
|
+
description: 'Model tries to escape the cwd; policy denies.',
|
|
42
|
+
task: 'escape',
|
|
43
|
+
script: [
|
|
44
|
+
[
|
|
45
|
+
{ kind: 'message_start' },
|
|
46
|
+
{ kind: 'tool_call_start', id: 'c1', name: 'write_file' },
|
|
47
|
+
{ kind: 'tool_call_delta', id: 'c1', argsJson: '{"path":"../escape.txt","content":"x"}' },
|
|
48
|
+
{ kind: 'tool_call_end', id: 'c1' },
|
|
49
|
+
{ kind: 'message_end', finishReason: 'tool_calls' },
|
|
50
|
+
],
|
|
51
|
+
[
|
|
52
|
+
{ kind: 'message_start' },
|
|
53
|
+
{ kind: 'text_delta', text: 'OK' },
|
|
54
|
+
{ kind: 'message_end', finishReason: 'stop' },
|
|
55
|
+
],
|
|
56
|
+
],
|
|
57
|
+
expectStatus: 'complete',
|
|
58
|
+
expectToolCalls: 1,
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
id: 't4-max-steps',
|
|
62
|
+
description: 'Model never finishes; runtime returns max_steps.',
|
|
63
|
+
task: 'loop forever',
|
|
64
|
+
script: Array.from({ length: 15 }, () => [
|
|
65
|
+
{ kind: 'message_start' },
|
|
66
|
+
{ kind: 'tool_call_start', id: 'c1', name: 'write_file' },
|
|
67
|
+
{ kind: 'tool_call_delta', id: 'c1', argsJson: '{"path":"a.txt","content":"x"}' },
|
|
68
|
+
{ kind: 'tool_call_end', id: 'c1' },
|
|
69
|
+
{ kind: 'message_end', finishReason: 'tool_calls' },
|
|
70
|
+
]),
|
|
71
|
+
expectStatus: 'max_steps',
|
|
72
|
+
expectToolCalls: 12, // maxSteps=12 in harness, one tool call per step
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
id: 't6-multitool',
|
|
76
|
+
description: 'Model issues two tool calls in one turn.',
|
|
77
|
+
task: 'create two files',
|
|
78
|
+
script: [
|
|
79
|
+
[
|
|
80
|
+
{ kind: 'message_start' },
|
|
81
|
+
{ kind: 'tool_call_start', id: 'c1', name: 'write_file' },
|
|
82
|
+
{ kind: 'tool_call_delta', id: 'c1', argsJson: '{"path":"a.txt","content":"1"}' },
|
|
83
|
+
{ kind: 'tool_call_end', id: 'c1' },
|
|
84
|
+
{ kind: 'tool_call_start', id: 'c2', name: 'write_file' },
|
|
85
|
+
{ kind: 'tool_call_delta', id: 'c2', argsJson: '{"path":"b.txt","content":"2"}' },
|
|
86
|
+
{ kind: 'tool_call_end', id: 'c2' },
|
|
87
|
+
{ kind: 'message_end', finishReason: 'tool_calls' },
|
|
88
|
+
],
|
|
89
|
+
[
|
|
90
|
+
{ kind: 'message_start' },
|
|
91
|
+
{ kind: 'text_delta', text: 'Both written.' },
|
|
92
|
+
{ kind: 'message_end', finishReason: 'stop' },
|
|
93
|
+
],
|
|
94
|
+
],
|
|
95
|
+
expectStatus: 'complete',
|
|
96
|
+
expectToolCalls: 2,
|
|
97
|
+
},
|
|
98
|
+
];
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Klyro — autonomous coding harness CLI.
|
|
4
|
+
*
|
|
5
|
+
* klyro start the TUI REPL
|
|
6
|
+
* klyro run "<prompt>" one-shot autonomous task, streams to stdout
|
|
7
|
+
* klyro chat "<prompt>" legacy one-shot streamed chat (kept for compat)
|
|
8
|
+
* klyro chat legacy interactive REPL (kept for compat)
|
|
9
|
+
* klyro eval <file> run scripted scenarios (JSONL)
|
|
10
|
+
*
|
|
11
|
+
* Provider: OpenAI-compatible /v1/chat/completions endpoint.
|
|
12
|
+
* Configure via env: KLYRO_BASE_URL, KLYRO_API_KEY, KLYRO_MODEL.
|
|
13
|
+
*/
|
|
14
|
+
export {};
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Klyro — autonomous coding harness CLI.
|
|
4
|
+
*
|
|
5
|
+
* klyro start the TUI REPL
|
|
6
|
+
* klyro run "<prompt>" one-shot autonomous task, streams to stdout
|
|
7
|
+
* klyro chat "<prompt>" legacy one-shot streamed chat (kept for compat)
|
|
8
|
+
* klyro chat legacy interactive REPL (kept for compat)
|
|
9
|
+
* klyro eval <file> run scripted scenarios (JSONL)
|
|
10
|
+
*
|
|
11
|
+
* Provider: OpenAI-compatible /v1/chat/completions endpoint.
|
|
12
|
+
* Configure via env: KLYRO_BASE_URL, KLYRO_API_KEY, KLYRO_MODEL.
|
|
13
|
+
*/
|
|
14
|
+
import { Command } from 'commander';
|
|
15
|
+
import { chat } from './chat.js';
|
|
16
|
+
import { repl } from './repl.js';
|
|
17
|
+
import { runOnce } from './cli/run.js';
|
|
18
|
+
import { runEval } from './cli/eval.js';
|
|
19
|
+
const VERSION = '0.1.0';
|
|
20
|
+
function parsePositiveInt(name, v) {
|
|
21
|
+
const n = Number(v);
|
|
22
|
+
if (!Number.isFinite(n) || n <= 0)
|
|
23
|
+
throw new Error(`invalid ${name}: ${v}`);
|
|
24
|
+
return n;
|
|
25
|
+
}
|
|
26
|
+
async function main() {
|
|
27
|
+
const program = new Command();
|
|
28
|
+
program
|
|
29
|
+
.name('klyro')
|
|
30
|
+
.description('Klyro — autonomous coding harness')
|
|
31
|
+
.version(VERSION, '-V, --version', 'Print the version number')
|
|
32
|
+
.helpOption('-h, --help', 'Print this help message')
|
|
33
|
+
.showHelpAfterError();
|
|
34
|
+
// Default action: TUI REPL. For now this falls back to the legacy REPL
|
|
35
|
+
// until the Ink-based TUI lands. The wiring point is cli/repl.ts.
|
|
36
|
+
program
|
|
37
|
+
.action(async () => {
|
|
38
|
+
await repl('You are a helpful assistant.');
|
|
39
|
+
});
|
|
40
|
+
program
|
|
41
|
+
.command('run <prompt>')
|
|
42
|
+
.description('Run a one-shot autonomous task. Streams text to stdout; tool calls to stderr.')
|
|
43
|
+
.option('-m, --model <id>', 'Model id (default: env KLYRO_MODEL)')
|
|
44
|
+
.option('--max-steps <n>', 'Max agent steps (default 30)', (v) => parsePositiveInt('--max-steps', v))
|
|
45
|
+
.option('--max-tokens <n>', 'Max output tokens per step', (v) => parsePositiveInt('--max-tokens', v))
|
|
46
|
+
.option('--temperature <n>', 'Sampling temperature', (v) => {
|
|
47
|
+
const n = Number(v);
|
|
48
|
+
if (!Number.isFinite(n) || n < 0 || n > 2)
|
|
49
|
+
throw new Error(`invalid --temperature: ${v}`);
|
|
50
|
+
return n;
|
|
51
|
+
})
|
|
52
|
+
.option('--timeout <ms>', 'Request timeout in ms (default: env KLYRO_TIMEOUT_MS or 60000)', (v) => parsePositiveInt('--timeout', v))
|
|
53
|
+
.option('--base-url <url>', 'Override KLYRO_BASE_URL')
|
|
54
|
+
.option('--api-key <key>', 'Override KLYRO_API_KEY')
|
|
55
|
+
.option('--output <mode>', 'Output mode: human (default), json (one JSON per line), silent')
|
|
56
|
+
.option('--provider <name>', 'Provider: openai (default) or anthropic')
|
|
57
|
+
.option('--dry-run', 'Print the prompt assembly (system, tools, task) and exit without calling the model')
|
|
58
|
+
.option('--resume <file>', 'Resume from a saved transcript JSON file (must have a "transcript" field)')
|
|
59
|
+
.action(async (prompt, opts) => {
|
|
60
|
+
const model = opts.model ?? process.env.KLYRO_MODEL;
|
|
61
|
+
if (!model) {
|
|
62
|
+
process.stderr.write('klyro: KLYRO_MODEL is not set (or pass --model)\n');
|
|
63
|
+
process.exit(2);
|
|
64
|
+
}
|
|
65
|
+
const output = (opts.output ?? 'human');
|
|
66
|
+
if (output !== 'human' && output !== 'json' && output !== 'silent') {
|
|
67
|
+
process.stderr.write(`klyro: invalid --output: ${output} (expected human|json|silent)\n`);
|
|
68
|
+
process.exit(2);
|
|
69
|
+
}
|
|
70
|
+
const provider = (opts.provider ?? 'openai');
|
|
71
|
+
if (provider !== 'openai' && provider !== 'anthropic') {
|
|
72
|
+
process.stderr.write(`klyro: invalid --provider: ${provider} (expected openai|anthropic)\n`);
|
|
73
|
+
process.exit(2);
|
|
74
|
+
}
|
|
75
|
+
try {
|
|
76
|
+
const code = await runOnce({
|
|
77
|
+
task: prompt,
|
|
78
|
+
cwd: process.cwd(),
|
|
79
|
+
model,
|
|
80
|
+
maxSteps: opts.maxSteps,
|
|
81
|
+
maxTokens: opts.maxTokens,
|
|
82
|
+
temperature: opts.temperature,
|
|
83
|
+
timeoutMs: opts.timeout,
|
|
84
|
+
baseUrl: opts.baseUrl,
|
|
85
|
+
apiKey: opts.apiKey,
|
|
86
|
+
output,
|
|
87
|
+
provider,
|
|
88
|
+
dryRun: !!opts.dryRun,
|
|
89
|
+
resumePath: opts.resume,
|
|
90
|
+
});
|
|
91
|
+
process.exit(code);
|
|
92
|
+
}
|
|
93
|
+
catch (err) {
|
|
94
|
+
process.stderr.write(`klyro: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
95
|
+
process.exit(2);
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
program
|
|
99
|
+
.command('chat [prompt]')
|
|
100
|
+
.description('Legacy streamed chat. Without a prompt, start an interactive REPL.')
|
|
101
|
+
.option('-s, --system <text>', 'System message', 'You are a helpful assistant.')
|
|
102
|
+
.option('-m, --model <id>', 'Override the model (default: env KLYRO_MODEL)')
|
|
103
|
+
.option('-t, --timeout <ms>', 'Request timeout in ms (default: env KLYRO_TIMEOUT_MS or 60000)', (v) => parsePositiveInt('-t/--timeout', v))
|
|
104
|
+
.action(async (prompt, opts) => {
|
|
105
|
+
if (!prompt) {
|
|
106
|
+
await repl(opts.system);
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
await chat(prompt, opts.system, opts.model, { timeoutMs: opts.timeout });
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
program
|
|
113
|
+
.command('eval <input>')
|
|
114
|
+
.description('Run scripted scenarios from a JSONL file against the agent runtime. Exits 0 if all pass, 1 otherwise.')
|
|
115
|
+
.option('--output <mode>', 'Output mode: human (default), json (one JSON per line)')
|
|
116
|
+
.action(async (input, opts) => {
|
|
117
|
+
const output = (opts.output ?? 'human');
|
|
118
|
+
const code = await runEval({ inputPath: input, output });
|
|
119
|
+
process.exit(code);
|
|
120
|
+
});
|
|
121
|
+
await program.parseAsync(process.argv);
|
|
122
|
+
}
|
|
123
|
+
main().catch((err) => {
|
|
124
|
+
// Last-resort: anything that escaped the command handlers lands here.
|
|
125
|
+
// eslint-disable-next-line no-console
|
|
126
|
+
console.error(`klyro: ${err instanceof Error ? err.message : String(err)}`);
|
|
127
|
+
process.exit(1);
|
|
128
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA;;;;;;;GAOG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,OAAO,GAAG,OAAO,CAAC;AAExB,KAAK,UAAU,IAAI;IACjB,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO;SACJ,IAAI,CAAC,MAAM,CAAC;SACZ,WAAW,CAAC,wBAAwB,CAAC;SACrC,OAAO,CAAC,OAAO,EAAE,eAAe,EAAE,0BAA0B,CAAC;SAC7D,UAAU,CAAC,YAAY,EAAE,yBAAyB,CAAC;SACnD,kBAAkB,EAAE,CAAC;IAExB,OAAO;SACJ,OAAO,CAAC,eAAe,CAAC;SACxB,WAAW,CAAC,qFAAqF,CAAC;SAClG,MAAM,CAAC,qBAAqB,EAAE,gBAAgB,EAAE,8BAA8B,CAAC;SAC/E,MAAM,CAAC,kBAAkB,EAAE,8CAA8C,CAAC;SAC1E,MAAM,CAAC,oBAAoB,EAAE,+DAA+D,EAAE,CAAC,CAAC,EAAE,EAAE;QACnG,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;QAC5E,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;SACD,MAAM,CAAC,KAAK,EAAE,MAA0B,EAAE,IAA0D,EAAE,EAAE;QACvG,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AACzC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,sEAAsE;IACtE,sCAAsC;IACtC,OAAO,CAAC,KAAK,CAAC,SAAS,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|