@oh-my-pi/pi-coding-agent 16.4.3 → 16.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/cli.js +3014 -2996
- package/dist/types/tiny/message-preproc.d.ts +63 -0
- package/dist/types/tiny/text.d.ts +1 -27
- package/package.json +12 -12
- package/scripts/bench-title-models.ts +332 -0
- package/scripts/build-binary.ts +12 -12
- package/src/auto-thinking/classifier.ts +2 -16
- package/src/cli.ts +11 -3
- package/src/prompts/system/title-system.md +11 -12
- package/src/session/agent-session.ts +1 -1
- package/src/tiny/message-preproc.ts +155 -0
- package/src/tiny/text.ts +12 -70
- package/src/tiny/worker.ts +6 -3
- package/src/tools/image-gen.ts +3 -4
- package/src/utils/title-generator.ts +5 -4
- package/src/prompts/system/tiny-title-system.md +0 -8
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Converts raw user text into bounded, low-noise input for tiny models.
|
|
3
|
+
*
|
|
4
|
+
* Tiny models copy literal noise verbatim and lose the task when only the head
|
|
5
|
+
* of a long message survives. The shared pipeline strips ANSI escapes, paired
|
|
6
|
+
* XML/tool envelopes, full commit hashes, and fenced code blocks, then preserves
|
|
7
|
+
* both ends with an explicit omission marker. Title generation, auto-thinking,
|
|
8
|
+
* and the title benchmark MUST use this same policy.
|
|
9
|
+
*/
|
|
10
|
+
/** Maximum characters emitted by {@link preprocessTinyMessage}. */
|
|
11
|
+
export declare const MAX_TINY_MESSAGE_CHARS = 2000;
|
|
12
|
+
/** Drop SGR ANSI escape sequences. */
|
|
13
|
+
export declare function stripAnsi(message: string): string;
|
|
14
|
+
/**
|
|
15
|
+
* Remove paired XML/HTML-ish blocks (`<user>…</user>`, `<think>…</think>`,
|
|
16
|
+
* tool envelopes). Self-closing and unpaired inline tags (`<Header/>`, a lone
|
|
17
|
+
* `<div>`) are left in place — only fully paired blocks, whose contents would
|
|
18
|
+
* otherwise dominate the title, are dropped.
|
|
19
|
+
*/
|
|
20
|
+
export declare function stripXmlBlocks(message: string): string;
|
|
21
|
+
/** Truncate full commit-hash-like hex runs (≥12 chars) to a short 7-char prefix. */
|
|
22
|
+
export declare function shortenHashes(message: string): string;
|
|
23
|
+
/**
|
|
24
|
+
* Middle-truncate cleaned text, preserving 2/3 of the available space from the
|
|
25
|
+
* head and 1/3 from the tail. The omission marker counts toward the bound.
|
|
26
|
+
*/
|
|
27
|
+
export declare function truncateTinyMessage(message: string): string;
|
|
28
|
+
/**
|
|
29
|
+
* Strip fenced code blocks from a message before titling.
|
|
30
|
+
*
|
|
31
|
+
* Small title models latch onto literal text inside code blocks — e.g. a pasted
|
|
32
|
+
* UI mockup containing "Welcome to Claude Code v2.1.158" yields that string as
|
|
33
|
+
* the title instead of the surrounding intent. Removing fenced blocks leaves the
|
|
34
|
+
* prose that actually describes the task. Inline code (single backticks) is kept
|
|
35
|
+
* — it is short, high-signal context like `/login`.
|
|
36
|
+
*
|
|
37
|
+
* Falls back to the original message when stripping leaves too little to title
|
|
38
|
+
* (a message that is essentially just a code block).
|
|
39
|
+
*/
|
|
40
|
+
export declare function stripCodeBlocks(message: string): string;
|
|
41
|
+
/** Clean noise from message content without applying the length bound. */
|
|
42
|
+
export declare function cleanTinyMessage(message: string): string;
|
|
43
|
+
/** Apply the shared tiny-model cleanup and middle-truncation policy. */
|
|
44
|
+
export declare function preprocessTinyMessage(message: string): string;
|
|
45
|
+
/** True when `message` is a preformatted replan context from
|
|
46
|
+
* {@link formatTitleConversationContext} — already cleaned per turn and
|
|
47
|
+
* bounded, so it must bypass {@link preprocessTinyMessage} (whose paired-tag
|
|
48
|
+
* stripping would consume the entire envelope). */
|
|
49
|
+
export declare function isPreformattedChatContext(message: string): boolean;
|
|
50
|
+
/** Drop the `<chat>`/`<user>`/`<assistant>`/`<think>` scaffolding, keeping turn
|
|
51
|
+
* text. Used for token-level signal checks on preformatted contexts. */
|
|
52
|
+
export declare function stripChatScaffolding(message: string): string;
|
|
53
|
+
/** Wrap a preprocessed user message for title generation. Preformatted replan
|
|
54
|
+
* contexts pass through untouched. */
|
|
55
|
+
export declare function formatTitleUserMessage(message: string): string;
|
|
56
|
+
/** One recent conversation turn supplied to title refresh after replanning. */
|
|
57
|
+
export interface TitleConversationTurn {
|
|
58
|
+
role: "user" | "assistant";
|
|
59
|
+
text?: string;
|
|
60
|
+
thinking?: string;
|
|
61
|
+
}
|
|
62
|
+
/** Format preprocessed recent context for title generation after a todo replan. */
|
|
63
|
+
export declare function formatTitleConversationContext(turns: readonly TitleConversationTurn[]): string;
|
|
@@ -1,29 +1,3 @@
|
|
|
1
|
-
export declare const MAX_TITLE_INPUT_CHARS = 2000;
|
|
2
|
-
export declare function truncateTitleInput(message: string): string;
|
|
3
|
-
/**
|
|
4
|
-
* Strip fenced code blocks from a message before titling.
|
|
5
|
-
*
|
|
6
|
-
* Small title models latch onto literal text inside code blocks — e.g. a pasted
|
|
7
|
-
* UI mockup containing "Welcome to Claude Code v2.1.158" yields that string as
|
|
8
|
-
* the title instead of the surrounding intent. Removing fenced blocks leaves the
|
|
9
|
-
* prose that actually describes the task. Inline code (single backticks) is kept
|
|
10
|
-
* — it is short, high-signal context like `/login`.
|
|
11
|
-
*
|
|
12
|
-
* Falls back to the original message when stripping leaves too little to title
|
|
13
|
-
* (a message that is essentially just a code block).
|
|
14
|
-
*/
|
|
15
|
-
export declare function stripCodeBlocks(message: string): string;
|
|
16
|
-
/** Prepare a raw user message for titling: drop code blocks, then bound length. */
|
|
17
|
-
export declare function prepareTitleInput(message: string): string;
|
|
18
|
-
export declare function formatTitleUserMessage(message: string): string;
|
|
19
|
-
/** Single recent conversation turn supplied to title refresh after replanning. */
|
|
20
|
-
export interface TitleConversationTurn {
|
|
21
|
-
role: "user" | "assistant";
|
|
22
|
-
text?: string;
|
|
23
|
-
thinking?: string;
|
|
24
|
-
}
|
|
25
|
-
/** Format recent user/assistant context for title generation after a todo replan. */
|
|
26
|
-
export declare function formatTitleConversationContext(turns: readonly TitleConversationTurn[]): string;
|
|
27
1
|
/**
|
|
28
2
|
* True when a first user message is too low-signal to title (greeting, ack,
|
|
29
3
|
* bare number, or empty once code/punctuation/emoji are stripped).
|
|
@@ -38,7 +12,7 @@ export declare function isLowSignalTitleInput(message: string): boolean;
|
|
|
38
12
|
* Sentinel a capable title model may emit when a message carries no concrete
|
|
39
13
|
* task. Treated as "no title yet" so the caller can defer titling. Backstop for
|
|
40
14
|
* the deterministic {@link isLowSignalTitleInput} filter; kept in sync with the
|
|
41
|
-
*
|
|
15
|
+
* `<title/>` instruction in `prompts/system/title-system.md`.
|
|
42
16
|
*/
|
|
43
17
|
export declare const NO_TITLE_SENTINEL = "none";
|
|
44
18
|
export declare function normalizeGeneratedTitle(value: string | null | undefined, sourceText?: string): string | null;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@oh-my-pi/pi-coding-agent",
|
|
4
|
-
"version": "16.4.
|
|
4
|
+
"version": "16.4.4",
|
|
5
5
|
"description": "Coding agent CLI with read, bash, edit, write tools and session management",
|
|
6
6
|
"homepage": "https://omp.sh",
|
|
7
7
|
"author": "Can Boluk",
|
|
@@ -52,17 +52,17 @@
|
|
|
52
52
|
"@agentclientprotocol/sdk": "0.25.0",
|
|
53
53
|
"@babel/parser": "^7.29.7",
|
|
54
54
|
"@mozilla/readability": "^0.6.0",
|
|
55
|
-
"@oh-my-pi/hashline": "16.4.
|
|
56
|
-
"@oh-my-pi/omp-stats": "16.4.
|
|
57
|
-
"@oh-my-pi/pi-agent-core": "16.4.
|
|
58
|
-
"@oh-my-pi/pi-ai": "16.4.
|
|
59
|
-
"@oh-my-pi/pi-catalog": "16.4.
|
|
60
|
-
"@oh-my-pi/pi-mnemopi": "16.4.
|
|
61
|
-
"@oh-my-pi/pi-natives": "16.4.
|
|
62
|
-
"@oh-my-pi/pi-tui": "16.4.
|
|
63
|
-
"@oh-my-pi/pi-utils": "16.4.
|
|
64
|
-
"@oh-my-pi/pi-wire": "16.4.
|
|
65
|
-
"@oh-my-pi/snapcompact": "16.4.
|
|
55
|
+
"@oh-my-pi/hashline": "16.4.4",
|
|
56
|
+
"@oh-my-pi/omp-stats": "16.4.4",
|
|
57
|
+
"@oh-my-pi/pi-agent-core": "16.4.4",
|
|
58
|
+
"@oh-my-pi/pi-ai": "16.4.4",
|
|
59
|
+
"@oh-my-pi/pi-catalog": "16.4.4",
|
|
60
|
+
"@oh-my-pi/pi-mnemopi": "16.4.4",
|
|
61
|
+
"@oh-my-pi/pi-natives": "16.4.4",
|
|
62
|
+
"@oh-my-pi/pi-tui": "16.4.4",
|
|
63
|
+
"@oh-my-pi/pi-utils": "16.4.4",
|
|
64
|
+
"@oh-my-pi/pi-wire": "16.4.4",
|
|
65
|
+
"@oh-my-pi/snapcompact": "16.4.4",
|
|
66
66
|
"@opentelemetry/api": "^1.9.1",
|
|
67
67
|
"@opentelemetry/context-async-hooks": "^2.7.1",
|
|
68
68
|
"@opentelemetry/exporter-trace-otlp-proto": "^0.218.0",
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
import { Database } from "bun:sqlite";
|
|
3
|
+
/**
|
|
4
|
+
* Title-generation benchmark harness.
|
|
5
|
+
*
|
|
6
|
+
* Samples random first-of-session messages from the local history DB, renders
|
|
7
|
+
* the shipped `title-system.md` prompt, and runs every message against a matrix
|
|
8
|
+
* of title models — the on-device ONNX models (LFM2 350M/700M, Gemma 270M) via
|
|
9
|
+
* the tiny-title worker, plus a remote Ollama model (Llama 3.2 3B by default).
|
|
10
|
+
* Each model lane runs concurrently; within a lane requests are sequential
|
|
11
|
+
* because the local worker serializes generation on one pipeline.
|
|
12
|
+
*
|
|
13
|
+
* Results (per-sample titles + latency, plus per-model summaries) are written
|
|
14
|
+
* to a timestamped JSON file so runs can be compared later.
|
|
15
|
+
*
|
|
16
|
+
* Usage:
|
|
17
|
+
* bun scripts/bench-title-models.ts
|
|
18
|
+
* bun scripts/bench-title-models.ts --count 30 --seed 42
|
|
19
|
+
* bun scripts/bench-title-models.ts --models lfm2-350m,gemma-270m
|
|
20
|
+
* bun scripts/bench-title-models.ts --ollama-url http://spark.internal:11434 --ollama-models llama3.2:3b,lfm2:2.6b
|
|
21
|
+
* bun scripts/bench-title-models.ts --db ~/.omp/agent/history.db --out bench.json
|
|
22
|
+
*/
|
|
23
|
+
import * as os from "node:os";
|
|
24
|
+
import * as path from "node:path";
|
|
25
|
+
import { prompt } from "@oh-my-pi/pi-utils";
|
|
26
|
+
import titleSystemPrompt from "../src/prompts/system/title-system.md" with { type: "text" };
|
|
27
|
+
import { preprocessTinyMessage } from "../src/tiny/message-preproc";
|
|
28
|
+
import { isTinyTitleLocalModelKey } from "../src/tiny/models";
|
|
29
|
+
import { normalizeGeneratedTitle } from "../src/tiny/text";
|
|
30
|
+
import { shutdownTinyTitleClient, tinyTitleClient } from "../src/tiny/title-client";
|
|
31
|
+
|
|
32
|
+
/** A sampled prompt with the cleaned text actually fed to the models. */
|
|
33
|
+
interface PreparedPrompt {
|
|
34
|
+
id: number;
|
|
35
|
+
raw: string;
|
|
36
|
+
input: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** One title produced for one input by one model, with wall-clock latency. */
|
|
40
|
+
interface BenchSample {
|
|
41
|
+
id: number;
|
|
42
|
+
input: string;
|
|
43
|
+
title: string | null;
|
|
44
|
+
ms: number;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** All samples for one model plus the aggregate quality/latency summary. */
|
|
48
|
+
interface BenchLane {
|
|
49
|
+
model: string;
|
|
50
|
+
transport: "local" | "ollama";
|
|
51
|
+
samples: BenchSample[];
|
|
52
|
+
summary: BenchSummary;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Aggregate stats for a lane; latency percentiles skip the cold first call. */
|
|
56
|
+
interface BenchSummary {
|
|
57
|
+
count: number;
|
|
58
|
+
nulls: number;
|
|
59
|
+
coldMs: number;
|
|
60
|
+
warmMeanMs: number;
|
|
61
|
+
warmMedianMs: number;
|
|
62
|
+
warmP95Ms: number;
|
|
63
|
+
lengthCompliant: string;
|
|
64
|
+
punctuationFree: string;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
interface BenchConfig {
|
|
68
|
+
dbPath: string;
|
|
69
|
+
count: number;
|
|
70
|
+
seed: number;
|
|
71
|
+
localModels: string[];
|
|
72
|
+
ollamaUrl: string | null;
|
|
73
|
+
ollamaModels: string[];
|
|
74
|
+
outPath: string;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const DEFAULT_LOCAL_MODELS = ["lfm2-350m", "lfm2-700m", "gemma-270m"];
|
|
78
|
+
const DEFAULT_OLLAMA_URL = "http://spark.internal:11434";
|
|
79
|
+
const DEFAULT_OLLAMA_MODELS = ["llama3.2:3b", "lfm2:2.6b"];
|
|
80
|
+
const MIN_INPUT_CHARS = 10;
|
|
81
|
+
const MAX_INPUT_CHARS = 800;
|
|
82
|
+
|
|
83
|
+
/** System prompt with examples (used for the capable Ollama model). */
|
|
84
|
+
const TITLE_PROMPT_WITH_EXAMPLES = prompt.render(titleSystemPrompt, { includeExamples: true });
|
|
85
|
+
/** Example-free prompt matching what the on-device worker ships to tiny models. */
|
|
86
|
+
const TITLE_PROMPT_NO_EXAMPLES = prompt.render(titleSystemPrompt, { includeExamples: false });
|
|
87
|
+
|
|
88
|
+
/** Deterministic mulberry32 PRNG so `--seed` reproduces a sample set. */
|
|
89
|
+
function createRng(seed: number): () => number {
|
|
90
|
+
let state = seed >>> 0;
|
|
91
|
+
return () => {
|
|
92
|
+
state |= 0;
|
|
93
|
+
state = (state + 0x6d2b79f5) | 0;
|
|
94
|
+
let t = Math.imul(state ^ (state >>> 15), 1 | state);
|
|
95
|
+
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
96
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Pick `count` distinct random first-of-session prompts within the size band. */
|
|
101
|
+
function sampleHistoryPrompts(dbPath: string, count: number, rng: () => number): { id: number; prompt: string }[] {
|
|
102
|
+
const db = new Database(dbPath, { readonly: true });
|
|
103
|
+
try {
|
|
104
|
+
const rows = db
|
|
105
|
+
.query(
|
|
106
|
+
`WITH firsts AS (
|
|
107
|
+
SELECT session_id, MIN(id) AS id FROM history
|
|
108
|
+
WHERE session_id IS NOT NULL
|
|
109
|
+
GROUP BY session_id
|
|
110
|
+
)
|
|
111
|
+
SELECT h.id AS id, h.prompt AS prompt
|
|
112
|
+
FROM history h JOIN firsts ON firsts.id = h.id
|
|
113
|
+
WHERE length(trim(h.prompt)) BETWEEN ? AND ?`,
|
|
114
|
+
)
|
|
115
|
+
.all(MIN_INPUT_CHARS, MAX_INPUT_CHARS) as { id: number; prompt: string }[];
|
|
116
|
+
const seen = new Set<string>();
|
|
117
|
+
const unique: { id: number; prompt: string }[] = [];
|
|
118
|
+
for (const row of rows) {
|
|
119
|
+
const key = row.prompt.trim();
|
|
120
|
+
if (seen.has(key)) continue;
|
|
121
|
+
seen.add(key);
|
|
122
|
+
unique.push(row);
|
|
123
|
+
}
|
|
124
|
+
// Fisher–Yates with the seeded RNG, then take the first `count`.
|
|
125
|
+
for (let i = unique.length - 1; i > 0; i--) {
|
|
126
|
+
const j = Math.floor(rng() * (i + 1));
|
|
127
|
+
[unique[i], unique[j]] = [unique[j], unique[i]];
|
|
128
|
+
}
|
|
129
|
+
return unique.slice(0, Math.min(count, unique.length));
|
|
130
|
+
} finally {
|
|
131
|
+
db.close();
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Run one local ONNX model over every prompt (sequential; worker is single-lane). */
|
|
136
|
+
async function runLocalLane(model: string, prompts: PreparedPrompt[]): Promise<BenchSample[]> {
|
|
137
|
+
const samples: BenchSample[] = [];
|
|
138
|
+
for (const item of prompts) {
|
|
139
|
+
const started = performance.now();
|
|
140
|
+
const title = await tinyTitleClient.generate(model, item.input, { systemPrompt: TITLE_PROMPT_NO_EXAMPLES });
|
|
141
|
+
samples.push({ id: item.id, input: item.input, title, ms: performance.now() - started });
|
|
142
|
+
}
|
|
143
|
+
return samples;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** Extract the `<title>` payload from a free-form chat completion. */
|
|
147
|
+
function parseChatTitle(text: string, sourceText: string): string | null {
|
|
148
|
+
if (!text || /<title\s*\/>/i.test(text)) return null;
|
|
149
|
+
const closed = /<title>([\s\S]*?)<\/title>/i.exec(text);
|
|
150
|
+
const open = closed ? null : /<title>([\s\S]*)/i.exec(text);
|
|
151
|
+
return normalizeGeneratedTitle(closed?.[1] ?? open?.[1] ?? text, sourceText);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Run one Ollama chat model over every prompt via the /api/chat endpoint. */
|
|
155
|
+
async function runOllamaLane(baseUrl: string, model: string, prompts: PreparedPrompt[]): Promise<BenchSample[]> {
|
|
156
|
+
const samples: BenchSample[] = [];
|
|
157
|
+
for (const item of prompts) {
|
|
158
|
+
const started = performance.now();
|
|
159
|
+
const response = await fetch(new URL("/api/chat", baseUrl), {
|
|
160
|
+
method: "POST",
|
|
161
|
+
headers: { "content-type": "application/json" },
|
|
162
|
+
body: JSON.stringify({
|
|
163
|
+
model,
|
|
164
|
+
stream: false,
|
|
165
|
+
keep_alive: "10m",
|
|
166
|
+
messages: [
|
|
167
|
+
{ role: "system", content: TITLE_PROMPT_WITH_EXAMPLES },
|
|
168
|
+
{ role: "user", content: `<user>\n${item.input}\n</user>` },
|
|
169
|
+
],
|
|
170
|
+
options: { temperature: 0, num_predict: 32 },
|
|
171
|
+
}),
|
|
172
|
+
});
|
|
173
|
+
if (!response.ok) throw new Error(`Ollama ${response.status}: ${await response.text()}`);
|
|
174
|
+
const payload = (await response.json()) as { message?: { content?: string } };
|
|
175
|
+
const raw = payload.message?.content ?? "";
|
|
176
|
+
samples.push({
|
|
177
|
+
id: item.id,
|
|
178
|
+
input: item.input,
|
|
179
|
+
title: parseChatTitle(raw, item.input),
|
|
180
|
+
ms: performance.now() - started,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
return samples;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Fold a lane's samples into latency percentiles and title-quality ratios. */
|
|
187
|
+
function summarize(samples: BenchSample[]): BenchSummary {
|
|
188
|
+
const warm = samples
|
|
189
|
+
.slice(1)
|
|
190
|
+
.map(sample => sample.ms)
|
|
191
|
+
.sort((a, b) => a - b);
|
|
192
|
+
const outputs = samples.filter(sample => sample.title !== null);
|
|
193
|
+
const percentile = (sorted: number[], q: number): number =>
|
|
194
|
+
sorted.length === 0 ? 0 : sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * q) - 1))];
|
|
195
|
+
const wordCompliant = outputs.filter(sample => {
|
|
196
|
+
const words = sample.title!.trim().split(/\s+/).length;
|
|
197
|
+
return words >= 3 && words <= 7;
|
|
198
|
+
}).length;
|
|
199
|
+
const punctuationFree = outputs.filter(sample => !/\p{P}/u.test(sample.title!)).length;
|
|
200
|
+
return {
|
|
201
|
+
count: samples.length,
|
|
202
|
+
nulls: samples.length - outputs.length,
|
|
203
|
+
coldMs: Number((samples[0]?.ms ?? 0).toFixed(1)),
|
|
204
|
+
warmMeanMs: Number((warm.reduce((sum, value) => sum + value, 0) / (warm.length || 1)).toFixed(1)),
|
|
205
|
+
warmMedianMs: Number(percentile(warm, 0.5).toFixed(1)),
|
|
206
|
+
warmP95Ms: Number(percentile(warm, 0.95).toFixed(1)),
|
|
207
|
+
lengthCompliant: `${wordCompliant}/${outputs.length}`,
|
|
208
|
+
punctuationFree: `${punctuationFree}/${outputs.length}`,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function parseArgs(argv: string[]): BenchConfig {
|
|
213
|
+
const get = (flag: string): string | undefined => {
|
|
214
|
+
const index = argv.indexOf(flag);
|
|
215
|
+
return index >= 0 ? argv[index + 1] : undefined;
|
|
216
|
+
};
|
|
217
|
+
const has = (flag: string): boolean => argv.includes(flag);
|
|
218
|
+
const modelsArg = get("--models");
|
|
219
|
+
const ollamaModelsArg = get("--ollama-models");
|
|
220
|
+
const ollamaUrlArg = get("--ollama-url");
|
|
221
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
222
|
+
return {
|
|
223
|
+
dbPath: (get("--db") ?? path.join(os.homedir(), ".omp/agent/history.db")).replace(/^~/, os.homedir()),
|
|
224
|
+
count: Number(get("--count") ?? 20),
|
|
225
|
+
seed: Number(get("--seed") ?? Date.now() & 0xffffffff),
|
|
226
|
+
localModels: modelsArg
|
|
227
|
+
? modelsArg
|
|
228
|
+
.split(",")
|
|
229
|
+
.map(model => model.trim())
|
|
230
|
+
.filter(Boolean)
|
|
231
|
+
: DEFAULT_LOCAL_MODELS,
|
|
232
|
+
ollamaUrl: has("--no-ollama") ? null : (ollamaUrlArg ?? DEFAULT_OLLAMA_URL),
|
|
233
|
+
ollamaModels: ollamaModelsArg
|
|
234
|
+
? ollamaModelsArg
|
|
235
|
+
.split(",")
|
|
236
|
+
.map(model => model.trim())
|
|
237
|
+
.filter(Boolean)
|
|
238
|
+
: DEFAULT_OLLAMA_MODELS,
|
|
239
|
+
outPath: get("--out") ?? path.join(os.tmpdir(), `title-bench-${stamp}.json`),
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
async function main(): Promise<void> {
|
|
244
|
+
const config = parseArgs(Bun.argv.slice(2));
|
|
245
|
+
const rng = createRng(config.seed);
|
|
246
|
+
const rows = sampleHistoryPrompts(config.dbPath, config.count, rng);
|
|
247
|
+
if (rows.length === 0) throw new Error(`No history prompts found in ${config.dbPath}`);
|
|
248
|
+
const prepared: PreparedPrompt[] = rows.map(row => ({
|
|
249
|
+
id: row.id,
|
|
250
|
+
raw: row.prompt,
|
|
251
|
+
input: preprocessTinyMessage(row.prompt),
|
|
252
|
+
}));
|
|
253
|
+
|
|
254
|
+
const invalidLocal = config.localModels.filter(model => !isTinyTitleLocalModelKey(model));
|
|
255
|
+
if (invalidLocal.length > 0) throw new Error(`Unknown local title model(s): ${invalidLocal.join(", ")}`);
|
|
256
|
+
|
|
257
|
+
console.info(`Benchmarking ${rows.length} prompts (seed ${config.seed}) from ${config.dbPath}`);
|
|
258
|
+
|
|
259
|
+
// Each model is its own concurrent lane; the local worker still serializes
|
|
260
|
+
// its own lanes internally, but the Ollama lane genuinely runs in parallel.
|
|
261
|
+
const laneTasks: Promise<BenchLane>[] = [
|
|
262
|
+
...config.localModels.map(async (model): Promise<BenchLane> => {
|
|
263
|
+
const samples = await runLocalLane(model, prepared);
|
|
264
|
+
return { model, transport: "local", samples, summary: summarize(samples) };
|
|
265
|
+
}),
|
|
266
|
+
];
|
|
267
|
+
if (config.ollamaUrl) {
|
|
268
|
+
const url = config.ollamaUrl;
|
|
269
|
+
for (const model of config.ollamaModels) {
|
|
270
|
+
laneTasks.push(
|
|
271
|
+
(async (): Promise<BenchLane> => {
|
|
272
|
+
const samples = await runOllamaLane(url, model, prepared);
|
|
273
|
+
return { model: `${model}@ollama`, transport: "ollama", samples, summary: summarize(samples) };
|
|
274
|
+
})(),
|
|
275
|
+
);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const settled = await Promise.allSettled(laneTasks);
|
|
280
|
+
const lanes: BenchLane[] = [];
|
|
281
|
+
for (const result of settled) {
|
|
282
|
+
if (result.status === "fulfilled") lanes.push(result.value);
|
|
283
|
+
else
|
|
284
|
+
console.error(
|
|
285
|
+
`Lane failed: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
|
|
286
|
+
);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
await shutdownTinyTitleClient();
|
|
290
|
+
|
|
291
|
+
// Prompt-centric view: each row is one input with every model's title beside it.
|
|
292
|
+
const matrix = prepared.map(item => {
|
|
293
|
+
const titles: Record<string, string> = {};
|
|
294
|
+
for (const lane of lanes) titles[lane.model] = lane.samples.find(sample => sample.id === item.id)?.title ?? "∅";
|
|
295
|
+
return { id: item.id, raw: item.raw, input: item.input, titles };
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
const report = {
|
|
299
|
+
generatedAt: new Date().toISOString(),
|
|
300
|
+
config: { ...config, prompts: prepared },
|
|
301
|
+
matrix,
|
|
302
|
+
lanes,
|
|
303
|
+
};
|
|
304
|
+
await Bun.write(config.outPath, JSON.stringify(report, null, 2));
|
|
305
|
+
|
|
306
|
+
for (const entry of matrix) {
|
|
307
|
+
console.info(`\n[#${entry.id}] ${entry.raw.replace(/\s+/g, " ").slice(0, 140)}`);
|
|
308
|
+
if (entry.input !== entry.raw.trim())
|
|
309
|
+
console.info(` (cleaned) ${entry.input.replace(/\s+/g, " ").slice(0, 140)}`);
|
|
310
|
+
console.table(Object.fromEntries(lanes.map(lane => [lane.model, { output: entry.titles[lane.model] }])));
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
console.info("\nSummary:");
|
|
314
|
+
console.table(
|
|
315
|
+
Object.fromEntries(
|
|
316
|
+
lanes.map(lane => [
|
|
317
|
+
lane.model,
|
|
318
|
+
{
|
|
319
|
+
cold: lane.summary.coldMs,
|
|
320
|
+
warmMean: lane.summary.warmMeanMs,
|
|
321
|
+
warmP95: lane.summary.warmP95Ms,
|
|
322
|
+
nulls: lane.summary.nulls,
|
|
323
|
+
len3to7: lane.summary.lengthCompliant,
|
|
324
|
+
punctFree: lane.summary.punctuationFree,
|
|
325
|
+
},
|
|
326
|
+
]),
|
|
327
|
+
),
|
|
328
|
+
);
|
|
329
|
+
console.info(`\nWrote ${config.outPath}`);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
await main();
|
package/scripts/build-binary.ts
CHANGED
|
@@ -7,14 +7,16 @@ import { compileCodingAgent } from "./compile-binary";
|
|
|
7
7
|
const packageDir = path.join(import.meta.dir, "..");
|
|
8
8
|
const repoRoot = path.join(packageDir, "..", "..");
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
/** Binary cross-compilation settings selected by `CROSS_TARGET`. */
|
|
11
|
+
export interface CrossBuild {
|
|
11
12
|
readonly id: string;
|
|
12
13
|
readonly platform: string;
|
|
13
14
|
readonly arch: string;
|
|
14
15
|
readonly target: Bun.Build.CompileTarget;
|
|
15
16
|
}
|
|
16
17
|
|
|
17
|
-
|
|
18
|
+
/** Resolves a CROSS_TARGET value to the Bun compile target used by local binary builds. */
|
|
19
|
+
export function resolveCrossBuild(value: string | undefined): CrossBuild | null {
|
|
18
20
|
switch (value) {
|
|
19
21
|
case undefined:
|
|
20
22
|
case "":
|
|
@@ -29,16 +31,12 @@ function resolveCrossBuild(value: string | undefined): CrossBuild | null {
|
|
|
29
31
|
return { id: value, platform: "linux", arch: "x64", target: "bun-linux-x64-baseline" };
|
|
30
32
|
case "win32-x64":
|
|
31
33
|
case "windows-x64":
|
|
32
|
-
return { id: value, platform: "win32", arch: "x64", target: "bun-windows-x64-
|
|
34
|
+
return { id: value, platform: "win32", arch: "x64", target: "bun-windows-x64-baseline" };
|
|
33
35
|
default:
|
|
34
36
|
throw new Error(`Unsupported CROSS_TARGET: ${value}`);
|
|
35
37
|
}
|
|
36
38
|
}
|
|
37
39
|
|
|
38
|
-
const crossBuild = resolveCrossBuild(Bun.env.CROSS_TARGET);
|
|
39
|
-
const outName = crossBuild ? `omp-${crossBuild.id}` : "omp";
|
|
40
|
-
const outputPath = path.join(packageDir, "dist", outName);
|
|
41
|
-
|
|
42
40
|
// Transformers.js is an optional, native-heavy dependency that is never bundled
|
|
43
41
|
// into the binary; the tiny-model worker `bun install`s it into a runtime cache
|
|
44
42
|
// on first use. The `catalog:` spec cannot be resolved from inside the compiled
|
|
@@ -55,7 +53,7 @@ if (
|
|
|
55
53
|
}
|
|
56
54
|
const transformersVersion = transformersManifest.version;
|
|
57
55
|
|
|
58
|
-
function shouldAdhocSignDarwinBinary(): boolean {
|
|
56
|
+
function shouldAdhocSignDarwinBinary(crossBuild: CrossBuild | null): boolean {
|
|
59
57
|
return process.platform === "darwin" && !crossBuild;
|
|
60
58
|
}
|
|
61
59
|
|
|
@@ -77,6 +75,9 @@ async function runCommand(
|
|
|
77
75
|
}
|
|
78
76
|
|
|
79
77
|
async function main(): Promise<void> {
|
|
78
|
+
const crossBuild = resolveCrossBuild(Bun.env.CROSS_TARGET);
|
|
79
|
+
const outName = crossBuild ? `omp-${crossBuild.id}` : "omp";
|
|
80
|
+
const outputPath = path.join(packageDir, "dist", outName);
|
|
80
81
|
// Generate inside the try so the finally always restores the empty checked-in
|
|
81
82
|
// placeholders (stats client archive, docs index) even on failure.
|
|
82
83
|
try {
|
|
@@ -99,11 +100,10 @@ async function main(): Promise<void> {
|
|
|
99
100
|
transformersVersion,
|
|
100
101
|
target: crossBuild?.target,
|
|
101
102
|
external: ["fastembed", "onnxruntime-node"],
|
|
102
|
-
skipBuiltinCodesign: shouldAdhocSignDarwinBinary(),
|
|
103
|
+
skipBuiltinCodesign: shouldAdhocSignDarwinBinary(crossBuild),
|
|
103
104
|
});
|
|
104
105
|
|
|
105
|
-
|
|
106
|
-
if (shouldAdhocSignDarwinBinary()) {
|
|
106
|
+
if (shouldAdhocSignDarwinBinary(crossBuild)) {
|
|
107
107
|
await runCommand(["codesign", "--force", "--sign", "-", outputPath]);
|
|
108
108
|
}
|
|
109
109
|
} finally {
|
|
@@ -115,4 +115,4 @@ async function main(): Promise<void> {
|
|
|
115
115
|
}
|
|
116
116
|
}
|
|
117
117
|
|
|
118
|
-
await main();
|
|
118
|
+
if (import.meta.main) await main();
|
|
@@ -22,6 +22,7 @@ import type { Settings } from "../config/settings";
|
|
|
22
22
|
import difficultySystemPrompt from "../prompts/system/auto-thinking-difficulty.md" with { type: "text" };
|
|
23
23
|
import difficultyLocalPrompt from "../prompts/system/auto-thinking-difficulty-local.md" with { type: "text" };
|
|
24
24
|
import { clampAutoThinkingEffort } from "../thinking";
|
|
25
|
+
import { preprocessTinyMessage } from "../tiny/message-preproc";
|
|
25
26
|
import {
|
|
26
27
|
isTinyMemoryLocalModelKey,
|
|
27
28
|
isTinyMemoryReasoningModelKey,
|
|
@@ -31,10 +32,6 @@ import { tinyModelClient } from "../tiny/title-client";
|
|
|
31
32
|
|
|
32
33
|
const DIFFICULTY_SYSTEM_PROMPT = prompt.render(difficultySystemPrompt);
|
|
33
34
|
|
|
34
|
-
/** Upper bound on prompt characters fed to the classifier. */
|
|
35
|
-
const MAX_INPUT_CHARS = 6000;
|
|
36
|
-
const HEAD_CHARS = 4000;
|
|
37
|
-
const TAIL_CHARS = 2000;
|
|
38
35
|
/** Local classifiers occasionally need more room for chat-template boilerplate. */
|
|
39
36
|
const LOCAL_ANSWER_MAX_TOKENS = 16;
|
|
40
37
|
/**
|
|
@@ -66,7 +63,7 @@ export async function classifyDifficulty(
|
|
|
66
63
|
deps: ClassifyDifficultyDeps,
|
|
67
64
|
): Promise<Effort | undefined> {
|
|
68
65
|
const backend = deps.settings.get("providers.autoThinkingModel");
|
|
69
|
-
const input =
|
|
66
|
+
const input = preprocessTinyMessage(promptText);
|
|
70
67
|
const effort =
|
|
71
68
|
backend === ONLINE_AUTO_THINKING_MODEL_KEY
|
|
72
69
|
? await classifyOnline(input, deps)
|
|
@@ -183,14 +180,3 @@ function extractText(content: AssistantMessage["content"]): string {
|
|
|
183
180
|
.join(" ")
|
|
184
181
|
.trim();
|
|
185
182
|
}
|
|
186
|
-
|
|
187
|
-
/**
|
|
188
|
-
* Bound the classifier input. Code blocks are kept (a large diff is signal), but
|
|
189
|
-
* very long prompts are head+tail trimmed so the intent (start) and any trailing
|
|
190
|
-
* error/stacktrace (end) both survive.
|
|
191
|
-
*/
|
|
192
|
-
function prepareClassifierInput(text: string): string {
|
|
193
|
-
const trimmed = text.trim();
|
|
194
|
-
if (trimmed.length <= MAX_INPUT_CHARS) return trimmed;
|
|
195
|
-
return `${trimmed.slice(0, HEAD_CHARS)}\n…\n${trimmed.slice(-TAIL_CHARS)}`;
|
|
196
|
-
}
|
package/src/cli.ts
CHANGED
|
@@ -37,6 +37,14 @@ if (Bun.semver.order(Bun.version, MIN_BUN_VERSION) < 0) {
|
|
|
37
37
|
|
|
38
38
|
process.title = APP_NAME;
|
|
39
39
|
|
|
40
|
+
// `Bun.build`-API compiled Windows executables report `import.meta.main ===
|
|
41
|
+
// false`: the standalone loader keys the entry module with native backslashes
|
|
42
|
+
// (`B:\~BUN\root\cli.js`) but registers the main path with forward slashes
|
|
43
|
+
// (`B:/~BUN/root/cli.js`), so Bun's internal match fails. `bun build --compile`
|
|
44
|
+
// CLI builds are unaffected. A compiled binary's entry module is by definition
|
|
45
|
+
// the process entry, so the define-folded PI_COMPILED marker stands in.
|
|
46
|
+
const isProcessEntry = import.meta.main || process.env.PI_COMPILED === "true";
|
|
47
|
+
|
|
40
48
|
// Worker-host entry declaration (Worker threads and worker subprocesses
|
|
41
49
|
// re-enter `Bun.main` with a hidden argv selector instead of loading separate
|
|
42
50
|
// worker entrypoints) happens inside `runCli` after profile bootstrap:
|
|
@@ -305,13 +313,13 @@ export async function runCli(argv: string[]): Promise<void> {
|
|
|
305
313
|
// Declare this module as the worker-host entry now that the active profile
|
|
306
314
|
// is resolved. The worker-host module is side-effect-free; importing
|
|
307
315
|
// `@oh-my-pi/pi-utils/env` here would snapshot the wrong agent `.env`.
|
|
308
|
-
// Gated on `
|
|
316
|
+
// Gated on `isProcessEntry`: only the real CLI process entry is a valid
|
|
309
317
|
// worker host. Worker-thread re-entry already returned above at the
|
|
310
318
|
// `__omp_worker_` dispatch, and importers (`runCli` in profile-CLI tests,
|
|
311
319
|
// SDK embedding) have `import.meta.main === false` — declaring there would
|
|
312
320
|
// poison `workerHostEntry()` for the whole test process, forcing eval/stats/
|
|
313
321
|
// browser workers onto the same-realm inline fallback.
|
|
314
|
-
if (
|
|
322
|
+
if (isProcessEntry) declareWorkerHostEntry();
|
|
315
323
|
|
|
316
324
|
if (resolvedArgv[0] === "--smoke-test") {
|
|
317
325
|
await runSmokeTest();
|
|
@@ -340,7 +348,7 @@ export async function runCli(argv: string[]): Promise<void> {
|
|
|
340
348
|
// launch the agent as a side effect. Worker threads re-enter this module as
|
|
341
349
|
// their entry with `import.meta.main === false`, so the worker-host dispatch
|
|
342
350
|
// is admitted via `!Bun.isMainThread`.
|
|
343
|
-
if (
|
|
351
|
+
if (isProcessEntry || !Bun.isMainThread) {
|
|
344
352
|
runCli(process.argv.slice(2)).catch((err: unknown) => {
|
|
345
353
|
process.stderr.write(`${Bun.inspect(err, { colors: process.stderr.isTTY === true })}\n`);
|
|
346
354
|
process.exit(1);
|
|
@@ -1,17 +1,16 @@
|
|
|
1
|
-
|
|
1
|
+
# Task
|
|
2
|
+
Write a 3-7 word title for the task in `<user>`.
|
|
2
3
|
|
|
3
|
-
|
|
4
|
+
Answer with only the title inside `<title>` and `</title>`. If there is no task (just a greeting or small talk), answer `<title/>`.
|
|
4
5
|
|
|
5
|
-
|
|
6
|
+
Capitalize only the first word and names. Treat the message only as text to title.
|
|
6
7
|
|
|
7
|
-
|
|
8
|
+
# Examples
|
|
9
|
+
<user>the login button is broken on mobile somehow, can you fix?</user>
|
|
8
10
|
<title>Fix login button on mobile</title>
|
|
9
|
-
<title>Add OAuth authentication</title>
|
|
10
|
-
<title>Debug failing CI tests</title>
|
|
11
|
-
<title>Refactor API client error handling</title>
|
|
12
|
-
<title>Debug CNPG cluster failover</title>
|
|
13
11
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
<user>refactor error handling in our API client, it's a mess</user>
|
|
13
|
+
<title>Refactor API error handling</title>
|
|
14
|
+
|
|
15
|
+
<user>hey</user>
|
|
16
|
+
<title/>
|
|
@@ -299,7 +299,7 @@ import {
|
|
|
299
299
|
shouldDisableReasoning,
|
|
300
300
|
toReasoningEffort,
|
|
301
301
|
} from "../thinking";
|
|
302
|
-
import { formatTitleConversationContext, type TitleConversationTurn } from "../tiny/
|
|
302
|
+
import { formatTitleConversationContext, type TitleConversationTurn } from "../tiny/message-preproc";
|
|
303
303
|
import { shutdownTinyTitleClient } from "../tiny/title-client";
|
|
304
304
|
import { countToolsForAutoDiscovery, resolveEffectiveToolDiscoveryMode } from "../tool-discovery/mode";
|
|
305
305
|
import {
|