@oh-my-pi/pi-coding-agent 16.2.5 → 16.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -1
- package/dist/cli.js +2991 -3011
- package/dist/types/cli/bench-cli.d.ts +6 -1
- package/dist/types/commands/bench.d.ts +8 -0
- package/dist/types/config/model-discovery.d.ts +6 -1
- package/dist/types/config/settings-schema.d.ts +1 -1
- package/dist/types/edit/index.d.ts +1 -0
- package/dist/types/edit/renderer.d.ts +4 -0
- package/dist/types/edit/snapshot-details.d.ts +33 -0
- package/dist/types/mcp/oauth-flow.d.ts +4 -6
- package/dist/types/modes/controllers/tool-args-reveal.d.ts +9 -2
- package/dist/types/session/agent-session.d.ts +4 -0
- package/dist/types/session/session-manager.d.ts +3 -4
- package/dist/types/web/search/providers/duckduckgo.d.ts +2 -2
- package/dist/types/web/search/types.d.ts +1 -1
- package/package.json +12 -12
- package/src/cli/args.ts +32 -1
- package/src/cli/bench-cli.ts +89 -21
- package/src/cli/web-search-cli.ts +6 -1
- package/src/commands/bench.ts +10 -1
- package/src/config/mcp-schema.json +1 -1
- package/src/config/model-discovery.ts +66 -8
- package/src/config/model-registry.ts +13 -6
- package/src/edit/hashline/execute.ts +15 -9
- package/src/edit/index.ts +19 -6
- package/src/edit/modes/patch.ts +3 -2
- package/src/edit/modes/replace.ts +3 -2
- package/src/edit/renderer.ts +4 -0
- package/src/edit/snapshot-details.ts +77 -0
- package/src/extensibility/plugins/legacy-pi-compat.ts +2 -2
- package/src/internal-urls/docs-index.generated.txt +1 -1
- package/src/mcp/oauth-flow.ts +10 -8
- package/src/mcp/transports/stdio.ts +9 -17
- package/src/modes/controllers/event-controller.ts +17 -8
- package/src/modes/controllers/tool-args-reveal.ts +100 -22
- package/src/prompts/bench.md +4 -10
- package/src/prompts/tools/irc.md +19 -29
- package/src/prompts/tools/job.md +8 -14
- package/src/prompts/tools/lsp.md +19 -30
- package/src/prompts/tools/task.md +42 -62
- package/src/sdk.ts +1 -0
- package/src/session/agent-session.ts +10 -0
- package/src/session/session-listing.ts +9 -8
- package/src/session/session-loader.ts +98 -3
- package/src/session/session-manager.ts +34 -4
- package/src/subprocess/worker-client.ts +12 -4
- package/src/tools/path-utils.ts +4 -2
- package/src/web/search/index.ts +14 -8
- package/src/web/search/providers/duckduckgo.ts +136 -78
- package/src/web/search/providers/gemini.ts +268 -185
- package/src/web/search/types.ts +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ResolvedThinkingLevel } from "@oh-my-pi/pi-agent-core";
|
|
2
|
-
import type { Api, ApiKeyResolver, AssistantMessageEventStream, Context, Model, SimpleStreamOptions } from "@oh-my-pi/pi-ai";
|
|
2
|
+
import type { Api, ApiKeyResolver, AssistantMessageEventStream, Context, Model, ServiceTier, SimpleStreamOptions } from "@oh-my-pi/pi-ai";
|
|
3
3
|
import { type CanonicalModelVariant } from "@oh-my-pi/pi-catalog/identity";
|
|
4
4
|
import type { ApiKeyResolverModel } from "../config/api-key-resolver";
|
|
5
5
|
import { type CanonicalModelQueryOptions } from "../config/model-registry";
|
|
@@ -10,7 +10,10 @@ export interface BenchCommandArgs {
|
|
|
10
10
|
runs?: number;
|
|
11
11
|
maxTokens?: number;
|
|
12
12
|
prompt?: string;
|
|
13
|
+
/** Service-tier setting value (`none` omits); overrides the configured `serviceTier` setting. */
|
|
14
|
+
serviceTier?: string;
|
|
13
15
|
json?: boolean;
|
|
16
|
+
par?: number;
|
|
14
17
|
};
|
|
15
18
|
}
|
|
16
19
|
export interface BenchModelRegistry {
|
|
@@ -62,6 +65,8 @@ export interface BenchSummary {
|
|
|
62
65
|
maxTokens: number;
|
|
63
66
|
models: BenchModelReport[];
|
|
64
67
|
failures: number;
|
|
68
|
+
/** Requested service tier passed to every request; absent when none was requested. Scoped tiers (`openai-only`/`claude-only`) may be dropped per-provider downstream. */
|
|
69
|
+
serviceTier?: ServiceTier;
|
|
65
70
|
}
|
|
66
71
|
type BenchStreamSimple = (model: Model<Api>, context: Context, options?: SimpleStreamOptions) => AssistantMessageEventStream;
|
|
67
72
|
export interface BenchDependencies {
|
|
@@ -20,9 +20,17 @@ export default class Bench extends Command {
|
|
|
20
20
|
prompt: import("@oh-my-pi/pi-utils/cli").FlagDescriptor<"string"> & {
|
|
21
21
|
description: string;
|
|
22
22
|
};
|
|
23
|
+
"service-tier": import("@oh-my-pi/pi-utils/cli").FlagDescriptor<"string"> & {
|
|
24
|
+
description: string;
|
|
25
|
+
options: readonly ["none", "auto", "default", "flex", "scale", "priority", "openai-only", "claude-only"];
|
|
26
|
+
};
|
|
23
27
|
json: import("@oh-my-pi/pi-utils/cli").FlagDescriptor<"boolean"> & {
|
|
24
28
|
description: string;
|
|
25
29
|
};
|
|
30
|
+
par: import("@oh-my-pi/pi-utils/cli").FlagDescriptor<"integer"> & {
|
|
31
|
+
description: string;
|
|
32
|
+
default: number;
|
|
33
|
+
};
|
|
26
34
|
};
|
|
27
35
|
static examples: string[];
|
|
28
36
|
run(): Promise<void>;
|
|
@@ -37,10 +37,14 @@ export interface DiscoveryContext {
|
|
|
37
37
|
*/
|
|
38
38
|
getBearerApiKeyResolver(provider: string): Promise<ApiKey | undefined>;
|
|
39
39
|
}
|
|
40
|
+
type LlamaCppDiscoveredModelRuntimeMetadata = {
|
|
41
|
+
contextWindow: number;
|
|
42
|
+
maxTokens: number;
|
|
43
|
+
};
|
|
40
44
|
export declare function discoverModelsByProviderType(providerConfig: DiscoveryProviderConfig, ctx: DiscoveryContext): Promise<Model<Api>[]>;
|
|
41
45
|
export declare function discoverOllamaModels(providerConfig: DiscoveryProviderConfig, ctx: DiscoveryContext): Promise<Model<Api>[]>;
|
|
42
46
|
export declare function discoverLlamaCppModels(providerConfig: DiscoveryProviderConfig, ctx: DiscoveryContext): Promise<Model<Api>[]>;
|
|
43
|
-
export declare function
|
|
47
|
+
export declare function discoverLlamaCppModelRuntimeMetadata(model: Pick<Model<Api>, "provider" | "id" | "baseUrl" | "headers">, ctx: DiscoveryContext): Promise<LlamaCppDiscoveredModelRuntimeMetadata | undefined>;
|
|
44
48
|
export declare function discoverOpenAIModelsList(providerConfig: DiscoveryProviderConfig, ctx: DiscoveryContext): Promise<Model<Api>[]>;
|
|
45
49
|
export declare function discoverLiteLLMModels(providerConfig: DiscoveryProviderConfig, ctx: DiscoveryContext): Promise<Model<Api>[]>;
|
|
46
50
|
/**
|
|
@@ -61,3 +65,4 @@ export declare function discoverLiteLLMModels(providerConfig: DiscoveryProviderC
|
|
|
61
65
|
export declare function discoverProxyModels(providerConfig: DiscoveryProviderConfig, ctx: DiscoveryContext): Promise<Model<Api>[]>;
|
|
62
66
|
export declare function normalizeLiteLLMDiscoveryBaseUrl(baseUrl?: string): string;
|
|
63
67
|
export declare function normalizeOpenAIModelsListBaseUrl(baseUrl?: string): string;
|
|
68
|
+
export {};
|
|
@@ -4494,7 +4494,7 @@ export declare const SETTINGS_SCHEMA: {
|
|
|
4494
4494
|
}, {
|
|
4495
4495
|
readonly value: "duckduckgo";
|
|
4496
4496
|
readonly label: "DuckDuckGo";
|
|
4497
|
-
readonly description: "
|
|
4497
|
+
readonly description: "Scrapes the DuckDuckGo HTML frontend (no API key)";
|
|
4498
4498
|
}];
|
|
4499
4499
|
};
|
|
4500
4500
|
};
|
|
@@ -18,6 +18,7 @@ export * from "./modes/patch";
|
|
|
18
18
|
export * from "./modes/replace";
|
|
19
19
|
export * from "./normalize";
|
|
20
20
|
export * from "./renderer";
|
|
21
|
+
export * from "./snapshot-details";
|
|
21
22
|
export * from "./streaming";
|
|
22
23
|
type TInput = typeof replaceEditSchema | typeof patchEditSchema | typeof hashlineEditParamsSchema | typeof applyPatchSchema;
|
|
23
24
|
type HashlineParams = typeof hashlineEditParamsSchema.infer;
|
|
@@ -29,6 +29,8 @@ export interface EditToolPerFileResult {
|
|
|
29
29
|
oldText?: string;
|
|
30
30
|
/** Source-of-truth content after the edit; `undefined` for delete operations. */
|
|
31
31
|
newText?: string;
|
|
32
|
+
/** True when {@link pruneOversizedEditSnapshots} dropped `oldText`/`newText` from this entry. Aggregators check this to suppress misleading combined snapshots when at least one entry of a multi-entry single-path edit was pruned. */
|
|
33
|
+
snapshotsPruned?: boolean;
|
|
32
34
|
/** Pre-move source path; set only when the edit moved/renamed the file. The header renders `sourcePath → path`. */
|
|
33
35
|
sourcePath?: string;
|
|
34
36
|
}
|
|
@@ -53,6 +55,8 @@ export interface EditToolDetails {
|
|
|
53
55
|
oldText?: string;
|
|
54
56
|
/** Source-of-truth content after the edit; `undefined` for delete operations. */
|
|
55
57
|
newText?: string;
|
|
58
|
+
/** True when {@link pruneOversizedEditSnapshots} dropped `oldText`/`newText` from this entry. Aggregators check this to suppress misleading combined snapshots when at least one entry of a multi-entry single-path edit was pruned. */
|
|
59
|
+
snapshotsPruned?: boolean;
|
|
56
60
|
/** Pre-move source path; set only when the edit moved/renamed the file. The header renders `sourcePath → path`. */
|
|
57
61
|
sourcePath?: string;
|
|
58
62
|
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bound the size of the `oldText` / `newText` snapshots that edit-tool results
|
|
3
|
+
* carry in `details`. These fields hold the full pre/post file content; for
|
|
4
|
+
* large files they balloon the per-turn JSONL line and the session file
|
|
5
|
+
* (300 KB+ each on the cases reported in #3786) without paying for any LLM
|
|
6
|
+
* context (provider serializers send only `content`, never `details`).
|
|
7
|
+
*
|
|
8
|
+
* Only consumer of the raw snapshots is the ACP event mapper, which builds a
|
|
9
|
+
* `diff` ToolCallContent for ACP clients (Zed). When the snapshots are pruned
|
|
10
|
+
* the mapper returns `undefined` for that file and the text content still
|
|
11
|
+
* flows — diff visualization degrades gracefully for over-threshold edits.
|
|
12
|
+
*/
|
|
13
|
+
import type { EditToolDetails, EditToolPerFileResult } from "./renderer";
|
|
14
|
+
/**
|
|
15
|
+
* Combined `oldText` + `newText` character budget for a single edit-tool
|
|
16
|
+
* result. Applies both per-entry (one file at a time) and as an aggregate
|
|
17
|
+
* across `perFileResults` (so a many-small-files batch can't accumulate
|
|
18
|
+
* unbounded snapshot bytes — see #3787 review).
|
|
19
|
+
*
|
|
20
|
+
* Picked so typical code-file edits keep ACP diff visualization while
|
|
21
|
+
* pathological cases (large generated files, full-file rewrites, or
|
|
22
|
+
* many-file batches) drop the raw snapshots before they hit the
|
|
23
|
+
* session JSONL.
|
|
24
|
+
*/
|
|
25
|
+
export declare const MAX_EDIT_SNAPSHOT_TEXT_CHARS = 32768;
|
|
26
|
+
/**
|
|
27
|
+
* Prune oversized `oldText` / `newText` from an edit-tool details payload,
|
|
28
|
+
* recursing into `perFileResults` when present. Per-file overload comes first
|
|
29
|
+
* so the more specific shape (required `path`) wins overload resolution at
|
|
30
|
+
* the per-file call sites.
|
|
31
|
+
*/
|
|
32
|
+
export declare function pruneOversizedEditSnapshots(details: EditToolPerFileResult): EditToolPerFileResult;
|
|
33
|
+
export declare function pruneOversizedEditSnapshots(details: EditToolDetails): EditToolDetails;
|
|
@@ -59,12 +59,10 @@ export interface MCPOAuthConfig {
|
|
|
59
59
|
/** OAuth scopes (space-separated) */
|
|
60
60
|
scopes?: string;
|
|
61
61
|
/**
|
|
62
|
-
* `prompt` parameter for the authorization request.
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
* (RFC 6749 §3.1 requires servers to ignore the param when unsupported).
|
|
67
|
-
* Set to `""` to omit the parameter entirely.
|
|
62
|
+
* `prompt` parameter for the authorization request. By default the parameter
|
|
63
|
+
* is omitted, matching the reference MCP SDK, except for `offline_access`
|
|
64
|
+
* requests where OIDC Core requires `prompt=consent` to issue refresh-token
|
|
65
|
+
* access. Set to `""` to omit the parameter entirely.
|
|
68
66
|
*/
|
|
69
67
|
prompt?: string;
|
|
70
68
|
/** Exact redirect URI to advertise to the provider */
|
|
@@ -6,13 +6,20 @@ type ToolArgsRevealControllerOptions = {
|
|
|
6
6
|
getSmoothStreaming(): boolean;
|
|
7
7
|
requestRender(): void;
|
|
8
8
|
};
|
|
9
|
+
type ToolArgsRevealTarget = {
|
|
10
|
+
rawInput: boolean;
|
|
11
|
+
exposeRawPartialJson: boolean;
|
|
12
|
+
fullArgs: Record<string, unknown>;
|
|
13
|
+
};
|
|
9
14
|
/**
|
|
10
15
|
* Paces streamed tool-call arguments the same way StreamingRevealController
|
|
11
16
|
* paces assistant text: providers that deliver `partialJson` in large batches
|
|
12
17
|
* (or throttle their partial parses) would otherwise make write/edit/bash
|
|
13
18
|
* streaming previews jump in chunks. Each pending tool call reveals its raw
|
|
14
19
|
* argument stream at the shared 30fps cadence with the same adaptive
|
|
15
|
-
* catch-up step
|
|
20
|
+
* catch-up step. JSON prefixes are parsed only when enough new bytes arrive to
|
|
21
|
+
* change renderer-visible fields, while raw-prefix consumers still receive
|
|
22
|
+
* fresh `__partialJson` on every reveal frame.
|
|
16
23
|
*
|
|
17
24
|
* Reveal units are UTF-16 code units of the raw stream, not graphemes —
|
|
18
25
|
* the prefix goes through a JSON parser rather than straight to the screen,
|
|
@@ -26,7 +33,7 @@ export declare class ToolArgsRevealController {
|
|
|
26
33
|
* args to render right now. With smoothing disabled the full target passes
|
|
27
34
|
* through in the caller's legacy shape (`{ ...args, __partialJson }`).
|
|
28
35
|
*/
|
|
29
|
-
setTarget(id: string, partialJson: string,
|
|
36
|
+
setTarget(id: string, partialJson: string, target: ToolArgsRevealTarget): Record<string, unknown>;
|
|
30
37
|
/** Attach the component future ticks push frames into. */
|
|
31
38
|
bind(id: string, component: ToolArgsRevealComponent): void;
|
|
32
39
|
/** Final arguments arrived (the JSON closed): drop the reveal so the
|
|
@@ -184,6 +184,8 @@ export interface AgentSessionConfig {
|
|
|
184
184
|
* main agent. Defaults to plain `streamSimple` when omitted.
|
|
185
185
|
*/
|
|
186
186
|
advisorStreamFn?: StreamFn;
|
|
187
|
+
/** Hint that OpenAI Codex requests should prefer websocket transport when supported. */
|
|
188
|
+
preferWebsockets?: boolean;
|
|
187
189
|
/** Provider payload hook used by the active session request path */
|
|
188
190
|
onPayload?: SimpleStreamOptions["onPayload"];
|
|
189
191
|
/** Provider response hook used by the active session request path */
|
|
@@ -470,6 +472,8 @@ export declare class AgentSession {
|
|
|
470
472
|
setSessionSwitchReconciler(reconciler: (() => Promise<void>) | null): void;
|
|
471
473
|
/** Provider-scoped mutable state store for transport/session caches. */
|
|
472
474
|
get providerSessionState(): Map<string, ProviderSessionState>;
|
|
475
|
+
/** Hint forwarded to provider calls that support websocket transport. */
|
|
476
|
+
get preferWebsockets(): boolean | undefined;
|
|
473
477
|
getHindsightSessionState(): HindsightSessionState | undefined;
|
|
474
478
|
setHindsightSessionState(state: HindsightSessionState | undefined): HindsightSessionState | undefined;
|
|
475
479
|
getMnemopiSessionState(): MnemopiSessionState | undefined;
|
|
@@ -75,10 +75,9 @@ export declare class SessionManager {
|
|
|
75
75
|
/** Flush pending writes. Call before switching sessions or on shutdown. */
|
|
76
76
|
flush(): Promise<void>;
|
|
77
77
|
/**
|
|
78
|
-
* Synchronously
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
* lands mid-`writeFileSync`.
|
|
78
|
+
* Synchronously makes the current append-only session durable. Avoid rewriting
|
|
79
|
+
* an already-current file: large restored sessions can contain GiB of compacted
|
|
80
|
+
* history, and Ctrl+C must not rebuild the whole JSONL string just to flush.
|
|
82
81
|
*/
|
|
83
82
|
flushSync(): void;
|
|
84
83
|
/** Flush, then close the append writer. */
|
|
@@ -2,9 +2,9 @@ import type { AuthStorage } from "@oh-my-pi/pi-ai";
|
|
|
2
2
|
import type { SearchResponse } from "../../../web/search/types";
|
|
3
3
|
import type { SearchParams } from "./base";
|
|
4
4
|
import { SearchProvider } from "./base";
|
|
5
|
-
/** Execute DuckDuckGo
|
|
5
|
+
/** Execute a DuckDuckGo web search via the no-JS HTML frontend. */
|
|
6
6
|
export declare function searchDuckDuckGo(params: SearchParams): Promise<SearchResponse>;
|
|
7
|
-
/** Search provider for DuckDuckGo
|
|
7
|
+
/** Search provider for DuckDuckGo (no API key required). */
|
|
8
8
|
export declare class DuckDuckGoProvider extends SearchProvider {
|
|
9
9
|
readonly id = "duckduckgo";
|
|
10
10
|
readonly label = "DuckDuckGo";
|
|
@@ -78,7 +78,7 @@ export declare const SEARCH_PROVIDER_OPTIONS: readonly [{
|
|
|
78
78
|
}, {
|
|
79
79
|
readonly value: "duckduckgo";
|
|
80
80
|
readonly label: "DuckDuckGo";
|
|
81
|
-
readonly description: "
|
|
81
|
+
readonly description: "Scrapes the DuckDuckGo HTML frontend (no API key)";
|
|
82
82
|
}];
|
|
83
83
|
/** Supported web search providers (every option except `auto`). */
|
|
84
84
|
export type SearchProviderId = Exclude<(typeof SEARCH_PROVIDER_OPTIONS)[number]["value"], "auto">;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@oh-my-pi/pi-coding-agent",
|
|
4
|
-
"version": "16.2.
|
|
4
|
+
"version": "16.2.6",
|
|
5
5
|
"description": "Coding agent CLI with read, bash, edit, write tools and session management",
|
|
6
6
|
"homepage": "https://omp.sh",
|
|
7
7
|
"author": "Can Boluk",
|
|
@@ -55,17 +55,17 @@
|
|
|
55
55
|
"@agentclientprotocol/sdk": "0.25.0",
|
|
56
56
|
"@babel/parser": "^7.29.7",
|
|
57
57
|
"@mozilla/readability": "^0.6.0",
|
|
58
|
-
"@oh-my-pi/hashline": "16.2.
|
|
59
|
-
"@oh-my-pi/omp-stats": "16.2.
|
|
60
|
-
"@oh-my-pi/pi-agent-core": "16.2.
|
|
61
|
-
"@oh-my-pi/pi-ai": "16.2.
|
|
62
|
-
"@oh-my-pi/pi-catalog": "16.2.
|
|
63
|
-
"@oh-my-pi/pi-mnemopi": "16.2.
|
|
64
|
-
"@oh-my-pi/pi-natives": "16.2.
|
|
65
|
-
"@oh-my-pi/pi-tui": "16.2.
|
|
66
|
-
"@oh-my-pi/pi-utils": "16.2.
|
|
67
|
-
"@oh-my-pi/pi-wire": "16.2.
|
|
68
|
-
"@oh-my-pi/snapcompact": "16.2.
|
|
58
|
+
"@oh-my-pi/hashline": "16.2.6",
|
|
59
|
+
"@oh-my-pi/omp-stats": "16.2.6",
|
|
60
|
+
"@oh-my-pi/pi-agent-core": "16.2.6",
|
|
61
|
+
"@oh-my-pi/pi-ai": "16.2.6",
|
|
62
|
+
"@oh-my-pi/pi-catalog": "16.2.6",
|
|
63
|
+
"@oh-my-pi/pi-mnemopi": "16.2.6",
|
|
64
|
+
"@oh-my-pi/pi-natives": "16.2.6",
|
|
65
|
+
"@oh-my-pi/pi-tui": "16.2.6",
|
|
66
|
+
"@oh-my-pi/pi-utils": "16.2.6",
|
|
67
|
+
"@oh-my-pi/pi-wire": "16.2.6",
|
|
68
|
+
"@oh-my-pi/snapcompact": "16.2.6",
|
|
69
69
|
"@opentelemetry/api": "^1.9.1",
|
|
70
70
|
"@opentelemetry/context-async-hooks": "^2.7.1",
|
|
71
71
|
"@opentelemetry/exporter-trace-otlp-proto": "^0.218.0",
|
package/src/cli/args.ts
CHANGED
|
@@ -94,6 +94,35 @@ const PARSE_DEPS: ParseDeps = {
|
|
|
94
94
|
thinkingEfforts: CLI_THINKING_LEVELS,
|
|
95
95
|
};
|
|
96
96
|
|
|
97
|
+
const WINDOWS_PATH_VALUE_FLAGS: ReadonlySet<string> = new Set(["--extension", "-e", "--hook"]);
|
|
98
|
+
const WINDOWS_PATH_START_RE =
|
|
99
|
+
/^(?:[A-Za-z]:[\\/]|\\\\[?]\\(?:[A-Za-z]:[\\/]|UNC[\\/])|\\\\[^\\/]+[\\/][^\\/]+[\\/]|\/\/[?]\/(?:[A-Za-z]:\/|UNC\/)|\/\/[^/]+\/[^/]+\/)/;
|
|
100
|
+
const WINDOWS_MODULE_PATH_SUFFIX_RE = /\.(?:[cm]?[jt]sx?)$/i;
|
|
101
|
+
|
|
102
|
+
function consumeBuiltInStringValue(flag: string, args: string[], valueIndex: number): { value: string; index: number } {
|
|
103
|
+
const value = args[valueIndex];
|
|
104
|
+
if (
|
|
105
|
+
value === undefined ||
|
|
106
|
+
!WINDOWS_PATH_VALUE_FLAGS.has(flag) ||
|
|
107
|
+
!WINDOWS_PATH_START_RE.test(value) ||
|
|
108
|
+
WINDOWS_MODULE_PATH_SUFFIX_RE.test(value)
|
|
109
|
+
) {
|
|
110
|
+
return { value: value ?? "", index: valueIndex };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
let candidate = value;
|
|
114
|
+
for (let index = valueIndex + 1; index < args.length; index++) {
|
|
115
|
+
const next = args[index];
|
|
116
|
+
if (next === PROFILE_BOOTSTRAP_BOUNDARY_ARG || next.startsWith("-")) break;
|
|
117
|
+
candidate += ` ${next}`;
|
|
118
|
+
if (WINDOWS_MODULE_PATH_SUFFIX_RE.test(candidate)) {
|
|
119
|
+
return { value: candidate, index };
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
return { value, index: valueIndex };
|
|
124
|
+
}
|
|
125
|
+
|
|
97
126
|
export function parseArgs(inputArgs: string[], extensionFlags?: Map<string, { type: "boolean" | "string" }>): Args {
|
|
98
127
|
// Work on a copy: the `--option=value` handling below splices the value
|
|
99
128
|
// into the array, and callers reuse the same argv (the post-extension
|
|
@@ -161,7 +190,9 @@ export function parseArgs(inputArgs: string[], extensionFlags?: Map<string, { ty
|
|
|
161
190
|
// here only when its boolean extension is NOT loaded) would otherwise swallow
|
|
162
191
|
// the marker as its value and drop the user's trailing message.
|
|
163
192
|
if (i + 1 < args.length && args[i + 1] !== PROFILE_BOOTSTRAP_BOUNDARY_ARG) {
|
|
164
|
-
|
|
193
|
+
const consumed = consumeBuiltInStringValue(arg, args, i + 1);
|
|
194
|
+
i = consumed.index;
|
|
195
|
+
STRING_SETTERS[arg](result, consumed.value, PARSE_DEPS);
|
|
165
196
|
}
|
|
166
197
|
} else if (OPTIONAL_VALUE_FLAGS.has(arg)) {
|
|
167
198
|
const config = OPTIONAL_FLAGS[arg];
|
package/src/cli/bench-cli.ts
CHANGED
|
@@ -8,6 +8,8 @@ import type {
|
|
|
8
8
|
Context,
|
|
9
9
|
Effort,
|
|
10
10
|
Model,
|
|
11
|
+
ProviderSessionState,
|
|
12
|
+
ServiceTier,
|
|
11
13
|
SimpleStreamOptions,
|
|
12
14
|
} from "@oh-my-pi/pi-ai";
|
|
13
15
|
import { streamSimple } from "@oh-my-pi/pi-ai";
|
|
@@ -23,12 +25,14 @@ import {
|
|
|
23
25
|
getModelMatchPreferences,
|
|
24
26
|
resolveCliModel,
|
|
25
27
|
} from "../config/model-resolver";
|
|
28
|
+
import { resolveServiceTierSetting } from "../config/service-tier";
|
|
26
29
|
import { Settings } from "../config/settings";
|
|
27
30
|
import benchPrompt from "../prompts/bench.md" with { type: "text" };
|
|
28
31
|
import { discoverAuthStorage, loadCliExtensionProviders } from "../sdk";
|
|
29
32
|
import { resolveThinkingLevelForModel, shouldDisableReasoning, toReasoningEffort } from "../thinking";
|
|
30
33
|
|
|
31
|
-
const DEFAULT_RUNS =
|
|
34
|
+
const DEFAULT_RUNS = 10;
|
|
35
|
+
const DEFAULT_PAR = 4;
|
|
32
36
|
const DEFAULT_MAX_TOKENS = 512;
|
|
33
37
|
const ERROR_WIDTH = 110;
|
|
34
38
|
const BENCH_PROMPT = benchPrompt.trim();
|
|
@@ -39,7 +43,10 @@ export interface BenchCommandArgs {
|
|
|
39
43
|
runs?: number;
|
|
40
44
|
maxTokens?: number;
|
|
41
45
|
prompt?: string;
|
|
46
|
+
/** Service-tier setting value (`none` omits); overrides the configured `serviceTier` setting. */
|
|
47
|
+
serviceTier?: string;
|
|
42
48
|
json?: boolean;
|
|
49
|
+
par?: number;
|
|
43
50
|
};
|
|
44
51
|
}
|
|
45
52
|
|
|
@@ -99,6 +106,8 @@ export interface BenchSummary {
|
|
|
99
106
|
maxTokens: number;
|
|
100
107
|
models: BenchModelReport[];
|
|
101
108
|
failures: number;
|
|
109
|
+
/** Requested service tier passed to every request; absent when none was requested. Scoped tiers (`openai-only`/`claude-only`) may be dropped per-provider downstream. */
|
|
110
|
+
serviceTier?: ServiceTier;
|
|
102
111
|
}
|
|
103
112
|
|
|
104
113
|
type BenchStreamSimple = (
|
|
@@ -131,6 +140,13 @@ function normalizePositiveInteger(name: string, value: number | undefined, fallb
|
|
|
131
140
|
return value;
|
|
132
141
|
}
|
|
133
142
|
|
|
143
|
+
function closeProviderSessionStates(providerSessionState: Map<string, ProviderSessionState>): void {
|
|
144
|
+
for (const state of providerSessionState.values()) {
|
|
145
|
+
state.close();
|
|
146
|
+
}
|
|
147
|
+
providerSessionState.clear();
|
|
148
|
+
}
|
|
149
|
+
|
|
134
150
|
function isFirstTokenEvent(event: AssistantMessageEvent): boolean {
|
|
135
151
|
switch (event.type) {
|
|
136
152
|
case "text_delta":
|
|
@@ -188,6 +204,8 @@ interface BenchRequestOptions {
|
|
|
188
204
|
reasoning?: Effort;
|
|
189
205
|
/** Only set for an explicit `:off` suffix — some endpoints reject disablement. */
|
|
190
206
|
disableReasoning?: boolean;
|
|
207
|
+
/** Requested service tier passed to `streamSimple`; absent omits the option. The provider layer applies scope/support gating before it reaches the wire. */
|
|
208
|
+
serviceTier?: ServiceTier;
|
|
191
209
|
}
|
|
192
210
|
|
|
193
211
|
async function runBenchRequest(
|
|
@@ -198,6 +216,7 @@ async function runBenchRequest(
|
|
|
198
216
|
): Promise<BenchRunResult> {
|
|
199
217
|
const startedAt = now();
|
|
200
218
|
let firstTokenAt: number | undefined;
|
|
219
|
+
const providerSessionState = new Map<string, ProviderSessionState>();
|
|
201
220
|
try {
|
|
202
221
|
const context: Context = {
|
|
203
222
|
// Codex's Responses endpoint 400s with "Instructions are required" when no
|
|
@@ -214,6 +233,9 @@ async function runBenchRequest(
|
|
|
214
233
|
: options.maxTokens,
|
|
215
234
|
reasoning: options.reasoning,
|
|
216
235
|
disableReasoning: options.disableReasoning,
|
|
236
|
+
serviceTier: options.serviceTier,
|
|
237
|
+
providerSessionState,
|
|
238
|
+
preferWebsockets: true,
|
|
217
239
|
// pi-ai opts every OpenRouter request into response caching (1h TTL).
|
|
218
240
|
// Bench sends a byte-identical request each run, so within the TTL
|
|
219
241
|
// OpenRouter replays the cached generation with zeroed usage — the run
|
|
@@ -270,6 +292,8 @@ async function runBenchRequest(
|
|
|
270
292
|
};
|
|
271
293
|
} catch (error) {
|
|
272
294
|
return { ok: false, error: getErrorMessage(error) };
|
|
295
|
+
} finally {
|
|
296
|
+
closeProviderSessionStates(providerSessionState);
|
|
273
297
|
}
|
|
274
298
|
}
|
|
275
299
|
|
|
@@ -469,10 +493,11 @@ function resolveBenchModels(
|
|
|
469
493
|
}
|
|
470
494
|
return resolved;
|
|
471
495
|
}
|
|
472
|
-
|
|
473
496
|
export async function runBenchCommand(command: BenchCommandArgs, deps: BenchDependencies = {}): Promise<BenchSummary> {
|
|
474
497
|
const runs = normalizePositiveInteger("runs", command.flags.runs, DEFAULT_RUNS);
|
|
475
498
|
const maxTokens = normalizePositiveInteger("max-tokens", command.flags.maxTokens, DEFAULT_MAX_TOKENS);
|
|
499
|
+
const par =
|
|
500
|
+
command.flags.par !== undefined ? normalizePositiveInteger("par", command.flags.par, DEFAULT_PAR) : DEFAULT_PAR;
|
|
476
501
|
const prompt = command.flags.prompt?.trim() || BENCH_PROMPT;
|
|
477
502
|
const json = command.flags.json === true;
|
|
478
503
|
const randomSessionId = deps.randomSessionId ?? (() => Bun.randomUUIDv7());
|
|
@@ -493,6 +518,12 @@ export async function runBenchCommand(command: BenchCommandArgs, deps: BenchDepe
|
|
|
493
518
|
const runtime = await (deps.createRuntime ?? createDefaultRuntime)();
|
|
494
519
|
try {
|
|
495
520
|
const targets = resolveBenchModels(command.models, runtime.modelRegistry, runtime.settings, writeStderr);
|
|
521
|
+
// Explicit `--service-tier` wins; otherwise fall back to the configured
|
|
522
|
+
// `serviceTier` setting (`none`/unset omits the wire field). Scope-aware
|
|
523
|
+
// gating to the model's provider happens downstream in the provider layer.
|
|
524
|
+
const serviceTierValue = command.flags.serviceTier ?? runtime.settings?.get("serviceTier");
|
|
525
|
+
const serviceTier = serviceTierValue ? resolveServiceTierSetting(serviceTierValue, undefined) : undefined;
|
|
526
|
+
if (!json && serviceTier) writeStdout(`${chalk.dim(`service tier: ${serviceTier}`)}\n`);
|
|
496
527
|
const reports: BenchModelReport[] = [];
|
|
497
528
|
for (const { selector, model, thinking } of targets) {
|
|
498
529
|
if (!json) {
|
|
@@ -501,21 +532,29 @@ export async function runBenchCommand(command: BenchCommandArgs, deps: BenchDepe
|
|
|
501
532
|
writeStdout(`${chalk.bold(resolvedModel)}${resolvedNote}\n`);
|
|
502
533
|
}
|
|
503
534
|
const results: BenchRunResult[] = [];
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
if (!json
|
|
517
|
-
|
|
518
|
-
|
|
535
|
+
|
|
536
|
+
// Preflight check: let's verify credentials before starting any runs.
|
|
537
|
+
// This matches the old sequential break behavior exactly and avoids launching/printing
|
|
538
|
+
// multiple failures.
|
|
539
|
+
const testSessionId = randomSessionId();
|
|
540
|
+
const preflightKey = await runtime.modelRegistry.getApiKey(model, testSessionId);
|
|
541
|
+
if (!preflightKey) {
|
|
542
|
+
const failure: BenchRunFailure = {
|
|
543
|
+
ok: false,
|
|
544
|
+
error: `No credentials for provider "${model.provider}". Run \`omp\` and use /login, or set the provider API key.`,
|
|
545
|
+
};
|
|
546
|
+
results.push(failure);
|
|
547
|
+
if (!json) writeStdout(`${formatRunLine(failure, 0, runs)}\n`);
|
|
548
|
+
reports.push(buildModelReport(selector, model, thinking, results));
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
// We will launch up to `par` workers/requests concurrently.
|
|
553
|
+
// To keep output clean, non-JSON output emits entries in correct index order.
|
|
554
|
+
let nextToPrint = 0;
|
|
555
|
+
|
|
556
|
+
const runWorker = async (index: number) => {
|
|
557
|
+
const sessionId = index === 0 ? testSessionId : randomSessionId();
|
|
519
558
|
const result = await runBenchRequest(
|
|
520
559
|
model,
|
|
521
560
|
{
|
|
@@ -525,20 +564,49 @@ export async function runBenchCommand(command: BenchCommandArgs, deps: BenchDepe
|
|
|
525
564
|
maxTokens,
|
|
526
565
|
reasoning: toReasoningEffort(thinking),
|
|
527
566
|
disableReasoning: shouldDisableReasoning(thinking) ? true : undefined,
|
|
567
|
+
serviceTier,
|
|
528
568
|
},
|
|
529
569
|
streamFn,
|
|
530
570
|
now,
|
|
531
571
|
);
|
|
532
|
-
results
|
|
572
|
+
results[index] = result;
|
|
573
|
+
};
|
|
574
|
+
|
|
575
|
+
// Concurrency-limited running pool
|
|
576
|
+
const queue = Array.from({ length: runs }, (_, i) => i);
|
|
577
|
+
const activeWorkers: Promise<void>[] = [];
|
|
578
|
+
|
|
579
|
+
const processNext = async (): Promise<void> => {
|
|
580
|
+
if (queue.length === 0) return;
|
|
581
|
+
const index = queue.shift()!;
|
|
582
|
+
|
|
583
|
+
// Pre-print a status update if requested and interactive
|
|
584
|
+
if (!json && interactive) {
|
|
585
|
+
writeStdout(chalk.dim(` … run ${index + 1}/${runs} streaming\n`));
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
await runWorker(index);
|
|
589
|
+
|
|
590
|
+
// Attempt to print completed results that are in-order
|
|
533
591
|
if (!json) {
|
|
534
|
-
|
|
535
|
-
|
|
592
|
+
while (nextToPrint < runs && results[nextToPrint] !== undefined) {
|
|
593
|
+
const res = results[nextToPrint];
|
|
594
|
+
writeStdout(`${formatRunLine(res, nextToPrint, runs)}\n`);
|
|
595
|
+
nextToPrint++;
|
|
596
|
+
}
|
|
536
597
|
}
|
|
598
|
+
|
|
599
|
+
await processNext();
|
|
600
|
+
};
|
|
601
|
+
|
|
602
|
+
for (let w = 0; w < Math.min(par, runs); w++) {
|
|
603
|
+
activeWorkers.push(processNext());
|
|
537
604
|
}
|
|
605
|
+
await Promise.all(activeWorkers);
|
|
538
606
|
reports.push(buildModelReport(selector, model, thinking, results));
|
|
539
607
|
}
|
|
540
608
|
const failures = reports.reduce((sum, report) => sum + report.results.filter(result => !result.ok).length, 0);
|
|
541
|
-
const summary: BenchSummary = { runs, maxTokens, models: reports, failures };
|
|
609
|
+
const summary: BenchSummary = { runs, maxTokens, models: reports, failures, serviceTier };
|
|
542
610
|
if (json) {
|
|
543
611
|
writeStdout(`${JSON.stringify(summary, null, 2)}\n`);
|
|
544
612
|
} else if (reports.length > 1 || runs > 1) {
|
|
@@ -4,8 +4,10 @@
|
|
|
4
4
|
* Handles `omp q`/`omp web-search` subcommands for testing web search providers.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import { APP_NAME } from "@oh-my-pi/pi-utils";
|
|
7
|
+
import { APP_NAME, getProjectDir } from "@oh-my-pi/pi-utils";
|
|
8
8
|
import chalk from "chalk";
|
|
9
|
+
import { applyProviderGlobalsFromSettings } from "../config/provider-globals";
|
|
10
|
+
import { Settings } from "../config/settings";
|
|
9
11
|
import { initTheme, theme } from "../modes/theme/theme";
|
|
10
12
|
import { runSearchQuery, type SearchQueryParams } from "../web/search/index";
|
|
11
13
|
import { SEARCH_PROVIDER_ORDER } from "../web/search/provider";
|
|
@@ -85,6 +87,9 @@ export async function runSearchCommand(cmd: SearchCommandArgs): Promise<void> {
|
|
|
85
87
|
process.exit(1);
|
|
86
88
|
}
|
|
87
89
|
|
|
90
|
+
const settings = await Settings.init({ cwd: getProjectDir() });
|
|
91
|
+
applyProviderGlobalsFromSettings(settings);
|
|
92
|
+
|
|
88
93
|
await initTheme();
|
|
89
94
|
|
|
90
95
|
const params: SearchQueryParams = {
|
package/src/commands/bench.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Args, Command, Flags } from "@oh-my-pi/pi-utils/cli";
|
|
2
2
|
import { runBenchCommand } from "../cli/bench-cli";
|
|
3
|
+
import { SERVICE_TIER_SETTING_VALUES } from "../config/service-tier";
|
|
3
4
|
|
|
4
5
|
export default class Bench extends Command {
|
|
5
6
|
static description =
|
|
@@ -14,16 +15,22 @@ export default class Bench extends Command {
|
|
|
14
15
|
};
|
|
15
16
|
|
|
16
17
|
static flags = {
|
|
17
|
-
runs: Flags.integer({ description: "Requests per model (results are averaged)", default:
|
|
18
|
+
runs: Flags.integer({ description: "Requests per model (results are averaged)", default: 10 }),
|
|
18
19
|
"max-tokens": Flags.integer({ description: "Max output tokens per request", default: 512 }),
|
|
19
20
|
prompt: Flags.string({ description: "Custom prompt text (default: bundled bench prompt)" }),
|
|
21
|
+
"service-tier": Flags.string({
|
|
22
|
+
description: "Service tier hint (default: configured `serviceTier` setting; `none` omits it)",
|
|
23
|
+
options: SERVICE_TIER_SETTING_VALUES,
|
|
24
|
+
}),
|
|
20
25
|
json: Flags.boolean({ description: "Output JSON" }),
|
|
26
|
+
par: Flags.integer({ description: "Execute runs with N parallel queries/requests", default: 4 }),
|
|
21
27
|
};
|
|
22
28
|
|
|
23
29
|
static examples = [
|
|
24
30
|
"# Compare two models\n omp bench anthropic/claude-opus-4-5 openai/gpt-5.2",
|
|
25
31
|
"# Fuzzy selectors work\n omp bench opus sonnet",
|
|
26
32
|
"# Average over 3 runs each\n omp bench opus gpt-5.2 --runs 3",
|
|
33
|
+
"# Force priority serving tier\n omp bench openai-codex/gpt-5.5:low --runs 10 --service-tier priority",
|
|
27
34
|
"# Machine-readable output\n omp bench opus --json",
|
|
28
35
|
];
|
|
29
36
|
|
|
@@ -35,7 +42,9 @@ export default class Bench extends Command {
|
|
|
35
42
|
runs: flags.runs,
|
|
36
43
|
maxTokens: flags["max-tokens"],
|
|
37
44
|
prompt: flags.prompt,
|
|
45
|
+
serviceTier: flags["service-tier"],
|
|
38
46
|
json: flags.json,
|
|
47
|
+
par: flags.par,
|
|
39
48
|
},
|
|
40
49
|
});
|
|
41
50
|
}
|
|
@@ -92,7 +92,7 @@
|
|
|
92
92
|
},
|
|
93
93
|
"prompt": {
|
|
94
94
|
"type": "string",
|
|
95
|
-
"description": "OAuth `prompt` parameter sent during authorization (default:
|
|
95
|
+
"description": "OAuth `prompt` parameter sent during authorization (default: omit unless the requested scope contains `offline_access`, which sends \"consent\"; set to \"\" to always omit)."
|
|
96
96
|
}
|
|
97
97
|
},
|
|
98
98
|
"description": "Explicit OAuth client settings for servers that need them during /mcp reauth or initial connect."
|