@f5-sales-demo/xcsh 20.19.2 → 20.19.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/config/model-registry.ts +38 -1
- package/src/config/model-resolver.ts +4 -20
- package/src/config/settings-schema.ts +2 -1
- package/src/internal-urls/build-info.generated.ts +8 -8
- package/src/internal-urls/console-catalog.generated.ts +2 -2
- package/src/internal-urls/docs-index.generated.ts +1 -1
- package/src/modes/controllers/login-model.ts +11 -1
- package/src/modes/controllers/selector-controller.ts +4 -163
- package/src/routing/presets.ts +2 -0
- package/src/routing/subscription-profiles.ts +31 -1
- package/src/web/search/provider.ts +3 -0
- package/src/web/search/providers/codex.ts +402 -0
- package/src/web/search/types.ts +2 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": "module",
|
|
3
3
|
"name": "@f5-sales-demo/xcsh",
|
|
4
|
-
"version": "20.19.
|
|
4
|
+
"version": "20.19.4",
|
|
5
5
|
"description": "Coding agent CLI with read, bash, edit, write tools and session management",
|
|
6
6
|
"homepage": "https://github.com/f5-sales-demo/xcsh",
|
|
7
7
|
"author": "Can Boluk",
|
|
@@ -61,13 +61,13 @@
|
|
|
61
61
|
},
|
|
62
62
|
"dependencies": {
|
|
63
63
|
"@agentclientprotocol/sdk": "1.3.0",
|
|
64
|
-
"@f5-sales-demo/pi-agent-core": "20.19.
|
|
65
|
-
"@f5-sales-demo/pi-ai": "20.19.
|
|
66
|
-
"@f5-sales-demo/pi-natives": "20.19.
|
|
67
|
-
"@f5-sales-demo/pi-resource-management": "20.19.
|
|
68
|
-
"@f5-sales-demo/pi-tui": "20.19.
|
|
69
|
-
"@f5-sales-demo/pi-utils": "20.19.
|
|
70
|
-
"@f5-sales-demo/xcsh-stats": "20.19.
|
|
64
|
+
"@f5-sales-demo/pi-agent-core": "20.19.4",
|
|
65
|
+
"@f5-sales-demo/pi-ai": "20.19.4",
|
|
66
|
+
"@f5-sales-demo/pi-natives": "20.19.4",
|
|
67
|
+
"@f5-sales-demo/pi-resource-management": "20.19.4",
|
|
68
|
+
"@f5-sales-demo/pi-tui": "20.19.4",
|
|
69
|
+
"@f5-sales-demo/pi-utils": "20.19.4",
|
|
70
|
+
"@f5-sales-demo/xcsh-stats": "20.19.4",
|
|
71
71
|
"@mozilla/readability": "^0.6",
|
|
72
72
|
"@sinclair/typebox": "^0.34",
|
|
73
73
|
"@xterm/headless": "^6.0",
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
type ModelRefreshStrategy,
|
|
17
17
|
type OAuthCredentials,
|
|
18
18
|
type OAuthLoginCallbacks,
|
|
19
|
+
openaiCodexModelManagerOptions,
|
|
19
20
|
PROVIDER_DESCRIPTORS,
|
|
20
21
|
readModelCache,
|
|
21
22
|
registerCustomApi,
|
|
@@ -31,7 +32,7 @@ import { type ConfigError, ConfigFile } from "../config";
|
|
|
31
32
|
import { hasLiteLLMEnv, probeAndUpgradeLiteLLMConfig, startupHealthCheck } from "../config/auto-config";
|
|
32
33
|
import { parseModelString, resolveProviderModelReference } from "../config/model-resolver";
|
|
33
34
|
import { isValidThemeColor, type ThemeColor } from "../modes/theme/theme";
|
|
34
|
-
import type { AuthStorage } from "../session/auth-storage";
|
|
35
|
+
import type { AuthStorage, OAuthCredential } from "../session/auth-storage";
|
|
35
36
|
import {
|
|
36
37
|
buildCanonicalModelIndex,
|
|
37
38
|
type CanonicalModelIndex,
|
|
@@ -552,6 +553,31 @@ function extractGoogleOAuthToken(value: string | undefined): string | undefined
|
|
|
552
553
|
return value;
|
|
553
554
|
}
|
|
554
555
|
|
|
556
|
+
function getOAuthCredentialsForProvider(authStorage: AuthStorage, provider: string): OAuthCredential[] {
|
|
557
|
+
const providerEntry = authStorage.getAll()[provider];
|
|
558
|
+
if (!providerEntry) {
|
|
559
|
+
return [];
|
|
560
|
+
}
|
|
561
|
+
const entries = Array.isArray(providerEntry) ? providerEntry : [providerEntry];
|
|
562
|
+
return entries.filter((entry): entry is OAuthCredential => entry.type === "oauth");
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
function resolveOAuthAccountIdForAccessToken(
|
|
566
|
+
authStorage: AuthStorage,
|
|
567
|
+
provider: string,
|
|
568
|
+
accessToken: string,
|
|
569
|
+
): string | undefined {
|
|
570
|
+
const oauthCredentials = getOAuthCredentialsForProvider(authStorage, provider);
|
|
571
|
+
const matchingCredential = oauthCredentials.find(credential => credential.access === accessToken);
|
|
572
|
+
if (matchingCredential) {
|
|
573
|
+
return matchingCredential.accountId;
|
|
574
|
+
}
|
|
575
|
+
if (oauthCredentials.length === 1) {
|
|
576
|
+
return oauthCredentials[0].accountId;
|
|
577
|
+
}
|
|
578
|
+
return undefined;
|
|
579
|
+
}
|
|
580
|
+
|
|
555
581
|
function mergeCompat(
|
|
556
582
|
baseCompat: Model<Api>["compat"],
|
|
557
583
|
overrideCompat: ModelOverride["compat"],
|
|
@@ -1390,6 +1416,17 @@ export class ModelRegistry {
|
|
|
1390
1416
|
endpoint: this.getProviderBaseUrl("google-gemini-cli"),
|
|
1391
1417
|
}),
|
|
1392
1418
|
},
|
|
1419
|
+
{
|
|
1420
|
+
providerId: "openai-codex",
|
|
1421
|
+
resolveKey: value => value,
|
|
1422
|
+
createOptions: accessToken => {
|
|
1423
|
+
const accountId = resolveOAuthAccountIdForAccessToken(this.authStorage, "openai-codex", accessToken);
|
|
1424
|
+
return openaiCodexModelManagerOptions({
|
|
1425
|
+
accessToken,
|
|
1426
|
+
accountId,
|
|
1427
|
+
});
|
|
1428
|
+
},
|
|
1429
|
+
},
|
|
1393
1430
|
];
|
|
1394
1431
|
const peekKey = (descriptor: { providerId: string }) => this.#peekApiKeyForProvider(descriptor.providerId);
|
|
1395
1432
|
// Special providers discover entitlement-scoped models through OAuth. Refresh their
|
|
@@ -1049,9 +1049,6 @@ export interface InitialModelResult {
|
|
|
1049
1049
|
fallbackMessage: string | undefined;
|
|
1050
1050
|
}
|
|
1051
1051
|
|
|
1052
|
-
export const UNSUPPORTED_OPENAI_CODEX_RECOVERY =
|
|
1053
|
-
"Saved OpenAI Codex subscription configuration is unsupported in xcsh. Use the official codex CLI for ChatGPT subscription access, set OPENAI_API_KEY for usage-based OpenAI API access in xcsh, select another provider, or remove the disabled legacy credential with /logout openai-codex.";
|
|
1054
|
-
|
|
1055
1052
|
/**
|
|
1056
1053
|
* Find the initial model to use based on priority:
|
|
1057
1054
|
* 1. CLI args (provider + model)
|
|
@@ -1112,8 +1109,7 @@ export async function findInitialModel(options: {
|
|
|
1112
1109
|
}
|
|
1113
1110
|
|
|
1114
1111
|
// 3. Try saved default from settings
|
|
1115
|
-
|
|
1116
|
-
if (defaultProvider && defaultModelId && !hasUnsupportedOpenAICodexDefault) {
|
|
1112
|
+
if (defaultProvider && defaultModelId) {
|
|
1117
1113
|
const found = modelRegistry.find(defaultProvider, defaultModelId);
|
|
1118
1114
|
if (found) {
|
|
1119
1115
|
model = found;
|
|
@@ -1131,28 +1127,16 @@ export async function findInitialModel(options: {
|
|
|
1131
1127
|
const defaultId = defaultModelPerProvider[provider];
|
|
1132
1128
|
const match = availableModels.find(m => m.provider === provider && m.id === defaultId);
|
|
1133
1129
|
if (match) {
|
|
1134
|
-
return {
|
|
1135
|
-
model: match,
|
|
1136
|
-
thinkingLevel: undefined,
|
|
1137
|
-
fallbackMessage: hasUnsupportedOpenAICodexDefault ? UNSUPPORTED_OPENAI_CODEX_RECOVERY : undefined,
|
|
1138
|
-
};
|
|
1130
|
+
return { model: match, thinkingLevel: undefined, fallbackMessage: undefined };
|
|
1139
1131
|
}
|
|
1140
1132
|
}
|
|
1141
1133
|
|
|
1142
1134
|
// If no default found, use first available
|
|
1143
|
-
return {
|
|
1144
|
-
model: availableModels[0],
|
|
1145
|
-
thinkingLevel: undefined,
|
|
1146
|
-
fallbackMessage: hasUnsupportedOpenAICodexDefault ? UNSUPPORTED_OPENAI_CODEX_RECOVERY : undefined,
|
|
1147
|
-
};
|
|
1135
|
+
return { model: availableModels[0], thinkingLevel: undefined, fallbackMessage: undefined };
|
|
1148
1136
|
}
|
|
1149
1137
|
|
|
1150
1138
|
// 5. No model found
|
|
1151
|
-
return {
|
|
1152
|
-
model: undefined,
|
|
1153
|
-
thinkingLevel: undefined,
|
|
1154
|
-
fallbackMessage: hasUnsupportedOpenAICodexDefault ? UNSUPPORTED_OPENAI_CODEX_RECOVERY : undefined,
|
|
1155
|
-
};
|
|
1139
|
+
return { model: undefined, thinkingLevel: undefined, fallbackMessage: undefined };
|
|
1156
1140
|
}
|
|
1157
1141
|
|
|
1158
1142
|
/**
|
|
@@ -564,7 +564,7 @@ export const SETTINGS_SCHEMA = {
|
|
|
564
564
|
|
|
565
565
|
"routing.profile": {
|
|
566
566
|
type: "enum",
|
|
567
|
-
values: ["none", "google-antigravity"] as const,
|
|
567
|
+
values: ["none", "google-antigravity", "openai-codex"] as const,
|
|
568
568
|
default: "none",
|
|
569
569
|
ui: {
|
|
570
570
|
tab: "model",
|
|
@@ -1988,6 +1988,7 @@ export const SETTINGS_SCHEMA = {
|
|
|
1988
1988
|
"perplexity",
|
|
1989
1989
|
"anthropic",
|
|
1990
1990
|
"gemini",
|
|
1991
|
+
"codex",
|
|
1991
1992
|
"tavily",
|
|
1992
1993
|
"kagi",
|
|
1993
1994
|
"synthetic",
|
|
@@ -17,17 +17,17 @@ export interface BuildInfo {
|
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
export const BUILD_INFO: BuildInfo = {
|
|
20
|
-
"version": "20.19.
|
|
21
|
-
"commit": "
|
|
22
|
-
"shortCommit": "
|
|
20
|
+
"version": "20.19.4",
|
|
21
|
+
"commit": "3674869155eeadc820a02323886cb5a594ddfc63",
|
|
22
|
+
"shortCommit": "3674869",
|
|
23
23
|
"branch": "main",
|
|
24
|
-
"tag": "v20.19.
|
|
25
|
-
"commitDate": "2026-08-
|
|
26
|
-
"buildDate": "2026-08-
|
|
24
|
+
"tag": "v20.19.4",
|
|
25
|
+
"commitDate": "2026-08-16T14:55:08Z",
|
|
26
|
+
"buildDate": "2026-08-16T15:23:23.053Z",
|
|
27
27
|
"dirty": true,
|
|
28
28
|
"prNumber": "",
|
|
29
29
|
"repoUrl": "https://github.com/f5-sales-demo/xcsh",
|
|
30
30
|
"repoSlug": "f5-sales-demo/xcsh",
|
|
31
|
-
"commitUrl": "https://github.com/f5-sales-demo/xcsh/commit/
|
|
32
|
-
"releaseUrl": "https://github.com/f5-sales-demo/xcsh/releases/tag/v20.19.
|
|
31
|
+
"commitUrl": "https://github.com/f5-sales-demo/xcsh/commit/3674869155eeadc820a02323886cb5a594ddfc63",
|
|
32
|
+
"releaseUrl": "https://github.com/f5-sales-demo/xcsh/releases/tag/v20.19.4"
|
|
33
33
|
};
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
import type { ConsoleCatalogData } from "./console-catalog-types";
|
|
4
4
|
|
|
5
|
-
export const CONSOLE_CATALOG_VERSION = "
|
|
5
|
+
export const CONSOLE_CATALOG_VERSION = "35f6e40dcaccc906474e2c28999f04d4bee4f519";
|
|
6
6
|
|
|
7
7
|
export const CONSOLE_CATALOG_DATA: ConsoleCatalogData = {
|
|
8
|
-
version: "
|
|
8
|
+
version: "35f6e40dcaccc906474e2c28999f04d4bee4f519",
|
|
9
9
|
workflows: {
|
|
10
10
|
"address-allocator/create":
|
|
11
11
|
'---\nschema: urn:xcsh:console:workflow:v1\nid: address-allocator-create\nlabel: Create IP Address Allocators\nresource: address-allocator\noperation: create\npreconditions:\n - user_logged_in\n - "role_minimum: admin"\nparams:\n name:\n required: true\n description: IP Address Allocators name (lowercase alphanumeric and hyphens)\n example: example-address-allocator\n address_allocator_mode:\n required: true\n description: Address Allocator Mode\n allocation_unit:\n required: false\n description: Allocation Unit\n default: 0\n address_pool:\n required: false\n description: Address Pool\n default: value\n address_allocation_scheme:\n required: false\n description: "Server-required: Field should be not nil"\n default: value\nsteps:\n - id: navigate-to-list\n action: navigate\n url: /web/workspaces/multi-cloud-network-connect/manage/networking/legacy_network_configuration/address_allocators\n wait_for: text(\'IP Address Allocators\')\n description: Navigate to IP Address Allocators list page\n - id: click-add-tab\n action: click\n selector: text(\'Add IP Address Allocator\')\n wait_for: textbox[name=\'Name\']\n description: Click Add IP Address Allocator to open the create form\n - id: fill-name\n action: fill\n selector: textbox[name=\'Name\']\n value: "{name}"\n description: Enter Name\n - id: select-address_allocator_mode\n action: select\n selector: listbox\n context: Address Allocator Mode section\n value: "{address_allocator_mode}"\n description: Select Address Allocator Mode\n - id: fill-allocation_unit\n action: fill\n selector: spinbutton[name=\'Allocation Unit\']\n value: "{allocation_unit}"\n description: Set Allocation Unit\n - id: fill-address_pool\n action: fill\n selector: ngx-datatable input.form-control\n context: Address Pool table\n value: "{address_pool}"\n description: Enter Address Pool in the existing table row (no Add Item needed — the table ships one empty row)\n - id: select-address_allocation_scheme\n action: select\n selector: listbox\n context: Address Allocation Scheme section\n value: "{address_allocation_scheme}"\n description: Select Address Allocation Scheme\n - id: save\n action: click\n selector: "[class*=\'save-bt\'],[class*=\'submit-button\']"\n context: footer\n wait_for: text(\'{name}\')\n wait_timeout_ms: 30000\n description: Save/submit the form (union selector matches save-bt OR submit-button)\npostconditions:\n - resource_list_page_visible\n - "resource_name_in_list: {name}"\nmetadata:\n confidence: inferred\n discovered_at: 2026-06-24\n console_version: "2025.06"\n notes: Auto-generated by scripts/generate-workflows.ts from api-specs-enriched field metadata.\n',
|
|
@@ -143,7 +143,7 @@ export const EMBEDDED_DOCS: Readonly<Record<string, string>> = {
|
|
|
143
143
|
"en/natives/natives-text-search-pipeline.md": "---\ntitle: Natives Text and Search Pipeline\ndescription: Native text search pipeline with grep, glob, and ripgrep-based file content indexing.\nsidebar:\n order: 6\n label: Text & search pipeline\n---\n\n# Natives Text/Search Pipeline\n\nThis document maps the `@f5-sales-demo/pi-natives` text/search surface (`grep`, `glob`, `text`, `highlight`) from TypeScript wrappers to Rust N-API exports and back to JS result objects.\n\nTerminology follows `docs/natives-architecture.md`:\n\n- **Wrapper**: TS API in `packages/natives/src/*`\n- **Rust module layer**: N-API exports in `crates/pi-natives/src/*`\n- **Shared scan cache**: `fs_cache`-backed directory-entry cache used by discovery/search flows\n\n## Implementation files\n\n- `packages/natives/src/grep/index.ts`\n- `packages/natives/src/grep/types.ts`\n- `packages/natives/src/glob/index.ts`\n- `packages/natives/src/glob/types.ts`\n- `packages/natives/src/text/index.ts`\n- `packages/natives/src/text/types.ts`\n- `packages/natives/src/highlight/index.ts`\n- `packages/natives/src/highlight/types.ts`\n- `crates/pi-natives/src/grep.rs`\n- `crates/pi-natives/src/glob.rs`\n- `crates/pi-natives/src/glob_util.rs`\n- `crates/pi-natives/src/fs_cache.rs`\n- `crates/pi-natives/src/text.rs`\n- `crates/pi-natives/src/highlight.rs`\n- `crates/pi-natives/src/fd.rs`\n\n## JS API ↔ Rust export mapping\n\n| JS wrapper API | Rust export (`#[napi]`, snake_case -> camelCase) | Rust module |\n| --- | --- | --- |\n| `grep(options, onMatch?)` | `grep` | `grep.rs` |\n| `searchContent(content, options)` | `search` | `grep.rs` |\n| `hasMatch(content, pattern, options?)` | `hasMatch` | `grep.rs` |\n| `fuzzyFind(options)` | `fuzzyFind` | `fd.rs` |\n| `glob(options, onMatch?)` | `glob` | `glob.rs` |\n| `invalidateFsScanCache(path?)` | `invalidateFsScanCache` | `fs_cache.rs` |\n| `wrapTextWithAnsi(text, width)` | `wrapTextWithAnsi` | `text.rs` |\n| `truncateToWidth(text, maxWidth, ellipsis, pad)` | `truncateToWidth` | `text.rs` |\n| `sliceWithWidth(line, startCol, length, strict?)` | `sliceWithWidth` | `text.rs` |\n| `extractSegments(line, beforeEnd, afterStart, afterLen, strictAfter)` | `extractSegments` | `text.rs` |\n| `sanitizeText(text)` | `sanitizeText` | `text.rs` |\n| `visibleWidth(text)` | `visibleWidth` | `text.rs` |\n| `highlightCode(code, lang, colors)` | `highlightCode` | `highlight.rs` |\n| `supportsLanguage(lang)` | `supportsLanguage` | `highlight.rs` |\n| `getSupportedLanguages()` | `getSupportedLanguages` | `highlight.rs` |\n\n## Pipeline overview by subsystem\n\n## 1) Regex search (`grep`, `searchContent`, `hasMatch`)\n\n### Input/options flow\n\n1. TS wrapper forwards options to native:\n - `grep/index.ts` passes `options` mostly unchanged and wraps callback from `(match) => void` to napi threadsafe callback shape `(err, match)`.\n - `searchContent` and `hasMatch` pass string/`Uint8Array` directly.\n2. Rust option structs in `grep.rs` deserialize camelCase fields (`ignoreCase`, `maxCount`, `contextBefore`, `contextAfter`, `maxColumns`, `timeoutMs`).\n3. `grep` creates `CancelToken` from `timeoutMs` + `AbortSignal` and runs inside `task::blocking(\"grep\", ...)`.\n\n### Execution branches\n\n- **In-memory branch (pure utility)**\n - `search` → `search_sync` → `run_search` on provided content bytes.\n - No filesystem scan, no `fs_cache`.\n- **Single-file branch (filesystem-dependent)**\n - `grep_sync` resolves path, checks metadata is file, streams up to `MAX_FILE_BYTES` per file (`4 MiB`) through ripgrep matcher.\n- **Directory branch (filesystem-dependent)**\n - Optional cache lookup via `fs_cache::get_or_scan` when `cache: true`.\n - Fresh scan via `fs_cache::force_rescan` when `cache: false`.\n - Optional empty-result recheck when cache age exceeds `empty_recheck_ms()`.\n - Entry filtering: file-only + optional glob filter (`glob_util`) + optional type filter mapping (`js`, `ts`, `rust`, etc.).\n\n### Search/collection semantics\n\n- Regex engine: `grep_regex::RegexMatcherBuilder` with `ignoreCase` and `multiline`.\n- Context resolution:\n - `contextBefore/contextAfter` override legacy `context`.\n - Non-content modes zero out context collection.\n- Output modes:\n - `content` => one `GrepMatch` per hit.\n - `count` and `filesWithMatches` both map to count-style entries (`lineNumber=0`, `line=\"\"`, `matchCount` set).\n- Limits:\n - Global `offset` and `maxCount` applied across files.\n - Parallel path is used only when `maxCount` is unset and `offset == 0`; otherwise sequential path preserves deterministic global offset/limit semantics.\n\n### Result shaping back to JS\n\n- Rust `SearchResult`/`GrepResult` fields map to TS types via N-API object field conversion.\n- Counters are clamped to `u32` before crossing N-API.\n- Optional booleans are omitted unless true in some paths (`limitReached`).\n- Streaming callback receives each shaped `GrepMatch` (content or count entry).\n\n### Failure behavior\n\n- `searchContent` returns `SearchResult.error` for regex/search failures instead of throwing.\n- `grep` rejects on hard errors (invalid path, invalid glob/regex, cancellation timeout/abort).\n- `hasMatch` returns `Result<bool>` and throws on invalid pattern/UTF-8 decoding errors.\n- File open/search errors in multi-file scans are skipped per-file; scan continues.\n\n### Malformed regex handling\n\n`grep.rs` sanitizes braces before regex compile:\n\n- Invalid repetition-like braces are escaped (`{`/`}` -> `\\{`/`\\}`) when they cannot form `{N}`, `{N,}`, `{N,M}`.\n- This prevents common literal-template fragments (for example `${platform}`) from failing as malformed repetition.\n- Remaining invalid regex syntax still returns a regex error.\n\n## 2) File discovery (`glob`) and fuzzy path search (`fuzzyFind`)\n\n`glob` and `fuzzyFind` share `fs_cache` scans; matching logic differs.\n\n### `glob` flow\n\n1. TS wrapper (`glob/index.ts`):\n - `path.resolve(options.path)`.\n - Defaults: `pattern=\"*\"`, `hidden=false`, `gitignore=true`, `recursive=true`.\n2. Rust `glob` builds `GlobConfig` and compiles pattern via `glob_util::compile_glob`.\n3. Entry source:\n - `cache=true` => `get_or_scan` + optional stale-empty `force_rescan`.\n - `cache=false` => `force_rescan(..., store=false)` (fresh only).\n4. Filtering:\n - Skip `.git` always.\n - Skip `node_modules` unless requested (`includeNodeModules` or pattern mentioning node_modules).\n - Apply glob match.\n - Apply file-type filter; symlink `file/dir` filters resolve target metadata.\n5. Optional sort by mtime desc (`sortByMtime`) before truncating to `maxResults`.\n\n### `fuzzyFind` flow (implemented in `fd.rs`)\n\n1. TS wrapper is exported from `grep` module, but Rust implementation lives in `fd.rs`.\n2. Shared scan source from `fs_cache` with same cache/no-cache split and stale-empty recheck policy.\n3. Scoring:\n - exact / starts-with / contains / subsequence-based fuzzy score\n - separator/punctuation-normalized scoring path\n - directory bonus and deterministic tie-break (`score desc`, then `path asc`)\n4. Symlink entries are excluded from fuzzy results.\n\n### Failure behavior\n\n- Invalid glob pattern => error from `glob_util::compile_glob`.\n- Search root must be an existing directory (`resolve_search_path`), otherwise error.\n- Cancellation/timeouts propagate as abort errors via `CancelToken::heartbeat()` checks in loops.\n\n### Malformed glob handling\n\n`glob_util::build_glob_pattern` is tolerant:\n\n- Normalizes `\\` to `/`.\n- Auto-prefixes simple recursive patterns with `**/` when `recursive=true`.\n- Auto-closes unbalanced `{...` alternation groups before compile.\n\n## 3) Shared scan/cache lifecycle (`fs_cache`)\n\n`fs_cache` stores scan results as normalized relative entries (`path`, `fileType`, optional `mtime`) keyed by:\n\n- canonical search root\n- `include_hidden`\n- `use_gitignore`\n\n### Cache state transitions\n\n1. **Miss / disabled**\n - TTL is `0` or key absent/expired -> fresh `collect_entries`.\n2. **Hit**\n - Entry age `< cache_ttl_ms()` -> return cached entries + `cache_age_ms`.\n3. **Stale-empty recheck** (caller policy in `glob`/`grep`/`fd`)\n - If query yields zero matches and `cache_age_ms >= empty_recheck_ms()`, force one rescan.\n4. **Invalidation**\n - `invalidateFsScanCache(path?)`:\n - no arg: clear all keys\n - path arg: remove keys whose root prefixes that target path\n\n### Stale-result tradeoff\n\n- Cache favors low-latency repeated scans over immediate consistency.\n- TTL window can return stale positives/negatives.\n- Empty-result recheck reduces stale negatives for older cached scans at the cost of one extra scan.\n- Explicit invalidation is the intended correctness hook after file mutations.\n\n## 4) ANSI text utilities (`text`)\n\nThese are pure, in-memory utilities (no filesystem scanning).\n\n### Boundaries and responsibilities\n\n- **`text.rs` owns terminal-cell semantics**:\n - ANSI sequence parsing\n - grapheme-aware width and slicing\n - wrap/truncate/sanitize behavior\n- **`grep.rs` line truncation (`maxColumns`) is separate**:\n - simple character-boundary truncation of matched lines with `...`\n - not ANSI-state-preserving and not terminal-cell width aware\n\n### Key behaviors\n\n- `wrapTextWithAnsi`: wraps by visible width, carries active SGR codes across wrapped lines.\n- `truncateToWidth`: visible-cell truncation with ellipsis policy (`Unicode`, `Ascii`, `Omit`), optional right padding, and fast-path returning original JS string when unchanged.\n- `sliceWithWidth`: column slicing with optional strict width enforcement.\n- `extractSegments`: extracts before/after segments around an overlay while restoring ANSI state for the `after` segment.\n- `sanitizeText`: strips ANSI escapes + control chars, drops lone surrogates, normalizes CR/LF by removing `\\r`.\n- `visibleWidth`: counts visible terminal cells (tabs use fixed `TAB_WIDTH` from Rust implementation).\n\n### Failure behavior\n\nText functions generally return deterministic transformed output; errors are limited to JS string conversion boundaries (N-API argument conversion failures).\n\n## 5) Syntax highlighting (`highlight`)\n\n`highlight.rs` is pure transformation (no FS, no cache).\n\n### Flow\n\n1. Wrapper forwards `code`, optional `lang`, and ANSI color palette.\n2. Rust resolves syntax by:\n - token/name lookup\n - extension lookup\n - alias table fallback (`ts/tsx/js -> JavaScript`, etc.)\n - fallback to plain text syntax when unresolved\n3. Parse each line with syntect `ParseState` and scope stack.\n4. Map scopes to 11 semantic color categories and inject/reset ANSI color codes.\n\n### Failure behavior\n\n- Per-line parse failure does not fail the call: that line is appended unhighlighted and processing continues.\n- Unknown/unsupported language falls back to plain text syntax.\n\n## Pure utility vs filesystem-dependent flows\n\n| Flow | Filesystem access | Shared cache | Notes |\n| --- | --- | --- | --- |\n| `searchContent` / `hasMatch` | No | No | regex on provided bytes/string only |\n| `text` module functions | No | No | ANSI/width/sanitization only |\n| `highlight` module functions | No | No | syntax + ANSI coloring only |\n| `glob` | Yes | Optional | directory scans + glob filtering |\n| `fuzzyFind` | Yes | Optional | directory scans + fuzzy scoring |\n| `grep` (file/dir path) | Yes | Optional (dir mode) | ripgrep over files, optional filters/callback |\n\n## End-to-end lifecycle summary\n\n1. Caller invokes TS wrapper with typed options.\n2. Wrapper normalizes defaults (notably `glob`) and forwards to `native.*` export.\n3. Rust validates/normalizes options and builds matcher/search config.\n4. For filesystem flows, entries are scanned (cache hit/miss/rescan) then filtered/scored.\n5. Worker loops periodically call cancel heartbeat; timeout/abort can terminate execution.\n6. Rust shapes outputs into N-API objects (`lineNumber`, `matchCount`, `limitReached`, etc.).\n7. TS wrapper returns typed JS objects (and optional per-match callbacks for `grep`/`glob`).\n",
|
|
144
144
|
"en/natives/porting-to-natives.md": "---\ntitle: Porting to pi-natives (N-API) — Field Notes\ndescription: Field notes for migrating Node.js child_process and shell code to the Rust N-API native layer.\nsidebar:\n order: 9\n label: Porting to pi-natives\n---\n\n# Porting to pi-natives (N-API) — Field Notes\n\nThis is a practical guide for moving hot paths into `crates/pi-natives` and wiring them through the JS bindings. It exists to avoid the same failures happening twice.\n\n## When to port\n\nPort when any of these are true:\n\n- The hot path runs in render loops, tight UI updates, or large batches.\n- JS allocations dominate (string churn, regex backtracking, large arrays).\n- You already have a JS baseline and can benchmark both versions side by side.\n- The work is CPU-bound or blocking I/O that can run on the libuv thread pool.\n- The work is async I/O that can run on Tokio's runtime (e.g., shell execution).\n\nAvoid ports that depend on JS-only state or dynamic imports. N-API exports should be pure, data-in/data-out. Long-running work should go through `task::blocking` (CPU-bound/blocking I/O) or `task::future` (async I/O) with cancellation.\n\n## Anatomy of a native export\n\n**Rust side:**\n\n- Implementation lives in `crates/pi-natives/src/<module>.rs`. If you add a new module, register it in `crates/pi-natives/src/lib.rs`.\n- Export with `#[napi]`; snake_case exports are converted to camelCase automatically. Use explicit `js_name` only for true aliases/non-default names. Use `#[napi(object)]` for structs.\n- Use `task::blocking(tag, cancel_token, work)` (see `crates/pi-natives/src/task.rs`) for CPU-bound or blocking work. Use `task::future(env, tag, work)` for async work that needs Tokio (e.g., shell sessions). Pass a `CancelToken` when you expose `timeoutMs` or `AbortSignal`.\n\n**JS side:**\n\n- `packages/natives/src/bindings.ts` holds the base `NativeBindings` interface.\n- `packages/natives/src/<module>/types.ts` defines TS types and augments `NativeBindings` via declaration merging.\n- `packages/natives/src/native.ts` imports each `<module>/types.ts` file to activate the declarations.\n- `packages/natives/src/<module>/index.ts` wraps the `native` binding from `packages/natives/src/native.ts`.\n- `packages/natives/src/native.ts` loads the addon and `validateNative` enforces required exports.\n- `packages/natives/src/index.ts` re-exports the wrapper for callers in `packages/*`.\n\n## Porting checklist\n\n1. **Add the Rust implementation**\n\n- Put the core logic in a plain Rust function.\n- If it’s a new module, add it to `crates/pi-natives/src/lib.rs`.\n- Expose it with `#[napi]` so the default snake_case -> camelCase mapping stays consistent.\n- Keep signatures owned and simple: `String`, `Vec<String>`, `Uint8Array`, or `Either<JsString, Uint8Array>` for large string/byte inputs.\n- For CPU-bound or blocking work, use `task::blocking`; for async work, use `task::future`. Pass a `CancelToken` and call `heartbeat()` inside long loops.\n\n2. **Wire JS bindings**\n\n- Add the types and `NativeBindings` augmentation in `packages/natives/src/<module>/types.ts`.\n- Import `./<module>/types` in `packages/natives/src/native.ts` to trigger declaration merging.\n- Add a wrapper in `packages/natives/src/<module>/index.ts` that calls `native`.\n- Re-export from `packages/natives/src/index.ts`.\n\n3. **Update native validation**\n\n- Add `checkFn(\"newExport\")` in `validateNative` (`packages/natives/src/native.ts`).\n\n4. **Add benchmarks**\n\n- Put benchmarks next to the owning package (`packages/tui/bench`, `packages/natives/bench`, or `packages/coding-agent/bench`).\n- Include a JS baseline and native version in the same run.\n- Use `Bun.nanoseconds()` and a fixed iteration count.\n- Keep the benchmark inputs small and realistic (actual data seen in the hot path).\n\n5. **Build the native binary**\n\n- `bun --cwd=packages/natives run build`\n- Use `bun --cwd=packages/natives run build` and set `PI_DEV=1` if you want loader diagnostics while testing.\n\n6. **Run the benchmark**\n\n- `bun run packages/<pkg>/bench/<bench>.ts` (or `bun --cwd=packages/natives run bench`)\n\n7. **Decide on usage**\n\n- If native is slower, **keep JS** and leave the native export unused.\n- If native is faster, switch call sites to the native wrapper.\n\n## Pain points and how to avoid them\n\n### 1) Stale `pi_natives.node` prevents new exports\n\nThe loader prefers the platform-tagged binary in `packages/natives/native` (`pi_natives.<platform>-<arch>.node`). `PI_DEV=1` now only enables loader diagnostics; it no longer switches to a separate dev addon filename. There is also a fallback `pi_natives.node`. Compiled binaries extract to `~/.xcsh/natives/<version>/pi_natives.<platform>-<arch>.node`. If any of these are stale, exports won’t update.\n\n**Fix:** remove the stale file before rebuilding.\n\n```bash\nrm packages/natives/native/pi_natives.linux-x64.node\nrm packages/natives/native/pi_natives.node\nbun --cwd=packages/natives run build\n```\n\nIf you’re running a compiled binary, delete the cached addon directory:\n\n```bash\nrm -rf ~/.xcsh/natives/<version>\n```\n\nThen verify the export exists in the binary:\n\n```bash\nbun -e 'const tag = `${process.platform}-${process.arch}`; const mod = require(`./packages/natives/native/pi_natives.${tag}.node`); console.log(Object.keys(mod).includes(\"newExport\"));'\n```\n\n### 2) “Missing exports” errors from `validateNative`\n\nThis is **good** — it prevents silent mismatches. When you see this:\n\n```\nNative addon missing exports ... Missing: visibleWidth\n```\n\nit means your binary is stale, the Rust export name (or explicit alias when used) doesn’t match the JS name, or the export never compiled in. Fix the build and the naming mismatch, don’t weaken validation.\n\n### 3) Rust signature mismatch\n\nKeep it simple and owned. `String`, `Vec<String>`, and `Uint8Array` work. Avoid references like `&str` in public exports. If you need structured data, wrap it in `#[napi(object)]` structs.\n\n### 4) Benchmarking mistakes\n\n- Don’t compare different inputs or allocations.\n- Keep JS and native using identical input arrays.\n- Run both in the same benchmark file to avoid skew.\n\n## Benchmark template\n\n```ts\nconst ITERATIONS = 2000;\n\nfunction bench(name: string, fn: () => void): number {\n const start = Bun.nanoseconds();\n for (let i = 0; i < ITERATIONS; i++) fn();\n const elapsed = (Bun.nanoseconds() - start) / 1e6;\n console.log(`${name}: ${elapsed.toFixed(2)}ms total (${(elapsed / ITERATIONS).toFixed(6)}ms/op)`);\n return elapsed;\n}\n\nbench(\"feature/js\", () => {\n jsImpl(sample);\n});\n\nbench(\"feature/native\", () => {\n nativeImpl(sample);\n});\n```\n\n## Verification checklist\n\n- `validateNative` passes (no missing exports).\n- `NativeBindings` is augmented in `packages/natives/src/<module>/types.ts` and the wrapper is re-exported in `packages/natives/src/index.ts`.\n- `Object.keys(require(...))` includes your new export.\n- Bench numbers recorded in the PR/notes.\n- Call site updated **only if** native is faster or equal.\n\n## Rule of thumb\n\n- If native is slower, **do not switch**. Keep the export for future work, but the TUI should stay on the faster path.\n- If native is faster, switch the call site and keep the benchmark in place to catch regressions.\n",
|
|
145
145
|
"en/providers/models.md": "---\ntitle: Model and Provider Configuration\ndescription: Model registry and provider configuration via models.yml with routing, fallback, and pricing.\nsidebar:\n order: 1\n label: Models & providers\n---\n\n# Model and Provider Configuration (`models.yml`)\n\nThis document describes how the coding-agent currently loads models, applies overrides, resolves credentials, and chooses models at runtime.\n\n## What controls model behavior\n\nPrimary implementation files:\n\n- `src/config/model-registry.ts` — loads built-in + custom models, provider overrides, runtime discovery, auth integration\n- `src/config/model-resolver.ts` — parses model patterns and selects initial/smol/slow models\n- `src/config/settings-schema.ts` — model-related settings (`modelRoles`, provider transport preferences)\n- `src/session/auth-storage.ts` — API key + OAuth resolution order\n- `packages/ai/src/models.ts` and `packages/ai/src/types.ts` — built-in providers/models and `Model`/`compat` types\n\n## Config file location and legacy behavior\n\nDefault config path:\n\n- `~/.xcsh/agent/models.yml`\n\nLegacy behavior still present:\n\n- If `models.yml` is missing and `models.json` exists at the same location, it is migrated to `models.yml`.\n- Explicit `.json` / `.jsonc` config paths are still supported when passed programmatically to `ModelRegistry`.\n\n## `models.yml` shape\n\n```yaml\nconfigVersion: 1 # optional — written by auto-config, used for migration detection\nproviders:\n <provider-id>:\n # provider-level config\nequivalence:\n overrides:\n <provider-id>/<model-id>: <canonical-model-id>\n exclude:\n - <provider-id>/<model-id>\n```\n\n`configVersion` is an optional integer written by the auto-config system. When present, xcsh uses it to detect outdated configs and auto-upgrade them.\n\n`provider-id` is the canonical provider key used across selection and auth lookup.\n\n`equivalence` is optional and configures canonical model grouping on top of concrete provider models:\n\n- `overrides` maps an exact concrete selector (`provider/modelId`) to an official upstream canonical id\n- `exclude` opts a concrete selector out of canonical grouping\n\n## Provider-level fields\n\n```yaml\nproviders:\n my-provider:\n baseUrl: https://api.example.com/v1\n apiKey: MY_PROVIDER_API_KEY\n api: openai-completions\n headers:\n X-Team: platform\n authHeader: true\n auth: apiKey\n discovery:\n type: ollama\n modelOverrides:\n some-model-id:\n name: Renamed model\n models:\n - id: some-model-id\n name: Some Model\n api: openai-completions\n reasoning: false\n input: [text]\n cost:\n input: 0\n output: 0\n cacheRead: 0\n cacheWrite: 0\n contextWindow: 128000\n maxTokens: 16384\n headers:\n X-Model: value\n compat:\n supportsStore: true\n supportsDeveloperRole: true\n supportsReasoningEffort: true\n maxTokensField: max_completion_tokens\n openRouterRouting:\n only: [anthropic]\n vercelGatewayRouting:\n order: [anthropic, openai]\n extraBody:\n gateway: m1-01\n controller: mlx\n```\n\n### Allowed provider/model `api` values\n\n- `openai-completions`\n- `openai-responses`\n- `openai-codex-responses`\n- `azure-openai-responses`\n- `anthropic-messages`\n- `google-generative-ai`\n- `google-vertex`\n\n### Allowed auth/discovery values\n\n- `auth`: `apiKey` (default) or `none`\n- `discovery.type`: `ollama`\n\n## Validation rules (current)\n\n### Full custom provider (`models` is non-empty)\n\nRequired:\n\n- `baseUrl`\n- `apiKey` unless `auth: none`\n- `api` at provider level or each model\n\n### Override-only provider (`models` missing or empty)\n\nMust define at least one of:\n\n- `baseUrl`\n- `modelOverrides`\n- `discovery`\n\n### Discovery\n\n- `discovery` requires provider-level `api`.\n\n### Model value checks\n\n- `id` required\n- `contextWindow` and `maxTokens` must be positive if provided\n\n## Merge and override order\n\nModelRegistry pipeline (on refresh):\n\n1. Load built-in providers/models from `@f5-sales-demo/pi-ai`.\n2. Load `models.yml` custom config.\n3. Apply provider overrides (`baseUrl`, `headers`) to built-in models.\n4. Apply `modelOverrides` (per provider + model id).\n5. Merge custom `models`:\n - same `provider + id` replaces existing\n - otherwise append\n6. Apply runtime-discovered models (currently Ollama and LM Studio), then re-apply model overrides.\n\n## Canonical model equivalence and coalescing\n\nThe registry keeps every concrete provider model and then builds a canonical layer above them.\n\nCanonical ids are official upstream ids only, for example:\n\n- `claude-opus-4-6`\n- `claude-haiku-4-5`\n- `gpt-5.3-codex`\n\n### `models.yml` equivalence config\n\nExample:\n\n```yaml\nproviders:\n zenmux:\n baseUrl: https://api.zenmux.example/v1\n apiKey: ZENMUX_API_KEY\n api: openai-codex-responses\n models:\n - id: codex\n name: Zenmux Codex\n reasoning: true\n input: [text]\n cost:\n input: 0\n output: 0\n cacheRead: 0\n cacheWrite: 0\n contextWindow: 200000\n maxTokens: 32768\n\nequivalence:\n overrides:\n zenmux/codex: gpt-5.3-codex\n p-codex/codex: gpt-5.3-codex\n exclude:\n - demo/codex-preview\n```\n\nBuild order for canonical grouping:\n\n1. exact user override from `equivalence.overrides`\n2. bundled official-id matches from built-in model metadata\n3. conservative heuristic normalization for gateway/provider variants\n4. fallback to the concrete model's own id\n\nCurrent heuristics are intentionally narrow:\n\n- embedded upstream prefixes can be stripped when present, for example `anthropic/...` or `openai/...`\n- dotted and dashed version variants can normalize only when they map to an existing official id, for example `4.6 -> 4-6`\n- ambiguous families or versions are not merged without a bundled match or explicit override\n\n### Canonical resolution behavior\n\nWhen multiple concrete variants share a canonical id, resolution uses:\n\n1. availability and auth\n2. `config.yml` `modelProviderOrder`\n3. existing registry/provider order if `modelProviderOrder` is unset\n\nDisabled or unauthenticated providers are skipped.\n\nSession state and transcripts continue to record the concrete provider/model that actually executed the turn.\n\nProvider defaults vs per-model overrides:\n\n- Provider `headers` are baseline.\n- Model `headers` override provider header keys.\n- `modelOverrides` can override model metadata (`name`, `reasoning`, `input`, `cost`, `contextWindow`, `maxTokens`, `headers`, `compat`, `contextPromotionTarget`).\n- `compat` is deep-merged for nested routing blocks (`openRouterRouting`, `vercelGatewayRouting`, `extraBody`).\n\n## Runtime discovery integration\n\n### Implicit Ollama discovery\n\nIf `ollama` is not explicitly configured, registry adds an implicit discoverable provider:\n\n- provider: `ollama`\n- api: `openai-completions`\n- base URL: `OLLAMA_BASE_URL` or `http://127.0.0.1:11434`\n- auth mode: keyless (`auth: none` behavior)\n\nRuntime discovery calls `GET /api/tags` on Ollama and synthesizes model entries with local defaults.\n\n### Implicit llama.cpp discovery\n\nIf `llama.cpp` is not explicitly configured, registry adds an implicit discoverable provider:\nNote: it's using the newer antropic messages api instead of the openai-competions.\n\n- provider: `llama.cpp`\n- api: `openai-responses`\n- base URL: `LLAMA_CPP_BASE_URL` or `http://127.0.0.1:8080`\n- auth mode: keyless (`auth: none` behavior)\n\nRuntime discovery calls `GET models` on llama.cpp and synthesizes model entries with local defaults.\n\n### Implicit LM Studio discovery\n\nIf `lm-studio` is not explicitly configured, registry adds an implicit discoverable provider:\n\n- provider: `lm-studio`\n- api: `openai-completions`\n- base URL: `LM_STUDIO_BASE_URL` or `http://127.0.0.1:1234/v1`\n- auth mode: keyless (`auth: none` behavior)\n\nRuntime discovery fetches models (`GET /models`) and synthesizes model entries with local defaults.\n\n### Explicit provider discovery\n\nYou can configure discovery yourself:\n\n```yaml\nproviders:\n ollama:\n baseUrl: http://127.0.0.1:11434\n api: openai-completions\n auth: none\n discovery:\n type: ollama\n \n llama.cpp:\n baseUrl: http://127.0.0.1:8080\n api: openai-responses\n auth: none\n discovery:\n type: llama.cpp\n```\n\n### Extension provider registration\n\nExtensions can register providers at runtime (`pi.registerProvider(...)`), including:\n\n- model replacement/append for a provider\n- custom stream handler registration for new API IDs\n- custom OAuth provider registration\n\n## Auth and API key resolution order\n\nWhen requesting a key for a provider, effective order is:\n\n1. Runtime override (CLI `--api-key`)\n2. Stored API key credential in `agent.db`\n3. Stored OAuth credential in `agent.db` (with refresh)\n4. Environment variable mapping (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, etc.)\n5. ModelRegistry fallback resolver (provider `apiKey` from `models.yml`, env-name-or-literal semantics)\n\n`models.yml` `apiKey` behavior:\n\n- Value is first treated as an environment variable name.\n- If no env var exists, the literal string is used as the token.\n\nIf `authHeader: true` and provider `apiKey` is set, models get:\n\n- `Authorization: Bearer <resolved-key>` header injected.\n\nKeyless providers:\n\n- Providers marked `auth: none` are treated as available without credentials.\n- `getApiKey*` returns `kNoAuth` for them.\n\n## Model availability vs all models\n\n- `getAll()` returns the loaded model registry (built-in + merged custom + discovered).\n- `getAvailable()` filters to models that are keyless or have resolvable auth.\n\nSo a model can exist in registry but not be selectable until auth is available.\n\n## Runtime model resolution\n\n### CLI and pattern parsing\n\n`model-resolver.ts` supports:\n\n- exact `provider/modelId`\n- exact canonical model id\n- exact model id (provider inferred)\n- fuzzy/substring matching\n- glob scope patterns in `--models` (e.g. `openai/*`, `*sonnet*`)\n- optional `:thinkingLevel` suffix (`off|minimal|low|medium|high|xhigh`)\n\n`--provider` is legacy; `--model` is preferred.\n\nResolution precedence for exact selectors:\n\n1. exact `provider/modelId` bypasses coalescing\n2. exact canonical id resolves through the canonical index\n3. exact bare concrete id still works\n4. fuzzy and glob matching run after the exact paths\n\n### Initial model selection priority\n\n`findInitialModel(...)` uses this order:\n\n1. explicit CLI provider+model\n2. first scoped model (if not resuming)\n3. saved default provider/model\n4. known provider defaults (e.g. OpenAI/Anthropic/etc.) among available models\n5. first available model\n\n### Role aliases and settings\n\nSupported model roles:\n\n- `default`, `smol`, `slow`, `plan`, `commit`\n\nRole aliases like `pi/smol` expand through `settings.modelRoles`. Each role value can also append a thinking selector such as `:minimal`, `:low`, `:medium`, or `:high`.\n\nIf a role points at another role, the target model still inherits normally and any explicit suffix on the referring role wins for that role-specific use.\n\nRelated settings:\n\n- `modelRoles` (record)\n- `enabledModels` (scoped pattern list)\n- `modelProviderOrder` (global canonical-provider precedence)\n- `providers.kimiApiFormat` (`openai` or `anthropic` request format)\n- `providers.openaiWebsockets` (`auto|off|on` websocket preference for OpenAI Codex transport)\n\n`modelRoles` may store either:\n\n- `provider/modelId` to pin a concrete provider variant\n- a canonical id such as `gpt-5.3-codex` to allow provider coalescing\n\nFor `enabledModels` and CLI `--models`:\n\n- exact canonical ids expand to all concrete variants in that canonical group\n- explicit `provider/modelId` entries stay exact\n- globs and fuzzy matches still operate on concrete models\n\n## `/model` and `--list-models`\n\nBoth surfaces keep provider-prefixed models visible and selectable.\n\nThey now also expose canonical/coalesced models:\n\n- `/model` includes a canonical view alongside provider tabs\n- `--list-models` prints a canonical section plus the concrete provider rows\n\nSelecting a canonical entry stores the canonical selector. Selecting a provider row stores the explicit `provider/modelId`.\n\n## Context promotion (model-level fallback chains)\n\nContext promotion is an overflow recovery mechanism for small-context variants (for example `*-spark`) that automatically promotes to a larger-context sibling when the API rejects a request with a context length error.\n\n### Trigger and order\n\nWhen a turn fails with a context overflow error (e.g. `context_length_exceeded`), `AgentSession` attempts promotion **before** falling back to compaction:\n\n1. If `contextPromotion.enabled` is true, resolve a promotion target (see below).\n2. If a target is found, switch to it and retry the request — no compaction needed.\n3. If no target is available, fall through to auto-compaction on the current model.\n\n### Target selection\n\nSelection is model-driven, not role-driven:\n\n1. `currentModel.contextPromotionTarget` (if configured)\n2. smallest larger-context model on the same provider + API\n\nCandidates are ignored unless credentials resolve (`ModelRegistry.getApiKey(...)`).\n\n### OpenAI Codex websocket handoff\n\nIf switching from/to `openai-codex-responses`, session provider state key `openai-codex-responses` is closed before model switch. This drops websocket transport state so the next turn starts clean on the promoted model.\n\n### Persistence behavior\n\nPromotion uses temporary switching (`setModelTemporary`):\n\n- recorded as a temporary `model_change` in session history\n- does not rewrite saved role mapping\n\n### Configuring explicit fallback chains\n\nConfigure fallback directly in model metadata via `contextPromotionTarget`.\n\n`contextPromotionTarget` accepts either:\n\n- `provider/model-id` (explicit)\n- `model-id` (resolved within current provider)\n\nExample (`models.yml`) for Spark -> non-Spark on the same provider:\n\n```yaml\nproviders:\n openai-codex:\n modelOverrides:\n gpt-5.3-codex-spark:\n contextPromotionTarget: openai-codex/gpt-5.3-codex\n```\n\nThe built-in model generator also assigns this automatically for `*-spark` models when a same-provider base model exists.\n\n## Compatibility and routing fields\n\n`models.yml` supports this `compat` subset:\n\n- `supportsStore`\n- `supportsDeveloperRole`\n- `supportsReasoningEffort`\n- `maxTokensField` (`max_completion_tokens` or `max_tokens`)\n- `openRouterRouting.only` / `openRouterRouting.order`\n- `vercelGatewayRouting.only` / `vercelGatewayRouting.order`\n\nThese are consumed by the OpenAI-completions transport logic and combined with URL-based auto-detection.\n\n## Practical examples\n\n### Local OpenAI-compatible endpoint (no auth)\n\n```yaml\nproviders:\n local-openai:\n baseUrl: http://127.0.0.1:8000/v1\n auth: none\n api: openai-completions\n models:\n - id: Qwen/Qwen2.5-Coder-32B-Instruct\n name: Qwen 2.5 Coder 32B (local)\n```\n\n### Hosted proxy with env-based key\n\n```yaml\nproviders:\n anthropic-proxy:\n baseUrl: https://proxy.example.com/anthropic\n apiKey: ANTHROPIC_PROXY_API_KEY\n api: anthropic-messages\n authHeader: true\n models:\n - id: claude-sonnet-4-20250514\n name: Claude Sonnet 4 (Proxy)\n reasoning: true\n input: [text, image]\n```\n\n### Override built-in provider route + model metadata\n\n```yaml\nproviders:\n openrouter:\n baseUrl: https://my-proxy.example.com/v1\n headers:\n X-Team: platform\n modelOverrides:\n anthropic/claude-sonnet-4:\n name: Sonnet 4 (Corp)\n compat:\n openRouterRouting:\n only: [anthropic]\n```\n\n## LiteLLM proxy auto-configuration\n\nWhen both `LITELLM_BASE_URL` and `LITELLM_API_KEY` environment variables are set, xcsh automatically manages `models.yml` configuration for the LiteLLM proxy.\n\n### First-run auto-generation\n\nIf `models.yml` does not exist and LiteLLM env vars are detected, xcsh generates it automatically:\n\n```yaml\n# Auto-generated by xcsh for LiteLLM proxy\n# API key resolved from LITELLM_API_KEY env var at runtime\nconfigVersion: 1\nproviders:\n anthropic:\n baseUrl: \"https://your-litellm-proxy.example.com/anthropic\"\n apiKey: LITELLM_API_KEY\n```\n\nA default `config.yml` is also generated with sensible image provider settings.\n\n### Startup self-healing\n\nOn every startup, `startupHealthCheck()` in the model registry runs the following checks:\n\n| Condition | Action |\n|-----------|--------|\n| `models.yml` missing | Auto-generate from env vars |\n| `models.yml` corrupt or unparseable | Backup to `.bak`, regenerate |\n| `baseUrl` doesn't match `LITELLM_BASE_URL` | Backup to `.bak`, regenerate with new URL |\n| `configVersion` missing or outdated | Backup to `.bak`, regenerate with current version |\n| Config is healthy | No action |\n\nAll repairs create `.bak` backups before overwriting. All operations are idempotent.\n\n### CLI command\n\n```bash\nxcsh setup litellm # Generate or fix LiteLLM config\nxcsh setup litellm --check # Validate without writing\nxcsh setup litellm --check --json # Machine-readable validation output\n```\n\n### Required environment variables\n\n| Variable | Purpose |\n|----------|---------|\n| `LITELLM_BASE_URL` | LiteLLM proxy URL (e.g. `https://your-proxy.example.com`). Must start with `http://` or `https://`. |\n| `LITELLM_API_KEY` | API key for the proxy. Referenced by name in generated config, resolved at runtime. |\n\nIf either variable is unset, auto-configuration is silently skipped.\n\n### Config versioning\n\nGenerated configs include a `configVersion` field. When the generated format changes in future releases, xcsh detects outdated configs and automatically upgrades them (with backup).\n\n## Legacy consumer caveat\n\nMost model configuration now flows through `models.yml` via `ModelRegistry`.\n\nOne notable legacy path remains: web-search Anthropic auth resolution still reads `~/.xcsh/agent/models.json` directly in `src/web/search/auth.ts`.\n\nIf you rely on that specific path, keep JSON compatibility in mind until that module is migrated.\n\n## Failure mode\n\nIf `models.yml` fails schema or validation checks:\n\n- If `LITELLM_BASE_URL` and `LITELLM_API_KEY` are set, the startup health check attempts auto-repair (backup corrupt file, regenerate from env vars). If repair succeeds, the registry reloads the fixed config.\n- If auto-repair is not possible (env vars unset, write failure), the registry keeps operating with built-in models.\n- Error is exposed via `ModelRegistry.getError()` and surfaced in UI/notifications.\n",
|
|
146
|
-
"en/providers/openai-api-access.md": "---\ntitle: OpenAI access\ndescription:
|
|
146
|
+
"en/providers/openai-api-access.md": "---\ntitle: OpenAI access\ndescription: Choose ChatGPT subscription or usage-based OpenAI Platform access in xcsh.\nsidebar:\n order: 4\n label: OpenAI access\n---\n\nxcsh offers two separate OpenAI providers. Start xcsh without a configured provider, or run `/login`, and choose the option that matches how you want access to be billed.\n\n## ChatGPT subscription\n\nChoose **ChatGPT Plus/Pro (Codex Subscription)**, or run `/login openai-codex`. xcsh opens the ChatGPT OAuth flow and stores the resulting credential in its credential database. It then discovers the models advertised for that ChatGPT account.\n\nWhen the account advertises the complete GPT-5.6 family, xcsh selects `openai-codex/gpt-5.6-terra` with medium reasoning and configures Luna, Terra, and Sol for its subscription routing roles.\n\nThe browser callback uses `http://localhost:1455/auth/callback`. Port 1455 must be available while login is running. If the browser cannot reach the callback directly, copy the complete redirect URL and submit it with `/login <redirect-url>` in the waiting xcsh session.\n\nCredentials disabled by the OpenAI OAuth regression in v20.19.1 are reactivated automatically. Credentials disabled because they were deleted, invalid, expired, or failed for another reason remain disabled.\n\n## Usage-based OpenAI Platform API\n\nChoose **OpenAI Responses API (usage-based API access)** for OpenAI Platform billing. Set `OPENAI_API_KEY` in the environment, then select an OpenAI model with `/model`.\n\n```sh\nexport OPENAI_API_KEY=\"your-platform-api-key\"\nxcsh\n```\n\nThe ChatGPT subscription and OpenAI Platform API choices use different credentials and billing. `/login openai` therefore explains the API-key setup; it does not start ChatGPT OAuth.\n",
|
|
147
147
|
"en/providers/provider-streaming-internals.md": "---\ntitle: Provider Streaming Internals\ndescription: Provider streaming implementation with SSE parsing, token counting, and backpressure handling.\nsidebar:\n order: 2\n label: Streaming internals\n---\n\n# Provider streaming internals\n\nThis document explains how token/tool streaming is normalized in `@f5-sales-demo/pi-ai`, then propagated through `@f5-sales-demo/pi-agent-core` and `coding-agent` session events.\n\n## End-to-end flow\n\n1. `streamSimple()` (`packages/ai/src/stream.ts`) maps generic options and dispatches to a provider stream function.\n2. Provider stream functions (`anthropic.ts`, `openai-responses.ts`, `google.ts`) translate provider-native stream events into the unified `AssistantMessageEvent` sequence.\n3. Each provider pushes events into `AssistantMessageEventStream` (`packages/ai/src/utils/event-stream.ts`), which throttles delta events and exposes:\n - async iteration for incremental updates\n - `result()` for final `AssistantMessage`\n4. `agentLoop` (`packages/agent/src/agent-loop.ts`) consumes those events, mutates in-flight assistant state, and emits `message_update` events carrying the raw `assistantMessageEvent`.\n5. `AgentSession` (`packages/coding-agent/src/session/agent-session.ts`) subscribes to agent events, persists messages, drives extension hooks, and applies session behaviors (retry, compaction, TTSR, streaming-edit abort checks).\n\n## Unified stream contract in `@f5-sales-demo/pi-ai`\n\nAll providers emit the same shape (`AssistantMessageEvent` in `packages/ai/src/types.ts`):\n\n- `start`\n- content block lifecycle triplets:\n - text: `text_start` → `text_delta`* → `text_end`\n - thinking: `thinking_start` → `thinking_delta`* → `thinking_end`\n - tool call: `toolcall_start` → `toolcall_delta`* → `toolcall_end`\n- terminal event:\n - `done` with `reason: \"stop\" | \"length\" | \"toolUse\"`\n - or `error` with `reason: \"aborted\" | \"error\"`\n\n`AssistantMessageEventStream` guarantees:\n\n- final result is resolved by terminal event (`done` or `error`)\n- deltas are batched/throttled (~50ms)\n- buffered deltas are flushed before non-delta events and before completion\n\n## Delta throttling and harmonization behavior\n\n`AssistantMessageEventStream` treats `text_delta`, `thinking_delta`, and `toolcall_delta` as mergeable events:\n\n- buffered deltas are merged only when **type + contentIndex** match\n- merge keeps the latest `partial` snapshot\n- non-delta events force immediate flush\n\nThis smooths high-frequency provider streams for TUI/event consumers, but is not provider backpressure: providers still produce at full speed, while the local stream buffers.\n\n## Provider normalization details\n\n## Anthropic (`anthropic-messages`)\n\nSource: `packages/ai/src/providers/anthropic.ts`\n\nNormalization points:\n\n- `message_start` initializes usage (input/output/cache tokens)\n- `content_block_start` maps to text/thinking/toolcall starts\n- `content_block_delta` maps:\n - `text_delta` → `text_delta`\n - `thinking_delta` → `thinking_delta`\n - `input_json_delta` → `toolcall_delta`\n - `signature_delta` updates `thinkingSignature` only (no event)\n- `content_block_stop` emits corresponding `*_end`\n- `message_delta.stop_reason` maps via `mapStopReason()`\n\nTool-call argument streaming:\n\n- each tool block carries internal `partialJson`\n- every JSON delta appends to `partialJson`\n- `arguments` are reparsed on each delta via `parseStreamingJson()`\n- `toolcall_end` reparses once more, then strips `partialJson`\n\n## OpenAI Responses (`openai-responses`)\n\nSource: `packages/ai/src/providers/openai-responses.ts`\n\nNormalization points:\n\n- `response.output_item.added` starts reasoning/text/function-call blocks\n- reasoning summary events (`response.reasoning_summary_text.delta`) become `thinking_delta`\n- output/refusal deltas become `text_delta`\n- `response.function_call_arguments.delta` becomes `toolcall_delta`\n- `response.output_item.done` emits `thinking_end` / `text_end` / `toolcall_end`\n- `response.completed` maps status to stop reason and usage\n\nTool-call argument streaming:\n\n- same `partialJson` accumulation pattern as Anthropic\n- providers that send only `response.function_call_arguments.done` still populate final args\n- tool call IDs are normalized as `\"<call_id>|<item_id>\"`\n\n## Google Generative AI (`google-generative-ai`)\n\nSource: `packages/ai/src/providers/google.ts`\n\nNormalization points:\n\n- iterates `candidate.content.parts`\n- text parts are split into thinking vs text by `isThinkingPart(part)`\n- block transitions close previous block before starting a new one\n- `part.functionCall` is treated as a complete tool call (start/delta/end emitted immediately)\n- finish reason mapped by `mapStopReason()` from `google-shared.ts`\n\nTool-call argument streaming:\n\n- function call args arrive as structured object, not incremental JSON text\n- implementation emits one synthetic `toolcall_delta` containing `JSON.stringify(arguments)`\n- no partial JSON parser needed for Google in this path\n\n## Partial tool-call JSON accumulation and recovery\n\nShared behavior for Anthropic/OpenAI Responses uses `parseStreamingJson()` (`packages/ai/src/utils/json-parse.ts`):\n\n1. try `JSON.parse`\n2. fallback to `partial-json` parser for incomplete fragments\n3. if both fail, return `{}`\n\nImplications:\n\n- malformed or truncated argument deltas do not crash stream processing immediately\n- in-progress `arguments` may temporarily be `{}`\n- later valid deltas can recover structured arguments because parsing is retried on every append\n- final `toolcall_end` performs one more parse attempt before emission\n\n## Stop reasons vs transport/runtime errors\n\nProvider stop reasons are mapped to normalized `stopReason`:\n\n- Anthropic: `end_turn`→`stop`, `max_tokens`→`length`, `tool_use`→`toolUse`, safety/refusal cases→`error`\n- OpenAI Responses: `completed`→`stop`, `incomplete`→`length`, `failed/cancelled`→`error`\n- Google: `STOP`→`stop`, `MAX_TOKENS`→`length`, safety/prohibited/malformed-function-call classes→`error`\n\nError semantics are split in two stages:\n\n1. **Model completion semantics** (provider reported finish reason/status)\n2. **Transport/runtime failure** (network/client/parser/abort exceptions)\n\nIf provider stream throws or signals failure, each provider wrapper catches and emits terminal `error` event with:\n\n- `stopReason = \"aborted\"` when abort signal is set\n- otherwise `stopReason = \"error\"`\n- `errorMessage = formatErrorMessageWithRetryAfter(error)`\n\n## Malformed chunk / SSE parse failure behavior\n\nFor these provider paths, chunk/SSE framing is handled by vendor SDK streams (Anthropic SDK, OpenAI SDK, Google SDK). This code does not implement a custom SSE decoder here.\n\nObserved behavior in current implementation:\n\n- malformed chunk/SSE parsing at SDK level surfaces as an exception or stream `error` event\n- provider wrapper converts that into unified terminal `error` event\n- no provider-specific resume/retry inside the stream function itself\n- higher-level retries are handled in `AgentSession` auto-retry logic (message-level retry, not stream-chunk replay)\n\n## Cancellation boundaries\n\nCancellation is layered:\n\n- AI provider request: `options.signal` is passed into provider client stream call.\n- Provider wrapper: after stream loop, aborted signal forces error path (`\"Request was aborted\"`).\n- Agent loop: checks `signal.aborted` before handling each provider event and can synthesize an aborted assistant message from the latest partial.\n- Session/agent controls: `AgentSession.abort()` -> `agent.abort()` -> shared abort controller cancellation.\n\nTool execution cancellation is separate from model stream cancellation:\n\n- tool runners use `AbortSignal.any([agentSignal, steeringAbortSignal])`\n- steering interrupts can abort remaining tool execution while preserving already-produced tool results\n\n## Backpressure boundaries\n\nThere is no hard backpressure mechanism between provider SDK stream and downstream consumers:\n\n- `EventStream` uses in-memory queues with no max size\n- throttling reduces UI update rate but does not slow provider intake\n- if consumers lag significantly, queued events can grow until completion\n\nCurrent design favors responsiveness and simple ordering over bounded-buffer flow control.\n\n## How stream events surface as agent/session events\n\n`agentLoop.streamAssistantResponse()` bridges `AssistantMessageEvent` to `AgentEvent`:\n\n- on `start`: pushes placeholder assistant message and emits `message_start`\n- on block events (`text_*`, `thinking_*`, `toolcall_*`): updates last assistant message, emits `message_update` with raw `assistantMessageEvent`\n- on terminal (`done`/`error`): resolves final message from `response.result()`, emits `message_end`\n\n`AgentSession` then consumes those events for session-level behaviors:\n\n- TTSR watches `message_update.assistantMessageEvent` for `text_delta` and `toolcall_delta`\n- streaming edit guard inspects `toolcall_delta`/`toolcall_end` on `edit` calls and can abort early\n- persistence writes finalized messages at `message_end`\n- auto-retry examines assistant `stopReason === \"error\"` plus `errorMessage` heuristics\n\n## Unified vs provider-specific responsibilities\n\nUnified (common contract):\n\n- event shape (`AssistantMessageEvent`)\n- final result extraction (`done`/`error`)\n- delta throttling + merge rules\n- agent/session event propagation model\n\nProvider-specific (not fully abstracted):\n\n- upstream event taxonomies and mapping logic\n- stop-reason translation tables\n- tool-call ID conventions\n- reasoning/thinking block semantics and signatures\n- usage token semantics and availability timing\n- message conversion constraints per API\n\n## Implementation files\n\n- [`../../ai/src/stream.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/ai/src/stream.ts) — provider dispatch, option mapping, API key/session plumbing.\n- [`../../ai/src/utils/event-stream.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/ai/src/utils/event-stream.ts) — generic stream queue + assistant delta throttling.\n- [`../../ai/src/utils/json-parse.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/ai/src/utils/json-parse.ts) — partial JSON parsing for streamed tool arguments.\n- [`../../ai/src/providers/anthropic.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/ai/src/providers/anthropic.ts) — Anthropic event translation and tool JSON delta accumulation.\n- [`../../ai/src/providers/openai-responses.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/ai/src/providers/openai-responses.ts) — OpenAI Responses event translation and status mapping.\n- [`../../ai/src/providers/google.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/ai/src/providers/google.ts) — Gemini stream chunk-to-block translation.\n- [`../../ai/src/providers/google-shared.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/ai/src/providers/google-shared.ts) — Gemini finish-reason mapping and shared conversion rules.\n- [`../../agent/src/agent-loop.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/agent/src/agent-loop.ts) — provider stream consumption and `message_update` bridging.\n- [`../src/session/agent-session.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/session/agent-session.ts) — session-level handling of streaming updates, abort, retry, and persistence.\n",
|
|
148
148
|
"en/providers/python-repl.md": "---\ntitle: Python Tool and IPython Runtime\ndescription: Python REPL tool runtime with IPython kernel management, execution, and output capture.\nsidebar:\n order: 3\n label: Python & IPython\n---\n\n# Python Tool and IPython Runtime\n\nThis document describes the current Python execution stack in `packages/coding-agent`.\nIt covers tool behavior, kernel/gateway lifecycle, environment handling, execution semantics, output rendering, and operational failure modes.\n\n## Scope and Key Files\n\n- Tool surface: `src/tools/python.ts`\n- Session/per-call kernel orchestration: `src/ipy/executor.ts`\n- Kernel protocol + gateway integration: `src/ipy/kernel.ts`\n- Shared local gateway coordinator: `src/ipy/gateway-coordinator.ts`\n- Interactive-mode renderer for user-triggered Python runs: `src/modes/components/python-execution.ts`\n- Runtime/env filtering and Python resolution: `src/ipy/runtime.ts`\n\n## What the Python tool is\n\nThe `python` tool executes one or more Python cells through a Jupyter Kernel Gateway-backed kernel (not by spawning `python -c` directly per cell).\n\nTool params:\n\n```ts\n{\n cells: Array<{ code: string; title?: string }>;\n timeout?: number; // seconds, clamped to 1..600, default 30\n cwd?: string;\n reset?: boolean; // reset kernel before first cell only\n}\n```\n\nThe tool is `concurrency = \"exclusive\"` for a session, so calls do not overlap.\n\n## Gateway lifecycle\n\n### Modes\n\nThere are two gateway paths:\n\n1. **External gateway** (`PI_PYTHON_GATEWAY_URL` set)\n - Uses the configured URL directly.\n - Optional auth with `PI_PYTHON_GATEWAY_TOKEN`.\n - No local gateway process is spawned or managed.\n\n2. **Local shared gateway** (default path)\n - Uses a single shared process coordinated under `~/.xcsh/agent/python-gateway`.\n - Metadata file: `gateway.json`\n - Lock file: `gateway.lock`\n - Spawn command:\n - `python -m kernel_gateway`\n - bound to `127.0.0.1:<allocated-port>`\n - startup health check: `GET /api/kernelspecs`\n\n### Local shared gateway coordination\n\n`acquireSharedGateway()`:\n\n- Takes a file lock (`gateway.lock`) with heartbeat.\n- Reuses `gateway.json` if PID is alive and health check passes.\n- Cleans stale info/PIDs when needed.\n- Starts a new gateway when no healthy one exists.\n\n`releaseSharedGateway()` is currently a no-op (kernel shutdown does not tear down shared gateway).\n\n`shutdownSharedGateway()` explicitly terminates the shared process and clears gateway metadata.\n\n### Important constraint\n\n`python.sharedGateway=false` is rejected at kernel start:\n\n- Error: `Shared Python gateway required; local gateways are disabled`\n- There is no per-process non-shared local gateway mode.\n\n## Kernel lifecycle\n\nEach execution uses a kernel created via `POST /api/kernels` on the selected gateway.\n\nKernel startup sequence:\n\n1. Availability check (`checkPythonKernelAvailability`)\n2. Create kernel (`/api/kernels`)\n3. Open websocket (`/api/kernels/:id/channels`)\n4. Initialize kernel env (`cwd`, env vars, `sys.path`)\n5. Execute `PYTHON_PRELUDE`\n6. Load extension modules from:\n - user: `~/.xcsh/agent/modules/*.py`\n - project: `<cwd>/.xcsh/modules/*.py` (overrides same-name user module)\n\nKernel shutdown:\n\n- Deletes remote kernel via `DELETE /api/kernels/:id`\n- Closes websocket\n- Calls shared gateway release hook (no-op today)\n\n## Session persistence semantics\n\n`python.kernelMode` controls kernel reuse:\n\n- `session` (default)\n - Reuses kernel sessions keyed by session identity + cwd.\n - Execution is serialized per session via a queue.\n - Idle sessions are evicted after 5 minutes.\n - At most 4 sessions; oldest is evicted on overflow.\n - Heartbeat checks detect dead kernels.\n - Auto-restart allowed once; repeated crash => hard failure.\n\n- `per-call`\n - Creates a fresh kernel for each execute request.\n - Shuts kernel down after the request.\n - No cross-call state persistence.\n\n### Multi-cell behavior in a single tool call\n\nCells run sequentially in the same kernel instance for that tool call.\n\nIf an intermediate cell fails:\n\n- Earlier cell state remains in memory.\n- Tool returns a targeted error indicating which cell failed.\n- Later cells are not executed.\n\n`reset=true` only applies to the first cell execution in that call.\n\n## Environment filtering and runtime resolution\n\nEnvironment is filtered before launching gateway/kernel runtime:\n\n- Allowlist includes core vars like `PATH`, `HOME`, locale vars, `VIRTUAL_ENV`, `PYTHONPATH`, etc.\n- Allow-prefixes: `LC_`, `XDG_`, `PI_`\n- Denylist strips common API keys (OpenAI/Anthropic/Gemini/etc.)\n\nRuntime selection order:\n\n1. Active/located venv (`VIRTUAL_ENV`, then `<cwd>/.venv`, `<cwd>/venv`)\n2. Managed venv at `~/.xcsh/python-env`\n3. `python` or `python3` on PATH\n\nWhen a venv is selected, its bin/Scripts path is prepended to `PATH`.\n\nKernel env initialization inside Python also:\n\n- `os.chdir(cwd)`\n- injects provided env map into `os.environ`\n- ensures cwd is in `sys.path`\n\n## Tool availability and mode selection\n\n`python.toolMode` (default `both`) + optional `PI_PY` override controls exposure:\n\n- `ipy-only`\n- `bash-only`\n- `both`\n\n`PI_PY` accepted values:\n\n- `0` / `bash` -> `bash-only`\n- `1` / `py` -> `ipy-only`\n- `mix` / `both` -> `both`\n\nIf Python preflight fails, tool creation degrades to bash-only for that session.\n\n## Execution flow and cancellation/timeout\n\n### Tool-level timeout\n\n`python` tool timeout is in seconds, default 30, clamped to `1..600`.\n\nThe tool combines:\n\n- caller abort signal\n- timeout abort signal\n\nwith `AbortSignal.any(...)`.\n\n### Kernel execution cancellation\n\nOn abort/timeout:\n\n- Execution is marked cancelled.\n- Kernel interrupt is attempted via REST (`POST /interrupt`) and control-channel `interrupt_request`.\n- Result includes `cancelled=true`.\n- Timeout path annotates output as `Command timed out after <n> seconds`.\n\n### stdin behavior\n\nInteractive stdin is not supported.\n\nIf kernel emits `input_request`:\n\n- Tool records `stdinRequested=true`\n- Emits explanatory text\n- Sends empty `input_reply`\n- Execution is treated as failure at executor layer\n\n## Output capture and rendering\n\n### Captured output classes\n\nFrom kernel messages:\n\n- `stream` -> plain text chunks\n- `display_data`/`execute_result` -> rich display handling\n- `error` -> traceback text\n- custom MIME `application/x-xcsh-status` -> structured status events\n\nDisplay MIME precedence:\n\n1. `text/markdown`\n2. `text/plain`\n3. `text/html` (converted to basic markdown)\n\nAdditionally captured as structured outputs:\n\n- `application/json` -> JSON tree data\n- `image/png` -> image payloads\n- `application/x-xcsh-status` -> status events\n\n### Storage and truncation\n\nOutput is streamed through `OutputSink` and may be persisted to artifact storage.\n\nTool results can include truncation metadata and `artifact://<id>` for full output recovery.\n\n### Renderer behavior\n\n- Tool renderer (`python.ts`):\n - shows code-cell blocks with per-cell status\n - collapsed preview defaults to 10 lines\n - supports expanded mode for full output and richer status detail\n- Interactive renderer (`python-execution.ts`):\n - used for user-triggered Python execution in TUI\n - collapsed preview defaults to 20 lines\n - clamps very long individual lines to 4000 chars for display safety\n - shows cancellation/error/truncation notices\n\n## External gateway support\n\nSet:\n\n```bash\nexport PI_PYTHON_GATEWAY_URL=\"http://127.0.0.1:8888\"\n# Optional:\nexport PI_PYTHON_GATEWAY_TOKEN=\"...\"\n```\n\nBehavior differences from local shared gateway:\n\n- No local gateway lock/info files\n- No local process spawn/termination\n- Health checks and kernel CRUD run against external endpoint\n- Auth failures are surfaced with explicit token guidance\n\n## Operational troubleshooting (current failure modes)\n\n- **Python tool not available**\n - Check `python.toolMode` / `PI_PY`.\n - If preflight fails, runtime falls back to bash-only.\n\n- **Kernel availability errors**\n - Local mode requires both `kernel_gateway` and `ipykernel` importable in resolved Python runtime.\n - Install with:\n\n ```bash\n python -m pip install jupyter_kernel_gateway ipykernel\n ```\n\n- **`python.sharedGateway=false` causes startup failure**\n - This is expected with current implementation.\n\n- **External gateway auth/reachability failures**\n - 401/403 -> set `PI_PYTHON_GATEWAY_TOKEN`.\n - timeout/unreachable -> verify URL/network and gateway health.\n\n- **Execution hangs then times out**\n - Increase tool `timeout` (max 600s) if workload is legitimate.\n - For stuck code, cancellation triggers kernel interrupt but user code may still need refactor.\n\n- **stdin/input prompts in Python code**\n - `input()` is not supported interactively in this runtime path; pass data programmatically.\n\n- **Resource exhaustion (`EMFILE` / too many open files)**\n - Session manager triggers shared-gateway recovery (session teardown + shared gateway restart).\n\n- **Working directory errors**\n - Tool validates `cwd` exists and is a directory before execution.\n\n## Relevant environment variables\n\n- `PI_PY` — tool exposure override (`bash-only`/`ipy-only`/`both` mapping above)\n- `PI_PYTHON_GATEWAY_URL` — use external gateway\n- `PI_PYTHON_GATEWAY_TOKEN` — optional external gateway auth token\n- `PI_PYTHON_SKIP_CHECK=1` — bypass Python preflight/warm checks\n- `PI_PYTHON_IPC_TRACE=1` — log kernel IPC send/receive traces\n- `PI_DEBUG_STARTUP=1` — emit startup-stage debug markers\n",
|
|
149
149
|
"en/runtime-tools/bash-tool-runtime.md": "---\ntitle: Bash Tool Runtime\ndescription: Bash tool runtime with shell process management, sandboxing, timeout, and output streaming.\nsidebar:\n order: 1\n label: Bash tool\n---\n\n# Bash tool runtime\n\nThis document describes the **`bash` tool** runtime path used by agent tool calls, from command normalization to execution, truncation/artifacts, and rendering.\n\nIt also calls out where behavior diverges in interactive TUI, print mode, RPC mode, and user-initiated bang (`!`) shell execution.\n\n## Scope and runtime surfaces\n\nThere are two different bash execution surfaces in coding-agent:\n\n1. **Tool-call surface** (`toolName: \"bash\"`): used when the model calls the bash tool.\n - Entry point: `BashTool.execute()`.\n2. **User bang-command surface** (`!cmd` from interactive input or RPC `bash` command): session-level helper path.\n - Entry point: `AgentSession.executeBash()`.\n\nBoth eventually use `executeBash()` in `src/exec/bash-executor.ts` for non-PTY execution, but only the tool-call path runs normalization/interception and tool renderer logic.\n\n## End-to-end tool-call pipeline\n\n## 1) Input normalization and parameter merge\n\n`BashTool.execute()` first normalizes the raw command via `normalizeBashCommand()`:\n\n- extracts trailing `| head -n N`, `| head -N`, `| tail -n N`, `| tail -N` into structured limits,\n- trims trailing/leading whitespace,\n- keeps internal whitespace intact.\n\nThen it merges extracted limits with explicit tool args:\n\n- explicit `head`/`tail` args override extracted values,\n- extracted values are fallback only.\n\n### Caveat\n\n`bash-normalize.ts` comments mention stripping `2>&1`, but current implementation does not remove it. Runtime behavior is still correct (stdout/stderr are already merged), but the normalization behavior is narrower than comments suggest.\n\n## 2) Optional interception (blocked-command path)\n\nIf `bashInterceptor.enabled` is true, `BashTool` loads rules from settings and runs `checkBashInterception()` against the normalized command.\n\nInterception behavior:\n\n- command is blocked **only** when:\n - regex rule matches, and\n - the suggested tool is present in `ctx.toolNames`.\n- invalid regex rules are silently skipped.\n- on block, `BashTool` throws `ToolError` with message:\n - `Blocked: ...`\n - original command included.\n\nDefault rule patterns (defined in code) target common misuses:\n\n- file readers (`cat`, `head`, `tail`, ...)\n- search tools (`grep`, `rg`, ...)\n- file finders (`find`, `fd`, ...)\n- in-place editors (`sed -i`, `perl -i`, `awk -i inplace`)\n- shell redirection writes (`echo ... > file`, heredoc redirection)\n\n### Caveat\n\n`InterceptionResult` includes `suggestedTool`, but `BashTool` currently surfaces only the message text (no structured suggested-tool field in `details`).\n\n## 3) CWD validation and timeout clamping\n\n`cwd` is resolved relative to session cwd (`resolveToCwd`), then validated via `stat`:\n\n- missing path -> `ToolError(\"Working directory does not exist: ...\")`\n- non-directory -> `ToolError(\"Working directory is not a directory: ...\")`\n\nTimeout is clamped to `[1, 3600]` seconds and converted to milliseconds.\n\n## 4) Artifact allocation\n\nBefore execution, the tool allocates an artifact path/id (best-effort) for truncated output storage.\n\n- artifact allocation failure is non-fatal (execution continues without artifact spill file),\n- artifact id/path are passed into execution path for full-output persistence on truncation.\n\n## 5) PTY vs non-PTY execution selection\n\n`BashTool` chooses PTY execution only when all are true:\n\n- `bash.virtualTerminal === \"on\"`\n- `PI_NO_PTY !== \"1\"`\n- tool context has UI (`ctx.hasUI === true` and `ctx.ui` set)\n\nOtherwise it uses non-interactive `executeBash()`.\n\nThat means print mode and non-UI RPC/tool contexts always use non-PTY.\n\n## Non-interactive execution engine (`executeBash`)\n\n## Shell session reuse model\n\n`executeBash()` caches native `Shell` instances in a process-global map keyed by:\n\n- shell path,\n- configured command prefix,\n- snapshot path,\n- serialized shell env,\n- optional agent session key.\n\nFor session-level executions, `AgentSession.executeBash()` passes `sessionKey: this.sessionId`, isolating reuse per session.\n\nTool-call path does **not** pass `sessionKey`, so reuse scope is based on shell config/snapshot/env.\n\n## Shell config and snapshot behavior\n\nAt each call, executor loads settings shell config (`shell`, `env`, optional `prefix`).\n\nIf selected shell includes `bash`, it attempts `getOrCreateSnapshot()`:\n\n- snapshot captures aliases/functions/options from user rc,\n- snapshot creation is best-effort,\n- failure falls back to no snapshot.\n\nIf `prefix` is configured, command becomes:\n\n```text\n<prefix> <command>\n```\n\n## Streaming and cancellation\n\n`Shell.run()` streams chunks to callback. Executor pipes each chunk into `OutputSink` and optional `onChunk` callback.\n\nCancellation:\n\n- aborted signal triggers `shellSession.abort(...)`,\n- timeout from native result is mapped to `cancelled: true` + annotation text,\n- explicit cancellation similarly returns `cancelled: true` + annotation.\n\nNo exception is thrown inside executor for timeout/cancel; it returns structured `BashResult` and lets caller map error semantics.\n\n## Interactive PTY path (`runInteractiveBashPty`)\n\nWhen PTY is enabled, tool runs `runInteractiveBashPty()` which opens an overlay console component and drives a native `PtySession`.\n\nBehavior highlights:\n\n- xterm-headless virtual terminal renders viewport in overlay,\n- keyboard input is normalized (including Kitty sequences and application cursor mode handling),\n- `esc` while running kills the PTY session,\n- terminal resize propagates to PTY (`session.resize(cols, rows)`).\n\nEnvironment hardening defaults are injected for unattended runs:\n\n- pagers disabled (`PAGER=cat`, `GIT_PAGER=cat`, etc.),\n- editor prompts disabled (`GIT_EDITOR=true`, `EDITOR=true`, ...),\n- terminal/auth prompts reduced (`GIT_TERMINAL_PROMPT=0`, `SSH_ASKPASS=/usr/bin/false`, `CI=1`),\n- package-manager/tool automation flags for non-interactive behavior.\n\nPTY output is normalized (`CRLF`/`CR` to `LF`, `sanitizeText`) and written into `OutputSink`, including artifact spill support.\n\nOn PTY startup/runtime error, sink receives `PTY error: ...` line and command finalizes with undefined exit code.\n\n## Output handling: streaming, truncation, artifact spill\n\nBoth PTY and non-PTY paths use `OutputSink`.\n\n## OutputSink semantics\n\n- keeps an in-memory UTF-8-safe tail buffer (`DEFAULT_MAX_BYTES`, currently 50KB),\n- tracks total bytes/lines seen,\n- if artifact path exists and output overflows (or file already active), writes full stream to artifact file,\n- when memory threshold overflows, trims in-memory buffer to tail (UTF-8 boundary safe),\n- marks `truncated` when overflow/file spill occurs.\n\n`dump()` returns:\n\n- `output` (possibly annotated prefix),\n- `truncated`,\n- `totalLines/totalBytes`,\n- `outputLines/outputBytes`,\n- `artifactId` if artifact file was active.\n\n### Long-output caveat\n\nRuntime truncation is byte-threshold based in `OutputSink` (50KB default). It does not enforce a hard 2000-line cap in this code path.\n\n## Live tool updates\n\nFor non-PTY execution, `BashTool` uses a separate `TailBuffer` for partial updates and emits `onUpdate` snapshots while command is running.\n\nFor PTY execution, live rendering is handled by custom UI overlay, not by `onUpdate` text chunks.\n\n## Result shaping, metadata, and error mapping\n\nAfter execution:\n\n1. `cancelled` handling:\n - if abort signal is aborted -> throw `ToolAbortError` (abort semantics),\n - else -> throw `ToolError` (treated as tool failure).\n2. PTY `timedOut` -> throw `ToolError`.\n3. apply head/tail filters to final output text (`applyHeadTail`, head then tail).\n4. empty output becomes `(no output)`.\n5. attach truncation metadata via `toolResult(...).truncationFromSummary(result, { direction: \"tail\" })`.\n6. exit-code mapping:\n - missing exit code -> `ToolError(\"... missing exit status\")`\n - non-zero exit -> `ToolError(\"... Command exited with code N\")`\n - zero exit -> success result.\n\nSuccess payload structure:\n\n- `content`: text output,\n- `details.meta.truncation` when truncated, including:\n - `direction`, `truncatedBy`, total/output line+byte counts,\n - `shownRange`,\n - `artifactId` when available.\n\nBecause built-in tools are wrapped with `wrapToolWithMetaNotice()`, truncation notice text is appended to final text content automatically (for example: `Full: artifact://<id>`).\n\n## Rendering paths\n\n## Tool-call renderer (`bashToolRenderer`)\n\n`bashToolRenderer` is used for tool-call messages (`toolCall` / `toolResult`):\n\n- collapsed mode shows visual-line-truncated preview,\n- expanded mode shows all currently available output text,\n- warning line includes truncation reason and `artifact://<id>` when truncated,\n- timeout value (from args) is shown in footer metadata line.\n\n### Caveat: full artifact expansion\n\n`BashRenderContext` has `isFullOutput`, but current renderer context builder does not set it for bash tool results. Expanded view still uses the text already in result content (tail/truncated output) unless another caller provides full artifact content.\n\n## User bang-command component (`BashExecutionComponent`)\n\n`BashExecutionComponent` is for user `!` commands in interactive mode (not model tool calls):\n\n- streams chunks live,\n- collapsed preview keeps last 20 logical lines,\n- line clamp at 4000 chars per line,\n- shows truncation + artifact warnings when metadata is present,\n- marks cancelled/error/exit state separately.\n\nThis component is wired by `CommandController.handleBashCommand()` and fed from `AgentSession.executeBash()`.\n\n## Mode-specific behavior differences\n\n| Surface | Entry path | PTY eligible | Live output UX | Error surfacing |\n| ------------------------------ | ----------------------------------------------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------ | ------------------------------------------------ |\n| Interactive tool call | `BashTool.execute` | Yes, when `bash.virtualTerminal=on` and UI exists and `PI_NO_PTY!=1` | PTY overlay (interactive) or streamed tail updates | Tool errors become `toolResult.isError` |\n| Print mode tool call | `BashTool.execute` | No (no UI context) | No TUI overlay; output appears in event stream/final assistant text flow | Same tool error mapping |\n| RPC tool call (agent tooling) | `BashTool.execute` | Usually no UI -> non-PTY | Structured tool events/results | Same tool error mapping |\n| Interactive bang command (`!`) | `AgentSession.executeBash` + `BashExecutionComponent` | No (uses executor directly) | Dedicated bash execution component | Controller catches exceptions and shows UI error |\n| RPC `bash` command | `rpc-mode` -> `session.executeBash` | No | Returns `BashResult` directly | Consumer handles returned fields |\n\n## Filesystem containment: what enforces it, and where\n\nThe bash tool's filesystem boundary is enforced below the command text — the shell's own `cd` and\nredirections are checked where they act, and spawned children are confined by the operating system. Which\nOS mechanism does that depends on the host, and on one host family there is **no OS mechanism at all**.\n\n`xcsh://about` reports the active backend for the machine you are on. This table is for planning a fleet\nbefore you get there.\n\n| Host | Kernel | Landlock ABI | Backend | Boundary |\n| ---- | ------ | ------------ | ------- | -------- |\n| macOS | — | — | `seatbelt` | OS-enforced |\n| RHEL 9 and derivatives | 5.14 | 1 | `scanner-only` | **command-text scan only** |\n| Ubuntu 22.04, stock GA kernel | 5.15 | 1 | `scanner-only` | **command-text scan only** |\n| Debian 12 | 6.1 | 2 | `landlock` | OS-enforced; `truncate(2)` ungoverned |\n| Ubuntu 22.04 HWE, Ubuntu 24.04 | 6.8 | 4 | `landlock` | OS-enforced |\n| Fedora current | 6.1x–7.x | 6–9 | `landlock` | OS-enforced |\n\nABI numbers are anchored on two measured hosts: kernel 6.8.0-azure reports ABI 4, kernel 7.1.3 reports\nABI 9. The rest follow the kernel-to-ABI mapping.\n\n### Why ABI 1 gets no OS boundary\n\n`LANDLOCK_ACCESS_FS_REFER` does not exist before ABI 2, and the kernel denies cross-directory `rename` and\n`link` whenever a ruleset handles *any* filesystem right. On ABI 1 there is therefore no way to permit\n`mv a/x b/x`, and no way for `git` to do its write-tmp-then-rename. Confining on ABI 1 would break ordinary\nwork, which this boundary's design forbids, so it is refused rather than degraded.\n\nThat is a deliberate trade and not a bug to work around: the alternative is a boundary that breaks `git`.\n\n### What scanner-only means in practice\n\nThe command-text scan is still there and still refuses out-of-tree paths, but it reads what was *written*\nrather than what the shell will *do*. A path assembled at runtime — `P=/other/customer/secrets; cat \"$P\"`\n— is not caught. Treat it as a statement of intent, not a guarantee.\n\n**If sessions on an ABI 1 host handle more than one customer's data, that is a materially weaker posture\nthan the macOS default**, and the remedy is operational rather than a code change: run a newer kernel\n(Ubuntu 22.04 HWE is the smallest step), or run those sessions in a container on a newer host.\n\n### Debian 12 / ABI 2\n\nLandlock confines every read and every write, but `LANDLOCK_ACCESS_FS_TRUNCATE` only exists from ABI 3, so\n`truncate(2)` on a path outside the boundary is not governed. It destroys rather than discloses, and is\nunreachable through `>`. `containmentStatus` reports this as `truncationUngoverned` and `xcsh://about`\nstates it, so the session knows.\n\n## Operational caveats\n\n- Interceptor only blocks commands when suggested tool is currently available in context.\n- If artifact allocation fails, truncation still occurs but no `artifact://` back-reference is available.\n- Shell session cache has no explicit eviction in this module; lifetime is process-scoped.\n- PTY and non-PTY timeout surfaces differ:\n - PTY exposes explicit `timedOut` result field,\n - non-PTY maps timeout into `cancelled + annotation` summary.\n\n## Implementation files\n\n- [`src/tools/bash.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/tools/bash.ts) — tool entrypoint, normalization/interception, PTY/non-PTY selection, result/error mapping, bash tool renderer.\n- [`src/tools/bash-normalize.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/tools/bash-normalize.ts) — command normalization and post-run head/tail filtering.\n- [`src/tools/bash-interceptor.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/tools/bash-interceptor.ts) — interceptor rule matching and blocked-command messages.\n- [`src/exec/bash-executor.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/exec/bash-executor.ts) — non-PTY executor, shell session reuse, cancellation wiring, output sink integration.\n- [`src/tools/bash-interactive.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/tools/bash-interactive.ts) — PTY runtime, overlay UI, input normalization, non-interactive env defaults.\n- [`src/session/streaming-output.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/session/streaming-output.ts) — `OutputSink` truncation/artifact spill and summary metadata.\n- [`src/tools/output-utils.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/tools/output-utils.ts) — artifact allocation helpers and streaming tail buffer.\n- [`src/tools/output-meta.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/tools/output-meta.ts) — truncation metadata shape + notice injection wrapper.\n- [`src/session/agent-session.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/session/agent-session.ts) — session-level `executeBash`, message recording, abort lifecycle.\n- [`src/modes/components/bash-execution.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/modes/components/bash-execution.ts) — interactive `!` command execution component.\n- [`src/modes/controllers/command-controller.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/modes/controllers/command-controller.ts) — wiring for interactive `!` command UI stream/update completion.\n- [`src/modes/rpc/rpc-mode.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/modes/rpc/rpc-mode.ts) — RPC `bash` and `abort_bash` command surface.\n- [`src/internal-urls/artifact-protocol.ts`](https://github.com/f5-sales-demo/xcsh/blob/main/packages/coding-agent/src/internal-urls/artifact-protocol.ts) — `artifact://<id>` resolution.\n",
|
|
@@ -40,6 +40,14 @@ export const GOOGLE_ANTIGRAVITY_LOGIN_MODEL_CHOICE: LoginModelChoice = {
|
|
|
40
40
|
thinkingLevel: ThinkingLevel.High,
|
|
41
41
|
};
|
|
42
42
|
|
|
43
|
+
export const OPENAI_CODEX_LOGIN_MODEL_CHOICE: LoginModelChoice = {
|
|
44
|
+
label: "GPT-5.6 Terra",
|
|
45
|
+
description: "Balanced OpenAI Codex subscription model with medium reasoning",
|
|
46
|
+
provider: "openai-codex",
|
|
47
|
+
modelId: "gpt-5.6-terra",
|
|
48
|
+
thinkingLevel: ThinkingLevel.Medium,
|
|
49
|
+
};
|
|
50
|
+
|
|
43
51
|
export function getAvailableLiteLLMLoginModelChoices(availableModelIds: readonly string[]): LiteLLMLoginModelChoice[] {
|
|
44
52
|
const available = new Set(availableModelIds);
|
|
45
53
|
return LITELLM_LOGIN_MODEL_CHOICES.filter(choice => available.has(choice.modelId));
|
|
@@ -106,7 +114,9 @@ export async function applyOAuthLoginModel(
|
|
|
106
114
|
const choice =
|
|
107
115
|
canonicalProvider === GOOGLE_ANTIGRAVITY_LOGIN_MODEL_CHOICE.provider
|
|
108
116
|
? GOOGLE_ANTIGRAVITY_LOGIN_MODEL_CHOICE
|
|
109
|
-
:
|
|
117
|
+
: canonicalProvider === OPENAI_CODEX_LOGIN_MODEL_CHOICE.provider
|
|
118
|
+
? OPENAI_CODEX_LOGIN_MODEL_CHOICE
|
|
119
|
+
: undefined;
|
|
110
120
|
if (!choice) return undefined;
|
|
111
121
|
const discovery = session.modelRegistry.getProviderDiscoveryState?.(canonicalProvider);
|
|
112
122
|
if (session.modelRegistry.getProviderDiscoveryState && (discovery?.status !== "ok" || discovery.stale)) {
|
|
@@ -4,7 +4,7 @@ import { ThinkingLevel } from "@f5-sales-demo/pi-agent-core";
|
|
|
4
4
|
import { getOAuthProviders, loginLiteLLM, type OAuthPrompt, type OAuthProvider } from "@f5-sales-demo/pi-ai";
|
|
5
5
|
import type { Component } from "@f5-sales-demo/pi-tui";
|
|
6
6
|
import { Loader, Spacer, Text } from "@f5-sales-demo/pi-tui";
|
|
7
|
-
import { getAgentDbPath, getAgentDir, getConfigDirName, getProjectDir
|
|
7
|
+
import { getAgentDbPath, getAgentDir, getConfigDirName, getProjectDir } from "@f5-sales-demo/pi-utils";
|
|
8
8
|
import { invalidate as invalidateFsCache } from "../../capability/fs";
|
|
9
9
|
import { probeLiteLLMConnection, readLiteLLMConfig } from "../../config/auto-config";
|
|
10
10
|
import { getRoleInfo } from "../../config/model-registry";
|
|
@@ -64,13 +64,13 @@ import { commitLiteLLMLogin } from "./litellm-login-transaction";
|
|
|
64
64
|
import {
|
|
65
65
|
applyOAuthLoginModel,
|
|
66
66
|
GOOGLE_ANTIGRAVITY_LOGIN_MODEL_CHOICE,
|
|
67
|
-
getAvailableLiteLLMLoginModelChoices,
|
|
68
67
|
LITELLM_LOGIN_MODEL_CHOICES,
|
|
69
68
|
type LiteLLMLoginModelChoice,
|
|
70
69
|
} from "./login-model";
|
|
71
70
|
|
|
72
71
|
const CALLBACK_SERVER_PROVIDERS = new Set<OAuthProvider>([
|
|
73
72
|
"anthropic",
|
|
73
|
+
"openai-codex",
|
|
74
74
|
"gitlab-duo",
|
|
75
75
|
"google-gemini-cli",
|
|
76
76
|
"google-antigravity",
|
|
@@ -1184,11 +1184,7 @@ export class SelectorController {
|
|
|
1184
1184
|
new Text(theme.fg("dim", "Set OPENAI_API_KEY, then select an OpenAI model with /model."), 1, 0),
|
|
1185
1185
|
);
|
|
1186
1186
|
this.ctx.chatContainer.addChild(
|
|
1187
|
-
new Text(
|
|
1188
|
-
theme.fg("dim", "For ChatGPT subscription access, use the official codex CLI (`codex login`)."),
|
|
1189
|
-
1,
|
|
1190
|
-
0,
|
|
1191
|
-
),
|
|
1187
|
+
new Text(theme.fg("dim", "For ChatGPT subscription access, choose ChatGPT Plus/Pro in /login."), 1, 0),
|
|
1192
1188
|
);
|
|
1193
1189
|
this.ctx.ui.requestRender();
|
|
1194
1190
|
}
|
|
@@ -1211,162 +1207,7 @@ export class SelectorController {
|
|
|
1211
1207
|
}
|
|
1212
1208
|
|
|
1213
1209
|
async showFirstRunLogin(): Promise<void> {
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
// Prompt helper: renders label + hint + input together in the editor
|
|
1217
|
-
// container so they're always visible below the welcome branding.
|
|
1218
|
-
const promptInput = async (message: string, placeholder?: string, secret = false): Promise<string | null> => {
|
|
1219
|
-
const { promise, resolve } = Promise.withResolvers<string | null>();
|
|
1220
|
-
const input = createLoginPromptInput({ secret });
|
|
1221
|
-
input.onSubmit = () => {
|
|
1222
|
-
const raw = input.getValue();
|
|
1223
|
-
const value = raw.replace(/\x1b\[\d{3}~/g, "").replace(/\x1b[[\]()][^\x1b]*/g, "");
|
|
1224
|
-
this.ctx.editorContainer.clear();
|
|
1225
|
-
this.ctx.editorContainer.addChild(this.ctx.editor);
|
|
1226
|
-
this.ctx.ui.setFocus(this.ctx.editor);
|
|
1227
|
-
resolve(value);
|
|
1228
|
-
};
|
|
1229
|
-
input.onEscape = () => {
|
|
1230
|
-
this.ctx.editorContainer.clear();
|
|
1231
|
-
this.ctx.editorContainer.addChild(this.ctx.editor);
|
|
1232
|
-
this.ctx.ui.setFocus(this.ctx.editor);
|
|
1233
|
-
resolve(null);
|
|
1234
|
-
};
|
|
1235
|
-
|
|
1236
|
-
// Pack label + hint + input into the editor area as a single block
|
|
1237
|
-
this.ctx.editorContainer.clear();
|
|
1238
|
-
this.ctx.editorContainer.addChild(new Spacer(1));
|
|
1239
|
-
this.ctx.editorContainer.addChild(new Text(theme.bold(theme.fg("text", ` ${message}`)), 0, 0));
|
|
1240
|
-
if (placeholder) {
|
|
1241
|
-
this.ctx.editorContainer.addChild(new Text(theme.fg("dim", ` ${placeholder}`), 0, 0));
|
|
1242
|
-
}
|
|
1243
|
-
this.ctx.editorContainer.addChild(new Spacer(1));
|
|
1244
|
-
this.ctx.editorContainer.addChild(input);
|
|
1245
|
-
this.ctx.ui.setFocus(input);
|
|
1246
|
-
this.ctx.ui.requestRender();
|
|
1247
|
-
return promise;
|
|
1248
|
-
};
|
|
1249
|
-
|
|
1250
|
-
try {
|
|
1251
|
-
// Step 1: URL prompt
|
|
1252
|
-
let baseUrl: string | null = null;
|
|
1253
|
-
while (!baseUrl) {
|
|
1254
|
-
const urlInput = await promptInput(t("login.wizard.urlPrompt"), t("login.wizard.urlPlaceholder"));
|
|
1255
|
-
if (urlInput === null) return; // Escape pressed
|
|
1256
|
-
const trimmed = urlInput.trim();
|
|
1257
|
-
if (!trimmed) continue;
|
|
1258
|
-
if (!trimmed.startsWith("http://") && !trimmed.startsWith("https://")) {
|
|
1259
|
-
this.ctx.chatContainer.addChild(
|
|
1260
|
-
new Text(theme.fg("error", "URL must start with http:// or https://"), 1, 0),
|
|
1261
|
-
);
|
|
1262
|
-
this.ctx.ui.requestRender();
|
|
1263
|
-
continue;
|
|
1264
|
-
}
|
|
1265
|
-
baseUrl = trimmed.replace(/\/+$/, "");
|
|
1266
|
-
}
|
|
1267
|
-
|
|
1268
|
-
// Auto-detect known providers by hostname
|
|
1269
|
-
try {
|
|
1270
|
-
const hostname = new URL(baseUrl).hostname.toLowerCase();
|
|
1271
|
-
const providerMap: Record<string, string> = {
|
|
1272
|
-
"api.anthropic.com": "anthropic",
|
|
1273
|
-
"api.openai.com": "openai",
|
|
1274
|
-
"api.together.xyz": "together",
|
|
1275
|
-
};
|
|
1276
|
-
const detectedProvider =
|
|
1277
|
-
providerMap[hostname] ?? (hostname.endsWith(".googleapis.com") ? "google-gemini-cli" : null);
|
|
1278
|
-
if (detectedProvider === "openai") {
|
|
1279
|
-
this.#showOpenAIApiKeyGuidance();
|
|
1280
|
-
return;
|
|
1281
|
-
}
|
|
1282
|
-
if (detectedProvider) {
|
|
1283
|
-
this.ctx.editorContainer.clear();
|
|
1284
|
-
this.ctx.editorContainer.addChild(new Spacer(1));
|
|
1285
|
-
this.ctx.editorContainer.addChild(
|
|
1286
|
-
new Text(theme.fg("dim", `Detected ${detectedProvider} — launching login…`), 1, 0),
|
|
1287
|
-
);
|
|
1288
|
-
this.ctx.ui.requestRender();
|
|
1289
|
-
await this.#handleOAuthLogin(detectedProvider);
|
|
1290
|
-
return;
|
|
1291
|
-
}
|
|
1292
|
-
} catch {
|
|
1293
|
-
// URL parse failed — continue with proxy flow
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
// Step 2: API Key prompt (for non-OAuth proxy)
|
|
1297
|
-
let apiKey: string | null = null;
|
|
1298
|
-
let probeSuccess = false;
|
|
1299
|
-
|
|
1300
|
-
while (!probeSuccess) {
|
|
1301
|
-
if (!apiKey) {
|
|
1302
|
-
const keyInput = await promptInput(
|
|
1303
|
-
t("login.wizard.apiKeyPrompt"),
|
|
1304
|
-
t("login.wizard.apiKeyPlaceholder"),
|
|
1305
|
-
true,
|
|
1306
|
-
);
|
|
1307
|
-
if (keyInput === null) return;
|
|
1308
|
-
const trimmedKey = keyInput.trim();
|
|
1309
|
-
if (!trimmedKey) continue;
|
|
1310
|
-
apiKey = trimmedKey;
|
|
1311
|
-
}
|
|
1312
|
-
|
|
1313
|
-
// Show connection status in editor area
|
|
1314
|
-
this.ctx.editorContainer.clear();
|
|
1315
|
-
this.ctx.editorContainer.addChild(new Spacer(1));
|
|
1316
|
-
this.ctx.editorContainer.addChild(
|
|
1317
|
-
new Text(theme.fg("dim", ` ${t("login.wizard.connecting", { url: baseUrl })}`), 0, 0),
|
|
1318
|
-
);
|
|
1319
|
-
this.ctx.ui.requestRender();
|
|
1320
|
-
|
|
1321
|
-
let probe = await probeLiteLLMConnection(baseUrl, apiKey);
|
|
1322
|
-
|
|
1323
|
-
// Auto-retry once on network errors
|
|
1324
|
-
if (!probe.reachable && probe.error && !/\b(401|403|Unauthorized|Forbidden)\b/i.test(probe.error)) {
|
|
1325
|
-
await Bun.sleep(1000);
|
|
1326
|
-
probe = await probeLiteLLMConnection(baseUrl, apiKey);
|
|
1327
|
-
}
|
|
1328
|
-
|
|
1329
|
-
if (probe.reachable) {
|
|
1330
|
-
const choice = await this.#showLiteLLMLoginModelSelector(
|
|
1331
|
-
getAvailableLiteLLMLoginModelChoices(probe.models),
|
|
1332
|
-
);
|
|
1333
|
-
if (!choice) return;
|
|
1334
|
-
const configPath = path.join(path.dirname(modelsPath), "config.yml");
|
|
1335
|
-
await commitLiteLLMLogin({
|
|
1336
|
-
modelsPath,
|
|
1337
|
-
configPath,
|
|
1338
|
-
credentials: { baseUrl, apiKey },
|
|
1339
|
-
probe,
|
|
1340
|
-
choice,
|
|
1341
|
-
session: this.ctx.session,
|
|
1342
|
-
});
|
|
1343
|
-
await this.ctx.refreshWelcomeAfterLogin();
|
|
1344
|
-
this.ctx.showStatus("LiteLLM configured. Use /model to switch models without logging in again.");
|
|
1345
|
-
probeSuccess = true;
|
|
1346
|
-
} else {
|
|
1347
|
-
const errorMsg = probe.error ?? "connection failed";
|
|
1348
|
-
|
|
1349
|
-
// Classify error and re-prompt the appropriate field
|
|
1350
|
-
const isAuthError = /\b(401|403|Unauthorized|Forbidden)\b/i.test(errorMsg);
|
|
1351
|
-
if (isAuthError) {
|
|
1352
|
-
apiKey = null;
|
|
1353
|
-
} else {
|
|
1354
|
-
const urlRetry = await promptInput(
|
|
1355
|
-
`${theme.status.error} ${t("login.wizard.failed", { error: errorMsg })}\n\n ${t("login.wizard.urlPrompt")}`,
|
|
1356
|
-
baseUrl,
|
|
1357
|
-
);
|
|
1358
|
-
if (urlRetry === null) return;
|
|
1359
|
-
const trimmedUrl = urlRetry.trim();
|
|
1360
|
-
if (trimmedUrl && (trimmedUrl.startsWith("http://") || trimmedUrl.startsWith("https://"))) {
|
|
1361
|
-
baseUrl = trimmedUrl.replace(/\/+$/, "");
|
|
1362
|
-
}
|
|
1363
|
-
apiKey = null;
|
|
1364
|
-
}
|
|
1365
|
-
}
|
|
1366
|
-
}
|
|
1367
|
-
} catch (error: unknown) {
|
|
1368
|
-
this.ctx.showError(`Login failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1369
|
-
}
|
|
1210
|
+
await this.showOAuthSelector("login");
|
|
1370
1211
|
}
|
|
1371
1212
|
|
|
1372
1213
|
async showOAuthSelector(mode: "login" | "logout", providerId?: string): Promise<void> {
|
package/src/routing/presets.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { OPENAI_CODEX_ROUTING_POOL } from "./subscription-profiles";
|
|
1
2
|
import type { RoutingPoolConfig } from "./types";
|
|
2
3
|
|
|
3
4
|
export const BUILTIN_ROUTING_PRESETS: Record<string, RoutingPoolConfig> = {
|
|
5
|
+
[OPENAI_CODEX_ROUTING_POOL.id]: OPENAI_CODEX_ROUTING_POOL,
|
|
4
6
|
"openai/gpt-5.6": {
|
|
5
7
|
id: "openai/gpt-5.6",
|
|
6
8
|
provider: "openai",
|
|
@@ -1,11 +1,28 @@
|
|
|
1
|
-
|
|
1
|
+
import type { RoutingPoolConfig } from "./types";
|
|
2
|
+
|
|
3
|
+
export type SubscriptionProfileId = "google-antigravity" | "openai-codex";
|
|
2
4
|
|
|
3
5
|
export interface SubscriptionRoutingProfile {
|
|
4
6
|
id: SubscriptionProfileId;
|
|
5
7
|
provider: string;
|
|
6
8
|
roles: Readonly<Record<"smol" | "default" | "slow" | "plan", string>>;
|
|
9
|
+
pool?: RoutingPoolConfig;
|
|
7
10
|
}
|
|
8
11
|
|
|
12
|
+
const OPENAI_CODEX_POOL: RoutingPoolConfig = {
|
|
13
|
+
id: "openai-codex/gpt-5.6",
|
|
14
|
+
provider: "openai-codex",
|
|
15
|
+
tiers: {
|
|
16
|
+
utility: "gpt-5.6-luna",
|
|
17
|
+
balanced: "gpt-5.6-terra",
|
|
18
|
+
frontier: "gpt-5.6-sol",
|
|
19
|
+
},
|
|
20
|
+
effortPolicy: {
|
|
21
|
+
byTier: { utility: "low", balanced: "medium", frontier: "high" },
|
|
22
|
+
frontierEscalation: { effort: "xhigh", minimumComplexityScore: 90 },
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
|
|
9
26
|
export const SUBSCRIPTION_ROUTING_PROFILES: Readonly<Record<SubscriptionProfileId, SubscriptionRoutingProfile>> = {
|
|
10
27
|
"google-antigravity": {
|
|
11
28
|
id: "google-antigravity",
|
|
@@ -17,6 +34,17 @@ export const SUBSCRIPTION_ROUTING_PROFILES: Readonly<Record<SubscriptionProfileI
|
|
|
17
34
|
plan: "google-antigravity/gemini-3.1-pro-high-vertex:high",
|
|
18
35
|
},
|
|
19
36
|
},
|
|
37
|
+
"openai-codex": {
|
|
38
|
+
id: "openai-codex",
|
|
39
|
+
provider: "openai-codex",
|
|
40
|
+
roles: {
|
|
41
|
+
smol: "openai-codex/gpt-5.6-luna:low",
|
|
42
|
+
default: "openai-codex/gpt-5.6-terra:medium",
|
|
43
|
+
slow: "openai-codex/gpt-5.6-sol:high",
|
|
44
|
+
plan: "openai-codex/gpt-5.6-sol:high",
|
|
45
|
+
},
|
|
46
|
+
pool: OPENAI_CODEX_POOL,
|
|
47
|
+
},
|
|
20
48
|
};
|
|
21
49
|
|
|
22
50
|
function modelSelector(roleSelector: string): string {
|
|
@@ -50,3 +78,5 @@ export function applySubscriptionProfileRoles(
|
|
|
50
78
|
}
|
|
51
79
|
return { applied: true, roles: { ...currentRoles, ...profile.roles }, missingModels: [] };
|
|
52
80
|
}
|
|
81
|
+
|
|
82
|
+
export const OPENAI_CODEX_ROUTING_POOL = OPENAI_CODEX_POOL;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { AnthropicProvider } from "./providers/anthropic";
|
|
2
2
|
import type { SearchProvider } from "./providers/base";
|
|
3
3
|
import { BraveProvider } from "./providers/brave";
|
|
4
|
+
import { CodexProvider } from "./providers/codex";
|
|
4
5
|
import { ExaProvider } from "./providers/exa";
|
|
5
6
|
import { FirecrawlProvider } from "./providers/firecrawl";
|
|
6
7
|
import { GeminiProvider } from "./providers/gemini";
|
|
@@ -26,6 +27,7 @@ const SEARCH_PROVIDERS: Record<SearchProviderId, SearchProvider> = {
|
|
|
26
27
|
zai: new ZaiProvider(),
|
|
27
28
|
anthropic: new AnthropicProvider(),
|
|
28
29
|
gemini: new GeminiProvider(),
|
|
30
|
+
codex: new CodexProvider(),
|
|
29
31
|
tavily: new TavilyProvider(),
|
|
30
32
|
parallel: new ParallelProvider(),
|
|
31
33
|
kagi: new KagiProvider(),
|
|
@@ -42,6 +44,7 @@ export const SEARCH_PROVIDER_ORDER: SearchProviderId[] = [
|
|
|
42
44
|
"kimi",
|
|
43
45
|
"anthropic",
|
|
44
46
|
"gemini",
|
|
47
|
+
"codex",
|
|
45
48
|
"zai",
|
|
46
49
|
"exa",
|
|
47
50
|
"parallel",
|
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenAI Codex Web Search Provider
|
|
3
|
+
*
|
|
4
|
+
* Uses Codex's built-in web_search tool via the Responses API.
|
|
5
|
+
* Requires OAuth credentials stored in agent.db for provider "openai-codex".
|
|
6
|
+
* Returns synthesized answers with web search sources.
|
|
7
|
+
*/
|
|
8
|
+
import * as os from "node:os";
|
|
9
|
+
import { $env, getAgentDbPath, readSseJson } from "@f5-sales-demo/pi-utils";
|
|
10
|
+
import packageJson from "../../../../package.json" with { type: "json" };
|
|
11
|
+
import { AgentStorage } from "../../../session/agent-storage";
|
|
12
|
+
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
|
13
|
+
import { SearchProviderError } from "../../../web/search/types";
|
|
14
|
+
import type { SearchParams } from "./base";
|
|
15
|
+
import { SearchProvider } from "./base";
|
|
16
|
+
|
|
17
|
+
const CODEX_BASE_URL = "https://chatgpt.com/backend-api";
|
|
18
|
+
const CODEX_RESPONSES_PATH = "/codex/responses";
|
|
19
|
+
const DEFAULT_MODEL = "gpt-5-codex-mini";
|
|
20
|
+
const JWT_CLAIM_PATH = "https://api.openai.com/auth";
|
|
21
|
+
const DEFAULT_INSTRUCTIONS =
|
|
22
|
+
"You are a helpful assistant with web search capabilities. Search the web to answer the user's question accurately and cite your sources.";
|
|
23
|
+
|
|
24
|
+
function getModel(): string {
|
|
25
|
+
const configuredModel = $env.PI_CODEX_WEB_SEARCH_MODEL?.trim();
|
|
26
|
+
return configuredModel ? configuredModel : DEFAULT_MODEL;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface CodexSearchParams {
|
|
30
|
+
signal?: AbortSignal;
|
|
31
|
+
query: string;
|
|
32
|
+
system_prompt?: string;
|
|
33
|
+
num_results?: number;
|
|
34
|
+
/** Search context size: controls how much web content to include */
|
|
35
|
+
search_context_size?: "low" | "medium" | "high";
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** OAuth credential stored in agent.db */
|
|
39
|
+
interface CodexOAuthCredential {
|
|
40
|
+
type: "oauth";
|
|
41
|
+
access: string;
|
|
42
|
+
refresh?: string;
|
|
43
|
+
expires: number;
|
|
44
|
+
accountId?: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** JWT payload structure for extracting account ID */
|
|
48
|
+
type JwtPayload = {
|
|
49
|
+
[key: string]: unknown;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
/** Codex API response structure */
|
|
53
|
+
interface CodexResponseItem {
|
|
54
|
+
type: string;
|
|
55
|
+
id?: string;
|
|
56
|
+
role?: string;
|
|
57
|
+
name?: string;
|
|
58
|
+
call_id?: string;
|
|
59
|
+
status?: string;
|
|
60
|
+
arguments?: string;
|
|
61
|
+
content?: CodexContentPart[];
|
|
62
|
+
summary?: Array<{ type: string; text: string }>;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
interface CodexContentPart {
|
|
66
|
+
type: string;
|
|
67
|
+
text?: string;
|
|
68
|
+
annotations?: CodexAnnotation[];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
interface CodexAnnotation {
|
|
72
|
+
type: string;
|
|
73
|
+
url?: string;
|
|
74
|
+
title?: string;
|
|
75
|
+
start_index?: number;
|
|
76
|
+
end_index?: number;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
interface CodexUsage {
|
|
80
|
+
input_tokens?: number;
|
|
81
|
+
output_tokens?: number;
|
|
82
|
+
total_tokens?: number;
|
|
83
|
+
input_tokens_details?: { cached_tokens?: number };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
interface CodexResponse {
|
|
87
|
+
id?: string;
|
|
88
|
+
model?: string;
|
|
89
|
+
status?: string;
|
|
90
|
+
usage?: CodexUsage;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function isImagePlaceholderAnswer(text: string): boolean {
|
|
94
|
+
return text.trim().toLowerCase() === "(see attached image)";
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Decodes a JWT token and extracts the payload.
|
|
99
|
+
* @param token - JWT token string
|
|
100
|
+
* @returns Decoded payload, or null if parsing fails
|
|
101
|
+
*/
|
|
102
|
+
function decodeJwt(token: string): JwtPayload | null {
|
|
103
|
+
try {
|
|
104
|
+
const parts = token.split(".");
|
|
105
|
+
if (parts.length !== 3) return null;
|
|
106
|
+
const payload = parts[1] ?? "";
|
|
107
|
+
const decoded = Buffer.from(payload, "base64").toString("utf-8");
|
|
108
|
+
return JSON.parse(decoded) as JwtPayload;
|
|
109
|
+
} catch {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Extracts account ID from a Codex access token.
|
|
116
|
+
* @param accessToken - JWT access token
|
|
117
|
+
* @returns Account ID string, or null if not found
|
|
118
|
+
*/
|
|
119
|
+
function getAccountId(accessToken: string): string | null {
|
|
120
|
+
const payload = decodeJwt(accessToken);
|
|
121
|
+
const auth = payload?.[JWT_CLAIM_PATH] as { chatgpt_account_id?: string } | undefined;
|
|
122
|
+
const accountId = auth?.chatgpt_account_id;
|
|
123
|
+
return typeof accountId === "string" && accountId.length > 0 ? accountId : null;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Finds valid Codex OAuth credentials from agent.db.
|
|
128
|
+
* Checks agent credentials and returns the first non-expired credential.
|
|
129
|
+
* @returns OAuth credential with access token and account ID, or null if none found
|
|
130
|
+
*/
|
|
131
|
+
async function findCodexAuth(): Promise<{ accessToken: string; accountId: string } | null> {
|
|
132
|
+
const expiryBuffer = 5 * 60 * 1000; // 5 minutes
|
|
133
|
+
const now = Date.now();
|
|
134
|
+
|
|
135
|
+
try {
|
|
136
|
+
const storage = await AgentStorage.open(getAgentDbPath());
|
|
137
|
+
const records = storage.listAuthCredentials("openai-codex");
|
|
138
|
+
|
|
139
|
+
for (const record of records) {
|
|
140
|
+
const credential = record.credential;
|
|
141
|
+
if (credential.type !== "oauth") continue;
|
|
142
|
+
|
|
143
|
+
const oauthCred = credential as CodexOAuthCredential;
|
|
144
|
+
if (!oauthCred.access) continue;
|
|
145
|
+
if (oauthCred.expires <= now + expiryBuffer) continue;
|
|
146
|
+
|
|
147
|
+
const accountId = oauthCred.accountId ?? getAccountId(oauthCred.access);
|
|
148
|
+
if (!accountId) continue;
|
|
149
|
+
|
|
150
|
+
return { accessToken: oauthCred.access, accountId };
|
|
151
|
+
}
|
|
152
|
+
} catch {
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Builds HTTP headers for Codex API requests.
|
|
161
|
+
* @param accessToken - OAuth access token
|
|
162
|
+
* @param accountId - ChatGPT account ID
|
|
163
|
+
* @returns Headers object for fetch requests
|
|
164
|
+
*/
|
|
165
|
+
function buildCodexHeaders(accessToken: string, accountId: string): Record<string, string> {
|
|
166
|
+
return {
|
|
167
|
+
Authorization: `Bearer ${accessToken}`,
|
|
168
|
+
"chatgpt-account-id": accountId,
|
|
169
|
+
"OpenAI-Beta": "responses=experimental",
|
|
170
|
+
originator: "pi",
|
|
171
|
+
"User-Agent": `pi/${packageJson.version} (${os.platform()} ${os.release()}; ${os.arch()})`,
|
|
172
|
+
Accept: "text/event-stream",
|
|
173
|
+
"Content-Type": "application/json",
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Calls the Codex Responses API with web search tool enabled.
|
|
179
|
+
* Streams the response and collects all events.
|
|
180
|
+
* @param auth - Authentication info (access token and account ID)
|
|
181
|
+
* @param query - Search query from the user
|
|
182
|
+
* @param options - Search options including system prompt and context size
|
|
183
|
+
* @returns Parsed response with answer, sources, and usage
|
|
184
|
+
* @throws {SearchProviderError} If the API request fails
|
|
185
|
+
*/
|
|
186
|
+
async function callCodexSearch(
|
|
187
|
+
auth: { accessToken: string; accountId: string },
|
|
188
|
+
query: string,
|
|
189
|
+
options: { signal?: AbortSignal; systemPrompt?: string; searchContextSize?: "low" | "medium" | "high" },
|
|
190
|
+
): Promise<{
|
|
191
|
+
answer: string;
|
|
192
|
+
sources: SearchSource[];
|
|
193
|
+
model: string;
|
|
194
|
+
requestId: string;
|
|
195
|
+
usage?: { inputTokens: number; outputTokens: number; totalTokens: number };
|
|
196
|
+
}> {
|
|
197
|
+
const url = `${CODEX_BASE_URL}${CODEX_RESPONSES_PATH}`;
|
|
198
|
+
const headers = buildCodexHeaders(auth.accessToken, auth.accountId);
|
|
199
|
+
|
|
200
|
+
const requestedModel = getModel();
|
|
201
|
+
|
|
202
|
+
const body: Record<string, unknown> = {
|
|
203
|
+
model: requestedModel,
|
|
204
|
+
stream: true,
|
|
205
|
+
store: false,
|
|
206
|
+
input: [
|
|
207
|
+
{
|
|
208
|
+
type: "message",
|
|
209
|
+
role: "user",
|
|
210
|
+
content: [{ type: "input_text", text: query }],
|
|
211
|
+
},
|
|
212
|
+
],
|
|
213
|
+
tools: [
|
|
214
|
+
{
|
|
215
|
+
type: "web_search",
|
|
216
|
+
search_context_size: options.searchContextSize ?? "high",
|
|
217
|
+
},
|
|
218
|
+
],
|
|
219
|
+
instructions: options.systemPrompt ?? DEFAULT_INSTRUCTIONS,
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
const response = await fetch(url, {
|
|
223
|
+
method: "POST",
|
|
224
|
+
headers,
|
|
225
|
+
body: JSON.stringify(body),
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
if (!response.ok) {
|
|
229
|
+
const errorText = await response.text();
|
|
230
|
+
throw new SearchProviderError("codex", `Codex API error (${response.status}): ${errorText}`, response.status);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
if (!response.body) {
|
|
234
|
+
throw new SearchProviderError("codex", "Codex API returned no response body", 500);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Parse SSE stream
|
|
238
|
+
const answerParts: string[] = [];
|
|
239
|
+
const streamedAnswerParts: string[] = [];
|
|
240
|
+
const sources: SearchSource[] = [];
|
|
241
|
+
let model = requestedModel;
|
|
242
|
+
let requestId = "";
|
|
243
|
+
let usage: { inputTokens: number; outputTokens: number; totalTokens: number } | undefined;
|
|
244
|
+
|
|
245
|
+
for await (const rawEvent of readSseJson<Record<string, unknown>>(response.body, options.signal)) {
|
|
246
|
+
const eventType = typeof rawEvent.type === "string" ? rawEvent.type : "";
|
|
247
|
+
if (!eventType) continue;
|
|
248
|
+
|
|
249
|
+
if (eventType === "response.output_text.delta") {
|
|
250
|
+
const delta = typeof rawEvent.delta === "string" ? rawEvent.delta : "";
|
|
251
|
+
if (delta) {
|
|
252
|
+
streamedAnswerParts.push(delta);
|
|
253
|
+
}
|
|
254
|
+
} else if (eventType === "response.output_item.done") {
|
|
255
|
+
const item = rawEvent.item as CodexResponseItem | undefined;
|
|
256
|
+
if (!item) continue;
|
|
257
|
+
|
|
258
|
+
// Handle text message content and extract sources from annotations
|
|
259
|
+
if (item.type === "message" && item.content) {
|
|
260
|
+
for (const part of item.content) {
|
|
261
|
+
if (part.type === "output_text" && part.text) {
|
|
262
|
+
answerParts.push(part.text);
|
|
263
|
+
|
|
264
|
+
// Extract sources from url_citation annotations
|
|
265
|
+
if (part.annotations) {
|
|
266
|
+
for (const annotation of part.annotations) {
|
|
267
|
+
if (annotation.type === "url_citation" && annotation.url) {
|
|
268
|
+
// Deduplicate by URL
|
|
269
|
+
if (!sources.some(s => s.url === annotation.url)) {
|
|
270
|
+
sources.push({
|
|
271
|
+
title: annotation.title ?? annotation.url,
|
|
272
|
+
url: annotation.url,
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Handle reasoning summary as part of answer
|
|
283
|
+
if (item.type === "reasoning" && item.summary) {
|
|
284
|
+
for (const part of item.summary) {
|
|
285
|
+
if (part.type === "summary_text" && part.text) {
|
|
286
|
+
answerParts.push(part.text);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
} else if (eventType === "response.completed" || eventType === "response.done") {
|
|
291
|
+
const resp = (rawEvent as { response?: CodexResponse }).response;
|
|
292
|
+
if (resp) {
|
|
293
|
+
if (resp.model) model = resp.model;
|
|
294
|
+
if (resp.id) requestId = resp.id;
|
|
295
|
+
if (resp.usage) {
|
|
296
|
+
const cachedTokens = resp.usage.input_tokens_details?.cached_tokens ?? 0;
|
|
297
|
+
usage = {
|
|
298
|
+
inputTokens: (resp.usage.input_tokens ?? 0) - cachedTokens,
|
|
299
|
+
outputTokens: resp.usage.output_tokens ?? 0,
|
|
300
|
+
totalTokens: resp.usage.total_tokens ?? 0,
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
} else if (eventType === "error") {
|
|
305
|
+
const code = (rawEvent as { code?: string }).code ?? "";
|
|
306
|
+
const message = (rawEvent as { message?: string }).message ?? "Unknown error";
|
|
307
|
+
throw new SearchProviderError("codex", `Codex error (${code}): ${message}`, 500);
|
|
308
|
+
} else if (eventType === "response.failed") {
|
|
309
|
+
const resp = (rawEvent as { response?: { error?: { message?: string } } }).response;
|
|
310
|
+
const errorMessage = resp?.error?.message ?? "Request failed";
|
|
311
|
+
throw new SearchProviderError("codex", `Codex request failed: ${errorMessage}`, 500);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const finalAnswer = answerParts.join("\n\n").trim();
|
|
316
|
+
const streamedAnswer = streamedAnswerParts.join("").trim();
|
|
317
|
+
const answer =
|
|
318
|
+
finalAnswer.length > 0 && !isImagePlaceholderAnswer(finalAnswer)
|
|
319
|
+
? finalAnswer
|
|
320
|
+
: streamedAnswer.length > 0
|
|
321
|
+
? streamedAnswer
|
|
322
|
+
: finalAnswer;
|
|
323
|
+
|
|
324
|
+
return {
|
|
325
|
+
answer,
|
|
326
|
+
sources,
|
|
327
|
+
model,
|
|
328
|
+
requestId,
|
|
329
|
+
usage,
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Executes a web search using OpenAI Codex's built-in web search tool.
|
|
335
|
+
* Requires OAuth credentials stored in agent.db for provider "openai-codex".
|
|
336
|
+
* @param params - Search parameters including query and optional settings
|
|
337
|
+
* @returns Search response with synthesized answer, sources, and usage
|
|
338
|
+
* @throws {Error} If no Codex OAuth credentials are configured
|
|
339
|
+
*/
|
|
340
|
+
export async function searchCodex(params: CodexSearchParams): Promise<SearchResponse> {
|
|
341
|
+
const auth = await findCodexAuth();
|
|
342
|
+
if (!auth) {
|
|
343
|
+
throw new Error(
|
|
344
|
+
"No Codex OAuth credentials found. Login with 'xcsh /login openai-codex' to enable Codex web search.",
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
const result = await callCodexSearch(auth, params.query, {
|
|
349
|
+
systemPrompt: params.system_prompt,
|
|
350
|
+
searchContextSize: params.search_context_size ?? "high",
|
|
351
|
+
});
|
|
352
|
+
|
|
353
|
+
let sources = result.sources;
|
|
354
|
+
|
|
355
|
+
// Apply num_results limit if specified
|
|
356
|
+
if (params.num_results && sources.length > params.num_results) {
|
|
357
|
+
sources = sources.slice(0, params.num_results);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
return {
|
|
361
|
+
provider: "codex",
|
|
362
|
+
answer: result.answer || undefined,
|
|
363
|
+
sources,
|
|
364
|
+
usage: result.usage
|
|
365
|
+
? {
|
|
366
|
+
inputTokens: result.usage.inputTokens,
|
|
367
|
+
outputTokens: result.usage.outputTokens,
|
|
368
|
+
totalTokens: result.usage.totalTokens,
|
|
369
|
+
}
|
|
370
|
+
: undefined,
|
|
371
|
+
model: result.model,
|
|
372
|
+
requestId: result.requestId,
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Checks if Codex web search is available.
|
|
378
|
+
* @returns True if valid OAuth credentials exist for openai-codex
|
|
379
|
+
*/
|
|
380
|
+
export async function hasCodexSearch(): Promise<boolean> {
|
|
381
|
+
const auth = await findCodexAuth();
|
|
382
|
+
return auth !== null;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/** Search provider for OpenAI Codex web search. */
|
|
386
|
+
export class CodexProvider extends SearchProvider {
|
|
387
|
+
readonly id = "codex";
|
|
388
|
+
readonly label = "Codex";
|
|
389
|
+
|
|
390
|
+
isAvailable(): Promise<boolean> {
|
|
391
|
+
return Promise.resolve(hasCodexSearch());
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
search(params: SearchParams): Promise<SearchResponse> {
|
|
395
|
+
return searchCodex({
|
|
396
|
+
signal: params.signal,
|
|
397
|
+
query: params.query,
|
|
398
|
+
system_prompt: params.systemPrompt,
|
|
399
|
+
num_results: params.numSearchResults ?? params.limit,
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
}
|
package/src/web/search/types.ts
CHANGED
|
@@ -14,6 +14,7 @@ export type SearchProviderId =
|
|
|
14
14
|
| "anthropic"
|
|
15
15
|
| "perplexity"
|
|
16
16
|
| "gemini"
|
|
17
|
+
| "codex"
|
|
17
18
|
| "tavily"
|
|
18
19
|
| "parallel"
|
|
19
20
|
| "kagi"
|
|
@@ -30,6 +31,7 @@ export function isSearchProviderId(value: string): value is SearchProviderId {
|
|
|
30
31
|
"anthropic",
|
|
31
32
|
"perplexity",
|
|
32
33
|
"gemini",
|
|
34
|
+
"codex",
|
|
33
35
|
"tavily",
|
|
34
36
|
"parallel",
|
|
35
37
|
"kagi",
|