@oh-my-pi/pi-coding-agent 16.0.4 → 16.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/dist/cli.js +341 -261
- package/dist/types/advisor/advise-tool.d.ts +9 -0
- package/dist/types/cli/args.d.ts +1 -0
- package/dist/types/cli/bench-cli.d.ts +6 -0
- package/dist/types/commands/launch.d.ts +3 -0
- package/dist/types/config/settings-schema.d.ts +91 -2
- package/dist/types/extensibility/extensions/runner.d.ts +5 -2
- package/dist/types/extensibility/extensions/types.d.ts +8 -7
- package/dist/types/extensibility/shared-events.d.ts +22 -1
- package/dist/types/main.d.ts +1 -0
- package/dist/types/modes/components/status-line/component.d.ts +1 -1
- package/dist/types/modes/components/status-line/context-thresholds.d.ts +0 -1
- package/dist/types/modes/rpc/rpc-types.d.ts +1 -1
- package/dist/types/modes/utils/context-usage.d.ts +12 -0
- package/dist/types/sdk.d.ts +3 -1
- package/dist/types/session/agent-session.d.ts +20 -0
- package/dist/types/session/session-persistence.d.ts +4 -0
- package/dist/types/tools/read.d.ts +1 -0
- package/dist/types/tui/code-cell.d.ts +2 -0
- package/dist/types/utils/image-vision-fallback.d.ts +28 -0
- package/dist/types/web/search/providers/base.d.ts +1 -0
- package/dist/types/web/search/providers/gemini.d.ts +1 -0
- package/package.json +12 -12
- package/src/advisor/__tests__/advisor.test.ts +59 -0
- package/src/advisor/advise-tool.ts +13 -0
- package/src/cli/args.ts +1 -0
- package/src/cli/bench-cli.ts +30 -7
- package/src/cli/flag-tables.ts +8 -0
- package/src/collab/host.ts +2 -2
- package/src/commands/launch.ts +3 -0
- package/src/config/settings-schema.ts +84 -2
- package/src/eval/py/runner.py +44 -0
- package/src/extensibility/extensions/runner.ts +20 -2
- package/src/extensibility/extensions/types.ts +16 -5
- package/src/extensibility/shared-events.ts +24 -0
- package/src/internal-urls/docs-index.generated.ts +7 -7
- package/src/main.ts +12 -5
- package/src/modes/components/branch-summary-message.ts +1 -0
- package/src/modes/components/collab-prompt-message.ts +9 -7
- package/src/modes/components/compaction-summary-message.ts +1 -0
- package/src/modes/components/custom-message.ts +1 -0
- package/src/modes/components/footer.ts +6 -5
- package/src/modes/components/hook-message.ts +1 -0
- package/src/modes/components/read-tool-group.ts +9 -3
- package/src/modes/components/skill-message.ts +1 -0
- package/src/modes/components/status-line/component.ts +131 -14
- package/src/modes/components/status-line/context-thresholds.ts +0 -1
- package/src/modes/components/todo-reminder.ts +1 -0
- package/src/modes/components/ttsr-notification.ts +1 -0
- package/src/modes/components/user-message.ts +6 -6
- package/src/modes/controllers/event-controller.ts +2 -7
- package/src/modes/controllers/selector-controller.ts +10 -3
- package/src/modes/interactive-mode.ts +4 -2
- package/src/modes/rpc/rpc-types.ts +1 -1
- package/src/modes/utils/context-usage.ts +28 -15
- package/src/prompts/tools/image-attachment-describe-system.md +8 -0
- package/src/prompts/tools/image-attachment-describe.md +10 -0
- package/src/sdk.ts +14 -18
- package/src/session/agent-session.ts +564 -231
- package/src/session/session-loader.ts +19 -32
- package/src/session/session-persistence.ts +27 -11
- package/src/ssh/connection-manager.ts +3 -2
- package/src/task/executor.ts +1 -1
- package/src/tools/image-gen.ts +67 -25
- package/src/tools/read.ts +28 -6
- package/src/tui/code-cell.ts +44 -3
- package/src/utils/image-vision-fallback.ts +197 -0
- package/src/web/search/index.ts +12 -0
- package/src/web/search/providers/base.ts +1 -0
- package/src/web/search/providers/gemini.ts +56 -18
|
@@ -4,7 +4,7 @@ import { BlobStore, isBlobRef, resolveImageData, resolveImageDataUrl } from "./b
|
|
|
4
4
|
import { buildSessionContext } from "./session-context";
|
|
5
5
|
import type { FileEntry, SessionEntry, SessionHeader } from "./session-entries";
|
|
6
6
|
import { migrateToCurrentVersion } from "./session-migrations";
|
|
7
|
-
import { isImageBlock } from "./session-persistence";
|
|
7
|
+
import { isImageBlock, isImageDataPayload } from "./session-persistence";
|
|
8
8
|
import { FileSessionStorage, type SessionStorage } from "./session-storage";
|
|
9
9
|
|
|
10
10
|
/** Exported for compaction.test.ts */
|
|
@@ -44,9 +44,19 @@ function hasImageUrl(value: unknown): value is { image_url: string } {
|
|
|
44
44
|
return typeof value === "object" && value !== null && "image_url" in value && typeof value.image_url === "string";
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
|
|
47
|
+
function shouldResolveImagePayload(value: unknown, key: string | undefined): value is { data: string } {
|
|
48
|
+
if (!isImageDataPayload(value) || !isBlobRef(value.data)) return false;
|
|
49
|
+
return (key === "content" && isImageBlock(value)) || key === "images";
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async function resolvePersistedBlobRefs(value: unknown, blobStore: BlobStore, key?: string): Promise<void> {
|
|
53
|
+
if (shouldResolveImagePayload(value, key)) {
|
|
54
|
+
value.data = await resolveImageData(blobStore, value.data);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
|
|
48
58
|
if (Array.isArray(value)) {
|
|
49
|
-
await Promise.all(value.map(item =>
|
|
59
|
+
await Promise.all(value.map(item => resolvePersistedBlobRefs(item, blobStore, key)));
|
|
50
60
|
return;
|
|
51
61
|
}
|
|
52
62
|
|
|
@@ -56,38 +66,15 @@ async function resolvePersistedImageUrlRefs(value: unknown, blobStore: BlobStore
|
|
|
56
66
|
value.image_url = await resolveImageDataUrl(blobStore, value.image_url);
|
|
57
67
|
}
|
|
58
68
|
|
|
59
|
-
await Promise.all(
|
|
69
|
+
await Promise.all(
|
|
70
|
+
Object.entries(value).map(([childKey, item]) => resolvePersistedBlobRefs(item, blobStore, childKey)),
|
|
71
|
+
);
|
|
60
72
|
}
|
|
61
73
|
|
|
62
74
|
export async function resolveBlobRefsInEntries(entries: FileEntry[], blobStore: BlobStore): Promise<void> {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
if (entry.type === "session") continue;
|
|
67
|
-
|
|
68
|
-
let contentArray: unknown[] | undefined;
|
|
69
|
-
if (entry.type === "message" && "content" in entry.message && Array.isArray(entry.message.content)) {
|
|
70
|
-
contentArray = entry.message.content;
|
|
71
|
-
} else if (entry.type === "custom_message" && Array.isArray(entry.content)) {
|
|
72
|
-
contentArray = entry.content;
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
if (contentArray) {
|
|
76
|
-
for (const block of contentArray) {
|
|
77
|
-
if (isImageBlock(block) && isBlobRef(block.data)) {
|
|
78
|
-
promises.push(
|
|
79
|
-
resolveImageData(blobStore, block.data).then(resolved => {
|
|
80
|
-
block.data = resolved;
|
|
81
|
-
}),
|
|
82
|
-
);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
promises.push(resolvePersistedImageUrlRefs(entry, blobStore));
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
await Promise.all(promises);
|
|
75
|
+
await Promise.all(
|
|
76
|
+
entries.filter(entry => entry.type !== "session").map(entry => resolvePersistedBlobRefs(entry, blobStore)),
|
|
77
|
+
);
|
|
91
78
|
}
|
|
92
79
|
|
|
93
80
|
/**
|
|
@@ -36,10 +36,33 @@ export function isImageBlock(value: unknown): value is { type: "image"; data: st
|
|
|
36
36
|
);
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
function isImageMimeType(value: unknown): value is string {
|
|
40
|
+
return typeof value === "string" && value.toLowerCase().startsWith("image/");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function isImageDataPayload(value: unknown): value is { data: string; mimeType?: string } {
|
|
44
|
+
return (
|
|
45
|
+
typeof value === "object" &&
|
|
46
|
+
value !== null &&
|
|
47
|
+
"data" in value &&
|
|
48
|
+
typeof (value as { data?: string }).data === "string" &&
|
|
49
|
+
(isImageBlock(value) || ("mimeType" in value && isImageMimeType((value as { mimeType?: unknown }).mimeType)))
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function shouldExternalizeImagePayload(
|
|
54
|
+
value: unknown,
|
|
55
|
+
key: string | undefined,
|
|
56
|
+
): value is { data: string; mimeType?: string } {
|
|
57
|
+
if (!isImageDataPayload(value)) return false;
|
|
58
|
+
if (isBlobRef(value.data) || value.data.length < BLOB_EXTERNALIZE_THRESHOLD) return false;
|
|
59
|
+
return (key === TEXT_CONTENT_KEY && isImageBlock(value)) || key === "images";
|
|
60
|
+
}
|
|
61
|
+
|
|
39
62
|
/**
|
|
40
63
|
* Recursively truncate large strings in an object for session persistence.
|
|
41
64
|
* - Truncates any oversized string fields (key-agnostic)
|
|
42
|
-
* -
|
|
65
|
+
* - Externalizes oversized image payloads to blob refs
|
|
43
66
|
* - Updates lineCount when content is truncated
|
|
44
67
|
* - Returns original object if no changes needed (structural sharing)
|
|
45
68
|
*
|
|
@@ -50,6 +73,9 @@ export function isImageBlock(value: unknown): value is { type: "image"; data: st
|
|
|
50
73
|
*/
|
|
51
74
|
function truncateForPersistence(obj: unknown, blobStore: BlobStore, key?: string): unknown {
|
|
52
75
|
if (obj === null || obj === undefined) return obj;
|
|
76
|
+
if (shouldExternalizeImagePayload(obj, key)) {
|
|
77
|
+
return { ...obj, data: externalizeImageDataSync(blobStore, obj.data, obj.mimeType) };
|
|
78
|
+
}
|
|
53
79
|
|
|
54
80
|
if (typeof obj === "string") {
|
|
55
81
|
if (key === "image_url" && isImageDataUrl(obj)) {
|
|
@@ -72,16 +98,6 @@ function truncateForPersistence(obj: unknown, blobStore: BlobStore, key?: string
|
|
|
72
98
|
const result: unknown[] = new Array(obj.length);
|
|
73
99
|
for (let i = 0; i < obj.length; i++) {
|
|
74
100
|
const item = obj[i];
|
|
75
|
-
if (
|
|
76
|
-
key === TEXT_CONTENT_KEY &&
|
|
77
|
-
isImageBlock(item) &&
|
|
78
|
-
!isBlobRef(item.data) &&
|
|
79
|
-
item.data.length >= BLOB_EXTERNALIZE_THRESHOLD
|
|
80
|
-
) {
|
|
81
|
-
changed = true;
|
|
82
|
-
result[i] = { ...item, data: externalizeImageDataSync(blobStore, item.data, item.mimeType) };
|
|
83
|
-
continue;
|
|
84
|
-
}
|
|
85
101
|
const newItem = truncateForPersistence(item, blobStore, key);
|
|
86
102
|
if (newItem !== item) changed = true;
|
|
87
103
|
result[i] = newItem;
|
|
@@ -65,7 +65,7 @@ async function deleteHostInfoFromDisk(hostName: string): Promise<void> {
|
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
async function validateKeyPermissions(keyPath?: string): Promise<void> {
|
|
68
|
+
async function validateKeyPermissions(keyPath?: string, platform: SshPlatform = process.platform): Promise<void> {
|
|
69
69
|
if (!keyPath) return;
|
|
70
70
|
let stats: fs.Stats;
|
|
71
71
|
try {
|
|
@@ -79,6 +79,7 @@ async function validateKeyPermissions(keyPath?: string): Promise<void> {
|
|
|
79
79
|
if (!stats.isFile()) {
|
|
80
80
|
throw new Error(`SSH key is not a file: ${keyPath}`);
|
|
81
81
|
}
|
|
82
|
+
if (platform === "win32") return;
|
|
82
83
|
const mode = stats.mode & 0o777;
|
|
83
84
|
if ((mode & 0o077) !== 0) {
|
|
84
85
|
throw new Error(`SSH key permissions must be 600 or stricter: ${keyPath}`);
|
|
@@ -402,7 +403,7 @@ export async function buildRemoteCommand(
|
|
|
402
403
|
command: string,
|
|
403
404
|
options?: SSHArgsOptions,
|
|
404
405
|
): Promise<string[]> {
|
|
405
|
-
await validateKeyPermissions(host.keyPath);
|
|
406
|
+
await validateKeyPermissions(host.keyPath, options?.platform);
|
|
406
407
|
return [...buildCommonArgs(host, options), buildSshTarget(host.username, host.host), command];
|
|
407
408
|
}
|
|
408
409
|
|
package/src/task/executor.ts
CHANGED
|
@@ -2117,8 +2117,8 @@ export async function runSubprocess(options: ExecutorOptions): Promise<SingleRes
|
|
|
2117
2117
|
void session.abort();
|
|
2118
2118
|
}
|
|
2119
2119
|
|
|
2120
|
+
const pendingExtensionMessages: Array<Promise<unknown>> = [];
|
|
2120
2121
|
const extensionRunner = session.extensionRunner;
|
|
2121
|
-
const pendingExtensionMessages: Promise<unknown>[] = [];
|
|
2122
2122
|
if (extensionRunner) {
|
|
2123
2123
|
extensionRunner.initialize(
|
|
2124
2124
|
{
|
package/src/tools/image-gen.ts
CHANGED
|
@@ -21,8 +21,8 @@ import {
|
|
|
21
21
|
} from "@oh-my-pi/pi-utils";
|
|
22
22
|
import { z } from "zod/v4";
|
|
23
23
|
import packageJson from "../../package.json" with { type: "json" };
|
|
24
|
-
|
|
25
24
|
import { isAuthenticated, type ModelRegistry } from "../config/model-registry";
|
|
25
|
+
import { settings } from "../config/settings";
|
|
26
26
|
import type { CustomTool } from "../extensibility/custom-tools/types";
|
|
27
27
|
import { ohMyPiXAIUserAgent, resolveXAIHttpCredentials } from "../lib/xai-http";
|
|
28
28
|
import imageGenDescription from "../prompts/tools/image-gen.md" with { type: "text" };
|
|
@@ -38,7 +38,8 @@ const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
|
38
38
|
const OPENAI_IMAGE_OUTPUT_FORMAT = "webp";
|
|
39
39
|
const OPENAI_IMAGE_MIME_TYPE = "image/webp";
|
|
40
40
|
|
|
41
|
-
const
|
|
41
|
+
const DEFAULT_ANTIGRAVITY_ENDPOINT_PROD = "https://daily-cloudcode-pa.googleapis.com";
|
|
42
|
+
const DEFAULT_ANTIGRAVITY_ENDPOINT_SANDBOX = "https://daily-cloudcode-pa.sandbox.googleapis.com";
|
|
42
43
|
const IMAGE_SYSTEM_INSTRUCTION =
|
|
43
44
|
"You are an AI image generator. Generate images based on user descriptions. Focus on creating high-quality, visually appealing images that match the user's request.";
|
|
44
45
|
|
|
@@ -1164,33 +1165,74 @@ export const imageGenTool: CustomTool<typeof imageGenSchema, ImageGenToolDetails
|
|
|
1164
1165
|
resolvedImages,
|
|
1165
1166
|
);
|
|
1166
1167
|
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
}
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
}
|
|
1168
|
+
let endpoints = [DEFAULT_ANTIGRAVITY_ENDPOINT_PROD, DEFAULT_ANTIGRAVITY_ENDPOINT_SANDBOX];
|
|
1169
|
+
try {
|
|
1170
|
+
const mode = settings.get("providers.antigravityEndpoint");
|
|
1171
|
+
if (mode === "production") {
|
|
1172
|
+
endpoints = [DEFAULT_ANTIGRAVITY_ENDPOINT_PROD];
|
|
1173
|
+
} else if (mode === "sandbox") {
|
|
1174
|
+
endpoints = [DEFAULT_ANTIGRAVITY_ENDPOINT_SANDBOX];
|
|
1175
|
+
}
|
|
1176
|
+
} catch {
|
|
1177
|
+
// Ignored
|
|
1178
|
+
}
|
|
1178
1179
|
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1180
|
+
let resp: Response | undefined;
|
|
1181
|
+
let lastError: Error | undefined;
|
|
1182
|
+
|
|
1183
|
+
for (let i = 0; i < endpoints.length; i++) {
|
|
1184
|
+
const endpoint = endpoints[i];
|
|
1185
|
+
const isLastEndpoint = i === endpoints.length - 1;
|
|
1182
1186
|
try {
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
+
resp = await fetchImpl(`${endpoint}/v1internal:streamGenerateContent?alt=sse`, {
|
|
1188
|
+
method: "POST",
|
|
1189
|
+
headers: {
|
|
1190
|
+
Authorization: `Bearer ${bearer}`,
|
|
1191
|
+
"Content-Type": "application/json",
|
|
1192
|
+
Accept: "text/event-stream",
|
|
1193
|
+
"User-Agent": getAntigravityUserAgent(),
|
|
1194
|
+
},
|
|
1195
|
+
body: JSON.stringify(requestBody),
|
|
1196
|
+
signal: requestSignal,
|
|
1197
|
+
});
|
|
1198
|
+
|
|
1199
|
+
if (resp.ok) {
|
|
1200
|
+
break;
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
const errorText = await resp.text();
|
|
1204
|
+
let message = errorText;
|
|
1205
|
+
try {
|
|
1206
|
+
const parsedErr = JSON.parse(errorText) as { error?: { message?: string } };
|
|
1207
|
+
message = parsedErr.error?.message ?? message;
|
|
1208
|
+
} catch {
|
|
1209
|
+
// Keep raw text.
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
lastError = new ProviderHttpError(
|
|
1213
|
+
`Antigravity image request failed (${resp.status}): ${message}`,
|
|
1214
|
+
resp.status,
|
|
1215
|
+
{ headers: resp.headers },
|
|
1216
|
+
);
|
|
1217
|
+
|
|
1218
|
+
if (resp.status === 429 || (resp.status >= 500 && resp.status < 600)) {
|
|
1219
|
+
if (!isLastEndpoint) {
|
|
1220
|
+
continue;
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
break;
|
|
1224
|
+
} catch (error) {
|
|
1225
|
+
lastError = error as Error;
|
|
1226
|
+
if (isLastEndpoint) {
|
|
1227
|
+
break;
|
|
1228
|
+
}
|
|
1187
1229
|
}
|
|
1188
|
-
throw new ProviderHttpError(
|
|
1189
|
-
`Antigravity image request failed (${resp.status}): ${message}`,
|
|
1190
|
-
resp.status,
|
|
1191
|
-
{ headers: resp.headers },
|
|
1192
|
-
);
|
|
1193
1230
|
}
|
|
1231
|
+
|
|
1232
|
+
if (!resp?.ok) {
|
|
1233
|
+
throw lastError ?? new Error("Antigravity image generation failed");
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1194
1236
|
return resp;
|
|
1195
1237
|
},
|
|
1196
1238
|
{ signal: requestSignal },
|
package/src/tools/read.ts
CHANGED
|
@@ -689,7 +689,11 @@ export interface ReadToolDetails {
|
|
|
689
689
|
/** Raw text + start line for user-visible TUI rendering, set when content is text-like.
|
|
690
690
|
* Mirrors the same lines the model receives but without hashline/line-number prefixes,
|
|
691
691
|
* so the TUI can render the file content with its own gutter without re-parsing the formatted text. */
|
|
692
|
-
displayContent?: {
|
|
692
|
+
displayContent?: {
|
|
693
|
+
text: string;
|
|
694
|
+
startLine: number;
|
|
695
|
+
lineNumbers?: Array<number | null>;
|
|
696
|
+
};
|
|
693
697
|
summary?: { lines: number; elidedSpans: number; elidedLines: number };
|
|
694
698
|
/** Number of unresolved git conflicts surfaced by this read (TUI uses for inline `⚠ N` badge). */
|
|
695
699
|
conflictCount?: number;
|
|
@@ -1058,7 +1062,12 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
1058
1062
|
let emittedHashlineHeader = false;
|
|
1059
1063
|
let seenLines: number[] | undefined;
|
|
1060
1064
|
const formatText = (content: string, startNum: number): string => {
|
|
1061
|
-
|
|
1065
|
+
const lineCount = countTextLines(content);
|
|
1066
|
+
details.displayContent = {
|
|
1067
|
+
text: content,
|
|
1068
|
+
startLine: startNum,
|
|
1069
|
+
lineNumbers: Array.from({ length: lineCount }, (_, i) => startNum + i),
|
|
1070
|
+
};
|
|
1062
1071
|
if (shouldAddHashLines) seenLines = contiguousLineNumbers(startNum, countTextLines(content));
|
|
1063
1072
|
const formatted = formatTextWithMode(content, startNum, shouldAddHashLines, shouldAddLineNumbers);
|
|
1064
1073
|
if (!hashContext || emittedHashlineHeader) return formatted;
|
|
@@ -1070,6 +1079,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
1070
1079
|
details.displayContent = {
|
|
1071
1080
|
text: lineEntriesToPlainText(entries, BRACKET_CONTEXT_ELLIPSIS),
|
|
1072
1081
|
startLine: firstLine?.kind === "line" ? firstLine.lineNumber : startNum,
|
|
1082
|
+
lineNumbers: entries.map(entry => (entry.kind === "line" ? entry.lineNumber : null)),
|
|
1073
1083
|
};
|
|
1074
1084
|
if (shouldAddHashLines) seenLines = lineNumbersFromEntries(entries);
|
|
1075
1085
|
const formatted = formatLineEntriesWithMode(entries, shouldAddHashLines, shouldAddLineNumbers);
|
|
@@ -1218,6 +1228,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
1218
1228
|
details.displayContent = {
|
|
1219
1229
|
text: lineEntriesToPlainText(entries, BRACKET_CONTEXT_ELLIPSIS),
|
|
1220
1230
|
startLine: firstLine.lineNumber,
|
|
1231
|
+
lineNumbers: entries.map(entry => (entry.kind === "line" ? entry.lineNumber : null)),
|
|
1221
1232
|
};
|
|
1222
1233
|
}
|
|
1223
1234
|
const formatted = formatLineEntriesWithMode(entries, shouldAddHashLines, shouldAddLineNumbers);
|
|
@@ -1255,7 +1266,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
1255
1266
|
): Promise<{
|
|
1256
1267
|
outputText: string;
|
|
1257
1268
|
columnTruncated: number;
|
|
1258
|
-
displayContent?: { text: string; startLine: number };
|
|
1269
|
+
displayContent?: { text: string; startLine: number; lineNumbers?: Array<number | null> };
|
|
1259
1270
|
bridgeResult?: AgentToolResult<ReadToolDetails>;
|
|
1260
1271
|
}> {
|
|
1261
1272
|
const rawSelector = isRawSelector(parsed);
|
|
@@ -1292,7 +1303,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
1292
1303
|
const displayLineByNumber = new Map<number, string>();
|
|
1293
1304
|
const fullLines = rawSelector ? undefined : await readBracketContextFullLines(absolutePath, fileSize);
|
|
1294
1305
|
let columnTruncated = 0;
|
|
1295
|
-
let displayContent: { text: string; startLine: number } | undefined;
|
|
1306
|
+
let displayContent: { text: string; startLine: number; lineNumbers?: Array<number | null> } | undefined;
|
|
1296
1307
|
|
|
1297
1308
|
for (const range of ranges) {
|
|
1298
1309
|
const rangeStart = range.startLine - 1; // 0-indexed
|
|
@@ -1375,6 +1386,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
1375
1386
|
displayContent = {
|
|
1376
1387
|
text: lineEntriesToPlainText(entries, BRACKET_CONTEXT_ELLIPSIS),
|
|
1377
1388
|
startLine: firstLine?.kind === "line" ? firstLine.lineNumber : (visibleSpans[0]?.startLine ?? 1),
|
|
1389
|
+
lineNumbers: entries.map(entry => (entry.kind === "line" ? entry.lineNumber : null)),
|
|
1378
1390
|
};
|
|
1379
1391
|
outputText = formatLineEntriesWithMode(entries, shouldAddHashLines, shouldAddLineNumbers);
|
|
1380
1392
|
} else {
|
|
@@ -2292,10 +2304,17 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
2292
2304
|
}
|
|
2293
2305
|
}
|
|
2294
2306
|
|
|
2295
|
-
let capturedDisplayContent:
|
|
2307
|
+
let capturedDisplayContent:
|
|
2308
|
+
| { text: string; startLine: number; lineNumbers?: Array<number | null> }
|
|
2309
|
+
| undefined;
|
|
2296
2310
|
let emittedHashlineHeader = false;
|
|
2297
2311
|
const formatText = (text: string, startNum: number): string => {
|
|
2298
|
-
|
|
2312
|
+
const lineCount = countTextLines(text);
|
|
2313
|
+
capturedDisplayContent = {
|
|
2314
|
+
text,
|
|
2315
|
+
startLine: startNum,
|
|
2316
|
+
lineNumbers: Array.from({ length: lineCount }, (_, i) => startNum + i),
|
|
2317
|
+
};
|
|
2299
2318
|
const formatted = formatTextWithMode(text, startNum, shouldAddHashLines, shouldAddLineNumbers);
|
|
2300
2319
|
if (!hashContext || emittedHashlineHeader) return formatted;
|
|
2301
2320
|
emittedHashlineHeader = true;
|
|
@@ -2322,6 +2341,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
2322
2341
|
capturedDisplayContent = {
|
|
2323
2342
|
text: lineEntriesToPlainText(entries, BRACKET_CONTEXT_ELLIPSIS),
|
|
2324
2343
|
startLine: firstLine?.kind === "line" ? firstLine.lineNumber : startLineDisplay,
|
|
2344
|
+
lineNumbers: entries.map(entry => (entry.kind === "line" ? entry.lineNumber : null)),
|
|
2325
2345
|
};
|
|
2326
2346
|
const formatted = formatLineEntriesWithMode(entries, shouldAddHashLines, shouldAddLineNumbers);
|
|
2327
2347
|
if (!hashContext || emittedHashlineHeader) return formatted;
|
|
@@ -2928,6 +2948,8 @@ export const readToolRenderer = {
|
|
|
2928
2948
|
status: "complete",
|
|
2929
2949
|
output: warningLines.length > 0 ? warningLines.join("\n") : undefined,
|
|
2930
2950
|
expanded,
|
|
2951
|
+
codeStartLine: details?.displayContent?.startLine,
|
|
2952
|
+
codeLineNumbers: details?.displayContent?.lineNumbers,
|
|
2931
2953
|
width,
|
|
2932
2954
|
},
|
|
2933
2955
|
uiTheme,
|
package/src/tui/code-cell.ts
CHANGED
|
@@ -33,6 +33,8 @@ export interface CodeCellOptions {
|
|
|
33
33
|
codeTail?: boolean;
|
|
34
34
|
expanded?: boolean;
|
|
35
35
|
width: number;
|
|
36
|
+
codeStartLine?: number;
|
|
37
|
+
codeLineNumbers?: Array<number | null>;
|
|
36
38
|
}
|
|
37
39
|
|
|
38
40
|
function getState(status?: CodeCellOptions["status"]): State | undefined {
|
|
@@ -99,7 +101,17 @@ function collapseCarriageReturns(line: string): string {
|
|
|
99
101
|
return idx < 0 ? line : line.slice(idx + 1);
|
|
100
102
|
}
|
|
101
103
|
export function renderCodeCell(options: CodeCellOptions, theme: Theme): string[] {
|
|
102
|
-
const {
|
|
104
|
+
const {
|
|
105
|
+
code,
|
|
106
|
+
language,
|
|
107
|
+
output,
|
|
108
|
+
expanded = false,
|
|
109
|
+
outputMaxLines = 6,
|
|
110
|
+
codeMaxLines = 12,
|
|
111
|
+
width,
|
|
112
|
+
codeStartLine,
|
|
113
|
+
codeLineNumbers,
|
|
114
|
+
} = options;
|
|
103
115
|
const { title, meta } = formatHeader(options, theme);
|
|
104
116
|
const state = getState(options.status);
|
|
105
117
|
|
|
@@ -111,16 +123,45 @@ export function renderCodeCell(options: CodeCellOptions, theme: Theme): string[]
|
|
|
111
123
|
const startIndex = tail ? rawCodeLines.length - maxCodeLines : 0;
|
|
112
124
|
const visibleCode = rawCodeLines.slice(startIndex, startIndex + maxCodeLines).join("\n");
|
|
113
125
|
const codeLines = highlightCode(visibleCode, language);
|
|
126
|
+
|
|
127
|
+
let visibleLineNumbers: Array<number | null> | undefined;
|
|
128
|
+
let lineNumberWidth = 0;
|
|
129
|
+
if (codeLineNumbers) {
|
|
130
|
+
visibleLineNumbers = codeLineNumbers.slice(startIndex, startIndex + maxCodeLines);
|
|
131
|
+
} else if (codeStartLine !== undefined) {
|
|
132
|
+
visibleLineNumbers = Array.from({ length: maxCodeLines }, (_, i) => codeStartLine + startIndex + i);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (visibleLineNumbers) {
|
|
136
|
+
const validLineNums = visibleLineNumbers.filter((n): n is number => n !== null && n !== undefined);
|
|
137
|
+
const maxVal = validLineNums.length > 0 ? Math.max(...validLineNums) : 0;
|
|
138
|
+
if (maxVal > 0) {
|
|
139
|
+
lineNumberWidth = Math.max(2, String(maxVal).length);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (lineNumberWidth > 0 && visibleLineNumbers) {
|
|
144
|
+
for (let i = 0; i < codeLines.length; i++) {
|
|
145
|
+
const lineNum = visibleLineNumbers[i];
|
|
146
|
+
const gutter =
|
|
147
|
+
lineNum !== null && lineNum !== undefined
|
|
148
|
+
? String(lineNum).padStart(lineNumberWidth, " ")
|
|
149
|
+
: " ".repeat(lineNumberWidth);
|
|
150
|
+
codeLines[i] = theme.fg("dim", `${gutter} `) + codeLines[i];
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
114
154
|
if (hiddenCodeLines > 0) {
|
|
115
155
|
const hint = formatExpandHint(theme, expanded, hiddenCodeLines > 0);
|
|
156
|
+
const gutterPad = lineNumberWidth > 0 ? " ".repeat(lineNumberWidth + 1) : "";
|
|
116
157
|
if (tail) {
|
|
117
158
|
// Earlier rows scrolled above the live tail window — mark them on top so
|
|
118
159
|
// the newest streamed line stays pinned to the bottom of the box.
|
|
119
160
|
const earlier = `… ${hiddenCodeLines} earlier line${hiddenCodeLines === 1 ? "" : "s"}${hint ? ` ${hint}` : ""}`;
|
|
120
|
-
codeLines.unshift(theme.fg("dim", earlier));
|
|
161
|
+
codeLines.unshift(theme.fg("dim", gutterPad + earlier));
|
|
121
162
|
} else {
|
|
122
163
|
const moreLine = `${formatMoreItems(hiddenCodeLines, "line")}${hint ? ` ${hint}` : ""}`;
|
|
123
|
-
codeLines.push(theme.fg("dim", moreLine));
|
|
164
|
+
codeLines.push(theme.fg("dim", gutterPad + moreLine));
|
|
124
165
|
}
|
|
125
166
|
}
|
|
126
167
|
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vision fallback for text-only models. When a user attaches an image to a model
|
|
3
|
+
* that cannot accept image input, this:
|
|
4
|
+
* 1. saves each image under the session `local://` root (for later analysis), and
|
|
5
|
+
* 2. asks a vision-capable model to describe it and injects that description as
|
|
6
|
+
* a text block in place of the image:
|
|
7
|
+
*
|
|
8
|
+
* <image path="local://image-<hash>.png">
|
|
9
|
+
* <description>
|
|
10
|
+
* </image>
|
|
11
|
+
*
|
|
12
|
+
* Without this the provider layer drops the image entirely (NON_VISION_IMAGE_PLACEHOLDER).
|
|
13
|
+
*/
|
|
14
|
+
import * as path from "node:path";
|
|
15
|
+
import {
|
|
16
|
+
type AgentTelemetry,
|
|
17
|
+
type AgentTelemetryConfig,
|
|
18
|
+
instrumentedCompleteSimple,
|
|
19
|
+
resolveTelemetry,
|
|
20
|
+
} from "@oh-my-pi/pi-agent-core";
|
|
21
|
+
import type { Api, completeSimple, ImageContent, Model, TextContent } from "@oh-my-pi/pi-ai";
|
|
22
|
+
import { logger, prompt, toError } from "@oh-my-pi/pi-utils";
|
|
23
|
+
import { extractTextContent } from "../commit/utils";
|
|
24
|
+
import type { ModelRegistry } from "../config/model-registry";
|
|
25
|
+
import { expandRoleAlias, getModelMatchPreferences, resolveModelFromString } from "../config/model-resolver";
|
|
26
|
+
import type { Settings } from "../config/settings";
|
|
27
|
+
import { type LocalProtocolOptions, resolveLocalRoot } from "../internal-urls";
|
|
28
|
+
import describeUserPrompt from "../prompts/tools/image-attachment-describe.md" with { type: "text" };
|
|
29
|
+
import describeSystemPrompt from "../prompts/tools/image-attachment-describe-system.md" with { type: "text" };
|
|
30
|
+
|
|
31
|
+
/** Telemetry tag for the oneshot vision-description calls. */
|
|
32
|
+
const ONESHOT_KIND = "image_attachment_describe";
|
|
33
|
+
|
|
34
|
+
const NO_VISION_MODEL_NOTE =
|
|
35
|
+
"[No vision-capable model is configured, so this image could not be described automatically. " +
|
|
36
|
+
"The image was saved; configure a vision model role (modelRoles.vision) and use the inspect_image tool to analyze it.]";
|
|
37
|
+
|
|
38
|
+
const DESCRIPTION_UNAVAILABLE_NOTE =
|
|
39
|
+
"[Image description unavailable: the vision model returned no usable text. The image was saved for further analysis.]";
|
|
40
|
+
|
|
41
|
+
/** Registry surface needed to resolve a vision model and authorize requests. */
|
|
42
|
+
export type VisionFallbackRegistry = Pick<ModelRegistry, "getAvailable" | "getApiKey" | "resolver"> &
|
|
43
|
+
Partial<Pick<ModelRegistry, "resolveCanonicalModel" | "getCanonicalVariants" | "getCanonicalId">>;
|
|
44
|
+
|
|
45
|
+
export interface DescribeAttachedImagesDeps {
|
|
46
|
+
/** Active (text-only) model the prompt is destined for. */
|
|
47
|
+
activeModel: Model<Api>;
|
|
48
|
+
modelRegistry: VisionFallbackRegistry;
|
|
49
|
+
settings: Settings;
|
|
50
|
+
/** Inputs for resolving the session-scoped `local://` root. */
|
|
51
|
+
localProtocolOptions: LocalProtocolOptions;
|
|
52
|
+
/** `provider/id` of the active model; a last-resort vision-model candidate (filtered to image-capable). */
|
|
53
|
+
activeModelString?: string;
|
|
54
|
+
telemetryConfig?: AgentTelemetryConfig;
|
|
55
|
+
sessionId?: string;
|
|
56
|
+
/** Test seam: overrides the underlying completeSimple call. */
|
|
57
|
+
completeImpl?: typeof completeSimple;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Map an image MIME type to a file extension for the saved artifact. */
|
|
61
|
+
function extensionForMime(mimeType: string): string {
|
|
62
|
+
const subtype = mimeType.split("/")[1]?.toLowerCase() ?? "";
|
|
63
|
+
switch (subtype) {
|
|
64
|
+
case "jpeg":
|
|
65
|
+
case "jpg":
|
|
66
|
+
return "jpg";
|
|
67
|
+
case "png":
|
|
68
|
+
return "png";
|
|
69
|
+
case "gif":
|
|
70
|
+
return "gif";
|
|
71
|
+
case "webp":
|
|
72
|
+
return "webp";
|
|
73
|
+
default: {
|
|
74
|
+
const sanitized = subtype.replace(/[^a-z0-9]/g, "");
|
|
75
|
+
return sanitized || "png";
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Content-addressed file name so re-pasting the same image reuses one artifact. */
|
|
81
|
+
function imageFileName(image: ImageContent): string {
|
|
82
|
+
const hash = Bun.hash(image.data).toString(16);
|
|
83
|
+
return `image-${hash}.${extensionForMime(image.mimeType)}`;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Persist an image under the local root; returns its `local://` URL. */
|
|
87
|
+
async function saveImage(image: ImageContent, localRoot: string): Promise<string> {
|
|
88
|
+
const fileName = imageFileName(image);
|
|
89
|
+
const filePath = path.join(localRoot, fileName);
|
|
90
|
+
// Content-addressed: identical bytes overwrite themselves harmlessly. Bun.write creates parent dirs.
|
|
91
|
+
await Bun.write(filePath, Buffer.from(image.data, "base64"));
|
|
92
|
+
return `local://${fileName}`;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function formatImageBlock(localUrl: string, description: string): string {
|
|
96
|
+
return `<image path="${localUrl}">\n${description}\n</image>`;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Resolve a vision-capable model, mirroring the inspect_image priority
|
|
101
|
+
* (`pi/vision` → `pi/default` → active → first image-capable available), but
|
|
102
|
+
* never returning a text-only model.
|
|
103
|
+
*/
|
|
104
|
+
function resolveVisionModel(deps: DescribeAttachedImagesDeps): Model<Api> | undefined {
|
|
105
|
+
const available = deps.modelRegistry.getAvailable();
|
|
106
|
+
if (available.length === 0) return undefined;
|
|
107
|
+
const preferences = getModelMatchPreferences(deps.settings);
|
|
108
|
+
const resolvePattern = (pattern: string | undefined): Model<Api> | undefined => {
|
|
109
|
+
if (!pattern) return undefined;
|
|
110
|
+
const expanded = expandRoleAlias(pattern, deps.settings);
|
|
111
|
+
const model = resolveModelFromString(expanded, available, preferences, deps.modelRegistry);
|
|
112
|
+
return model?.input.includes("image") ? model : undefined;
|
|
113
|
+
};
|
|
114
|
+
return (
|
|
115
|
+
resolvePattern("pi/vision") ??
|
|
116
|
+
resolvePattern("pi/default") ??
|
|
117
|
+
resolvePattern(deps.activeModelString) ??
|
|
118
|
+
available.find(model => model.input.includes("image"))
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Run one vision-description round-trip; returns trimmed text or `null` on any failure. */
|
|
123
|
+
async function describeImage(
|
|
124
|
+
image: ImageContent,
|
|
125
|
+
visionModel: Model<Api>,
|
|
126
|
+
deps: DescribeAttachedImagesDeps,
|
|
127
|
+
telemetry: AgentTelemetry | undefined,
|
|
128
|
+
signal: AbortSignal | undefined,
|
|
129
|
+
): Promise<string | null> {
|
|
130
|
+
try {
|
|
131
|
+
const response = await instrumentedCompleteSimple(
|
|
132
|
+
visionModel,
|
|
133
|
+
{
|
|
134
|
+
systemPrompt: [prompt.render(describeSystemPrompt)],
|
|
135
|
+
messages: [
|
|
136
|
+
{
|
|
137
|
+
role: "user",
|
|
138
|
+
content: [
|
|
139
|
+
{ type: "image", data: image.data, mimeType: image.mimeType },
|
|
140
|
+
{ type: "text", text: prompt.render(describeUserPrompt) },
|
|
141
|
+
],
|
|
142
|
+
timestamp: Date.now(),
|
|
143
|
+
},
|
|
144
|
+
],
|
|
145
|
+
},
|
|
146
|
+
{ apiKey: deps.modelRegistry.resolver(visionModel, deps.sessionId), signal },
|
|
147
|
+
{ telemetry, oneshotKind: ONESHOT_KIND, completeImpl: deps.completeImpl },
|
|
148
|
+
);
|
|
149
|
+
if (response.stopReason === "error" || response.stopReason === "aborted") {
|
|
150
|
+
logger.warn("image attachment description did not complete", {
|
|
151
|
+
stopReason: response.stopReason,
|
|
152
|
+
model: `${visionModel.provider}/${visionModel.id}`,
|
|
153
|
+
});
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
const text = extractTextContent(response).trim();
|
|
157
|
+
return text.length > 0 ? text : null;
|
|
158
|
+
} catch (err) {
|
|
159
|
+
logger.warn("image attachment description failed", {
|
|
160
|
+
error: toError(err).message,
|
|
161
|
+
model: `${visionModel.provider}/${visionModel.id}`,
|
|
162
|
+
});
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Save each attached image under `local://` and replace it with a descriptive
|
|
169
|
+
* text block. Returns one {@link TextContent} per input image, in order. Never
|
|
170
|
+
* throws for an individual image: a failed description falls back to a note while
|
|
171
|
+
* the saved-path block is still emitted.
|
|
172
|
+
*/
|
|
173
|
+
export async function describeAttachedImagesForTextModel(
|
|
174
|
+
images: readonly ImageContent[],
|
|
175
|
+
deps: DescribeAttachedImagesDeps,
|
|
176
|
+
signal?: AbortSignal,
|
|
177
|
+
): Promise<TextContent[]> {
|
|
178
|
+
const localRoot = resolveLocalRoot(deps.localProtocolOptions);
|
|
179
|
+
const visionModel = resolveVisionModel(deps);
|
|
180
|
+
const apiKey = visionModel ? await deps.modelRegistry.getApiKey(visionModel, deps.sessionId) : undefined;
|
|
181
|
+
const canDescribe = Boolean(visionModel && apiKey);
|
|
182
|
+
const telemetry = resolveTelemetry(deps.telemetryConfig, deps.sessionId);
|
|
183
|
+
|
|
184
|
+
return Promise.all(
|
|
185
|
+
images.map(async (image): Promise<TextContent> => {
|
|
186
|
+
const localUrl = await saveImage(image, localRoot);
|
|
187
|
+
let description: string;
|
|
188
|
+
if (canDescribe && visionModel) {
|
|
189
|
+
description =
|
|
190
|
+
(await describeImage(image, visionModel, deps, telemetry, signal)) ?? DESCRIPTION_UNAVAILABLE_NOTE;
|
|
191
|
+
} else {
|
|
192
|
+
description = NO_VISION_MODEL_NOTE;
|
|
193
|
+
}
|
|
194
|
+
return { type: "text", text: formatImageBlock(localUrl, description) };
|
|
195
|
+
}),
|
|
196
|
+
);
|
|
197
|
+
}
|