@oh-my-pi/pi-coding-agent 17.3.2 → 17.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/{CHANGELOG-fr2awajz.md → CHANGELOG-trcc215s.md} +21 -0
- package/dist/cli.js +3558 -3530
- package/dist/docs-index.generated.txt +1 -1
- package/dist/types/markit/converters/pdf/index.d.ts +2 -1
- package/dist/types/session/turn-recovery.d.ts +1 -1
- package/dist/types/tools/read-pdf.d.ts +15 -0
- package/dist/types/utils/external-editor.d.ts +7 -0
- package/dist/types/utils/markit.d.ts +3 -4
- package/package.json +13 -16
- package/scripts/build-binary.ts +0 -2
- package/scripts/bundle-dist.ts +0 -1
- package/src/cli/read-cli.ts +2 -0
- package/src/markit/NOTICE +8 -8
- package/src/markit/converters/pdf/index.ts +14 -126
- package/src/mcp/client.ts +8 -9
- package/src/modes/controllers/event-controller.ts +57 -11
- package/src/modes/rpc/rpc-client.ts +7 -0
- package/src/modes/utils/hotkeys-markdown.ts +1 -1
- package/src/prompts/system/empty-stop-retry.md +1 -1
- package/src/session/agent-session.ts +8 -1
- package/src/session/turn-recovery.ts +59 -55
- package/src/tools/read-pdf.ts +135 -0
- package/src/tools/read.ts +63 -37
- package/src/utils/external-editor.ts +24 -5
- package/src/utils/markit.ts +6 -44
- package/dist/types/markit/converters/pdf/columns.d.ts +0 -35
- package/dist/types/markit/converters/pdf/extract.d.ts +0 -10
- package/dist/types/markit/converters/pdf/grid.d.ts +0 -25
- package/dist/types/markit/converters/pdf/headers.d.ts +0 -24
- package/dist/types/markit/converters/pdf/render.d.ts +0 -24
- package/dist/types/markit/converters/pdf/types.d.ts +0 -75
- package/dist/types/tools/read-pdf-images.d.ts +0 -12
- package/dist/types/utils/mupdf-wasm-embed.d.ts +0 -1
- package/scripts/embed-mupdf-wasm.ts +0 -67
- package/src/markit/converters/pdf/columns.ts +0 -103
- package/src/markit/converters/pdf/extract.ts +0 -598
- package/src/markit/converters/pdf/grid.ts +0 -780
- package/src/markit/converters/pdf/headers.ts +0 -106
- package/src/markit/converters/pdf/render.ts +0 -501
- package/src/markit/converters/pdf/types.ts +0 -84
- package/src/tools/read-pdf-images.ts +0 -250
- package/src/utils/mupdf-wasm-embed.ts +0 -12
|
@@ -395,7 +395,7 @@ export class TurnRecovery {
|
|
|
395
395
|
}
|
|
396
396
|
|
|
397
397
|
/** Handles empty terminal assistant turns and schedules bounded recovery. */
|
|
398
|
-
handleEmptyAssistantStop(message: AssistantMessage): Promise<
|
|
398
|
+
handleEmptyAssistantStop(message: AssistantMessage): Promise<"continue" | "terminal" | undefined> {
|
|
399
399
|
return this.#handleEmptyAssistantStop(message);
|
|
400
400
|
}
|
|
401
401
|
|
|
@@ -648,24 +648,37 @@ export class TurnRecovery {
|
|
|
648
648
|
return retryErrors;
|
|
649
649
|
}
|
|
650
650
|
|
|
651
|
-
|
|
652
|
-
if (
|
|
651
|
+
#isRecoverableProviderEmptyOutput(message: AssistantMessage): boolean {
|
|
652
|
+
if (message.stopReason !== "error") return false;
|
|
653
|
+
const id = this.#classifyRetryMessage(message);
|
|
654
|
+
if (!AIError.is(id, AIError.Flag.EmptyResponse)) return false;
|
|
655
|
+
return message.content.every(
|
|
656
|
+
block => block.type === "thinking" || (block.type === "text" && !hasNonWhitespace(block.text)),
|
|
657
|
+
);
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
async #handleEmptyAssistantStop(assistantMessage: AssistantMessage): Promise<"continue" | "terminal" | undefined> {
|
|
661
|
+
const providerEmptyOutput = this.#isRecoverableProviderEmptyOutput(assistantMessage);
|
|
662
|
+
if (!isEmptyAssistantStop(assistantMessage) && !providerEmptyOutput) {
|
|
653
663
|
this.#emptyStopRetryCount = 0;
|
|
654
|
-
return
|
|
664
|
+
return undefined;
|
|
655
665
|
}
|
|
656
666
|
|
|
657
667
|
if (this.#acceptTerminalEmptyStopForPrompt && assistantMessage.stopReason === "stop") {
|
|
658
668
|
this.#acceptTerminalEmptyStopForPrompt = false;
|
|
659
669
|
this.#discardAcceptedTerminalEmptyStop(assistantMessage);
|
|
660
670
|
this.#emptyStopRetryCount = 0;
|
|
661
|
-
return
|
|
671
|
+
return undefined;
|
|
662
672
|
}
|
|
663
673
|
|
|
664
674
|
this.#emptyStopRetryCount++;
|
|
665
675
|
if (this.#emptyStopRetryCount > EMPTY_STOP_MAX_RETRIES) {
|
|
666
676
|
const attempts = this.#emptyStopRetryCount - 1;
|
|
667
|
-
const finalError =
|
|
668
|
-
"Assistant returned
|
|
677
|
+
const finalError = providerEmptyOutput
|
|
678
|
+
? "Assistant returned no final output after retry cap; try switching models"
|
|
679
|
+
: "Assistant returned empty stop after retry cap; try switching models or `/shake images` to remove archived frames";
|
|
680
|
+
assistantMessage.errorMessage = finalError;
|
|
681
|
+
if (providerEmptyOutput) assistantMessage.errorId = AIError.create();
|
|
669
682
|
logger.warn(finalError, {
|
|
670
683
|
attempts,
|
|
671
684
|
model: assistantMessage.model,
|
|
@@ -680,12 +693,12 @@ export class TurnRecovery {
|
|
|
680
693
|
this.#clearPendingRetryErrors();
|
|
681
694
|
this.#retryAttempt = 0;
|
|
682
695
|
this.resolveRetry();
|
|
683
|
-
// A
|
|
684
|
-
// can anchor the next prompt at the full failed-request size
|
|
685
|
-
// compaction at the same boundary. Remove every capped
|
|
686
|
-
// orphans still need this for Anthropic
|
|
696
|
+
// A turn with no actionable output carries no transcript value, while its
|
|
697
|
+
// provider usage can anchor the next prompt at the full failed-request size
|
|
698
|
+
// and re-trigger compaction at the same boundary. Remove every capped
|
|
699
|
+
// empty output; toolUse orphans still need this for Anthropic history.
|
|
687
700
|
await this.dropPersistedAssistantTurn(assistantMessage);
|
|
688
|
-
return
|
|
701
|
+
return "terminal";
|
|
689
702
|
}
|
|
690
703
|
this.discardAssistantTurn(assistantMessage);
|
|
691
704
|
this.#host.agent.appendMessage({
|
|
@@ -695,7 +708,7 @@ export class TurnRecovery {
|
|
|
695
708
|
timestamp: Date.now(),
|
|
696
709
|
});
|
|
697
710
|
this.#host.scheduleAgentContinue({ generation: this.#host.promptGeneration() });
|
|
698
|
-
return
|
|
711
|
+
return "continue";
|
|
699
712
|
}
|
|
700
713
|
|
|
701
714
|
#emptyStopRetryReminder(): string {
|
|
@@ -1019,52 +1032,39 @@ export class TurnRecovery {
|
|
|
1019
1032
|
if (this.#isUsagePreflightBlocked(message)) return false;
|
|
1020
1033
|
|
|
1021
1034
|
const id = this.#classifyRetryMessage(message);
|
|
1022
|
-
// Context overflow is handled by compaction, not retry
|
|
1035
|
+
// Context overflow is handled by compaction, not retry.
|
|
1023
1036
|
const contextWindow = this.#host.model()?.contextWindow ?? 0;
|
|
1024
1037
|
if (AIError.isContextOverflow(message, contextWindow)) return false;
|
|
1025
1038
|
|
|
1026
1039
|
// Credential rotation and classifier fallbacks are safe only before
|
|
1027
1040
|
// committed text, images, tool calls, or server tools. Thinking-only
|
|
1028
|
-
// output remains replay-safe.
|
|
1029
|
-
//
|
|
1030
|
-
//
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1041
|
+
// output remains replay-safe. A classifier refusal or malformed-function
|
|
1042
|
+
// response may also be replayed when every emitted tool call is paired
|
|
1043
|
+
// with positive proof that it never executed.
|
|
1044
|
+
const replaySafeUnexecutedTools =
|
|
1045
|
+
(this.isClassifierRefusal(message) || AIError.is(id, AIError.Flag.MalformedFunctionCall)) &&
|
|
1046
|
+
this.#unexecutedToolCallsReplaySafe(message);
|
|
1047
|
+
if (this.#hasReplayUnsafeOutput(message) && !replaySafeUnexecutedTools) return false;
|
|
1034
1048
|
if (AIError.is(id, AIError.Flag.AccountPolicy) || this.isClassifierRefusal(message)) return true;
|
|
1035
1049
|
return AIError.retriable(id);
|
|
1036
1050
|
}
|
|
1037
1051
|
|
|
1038
1052
|
/**
|
|
1039
|
-
* True when
|
|
1040
|
-
*
|
|
1053
|
+
* True when every emitted tool call provably never executed and no other
|
|
1054
|
+
* replay-unsafe output exists. The caller restricts this exception to
|
|
1055
|
+
* classifier refusals and malformed-function responses.
|
|
1041
1056
|
*
|
|
1042
|
-
*
|
|
1043
|
-
*
|
|
1044
|
-
*
|
|
1045
|
-
*
|
|
1046
|
-
* likely have served just ended the turn.
|
|
1057
|
+
* Gemini can report `MALFORMED_FUNCTION_CALL` after streaming an earlier,
|
|
1058
|
+
* well-formed call. Anthropic classifiers can likewise refuse after a call.
|
|
1059
|
+
* The agent loop pairs each emitted-but-unrun call with a synthetic
|
|
1060
|
+
* `executed: false` result, which proves `tool.execute()` never ran.
|
|
1047
1061
|
*
|
|
1048
|
-
*
|
|
1049
|
-
*
|
|
1050
|
-
*
|
|
1051
|
-
*
|
|
1052
|
-
* lifted only when ALL of the following hold, and any uncertainty (assistant
|
|
1053
|
-
* message missing from state, a call with no result, a non-synthetic result, an
|
|
1054
|
-
* `executed` that is not exactly `false`) keeps it in place:
|
|
1055
|
-
*
|
|
1056
|
-
* - the stop is a classifier refusal/sensitivity stop;
|
|
1057
|
-
* - the only replay-unsafe blocks are tool calls — an `image`, an
|
|
1058
|
-
* `anthropicServerTool`, or committed non-whitespace text has already rendered
|
|
1059
|
-
* or has side effects, so replaying would duplicate it;
|
|
1060
|
-
* - at least one tool call was emitted (otherwise the plain refusal path already
|
|
1061
|
-
* handles it);
|
|
1062
|
-
* - every emitted call id has a result after the assistant message in state, and
|
|
1063
|
-
* every such result is synthetic with `executed === false`.
|
|
1062
|
+
* Any uncertainty keeps the replay veto in place: the assistant must exist
|
|
1063
|
+
* in state, every call must have a later synthetic result, every result must
|
|
1064
|
+
* say `executed === false`, and the turn must contain no image, server tool,
|
|
1065
|
+
* or committed non-whitespace text.
|
|
1064
1066
|
*/
|
|
1065
|
-
#
|
|
1066
|
-
if (!this.isClassifierRefusal(message)) return false;
|
|
1067
|
-
|
|
1067
|
+
#unexecutedToolCallsReplaySafe(message: AssistantMessage): boolean {
|
|
1068
1068
|
const emittedToolCallIds = new Set<string>();
|
|
1069
1069
|
for (const block of message.content) {
|
|
1070
1070
|
if (block.type === "toolCall") {
|
|
@@ -1076,7 +1076,7 @@ export class TurnRecovery {
|
|
|
1076
1076
|
}
|
|
1077
1077
|
if (emittedToolCallIds.size === 0) return false;
|
|
1078
1078
|
|
|
1079
|
-
// The
|
|
1079
|
+
// The errored assistant message is NOT the tail of state: the agent loop
|
|
1080
1080
|
// appends the synthetic results after it before the turn ends, so locate it
|
|
1081
1081
|
// by walking backwards exactly as `classifyResolvedInterruptedToolTurn` does.
|
|
1082
1082
|
const messages = this.#host.agent.state.messages;
|
|
@@ -1803,6 +1803,10 @@ export class TurnRecovery {
|
|
|
1803
1803
|
|
|
1804
1804
|
const errorMessage = message.errorMessage || "Unknown error";
|
|
1805
1805
|
const id = this.#classifyRetryMessage(message);
|
|
1806
|
+
const preserveFailedTurn =
|
|
1807
|
+
options?.preserveFailedTurn === true ||
|
|
1808
|
+
((classifierRefusal || AIError.is(id, AIError.Flag.MalformedFunctionCall)) &&
|
|
1809
|
+
this.#unexecutedToolCallsReplaySafe(message));
|
|
1806
1810
|
const rateLimitReason = parseRateLimitReason(errorMessage);
|
|
1807
1811
|
const staleOpenAIResponsesReplayError = AIError.is(id, AIError.Flag.StaleResponsesItem);
|
|
1808
1812
|
const accountPolicyDenial = AIError.is(id, AIError.Flag.AccountPolicy);
|
|
@@ -2013,9 +2017,10 @@ export class TurnRecovery {
|
|
|
2013
2017
|
errorId: message.errorId,
|
|
2014
2018
|
});
|
|
2015
2019
|
|
|
2016
|
-
// Resolved stream-stall tools
|
|
2017
|
-
//
|
|
2018
|
-
|
|
2020
|
+
// Resolved stream-stall tools and proven-unexecuted malformed/refused
|
|
2021
|
+
// calls keep their assistant/result pair. Continuation then sees explicit
|
|
2022
|
+
// synthetic results and cannot repeat a side effect.
|
|
2023
|
+
if (!preserveFailedTurn) {
|
|
2019
2024
|
this.removeAssistantMessageFromActiveContext(message, "auto-retry");
|
|
2020
2025
|
}
|
|
2021
2026
|
|
|
@@ -2058,11 +2063,10 @@ export class TurnRecovery {
|
|
|
2058
2063
|
// rejects any assistant tail, so a missed removal fails the scheduled
|
|
2059
2064
|
// retry locally before a provider request is ever made. Re-check the
|
|
2060
2065
|
// tail after the backoff (covering rebuilds during the sleep too) and
|
|
2061
|
-
// strip a still-failed assistant tail by position. Never
|
|
2062
|
-
//
|
|
2063
|
-
//
|
|
2064
|
-
|
|
2065
|
-
if (!options?.preserveFailedTurn && this.#host.promptGeneration() === generation) {
|
|
2066
|
+
// strip a still-failed assistant tail by position. Never when preserving
|
|
2067
|
+
// the failed turn — the kept turn ends in synthetic tool results that
|
|
2068
|
+
// continue() accepts — and never once a newer prompt owns the session.
|
|
2069
|
+
if (!preserveFailedTurn && this.#host.promptGeneration() === generation) {
|
|
2066
2070
|
this.#stripFailedAssistantTail();
|
|
2067
2071
|
}
|
|
2068
2072
|
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { pathToFileURL } from "node:url";
|
|
2
|
+
import { untilAborted } from "@oh-my-pi/pi-utils";
|
|
3
|
+
import type { ToolSession } from "../sdk";
|
|
4
|
+
import type { BrowserHandle } from "./browser/registry";
|
|
5
|
+
import type { ScreenshotResult } from "./browser/tab-protocol";
|
|
6
|
+
import { ToolAbortError, ToolError } from "./tool-errors";
|
|
7
|
+
|
|
8
|
+
const PDF_IMAGE_MEMBER_RE = /^(.*\.pdf):(.*)$/i;
|
|
9
|
+
const PDF_PAGE_MEMBER_RE = /^(?:p|page[-_]?)(\d+)(?:[-_].*)?\.png$/i;
|
|
10
|
+
const PDF_RENDER_TIMEOUT_MS = 30_000;
|
|
11
|
+
|
|
12
|
+
// Chromium's PDF plugin paints in an out-of-process frame after navigation has
|
|
13
|
+
// completed. Wait for document dimensions, then cross compositor boundaries
|
|
14
|
+
// before capturing; otherwise the screenshot can contain only the viewer shell.
|
|
15
|
+
const PDF_SCREENSHOT_CODE = `
|
|
16
|
+
let viewerFrame;
|
|
17
|
+
await wait(async () => {
|
|
18
|
+
for (const frame of page.frames()) {
|
|
19
|
+
try {
|
|
20
|
+
const loaded = await frame.evaluate(() => {
|
|
21
|
+
const viewer = document.querySelector("pdf-viewer");
|
|
22
|
+
const toolbar = viewer?.shadowRoot?.querySelector("viewer-toolbar");
|
|
23
|
+
const pageLength = toolbar
|
|
24
|
+
?.shadowRoot?.querySelector("viewer-page-selector")
|
|
25
|
+
?.shadowRoot?.querySelector("#pagelength")
|
|
26
|
+
?.textContent;
|
|
27
|
+
if (Number(pageLength) > 0 && !toolbar?.hasAttribute("loading_")) return true;
|
|
28
|
+
|
|
29
|
+
const plugin = document.querySelector('embed[type="application/x-google-chrome-pdf"]');
|
|
30
|
+
const sizer = document.querySelector("#sizer");
|
|
31
|
+
return plugin !== null && sizer !== null && sizer.clientWidth > 0 && sizer.clientHeight > 0;
|
|
32
|
+
});
|
|
33
|
+
if (loaded) {
|
|
34
|
+
viewerFrame = frame;
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
} catch {}
|
|
38
|
+
}
|
|
39
|
+
return false;
|
|
40
|
+
});
|
|
41
|
+
await page.screenshot({ type: "png" });
|
|
42
|
+
await viewerFrame.evaluate(() => {
|
|
43
|
+
const { promise, resolve } = Promise.withResolvers();
|
|
44
|
+
requestAnimationFrame(() =>
|
|
45
|
+
requestAnimationFrame(() =>
|
|
46
|
+
requestAnimationFrame(() => requestAnimationFrame(resolve)),
|
|
47
|
+
),
|
|
48
|
+
);
|
|
49
|
+
return promise;
|
|
50
|
+
});
|
|
51
|
+
return await tab.screenshot({ fullPage: true, silent: true });
|
|
52
|
+
`;
|
|
53
|
+
|
|
54
|
+
/** A legacy PDF image-member path interpreted as a page screenshot request. */
|
|
55
|
+
export interface PdfImageReadTarget {
|
|
56
|
+
/** PDF path before the member delimiter. */
|
|
57
|
+
pdfPath: string;
|
|
58
|
+
/** Original member text after the delimiter. */
|
|
59
|
+
member: string;
|
|
60
|
+
/** One-indexed page inferred from names such as `p2-img0.png`; defaults to page 1. */
|
|
61
|
+
page: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Parse a former PDF image-member path as a Chromium page screenshot request. */
|
|
65
|
+
export function splitPdfImageReadPath(readPath: string): PdfImageReadTarget | null {
|
|
66
|
+
const match = PDF_IMAGE_MEMBER_RE.exec(readPath);
|
|
67
|
+
const pdfPath = match?.[1];
|
|
68
|
+
const member = match?.[2];
|
|
69
|
+
if (!pdfPath || member === undefined) return null;
|
|
70
|
+
const pageText = PDF_PAGE_MEMBER_RE.exec(member)?.[1];
|
|
71
|
+
const parsedPage = pageText === undefined ? 1 : Number(pageText);
|
|
72
|
+
const page = Number.isSafeInteger(parsedPage) && parsedPage > 0 ? parsedPage : 1;
|
|
73
|
+
return { pdfPath, member, page };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Render one PDF page through the browser tool's shared headless Chromium. */
|
|
77
|
+
export async function renderPdfPageScreenshot(
|
|
78
|
+
session: ToolSession,
|
|
79
|
+
absolutePdfPath: string,
|
|
80
|
+
page: number,
|
|
81
|
+
signal?: AbortSignal,
|
|
82
|
+
): Promise<ScreenshotResult> {
|
|
83
|
+
const [{ acquireBrowser, holdBrowser, releaseBrowser }, { acquireTab, releaseTab, runInTab }] = await Promise.all([
|
|
84
|
+
import("./browser/registry"),
|
|
85
|
+
import("./browser/tab-supervisor"),
|
|
86
|
+
]);
|
|
87
|
+
const timeoutSignal = AbortSignal.timeout(PDF_RENDER_TIMEOUT_MS);
|
|
88
|
+
const renderSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
|
89
|
+
const tabName = `read-pdf-${Bun.randomUUIDv7()}`;
|
|
90
|
+
const url = pathToFileURL(absolutePdfPath);
|
|
91
|
+
url.hash = `page=${page}&toolbar=0&navpanes=0&view=Fit`;
|
|
92
|
+
|
|
93
|
+
let browserLease = false;
|
|
94
|
+
let tabOpened = false;
|
|
95
|
+
let browser: BrowserHandle | undefined;
|
|
96
|
+
try {
|
|
97
|
+
const acquiredBrowser = await untilAborted(renderSignal, () =>
|
|
98
|
+
acquireBrowser({ kind: "headless", headless: true }, { cwd: session.cwd, signal: renderSignal }),
|
|
99
|
+
);
|
|
100
|
+
browser = acquiredBrowser;
|
|
101
|
+
holdBrowser(acquiredBrowser);
|
|
102
|
+
browserLease = true;
|
|
103
|
+
await untilAborted(renderSignal, () =>
|
|
104
|
+
acquireTab(tabName, acquiredBrowser, {
|
|
105
|
+
url: url.href,
|
|
106
|
+
waitUntil: "load",
|
|
107
|
+
timeoutMs: PDF_RENDER_TIMEOUT_MS,
|
|
108
|
+
signal: renderSignal,
|
|
109
|
+
ownerSessionId: session.getSessionId?.() ?? undefined,
|
|
110
|
+
}),
|
|
111
|
+
);
|
|
112
|
+
tabOpened = true;
|
|
113
|
+
await releaseBrowser(acquiredBrowser, { kill: false });
|
|
114
|
+
browserLease = false;
|
|
115
|
+
|
|
116
|
+
const result = await runInTab(tabName, {
|
|
117
|
+
code: PDF_SCREENSHOT_CODE,
|
|
118
|
+
timeoutMs: PDF_RENDER_TIMEOUT_MS,
|
|
119
|
+
signal: renderSignal,
|
|
120
|
+
session,
|
|
121
|
+
});
|
|
122
|
+
const screenshot = result.screenshots.at(-1);
|
|
123
|
+
if (!screenshot) throw new ToolError(`Chromium did not capture PDF page ${page}.`);
|
|
124
|
+
return screenshot;
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (signal?.aborted) throw new ToolAbortError();
|
|
127
|
+
if (timeoutSignal.aborted) {
|
|
128
|
+
throw new ToolError(`Timed out rendering PDF page ${page} in Chromium.`);
|
|
129
|
+
}
|
|
130
|
+
throw error;
|
|
131
|
+
} finally {
|
|
132
|
+
if (tabOpened) await releaseTab(tabName, { kill: false });
|
|
133
|
+
if (browserLease && browser) await releaseBrowser(browser, { kill: false });
|
|
134
|
+
}
|
|
135
|
+
}
|
package/src/tools/read.ts
CHANGED
|
@@ -100,7 +100,7 @@ import {
|
|
|
100
100
|
isRemoteMountPath,
|
|
101
101
|
type SuffixMatchCache,
|
|
102
102
|
} from "./read-path-resolution";
|
|
103
|
-
import {
|
|
103
|
+
import { type PdfImageReadTarget, renderPdfPageScreenshot, splitPdfImageReadPath } from "./read-pdf";
|
|
104
104
|
import { isMultiRange, isRawSelector, type ParsedSelector, parseSel, selToOffsetLimit } from "./read-selector";
|
|
105
105
|
import { readSqlite, resolveSqliteReadPath } from "./read-sqlite";
|
|
106
106
|
import { isProseSummaryPath, renderSummary, routeReadThroughBridge, trySummarize } from "./read-summary";
|
|
@@ -398,8 +398,13 @@ type ReadParams = ReadToolInput;
|
|
|
398
398
|
*/
|
|
399
399
|
export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
400
400
|
readonly name = "read";
|
|
401
|
-
readonly approval = (args: unknown): ToolTier =>
|
|
402
|
-
|
|
401
|
+
readonly approval = (args: unknown): ToolTier => {
|
|
402
|
+
let readPath = "";
|
|
403
|
+
if (args && typeof args === "object" && "path" in args) readPath = String(args.path ?? "");
|
|
404
|
+
if (pathTargetsSsh(readPath)) return "exec";
|
|
405
|
+
const target = splitPathAndSel(readPath);
|
|
406
|
+
return target.sel === undefined && splitPdfImageReadPath(readPath) ? "exec" : "read";
|
|
407
|
+
};
|
|
403
408
|
readonly label = "Read";
|
|
404
409
|
readonly loadMode = "essential";
|
|
405
410
|
description: string;
|
|
@@ -546,6 +551,40 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
546
551
|
return toolResult<ReadToolDetails>({ notes, displayReadTargets }).content(content).done();
|
|
547
552
|
}
|
|
548
553
|
|
|
554
|
+
async #readPdfPageScreenshot(options: {
|
|
555
|
+
readPath: string;
|
|
556
|
+
absolutePdfPath: string;
|
|
557
|
+
page: number;
|
|
558
|
+
pdfFileSize: number;
|
|
559
|
+
suffixResolution?: { from: string; to: string };
|
|
560
|
+
signal?: AbortSignal;
|
|
561
|
+
}): Promise<AgentToolResult<ReadToolDetails>> {
|
|
562
|
+
const { readPath, absolutePdfPath, page, pdfFileSize, suffixResolution, signal } = options;
|
|
563
|
+
const screenshot = await renderPdfPageScreenshot(this.session, absolutePdfPath, page, signal);
|
|
564
|
+
const screenshotFile = Bun.file(screenshot.dest);
|
|
565
|
+
const screenshotMetadata = await readImageMetadata(screenshot.dest);
|
|
566
|
+
const loaded = await this.#loadImageContent({
|
|
567
|
+
readPath,
|
|
568
|
+
absolutePath: screenshot.dest,
|
|
569
|
+
mimeType: screenshot.mimeType,
|
|
570
|
+
imageMetadata: screenshotMetadata,
|
|
571
|
+
fileSize: screenshotFile.size,
|
|
572
|
+
});
|
|
573
|
+
if (suffixResolution) {
|
|
574
|
+
const firstText = loaded.content.find((entry): entry is TextContent => entry.type === "text");
|
|
575
|
+
if (firstText) firstText.text = prependSuffixResolutionNotice(firstText.text, suffixResolution);
|
|
576
|
+
}
|
|
577
|
+
const image = loaded.content.find((entry): entry is ImageContent => entry.type === "image");
|
|
578
|
+
const details: ReadToolDetails = {
|
|
579
|
+
...loaded.details,
|
|
580
|
+
resolvedPath: absolutePdfPath,
|
|
581
|
+
contentType: image?.mimeType ?? screenshot.mimeType,
|
|
582
|
+
fileSize: pdfFileSize,
|
|
583
|
+
suffixResolution,
|
|
584
|
+
};
|
|
585
|
+
return toolResult(details).content(loaded.content).sourcePath(loaded.sourcePath).done();
|
|
586
|
+
}
|
|
587
|
+
|
|
549
588
|
/**
|
|
550
589
|
* Build content blocks for an on-disk image file: an `inspect_image`
|
|
551
590
|
* metadata note when inspection is active, otherwise the decoded image
|
|
@@ -892,7 +931,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
892
931
|
|
|
893
932
|
// Prefer a literal filesystem match over selector interpretation so real
|
|
894
933
|
// POSIX filenames containing selector-looking suffixes win over structured
|
|
895
|
-
// archive / sqlite /
|
|
934
|
+
// archive / sqlite / unsupported PDF-image dispatch. A selector promoted from local://
|
|
896
935
|
// remains separate so it cannot be mistaken for part of the resolved path.
|
|
897
936
|
const literalSplit =
|
|
898
937
|
promotedSelector === undefined
|
|
@@ -903,6 +942,8 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
903
942
|
? readPath.includes(":") && (await probeLiteralPathExists(readPath, this.session.cwd)) !== "missing"
|
|
904
943
|
: literalSplit.sel === undefined && splitPathAndSel(readPath).sel !== undefined;
|
|
905
944
|
|
|
945
|
+
let pdfImageRead: PdfImageReadTarget | null = null;
|
|
946
|
+
|
|
906
947
|
if (!rawPathIsLiteral) {
|
|
907
948
|
const archivePath = await resolveArchiveReadPath(this.session, readPath, suffixCache, signal);
|
|
908
949
|
if (archivePath) {
|
|
@@ -925,39 +966,14 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
925
966
|
return readSqlite(sqlitePath, signal);
|
|
926
967
|
}
|
|
927
968
|
|
|
928
|
-
const
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
const stat = await Bun.file(absolutePdfPath).stat();
|
|
934
|
-
if (stat.isDirectory())
|
|
935
|
-
throw new ToolError(`Path '${pdfImageMemberPath.pdfPath}' is a directory, not a PDF file`);
|
|
936
|
-
} catch (error) {
|
|
937
|
-
if (!isNotFoundError(error) || isRemoteMountPath(absolutePdfPath)) throw error;
|
|
938
|
-
const suffixMatch = await findSuffixMatchCached(
|
|
939
|
-
this.session,
|
|
940
|
-
suffixCache,
|
|
941
|
-
pdfImageMemberPath.pdfPath,
|
|
942
|
-
signal,
|
|
943
|
-
);
|
|
944
|
-
if (!suffixMatch) throw new ToolError(`Path '${pdfImageMemberPath.pdfPath}' not found`);
|
|
945
|
-
absolutePdfPath = suffixMatch.absolutePath;
|
|
946
|
-
suffixResolution = { from: pdfImageMemberPath.pdfPath, to: suffixMatch.displayPath };
|
|
947
|
-
}
|
|
948
|
-
return readPdfImageMember(
|
|
949
|
-
this.session,
|
|
950
|
-
this.#autoResizeImages,
|
|
951
|
-
absolutePdfPath,
|
|
952
|
-
pdfImageMemberPath.pdfPath,
|
|
953
|
-
pdfImageMemberPath.member,
|
|
954
|
-
suffixResolution,
|
|
955
|
-
signal,
|
|
956
|
-
);
|
|
957
|
-
}
|
|
969
|
+
const pdfCandidate = literalSplit.sel === undefined ? splitPdfImageReadPath(readPath) : null;
|
|
970
|
+
pdfImageRead =
|
|
971
|
+
pdfCandidate && (await probeLiteralPathExists(readPath, this.session.cwd)) === "missing"
|
|
972
|
+
? pdfCandidate
|
|
973
|
+
: null;
|
|
958
974
|
}
|
|
959
975
|
|
|
960
|
-
const localTarget = literalSplit;
|
|
976
|
+
const localTarget = pdfImageRead ? { path: pdfImageRead.pdfPath, sel: undefined } : literalSplit;
|
|
961
977
|
const localReadPath = localTarget.path;
|
|
962
978
|
const parsed = parseSel(localTarget.sel);
|
|
963
979
|
|
|
@@ -1036,6 +1052,17 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
1036
1052
|
return this.#readFileConflicts(absolutePath, suffixResolution, signal);
|
|
1037
1053
|
}
|
|
1038
1054
|
|
|
1055
|
+
if (pdfImageRead) {
|
|
1056
|
+
return this.#readPdfPageScreenshot({
|
|
1057
|
+
readPath,
|
|
1058
|
+
absolutePdfPath: absolutePath,
|
|
1059
|
+
page: pdfImageRead.page,
|
|
1060
|
+
pdfFileSize: fileSize,
|
|
1061
|
+
suffixResolution,
|
|
1062
|
+
signal,
|
|
1063
|
+
});
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1039
1066
|
const imageMetadata = await readImageMetadata(absolutePath);
|
|
1040
1067
|
const mimeType = imageMetadata?.mimeType;
|
|
1041
1068
|
const ext = path.extname(absolutePath).toLowerCase();
|
|
@@ -1102,8 +1129,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
|
|
1102
1129
|
// Convert document via markit.
|
|
1103
1130
|
const result = await convertFileWithMarkit(absolutePath, signal);
|
|
1104
1131
|
if (result.ok) {
|
|
1105
|
-
const renderedContent =
|
|
1106
|
-
ext === ".pdf" ? rewritePdfImagePlaceholders(result.content, resolvedDisplayPath) : result.content;
|
|
1132
|
+
const renderedContent = result.content;
|
|
1107
1133
|
// Route the converted markdown through the in-memory text builder
|
|
1108
1134
|
// so line-range selectors (`file.pdf:50-100`, `:5-16,40-80`) and
|
|
1109
1135
|
// raw mode apply against the converted output. Without this,
|
|
@@ -33,6 +33,27 @@ export interface OpenInEditorOptions {
|
|
|
33
33
|
trimTrailingNewline?: boolean;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
+
/** Subprocess argv and Windows quoting mode used to launch an external editor. */
|
|
37
|
+
export interface EditorSpawnCommand {
|
|
38
|
+
cmd: string[];
|
|
39
|
+
windowsVerbatimArguments: boolean;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Resolves shell argv without letting the host runtime re-quote the editor command. */
|
|
43
|
+
export function resolveEditorSpawnCommand(
|
|
44
|
+
editorCmd: string,
|
|
45
|
+
tmpFile: string,
|
|
46
|
+
platform: NodeJS.Platform = process.platform,
|
|
47
|
+
): EditorSpawnCommand {
|
|
48
|
+
const windows = platform === "win32";
|
|
49
|
+
// cmd.exe strips the outer /s /c quote pair; Bun must pass the embedded
|
|
50
|
+
// editor/path quotes verbatim instead of applying argv escaping to them.
|
|
51
|
+
const cmd = windows
|
|
52
|
+
? ["cmd.exe", "/d", "/s", "/c", `"${editorCmd} "${tmpFile}""`]
|
|
53
|
+
: [$which("sh") ?? "sh", "-c", `${editorCmd} "$1"`, "sh", tmpFile];
|
|
54
|
+
return { cmd, windowsVerbatimArguments: windows };
|
|
55
|
+
}
|
|
56
|
+
|
|
36
57
|
/**
|
|
37
58
|
* Opens `content` in the user's external editor and returns the edited text.
|
|
38
59
|
* Returns `null` if the editor exits with a non-zero code.
|
|
@@ -50,15 +71,13 @@ export async function openInEditor(
|
|
|
50
71
|
try {
|
|
51
72
|
await Bun.write(tmpFile, content);
|
|
52
73
|
|
|
74
|
+
const spawnCommand = resolveEditorSpawnCommand(editorCmd, tmpFile);
|
|
53
75
|
const [stdin, stdout, stderr] = options?.stdio ?? ["inherit", "inherit", "inherit"];
|
|
54
|
-
const cmd
|
|
55
|
-
process.platform === "win32"
|
|
56
|
-
? ["cmd", "/c", `${editorCmd} "${tmpFile}"`]
|
|
57
|
-
: [$which("sh") ?? "sh", "-c", `${editorCmd} "$1"`, "sh", tmpFile];
|
|
58
|
-
const child = Bun.spawn(cmd, {
|
|
76
|
+
const child = Bun.spawn(spawnCommand.cmd, {
|
|
59
77
|
stdin,
|
|
60
78
|
stdout,
|
|
61
79
|
stderr,
|
|
80
|
+
windowsVerbatimArguments: spawnCommand.windowsVerbatimArguments,
|
|
62
81
|
});
|
|
63
82
|
const exitCode = await child.exited;
|
|
64
83
|
if (exitCode === 0) {
|
package/src/utils/markit.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as path from "node:path";
|
|
2
|
-
import {
|
|
2
|
+
import { untilAborted } from "@oh-my-pi/pi-utils";
|
|
3
3
|
import type { ConversionResult, Markit, StreamInfo } from "../markit";
|
|
4
4
|
import { ToolAbortError } from "../tools/tool-errors";
|
|
5
5
|
import {
|
|
@@ -8,7 +8,6 @@ import {
|
|
|
8
8
|
readMarkitConversionCache,
|
|
9
9
|
writeMarkitConversionCache,
|
|
10
10
|
} from "./markit-cache";
|
|
11
|
-
import { loadEmbeddedMupdfWasm } from "./mupdf-wasm-embed";
|
|
12
11
|
|
|
13
12
|
/**
|
|
14
13
|
* File extensions markit can actually convert to markdown — one per registered
|
|
@@ -31,53 +30,16 @@ export interface MarkitConversionResult {
|
|
|
31
30
|
|
|
32
31
|
export interface MarkitFileConversionOptions {
|
|
33
32
|
/**
|
|
34
|
-
* Directory
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
* placeholder comment instead.
|
|
33
|
+
* Directory converters may use for extracted image or diagram files. Since
|
|
34
|
+
* those files are conversion side effects, conversions using this option
|
|
35
|
+
* bypass the markdown cache.
|
|
38
36
|
*/
|
|
39
37
|
imageDir?: string;
|
|
40
38
|
}
|
|
41
39
|
|
|
42
|
-
interface MuPdfWasmModuleConfig {
|
|
43
|
-
print?: (...values: unknown[]) => void;
|
|
44
|
-
printErr?: (...values: unknown[]) => void;
|
|
45
|
-
wasmBinary?: Uint8Array;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function logMuPdfWasmOutput(stream: "stdout" | "stderr", values: unknown[]): void {
|
|
49
|
-
const message = values.length === 1 && typeof values[0] === "string" ? values[0] : values.map(String).join(" ");
|
|
50
|
-
logger.debug("mupdf wasm output", { stream, message });
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
// `$libmupdf_wasm_Module` is declared globally (as `any`) by the mupdf package.
|
|
54
|
-
// Install print hooks before the WASM module initializes so its stdout/stderr
|
|
55
|
-
// route to the file logger instead of corrupting the TUI.
|
|
56
|
-
function installMuPdfWasmLogger(): void {
|
|
57
|
-
const moduleConfig: MuPdfWasmModuleConfig = globalThis.$libmupdf_wasm_Module ?? {};
|
|
58
|
-
moduleConfig.print = (...values: unknown[]) => logMuPdfWasmOutput("stdout", values);
|
|
59
|
-
moduleConfig.printErr = (...values: unknown[]) => logMuPdfWasmOutput("stderr", values);
|
|
60
|
-
globalThis.$libmupdf_wasm_Module = moduleConfig;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
// Hand the WASM module its bytes directly when the compiled binary embedded them
|
|
64
|
-
// (scripts/embed-mupdf-wasm.ts); a single-file binary has no node_modules for
|
|
65
|
-
// mupdf to read `mupdf-wasm.wasm` from. Source/npm builds get undefined here and
|
|
66
|
-
// mupdf loads its own wasm. Must run before the mupdf module evaluates.
|
|
67
|
-
function installEmbeddedMupdfWasm(): void {
|
|
68
|
-
const wasmBinary = loadEmbeddedMupdfWasm();
|
|
69
|
-
if (!wasmBinary) return;
|
|
70
|
-
const moduleConfig: MuPdfWasmModuleConfig = globalThis.$libmupdf_wasm_Module ?? {};
|
|
71
|
-
moduleConfig.wasmBinary = wasmBinary;
|
|
72
|
-
globalThis.$libmupdf_wasm_Module = moduleConfig;
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
installMuPdfWasmLogger();
|
|
76
|
-
|
|
77
40
|
let markit: () => Markit | Promise<Markit> = async () => {
|
|
78
|
-
// Lazy: keep the document engine
|
|
79
|
-
//
|
|
80
|
-
installEmbeddedMupdfWasm();
|
|
41
|
+
// Lazy: keep the document engine off the startup import graph — it loads
|
|
42
|
+
// only when a document is first converted.
|
|
81
43
|
const promise = import("../markit").then(({ Markit }) => {
|
|
82
44
|
const instance = new Markit();
|
|
83
45
|
markit = () => instance;
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Multi-column layout detection and text box reordering.
|
|
3
|
-
*
|
|
4
|
-
* Many PDFs (legal documents, datasheets, academic papers) use two-column
|
|
5
|
-
* layouts. Without column detection, text boxes are ordered by Y position
|
|
6
|
-
* only, interleaving left and right column content.
|
|
7
|
-
*
|
|
8
|
-
* Algorithm:
|
|
9
|
-
* 1. Collect left edges of all text boxes on the page
|
|
10
|
-
* 2. Find the largest horizontal gap between consecutive left edges
|
|
11
|
-
* 3. If gap > MIN_GAP_RATIO of the text width and both sides have
|
|
12
|
-
* enough boxes → multi-column detected
|
|
13
|
-
* 4. Assign each text box to a column based on its center X
|
|
14
|
-
* 5. Return columns in reading order (left-to-right, top-to-bottom)
|
|
15
|
-
*
|
|
16
|
-
* This only detects the column structure. The caller is responsible for
|
|
17
|
-
* processing each column's text boxes independently (table detection,
|
|
18
|
-
* rendering, etc.).
|
|
19
|
-
*/
|
|
20
|
-
import type { TextBox } from "./types.js";
|
|
21
|
-
export interface ColumnLayout {
|
|
22
|
-
/** Number of columns detected (1 = single column, 2+ = multi-column). */
|
|
23
|
-
columnCount: number;
|
|
24
|
-
/** Text boxes grouped by column, in reading order (left to right). */
|
|
25
|
-
columns: TextBox[][];
|
|
26
|
-
/** X positions of column boundaries (between columns). */
|
|
27
|
-
boundaries: number[];
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* Detect column layout and return text boxes grouped by column.
|
|
31
|
-
*
|
|
32
|
-
* For single-column pages, returns all boxes in one group.
|
|
33
|
-
* For multi-column pages, returns boxes split by column in reading order.
|
|
34
|
-
*/
|
|
35
|
-
export declare function detectColumns(textBoxes: TextBox[]): ColumnLayout;
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { ImageRegion, PageContent } from "./types.js";
|
|
2
|
-
/**
|
|
3
|
-
* Render an image region from a PDF page as a PNG buffer.
|
|
4
|
-
* Uses mupdf's DrawDevice to render just the cropped area at 2x resolution.
|
|
5
|
-
*/
|
|
6
|
-
export declare function renderImageRegion(input: Uint8Array, region: ImageRegion): Promise<Uint8Array>;
|
|
7
|
-
/**
|
|
8
|
-
* Extract text boxes and vector segments from all pages of a PDF buffer.
|
|
9
|
-
*/
|
|
10
|
-
export declare function extractPages(input: Uint8Array): Promise<PageContent[]>;
|