newmark-agent 0.5.11 → 0.5.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/conversation-utility-host.bundle.cjs +75356 -26589
- package/dist/core/agent.d.ts +43 -18
- package/dist/core/agent.js +321 -69
- package/dist/core/browserUse.d.ts +7 -0
- package/dist/core/browserUse.js +24 -7
- package/dist/core/conversationKernel.d.ts +4 -0
- package/dist/core/conversationKernel.js +65 -3
- package/dist/core/displayImages.d.ts +10 -0
- package/dist/core/displayImages.js +31 -8
- package/dist/core/electronBrowserUseHost.d.ts +4 -0
- package/dist/core/electronBrowserUseHost.js +32 -6
- package/dist/core/electronUtilityAgentClient.d.ts +1 -0
- package/dist/core/electronUtilityAgentClient.js +70 -2
- package/dist/core/runtimeDiagnostics.d.ts +18 -0
- package/dist/core/runtimeDiagnostics.js +89 -0
- package/dist/core/runtimeLifecycle.d.ts +7 -1
- package/dist/core/runtimeLifecycle.js +29 -0
- package/dist/core/searchMcpPool.d.ts +80 -0
- package/dist/core/searchMcpPool.js +419 -0
- package/dist/main.js +72 -1
- package/dist/server.js +35 -0
- package/dist/tools/index.d.ts +17 -1
- package/dist/tools/index.js +203 -64
- package/dist/tui/src/data.js +2 -2
- package/dist/ui/index.html +208 -26
- package/dist/wsl-agent-host.bundle.cjs +75333 -26566
- package/package.json +11 -2
package/dist/server.js
CHANGED
|
@@ -56,6 +56,7 @@ const nativeBash_1 = require("./core/nativeBash");
|
|
|
56
56
|
const installUpdate_1 = require("./core/installUpdate");
|
|
57
57
|
const mobilePairing_1 = require("./core/mobilePairing");
|
|
58
58
|
const workEventCoalescer_1 = require("./core/workEventCoalescer");
|
|
59
|
+
const searchMcpPool_1 = require("./core/searchMcpPool");
|
|
59
60
|
const PORT = 47890;
|
|
60
61
|
let agent = null;
|
|
61
62
|
let automation = null;
|
|
@@ -995,6 +996,40 @@ async function handleApi(req, res, body) {
|
|
|
995
996
|
});
|
|
996
997
|
return;
|
|
997
998
|
}
|
|
999
|
+
case '/api/mobile/web-search': {
|
|
1000
|
+
const input = JSON.parse(body || '{}');
|
|
1001
|
+
const query = String(input.query || '').trim();
|
|
1002
|
+
if (!query) {
|
|
1003
|
+
mobileJson(res, { ok: false, error: 'query is required' }, 400);
|
|
1004
|
+
return;
|
|
1005
|
+
}
|
|
1006
|
+
if (query.length > 2_000) {
|
|
1007
|
+
mobileJson(res, { ok: false, error: 'query is too long' }, 400);
|
|
1008
|
+
return;
|
|
1009
|
+
}
|
|
1010
|
+
const result = await agent.tools.webSearchDetailed(query);
|
|
1011
|
+
mobileJson(res, result);
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
1014
|
+
case '/api/mobile/web-search-mcp': {
|
|
1015
|
+
const input = JSON.parse(body || '{}');
|
|
1016
|
+
const query = String(input.query || '').trim();
|
|
1017
|
+
if (!query) {
|
|
1018
|
+
mobileJson(res, { ok: false, error: 'query is required' }, 400);
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
if (query.length > 2_000) {
|
|
1022
|
+
mobileJson(res, { ok: false, error: 'query is too long' }, 400);
|
|
1023
|
+
return;
|
|
1024
|
+
}
|
|
1025
|
+
const result = await agent.tools.webSearchMcpOnly(query);
|
|
1026
|
+
mobileJson(res, result);
|
|
1027
|
+
return;
|
|
1028
|
+
}
|
|
1029
|
+
case '/api/mobile/search-mcp-manifest': {
|
|
1030
|
+
mobileJson(res, { version: 1, endpoints: (0, searchMcpPool_1.publicSearchMcpManifest)(appRoot) });
|
|
1031
|
+
return;
|
|
1032
|
+
}
|
|
998
1033
|
case '/api/mobile/state': {
|
|
999
1034
|
const active = agent.getConversationSnapshot(agent.activeConversationId, { window: 200 });
|
|
1000
1035
|
// 完整对话信息:workRuns 用持久化记录透出(含被中断的构建,不依赖运行时内存)
|
package/dist/tools/index.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { NewmarkToolDefinition, NewmarkToolResult } from '../core/compat';
|
|
|
3
3
|
import { SshManager } from '../core/ssh';
|
|
4
4
|
import { WorkspaceManager } from '../core/workspace';
|
|
5
5
|
import { LocalOcrResult } from '../core/localOcr';
|
|
6
|
+
import { SearchMcpAttempt } from '../core/searchMcpPool';
|
|
6
7
|
export interface ToolExecutionContext {
|
|
7
8
|
mode?: string;
|
|
8
9
|
workspacePath?: string;
|
|
@@ -21,6 +22,14 @@ export interface ToolHostProfile {
|
|
|
21
22
|
electronBrowser: boolean;
|
|
22
23
|
windowsComputerUse: boolean;
|
|
23
24
|
}
|
|
25
|
+
export interface WebSearchResult {
|
|
26
|
+
invocationId: string;
|
|
27
|
+
checkedAt: string;
|
|
28
|
+
ok: boolean;
|
|
29
|
+
provider: string;
|
|
30
|
+
text: string;
|
|
31
|
+
attempts: SearchMcpAttempt[];
|
|
32
|
+
}
|
|
24
33
|
export declare class ToolExecutor {
|
|
25
34
|
private config;
|
|
26
35
|
private ssh?;
|
|
@@ -28,11 +37,17 @@ export declare class ToolExecutor {
|
|
|
28
37
|
private root;
|
|
29
38
|
private readonly localOcr;
|
|
30
39
|
private readonly argumentValidators;
|
|
40
|
+
private readonly searchMcpPool;
|
|
31
41
|
private hostProfile;
|
|
32
42
|
constructor(root: string, config: ConfigManager, ssh?: SshManager | undefined, workspace?: WorkspaceManager | undefined);
|
|
33
|
-
webSearch(query: string): Promise<string>;
|
|
43
|
+
webSearch(query: string, signal?: AbortSignal): Promise<string>;
|
|
44
|
+
webSearchDetailed(query: string, signal?: AbortSignal): Promise<WebSearchResult>;
|
|
45
|
+
/** Search the configured MCP pool without entering any HTTP fallback. */
|
|
46
|
+
webSearchMcpOnly(query: string, signal?: AbortSignal): Promise<WebSearchResult>;
|
|
34
47
|
/** OCR entry point for the runtime's final visual fallback. */
|
|
35
48
|
finalVisualFallbackOcr(dataUrl: string, signal?: AbortSignal): Promise<LocalOcrResult>;
|
|
49
|
+
private readPdfFile;
|
|
50
|
+
private extractPdfText;
|
|
36
51
|
setHostProfile(profile: ToolHostProfile): void;
|
|
37
52
|
definitions(mode?: string): unknown[];
|
|
38
53
|
canonicalDefinitions(mode?: string): NewmarkToolDefinition[];
|
|
@@ -70,6 +85,7 @@ export declare class ToolExecutor {
|
|
|
70
85
|
private proxyEnvironment;
|
|
71
86
|
private proxyExec;
|
|
72
87
|
private wsearch;
|
|
88
|
+
private wsearchDetailed;
|
|
73
89
|
private wfetch;
|
|
74
90
|
private browserRun;
|
|
75
91
|
private formatBrowserResult;
|
package/dist/tools/index.js
CHANGED
|
@@ -60,6 +60,7 @@ const toolArgumentValidator_1 = require("../core/toolArgumentValidator");
|
|
|
60
60
|
const localOcr_1 = require("../core/localOcr");
|
|
61
61
|
const visualTextFallback_1 = require("../core/visualTextFallback");
|
|
62
62
|
const computerUseSession_1 = require("../core/computerUseSession");
|
|
63
|
+
const searchMcpPool_1 = require("../core/searchMcpPool");
|
|
63
64
|
function normalizeComputerUseAction(action) {
|
|
64
65
|
return String(action || '').trim().toLowerCase();
|
|
65
66
|
}
|
|
@@ -149,7 +150,7 @@ function decodePdfLiteral(input) {
|
|
|
149
150
|
.replace(/\\t/g, '\t')
|
|
150
151
|
.replace(/\\([0-7]{1,3})/g, (_match, octal) => String.fromCharCode(parseInt(octal, 8)));
|
|
151
152
|
}
|
|
152
|
-
function
|
|
153
|
+
function extractPdfTextLayerLegacy(buffer) {
|
|
153
154
|
const binary = buffer.toString('latin1');
|
|
154
155
|
const chunks = [];
|
|
155
156
|
for (const match of binary.matchAll(/\((?:\\.|[^\\)]){1,4000}\)\s*Tj/g)) {
|
|
@@ -171,6 +172,61 @@ function extractPdfTextLayer(buffer) {
|
|
|
171
172
|
}
|
|
172
173
|
return chunks.join(' ').replace(/\s+/g, ' ').trim().slice(0, 100_000);
|
|
173
174
|
}
|
|
175
|
+
async function extractPdfTextLayer(buffer, maxChars, signal) {
|
|
176
|
+
if (signal?.aborted)
|
|
177
|
+
throw abortReason(signal);
|
|
178
|
+
let loadingTask = null;
|
|
179
|
+
try {
|
|
180
|
+
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
|
181
|
+
loadingTask = pdfjs.getDocument({
|
|
182
|
+
data: new Uint8Array(buffer),
|
|
183
|
+
isEvalSupported: false,
|
|
184
|
+
useSystemFonts: true,
|
|
185
|
+
stopAtErrors: false,
|
|
186
|
+
});
|
|
187
|
+
const abortPromise = signal ? new Promise((_resolve, reject) => {
|
|
188
|
+
signal.addEventListener('abort', () => reject(abortReason(signal)), { once: true });
|
|
189
|
+
}) : null;
|
|
190
|
+
const document = await (abortPromise ? Promise.race([loadingTask.promise, abortPromise]) : loadingTask.promise);
|
|
191
|
+
const chunks = [];
|
|
192
|
+
let length = 0;
|
|
193
|
+
for (let pageNumber = 1; pageNumber <= document.numPages && length < maxChars; pageNumber += 1) {
|
|
194
|
+
if (signal?.aborted)
|
|
195
|
+
throw abortReason(signal);
|
|
196
|
+
const page = await document.getPage(pageNumber);
|
|
197
|
+
const content = await page.getTextContent({ disableNormalization: false });
|
|
198
|
+
const pageText = (Array.isArray(content.items) ? content.items : [])
|
|
199
|
+
.map((item) => `${String(item.str || '')}${item.hasEOL ? '\n' : ' '}`)
|
|
200
|
+
.join('')
|
|
201
|
+
.replace(/[ \t]+\n/g, '\n')
|
|
202
|
+
.replace(/[ \t]{2,}/g, ' ')
|
|
203
|
+
.trim();
|
|
204
|
+
if (pageText) {
|
|
205
|
+
const bounded = pageText.slice(0, Math.max(0, maxChars - length));
|
|
206
|
+
chunks.push(bounded);
|
|
207
|
+
length += bounded.length + 1;
|
|
208
|
+
}
|
|
209
|
+
page.cleanup();
|
|
210
|
+
}
|
|
211
|
+
return chunks.join('\n').trim().slice(0, maxChars);
|
|
212
|
+
}
|
|
213
|
+
catch (error) {
|
|
214
|
+
if (signal?.aborted)
|
|
215
|
+
throw abortReason(signal);
|
|
216
|
+
// Retain the bounded legacy decoder for minimal/malformed fixture PDFs and
|
|
217
|
+
// old documents whose streams pdf.js rejects. It never opens the browser.
|
|
218
|
+
const fallback = extractPdfTextLayerLegacy(buffer).slice(0, maxChars);
|
|
219
|
+
if (fallback)
|
|
220
|
+
return fallback;
|
|
221
|
+
return '';
|
|
222
|
+
}
|
|
223
|
+
finally {
|
|
224
|
+
try {
|
|
225
|
+
await loadingTask?.destroy();
|
|
226
|
+
}
|
|
227
|
+
catch { }
|
|
228
|
+
}
|
|
229
|
+
}
|
|
174
230
|
async function abortableToolDelay(durationMs, signal) {
|
|
175
231
|
if (signal?.aborted)
|
|
176
232
|
throw abortReason(signal);
|
|
@@ -219,6 +275,7 @@ class ToolExecutor {
|
|
|
219
275
|
root;
|
|
220
276
|
localOcr;
|
|
221
277
|
argumentValidators = new toolArgumentValidator_1.ToolArgumentValidatorRegistry();
|
|
278
|
+
searchMcpPool;
|
|
222
279
|
hostProfile = {
|
|
223
280
|
kind: 'desktop',
|
|
224
281
|
platform: process.platform,
|
|
@@ -231,14 +288,38 @@ class ToolExecutor {
|
|
|
231
288
|
this.workspace = workspace;
|
|
232
289
|
this.root = root;
|
|
233
290
|
this.localOcr = new localOcr_1.LocalOcrEngine(root);
|
|
291
|
+
this.searchMcpPool = new searchMcpPool_1.SearchMcpPool(root);
|
|
234
292
|
}
|
|
235
|
-
async webSearch(query) {
|
|
236
|
-
return this.
|
|
293
|
+
async webSearch(query, signal) {
|
|
294
|
+
return (await this.webSearchDetailed(query, signal)).text;
|
|
295
|
+
}
|
|
296
|
+
async webSearchDetailed(query, signal) {
|
|
297
|
+
return this.wsearchDetailed(query, signal);
|
|
298
|
+
}
|
|
299
|
+
/** Search the configured MCP pool without entering any HTTP fallback. */
|
|
300
|
+
async webSearchMcpOnly(query, signal) {
|
|
301
|
+
const result = await this.searchMcpPool.search(query, signal);
|
|
302
|
+
return {
|
|
303
|
+
invocationId: result.invocationId,
|
|
304
|
+
checkedAt: result.checkedAt,
|
|
305
|
+
ok: result.ok,
|
|
306
|
+
provider: result.provider || '',
|
|
307
|
+
text: result.text || '',
|
|
308
|
+
attempts: result.attempts,
|
|
309
|
+
};
|
|
237
310
|
}
|
|
238
311
|
/** OCR entry point for the runtime's final visual fallback. */
|
|
239
312
|
async finalVisualFallbackOcr(dataUrl, signal) {
|
|
240
313
|
return await this.localOcr.recognizeDataUrl(dataUrl, signal, 'sparse-ui');
|
|
241
314
|
}
|
|
315
|
+
async readPdfFile(pdfPath, signal) {
|
|
316
|
+
if (signal?.aborted)
|
|
317
|
+
throw abortReason(signal);
|
|
318
|
+
return await fs.promises.readFile(pdfPath, signal ? { signal } : undefined);
|
|
319
|
+
}
|
|
320
|
+
async extractPdfText(buffer, maxChars, signal) {
|
|
321
|
+
return await extractPdfTextLayer(buffer, maxChars, signal);
|
|
322
|
+
}
|
|
242
323
|
setHostProfile(profile) {
|
|
243
324
|
this.hostProfile = { ...profile };
|
|
244
325
|
}
|
|
@@ -282,8 +363,9 @@ class ToolExecutor {
|
|
|
282
363
|
t('browser_forward', 'Navigate the controlled browser forward.', {}, []),
|
|
283
364
|
t('browser_reload', 'Reload the controlled browser.', {}, []),
|
|
284
365
|
t('browser_cdp', 'Run a raw Chrome DevTools Protocol command against the controlled browser. Advanced use only.', { method: { type: 'string' }, params: { type: 'object' } }, ['method']),
|
|
285
|
-
t('browser_use', 'Native observe-then-act control
|
|
366
|
+
t('browser_use', 'Native observe-then-act browser control. visible defaults to true and uses the right-sidebar built-in browser. Set visible=false to run on an independent host-owned background page that never connects to, displays, or renders the right-sidebar webview. Keep the same visible value throughout one observe/action sequence. Receipts are owner/runtime/surface scoped; stale observations are rejected.', {
|
|
286
367
|
action: { type: 'string', enum: browserUseActions },
|
|
368
|
+
visible: { type: 'boolean', description: 'Whether to bind to the visible right-sidebar browser. Defaults to true. false uses an independent background execution surface and does not create or touch the sidebar webview.' },
|
|
287
369
|
action_id: { type: 'string', description: 'Unique idempotency id for this action. Reusing it returns the original receipt without repeating the action.' },
|
|
288
370
|
page_generation: { type: 'number', description: 'Generation returned by the latest observe receipt.' },
|
|
289
371
|
observation_id: { type: 'string', description: 'Opaque observation capability returned by the latest observe receipt.' },
|
|
@@ -346,8 +428,9 @@ class ToolExecutor {
|
|
|
346
428
|
},
|
|
347
429
|
},
|
|
348
430
|
}, ['action']),
|
|
349
|
-
t('image_inspect', 'Inspect a durable user-submitted image by stable attachment_id
|
|
350
|
-
action: { type: 'string', enum: ['source_info', 'crop'] },
|
|
431
|
+
t('image_inspect', 'Inspect a durable user-submitted image by stable attachment_id or latest-message image_index, or send one active-workspace PNG/JPEG to the current validated vision model with action=inspect. Use source_info first when submitted-image dimensions are unknown, then crop with pixel coordinates. Workspace observations and derived crops are current-turn only; image bytes never enter durable tool history.', {
|
|
432
|
+
action: { type: 'string', enum: ['source_info', 'crop', 'inspect'] },
|
|
433
|
+
path: { type: 'string', description: 'For action=inspect, a workspace-relative PNG/JPEG path. Absolute paths are accepted only when they remain inside the active workspace.' },
|
|
351
434
|
attachment_id: { type: 'string', description: 'Stable user-image attachment id from the visible conversation. Prefer this when revisiting an older submitted image.' },
|
|
352
435
|
image_index: { type: 'number', description: '1-based image index in the latest user message containing submitted images. Defaults to 1.' },
|
|
353
436
|
x: { type: 'number', description: 'Crop left edge in source-image pixels.' },
|
|
@@ -366,10 +449,11 @@ class ToolExecutor {
|
|
|
366
449
|
path: { type: 'string', description: 'For source=image, a workspace PNG/JPEG/BMP path.' },
|
|
367
450
|
fallback_reason: { type: 'string', enum: ['vision_unavailable', 'vision_failed'] },
|
|
368
451
|
}, ['source', 'fallback_reason']),
|
|
369
|
-
t('pdf_read', 'Read a PDF with enforced fallback order: embedded text layer first; if unreadable, render the requested page in Newmark Browser and send a screenshot to a validated vision model; use bundled Chinese/English OCR only when vision is unavailable, or later through ocr_read after vision failed. Designed for scanned PDFs, not layout/table reconstruction.', {
|
|
452
|
+
t('pdf_read', 'Read a PDF with enforced fallback order: embedded text layer first; if unreadable, render the requested page in Newmark Browser and send a screenshot to a validated vision model; use bundled Chinese/English OCR only when vision is unavailable, or later through ocr_read after vision failed. One cumulative timeout covers the entire PDF read, including file I/O, pdf.js parsing, and rendered-page observation. Designed for scanned PDFs, not layout/table reconstruction.', {
|
|
370
453
|
path: { type: 'string', description: 'Workspace PDF path.' },
|
|
371
454
|
page: { type: 'number', minimum: 1, maximum: 100, description: 'Page to render when no usable text layer exists. Defaults to 1.' },
|
|
372
455
|
max_chars: { type: 'number', minimum: 500, maximum: 100000 },
|
|
456
|
+
timeout_ms: { type: 'number', minimum: 1000, maximum: 120000, description: 'Bounded entire PDF read timeout. Defaults to 30000 ms. A timeout returns a recoverable tool result and does not abort the Agent run.' },
|
|
373
457
|
}, ['path']),
|
|
374
458
|
t('terminal_takeover', 'Take over a persistent owner-scoped PTY session that is independent from the one-shot bash tool. Actions: start creates/reuses a named PTY, write sends a command to the same session, read returns its output buffer, resize updates PTY geometry, detach releases the UI attachment without stopping the shell, stop interrupts it, list shows sessions. Use this when the user wants continuous terminal state such as cd/env/process context or interactive TTY programs.', {
|
|
375
459
|
action: { type: 'string', enum: ['start', 'write', 'read', 'resize', 'detach', 'stop', 'list'] },
|
|
@@ -517,6 +601,7 @@ class ToolExecutor {
|
|
|
517
601
|
copy.function.description = 'Plan read-only browser: observe, navigate, wait, extract only.';
|
|
518
602
|
copy.function.parameters.properties = {
|
|
519
603
|
action: { type: 'string', enum: [...toolPolicy_1.PLAN_BROWSER_USE_ACTIONS] },
|
|
604
|
+
visible: copy.function.parameters.properties.visible,
|
|
520
605
|
action_id: copy.function.parameters.properties.action_id,
|
|
521
606
|
page_generation: copy.function.parameters.properties.page_generation,
|
|
522
607
|
observation_id: copy.function.parameters.properties.observation_id,
|
|
@@ -688,6 +773,7 @@ class ToolExecutor {
|
|
|
688
773
|
const scope = browserUseScope(context, wsPath);
|
|
689
774
|
const request = {
|
|
690
775
|
...scope,
|
|
776
|
+
visible: typeof args.visible === 'boolean' ? args.visible : true,
|
|
691
777
|
action: String(args.action || '').trim().toLowerCase(),
|
|
692
778
|
...(g('action_id') ? { actionId: g('action_id') } : {}),
|
|
693
779
|
...(args.page_generation !== undefined ? { pageGeneration: Number(args.page_generation) } : {}),
|
|
@@ -760,47 +846,79 @@ class ToolExecutor {
|
|
|
760
846
|
const pdfPath = resolve(g('path'));
|
|
761
847
|
if (path.extname(pdfPath).toLowerCase() !== '.pdf')
|
|
762
848
|
return '[pdf_read error] path must end in .pdf.';
|
|
763
|
-
const stat = fs.statSync(pdfPath);
|
|
764
|
-
if (!stat.isFile() || stat.size <= 0 || stat.size > 250 * 1024 * 1024) {
|
|
765
|
-
return '[pdf_read error] PDF must be a regular file no larger than 250 MB.';
|
|
766
|
-
}
|
|
767
849
|
const maxChars = Math.max(500, Math.min(100_000, Number(args.max_chars || 50_000)));
|
|
768
|
-
const
|
|
769
|
-
const
|
|
770
|
-
|
|
850
|
+
const page = Math.max(1, Math.min(100, Math.floor(Number(args.page || 1))));
|
|
851
|
+
const timeoutMs = Math.max(1000, Math.min(120_000, Number(args.timeout_ms || 30_000)));
|
|
852
|
+
const guard = abortGuard(context.signal, timeoutMs);
|
|
853
|
+
let stage = 'file_stat';
|
|
854
|
+
try {
|
|
855
|
+
const guardedContext = { ...context, signal: guard.signal };
|
|
856
|
+
const stat = await fs.promises.stat(pdfPath);
|
|
857
|
+
if (!stat.isFile() || stat.size <= 0 || stat.size > 250 * 1024 * 1024) {
|
|
858
|
+
return '[pdf_read error] PDF must be a regular file no larger than 250 MB.';
|
|
859
|
+
}
|
|
860
|
+
stage = 'file_read';
|
|
861
|
+
const bytes = await this.readPdfFile(pdfPath, guard.signal);
|
|
862
|
+
stage = 'text_parse';
|
|
863
|
+
const textLayer = await this.extractPdfText(bytes, maxChars, guard.signal);
|
|
864
|
+
const readableCount = (textLayer.match(/[A-Za-z0-9\u3400-\u9fff]/g) || []).length;
|
|
865
|
+
if (readableCount >= 20) {
|
|
866
|
+
return JSON.stringify({
|
|
867
|
+
ok: true,
|
|
868
|
+
source: 'pdf_text_layer',
|
|
869
|
+
recognition_order: 'text>vision>local_ocr',
|
|
870
|
+
text: textLayer,
|
|
871
|
+
truncated: textLayer.length >= maxChars,
|
|
872
|
+
}, null, 2);
|
|
873
|
+
}
|
|
874
|
+
stage = 'rendered_page_observation';
|
|
875
|
+
const url = `${(0, url_1.pathToFileURL)(pdfPath).toString()}#page=${page}&zoom=page-fit`;
|
|
876
|
+
const opened = await this.browserRun({ action: 'open', url }, guard.signal, guardedContext, wsPath);
|
|
877
|
+
if (!opened.includes('[browser:open] OK')) {
|
|
878
|
+
return JSON.stringify({ ok: false, source: 'pdf_render', code: 'pdf_open_failed', error: opened || 'Unable to open PDF.' }, null, 2);
|
|
879
|
+
}
|
|
880
|
+
await abortableToolDelay(900, guard.signal);
|
|
881
|
+
const observed = await this.execute('browser_use', JSON.stringify({
|
|
882
|
+
action: 'observe',
|
|
883
|
+
action_id: `pdf-read-${crypto.randomUUID()}`,
|
|
884
|
+
max_chars: maxChars,
|
|
885
|
+
max_refs: 80,
|
|
886
|
+
}), wsPath, guardedContext);
|
|
887
|
+
if (guard.signal.aborted)
|
|
888
|
+
throw abortReason(guard.signal);
|
|
889
|
+
let parsed = observed;
|
|
890
|
+
try {
|
|
891
|
+
parsed = JSON.parse(observed);
|
|
892
|
+
}
|
|
893
|
+
catch { }
|
|
771
894
|
return JSON.stringify({
|
|
772
895
|
ok: true,
|
|
773
|
-
source: '
|
|
896
|
+
source: 'pdf_rendered_page',
|
|
897
|
+
page,
|
|
774
898
|
recognition_order: 'text>vision>local_ocr',
|
|
775
|
-
|
|
776
|
-
truncated: textLayer.length >= maxChars,
|
|
899
|
+
result: parsed,
|
|
777
900
|
}, null, 2);
|
|
778
901
|
}
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
902
|
+
catch (error) {
|
|
903
|
+
if (context.signal?.aborted)
|
|
904
|
+
throw abortReason(context.signal);
|
|
905
|
+
if (guard.signal.aborted) {
|
|
906
|
+
return JSON.stringify({
|
|
907
|
+
ok: false,
|
|
908
|
+
source: stage === 'rendered_page_observation' ? 'pdf_rendered_page' : 'pdf_read',
|
|
909
|
+
code: 'pdf_read_timeout',
|
|
910
|
+
stage,
|
|
911
|
+
page,
|
|
912
|
+
timeout_ms: timeoutMs,
|
|
913
|
+
recoverable: true,
|
|
914
|
+
error: `PDF read timed out during ${stage} after ${timeoutMs} ms. The Agent run can continue or retry with a larger timeout_ms.`,
|
|
915
|
+
}, null, 2);
|
|
916
|
+
}
|
|
917
|
+
throw error;
|
|
784
918
|
}
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
action: 'observe',
|
|
788
|
-
action_id: `pdf-read-${crypto.randomUUID()}`,
|
|
789
|
-
max_chars: maxChars,
|
|
790
|
-
max_refs: 80,
|
|
791
|
-
}), wsPath, context);
|
|
792
|
-
let parsed = observed;
|
|
793
|
-
try {
|
|
794
|
-
parsed = JSON.parse(observed);
|
|
919
|
+
finally {
|
|
920
|
+
guard.dispose();
|
|
795
921
|
}
|
|
796
|
-
catch { }
|
|
797
|
-
return JSON.stringify({
|
|
798
|
-
ok: true,
|
|
799
|
-
source: 'pdf_rendered_page',
|
|
800
|
-
page,
|
|
801
|
-
recognition_order: 'text>vision>local_ocr',
|
|
802
|
-
result: parsed,
|
|
803
|
-
}, null, 2);
|
|
804
922
|
}
|
|
805
923
|
case 'screen_capture': {
|
|
806
924
|
const target = g('target').toLowerCase() === 'application' ? 'application' : 'desktop';
|
|
@@ -1357,6 +1475,9 @@ class ToolExecutor {
|
|
|
1357
1475
|
}
|
|
1358
1476
|
}
|
|
1359
1477
|
async wsearch(query, signal) {
|
|
1478
|
+
return (await this.wsearchDetailed(query, signal)).text;
|
|
1479
|
+
}
|
|
1480
|
+
async wsearchDetailed(query, signal) {
|
|
1360
1481
|
const clean = (s) => s
|
|
1361
1482
|
.replace(/<[^>]+>/g, ' ')
|
|
1362
1483
|
.replace(/&/g, '&')
|
|
@@ -1367,34 +1488,23 @@ class ToolExecutor {
|
|
|
1367
1488
|
.replace(/\s+/g, ' ')
|
|
1368
1489
|
.trim();
|
|
1369
1490
|
const errors = [];
|
|
1491
|
+
let mcp = { invocationId: '', checkedAt: '', ok: false, attempts: [] };
|
|
1370
1492
|
try {
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
try {
|
|
1375
|
-
const resp = await this.proxyFetch(url, {
|
|
1376
|
-
headers: { 'User-Agent': 'NewmarkAgent/1.0' },
|
|
1377
|
-
signal: guard.signal,
|
|
1378
|
-
});
|
|
1379
|
-
html = await resp.text();
|
|
1493
|
+
mcp = await this.webSearchMcpOnly(query, signal);
|
|
1494
|
+
if (mcp.ok && mcp.text) {
|
|
1495
|
+
return { invocationId: mcp.invocationId, checkedAt: mcp.checkedAt, ok: true, provider: mcp.provider || 'MCP search', text: mcp.text, attempts: mcp.attempts };
|
|
1380
1496
|
}
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
let m;
|
|
1387
|
-
while ((m = re.exec(html)) !== null && results.length < 8) {
|
|
1388
|
-
results.push(`${clean(m[2])}\n${clean(m[1])}\n${clean(m[3])}`);
|
|
1497
|
+
for (const attempt of mcp.attempts) {
|
|
1498
|
+
if (attempt.status === 'error')
|
|
1499
|
+
errors.push(`${attempt.name}: ${attempt.error || 'failed'}`);
|
|
1500
|
+
else if (attempt.status === 'empty')
|
|
1501
|
+
errors.push(`${attempt.name}: no results`);
|
|
1389
1502
|
}
|
|
1390
|
-
if (results.length > 0)
|
|
1391
|
-
return results.join('\n\n');
|
|
1392
|
-
errors.push('DuckDuckGo returned no parseable results');
|
|
1393
1503
|
}
|
|
1394
|
-
catch (
|
|
1504
|
+
catch (error) {
|
|
1395
1505
|
if (signal?.aborted)
|
|
1396
1506
|
throw abortReason(signal);
|
|
1397
|
-
errors.push(`
|
|
1507
|
+
errors.push(`Search MCP pool: ${error instanceof Error ? error.message : String(error)}`);
|
|
1398
1508
|
}
|
|
1399
1509
|
try {
|
|
1400
1510
|
const url = `https://www.bing.com/search?q=${encodeURIComponent(query)}`;
|
|
@@ -1422,7 +1532,7 @@ class ToolExecutor {
|
|
|
1422
1532
|
results.push(`${clean(title[2])}\n${clean(title[1])}\n${snippet ? clean(snippet[1]) : ''}`.trim());
|
|
1423
1533
|
}
|
|
1424
1534
|
if (results.length > 0)
|
|
1425
|
-
return results.join('\n\n');
|
|
1535
|
+
return { invocationId: mcp.invocationId, checkedAt: mcp.checkedAt, ok: true, provider: 'Bing HTTP', text: results.join('\n\n'), attempts: mcp.attempts };
|
|
1426
1536
|
errors.push('Bing returned no parseable results');
|
|
1427
1537
|
}
|
|
1428
1538
|
catch (e) {
|
|
@@ -1430,7 +1540,36 @@ class ToolExecutor {
|
|
|
1430
1540
|
throw abortReason(signal);
|
|
1431
1541
|
errors.push(`Bing: ${e instanceof Error ? e.message : String(e)}`);
|
|
1432
1542
|
}
|
|
1433
|
-
|
|
1543
|
+
try {
|
|
1544
|
+
const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
1545
|
+
const guard = abortGuard(signal, 15000);
|
|
1546
|
+
let html = '';
|
|
1547
|
+
try {
|
|
1548
|
+
const resp = await this.proxyFetch(url, {
|
|
1549
|
+
headers: { 'User-Agent': 'NewmarkAgent/1.0' },
|
|
1550
|
+
signal: guard.signal,
|
|
1551
|
+
});
|
|
1552
|
+
html = await resp.text();
|
|
1553
|
+
}
|
|
1554
|
+
finally {
|
|
1555
|
+
guard.dispose();
|
|
1556
|
+
}
|
|
1557
|
+
const re = /<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>(.*?)<\/a>[\s\S]*?class="result__snippet">(.*?)<\/a>/g;
|
|
1558
|
+
const results = [];
|
|
1559
|
+
let match;
|
|
1560
|
+
while ((match = re.exec(html)) !== null && results.length < 8) {
|
|
1561
|
+
results.push(`${clean(match[2])}\n${clean(match[1])}\n${clean(match[3])}`);
|
|
1562
|
+
}
|
|
1563
|
+
if (results.length > 0)
|
|
1564
|
+
return { invocationId: mcp.invocationId, checkedAt: mcp.checkedAt, ok: true, provider: 'DuckDuckGo HTTP', text: results.join('\n\n'), attempts: mcp.attempts };
|
|
1565
|
+
errors.push('DuckDuckGo returned no parseable results');
|
|
1566
|
+
}
|
|
1567
|
+
catch (e) {
|
|
1568
|
+
if (signal?.aborted)
|
|
1569
|
+
throw abortReason(signal);
|
|
1570
|
+
errors.push(`DuckDuckGo: ${e instanceof Error ? e.message : String(e)}`);
|
|
1571
|
+
}
|
|
1572
|
+
return { invocationId: mcp.invocationId, checkedAt: mcp.checkedAt, ok: false, provider: '', text: `[web_search] No results. ${errors.join('; ')}`, attempts: mcp.attempts };
|
|
1434
1573
|
}
|
|
1435
1574
|
async wfetch(url, signal) {
|
|
1436
1575
|
try {
|
package/dist/tui/src/data.js
CHANGED
|
@@ -20,7 +20,7 @@ const navigation = [
|
|
|
20
20
|
const workspace = {
|
|
21
21
|
id: "workspace-newmark-agent-demo",
|
|
22
22
|
name: "Newmark Agent",
|
|
23
|
-
path: "C:\\Users\\
|
|
23
|
+
path: "C:\\Users\\DemoUser\\Projects\\Newmark Agent",
|
|
24
24
|
isInternal: false,
|
|
25
25
|
hostBinding: "demo-host",
|
|
26
26
|
icon: "[W]",
|
|
@@ -40,7 +40,7 @@ const workspaces = [
|
|
|
40
40
|
{
|
|
41
41
|
id: "workspace-condensed-lab-demo",
|
|
42
42
|
name: "Condensed Lab",
|
|
43
|
-
path: "C:\\Users\\
|
|
43
|
+
path: "C:\\Users\\DemoUser\\Projects\\Condensed Lab",
|
|
44
44
|
isInternal: false,
|
|
45
45
|
hostBinding: "demo-host",
|
|
46
46
|
icon: "[W]",
|