newmark-agent 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/conversation-utility-host.bundle.cjs +116 -31
- package/dist/core/agent.d.ts +7 -0
- package/dist/core/agent.js +63 -2
- package/dist/core/agentKernelRunner.js +18 -1
- package/dist/core/config.js +16 -4
- package/dist/llm/provider.js +31 -13
- package/dist/main.js +0 -12
- package/dist/server.js +0 -12
- package/dist/tools/computerUse.js +5 -10
- package/dist/tools/index.d.ts +3 -0
- package/dist/tools/index.js +7 -3
- package/dist/ui/index.html +4 -56
- package/dist/wsl-agent-host.bundle.cjs +116 -31
- package/package.json +1 -1
|
@@ -327672,7 +327672,9 @@ var ConfigManager = class {
|
|
|
327672
327672
|
this.backupConfig(cp, "invalid-shape");
|
|
327673
327673
|
return this.writeRecoveredConfig(cp);
|
|
327674
327674
|
}
|
|
327675
|
-
|
|
327675
|
+
const providerIdsMigrated = migrateProviderIdsInConfig(normalized);
|
|
327676
|
+
const marqueeConfigRemoved = removeDeprecatedMarqueeConfig(normalized);
|
|
327677
|
+
if (providerIdsMigrated || marqueeConfigRemoved) {
|
|
327676
327678
|
try {
|
|
327677
327679
|
if (!this.readOnly) fs3.writeFileSync(cp, JSON.stringify(normalized, null, 2), "utf-8");
|
|
327678
327680
|
} catch {
|
|
@@ -327991,6 +327993,18 @@ function normalizeConfigShape(raw, withDefaults) {
|
|
|
327991
327993
|
}
|
|
327992
327994
|
return base2;
|
|
327993
327995
|
}
|
|
327996
|
+
function removeDeprecatedMarqueeConfig(config) {
|
|
327997
|
+
const ui = config.ui;
|
|
327998
|
+
if (!ui) return false;
|
|
327999
|
+
let changed = false;
|
|
328000
|
+
for (const key3 of ["gradient_colors", "gradient_speed", "gradient_width"]) {
|
|
328001
|
+
if (Object.prototype.hasOwnProperty.call(ui, key3)) {
|
|
328002
|
+
delete ui[key3];
|
|
328003
|
+
changed = true;
|
|
328004
|
+
}
|
|
328005
|
+
}
|
|
328006
|
+
return changed;
|
|
328007
|
+
}
|
|
327994
328008
|
function isConfigEntry(value) {
|
|
327995
328009
|
return !!value && typeof value === "object" && !Array.isArray(value) && Object.prototype.hasOwnProperty.call(value, "value");
|
|
327996
328010
|
}
|
|
@@ -328358,9 +328372,6 @@ function defaultConfig() {
|
|
|
328358
328372
|
auto_download: { _description: "Auto-download skills", _type: "choice", _values: ["aggressive", "conservative", "disabled"], value: "conservative" }
|
|
328359
328373
|
},
|
|
328360
328374
|
ui: {
|
|
328361
|
-
gradient_colors: { _description: "Gradient colors (hex)", _type: "array", value: ["#00ff88", "#00ccff", "#aa44ff", "#ff4488"] },
|
|
328362
|
-
gradient_speed: { _description: "Animation speed 1-5", _type: "range", _min: 1, _max: 5, value: 2 },
|
|
328363
|
-
gradient_width: { _description: "Border width", _type: "integer", _min: 1, _max: 4, value: 2 },
|
|
328364
328375
|
glass_alpha: { _description: "Glass opacity", _type: "range", _min: 0, _max: 1, value: 0.85 },
|
|
328365
328376
|
show_mode_label: { _description: "Show mode on hover", _type: "boolean", value: true },
|
|
328366
328377
|
left_panel_collapsed: { _description: "Left panel collapsed", _type: "boolean", value: false },
|
|
@@ -329396,7 +329407,7 @@ function createProviderAdapter(providerId, apiMode) {
|
|
|
329396
329407
|
}
|
|
329397
329408
|
|
|
329398
329409
|
// src/llm/provider.ts
|
|
329399
|
-
var DEFAULT_PROVIDER_REQUEST_TIMEOUT_MS =
|
|
329410
|
+
var DEFAULT_PROVIDER_REQUEST_TIMEOUT_MS = 0;
|
|
329400
329411
|
var MIN_PROVIDER_REQUEST_TIMEOUT_MS = 50;
|
|
329401
329412
|
function providerTimeoutError(timeoutMs) {
|
|
329402
329413
|
const error = new Error(`Provider request timed out after ${timeoutMs}ms`);
|
|
@@ -329457,11 +329468,13 @@ var LLMProvider = class _LLMProvider {
|
|
|
329457
329468
|
static powershellTransport = null;
|
|
329458
329469
|
temperatureUnsupported = /* @__PURE__ */ new Set();
|
|
329459
329470
|
effectiveRequestTimeout(timeoutMs) {
|
|
329460
|
-
const requested = Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs :
|
|
329461
|
-
const configured = Number.isFinite(this.requestTimeoutMs) && this.requestTimeoutMs > 0 ? this.requestTimeoutMs :
|
|
329471
|
+
const requested = Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 0;
|
|
329472
|
+
const configured = Number.isFinite(this.requestTimeoutMs) && this.requestTimeoutMs > 0 ? this.requestTimeoutMs : 0;
|
|
329473
|
+
if (requested <= 0 || configured <= 0) return 0;
|
|
329462
329474
|
return Math.max(MIN_PROVIDER_REQUEST_TIMEOUT_MS, Math.min(requested, configured));
|
|
329463
329475
|
}
|
|
329464
329476
|
async withRequestTimeout(promise, timeoutMs, signal) {
|
|
329477
|
+
if (timeoutMs <= 0) return await abortable(promise, signal);
|
|
329465
329478
|
let timer;
|
|
329466
329479
|
const timeoutPromise = new Promise((_3, reject) => {
|
|
329467
329480
|
timer = setTimeout(() => reject(providerTimeoutError(timeoutMs)), timeoutMs);
|
|
@@ -329661,7 +329674,7 @@ var LLMProvider = class _LLMProvider {
|
|
|
329661
329674
|
const forwardAbort = () => abort.abort(signal?.reason);
|
|
329662
329675
|
if (signal?.aborted) forwardAbort();
|
|
329663
329676
|
else signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
329664
|
-
const timer = setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout);
|
|
329677
|
+
const timer = effectiveTimeout > 0 ? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout) : void 0;
|
|
329665
329678
|
try {
|
|
329666
329679
|
const response = await fetch(url, {
|
|
329667
329680
|
method: "POST",
|
|
@@ -329690,7 +329703,7 @@ var LLMProvider = class _LLMProvider {
|
|
|
329690
329703
|
async getJsonWithFetchFallback(url, headers, timeoutMs = 3e4) {
|
|
329691
329704
|
const effectiveTimeout = this.effectiveRequestTimeout(timeoutMs);
|
|
329692
329705
|
const abort = new AbortController();
|
|
329693
|
-
const timer = setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout);
|
|
329706
|
+
const timer = effectiveTimeout > 0 ? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout) : void 0;
|
|
329694
329707
|
try {
|
|
329695
329708
|
const response = await fetch(url, { method: "GET", headers, signal: abort.signal });
|
|
329696
329709
|
return response;
|
|
@@ -329764,9 +329777,11 @@ var LLMProvider = class _LLMProvider {
|
|
|
329764
329777
|
else fail(new Error("Node HTTP response closed before completion"));
|
|
329765
329778
|
});
|
|
329766
329779
|
});
|
|
329767
|
-
|
|
329768
|
-
req.
|
|
329769
|
-
|
|
329780
|
+
if (effectiveTimeout > 0) {
|
|
329781
|
+
req.setTimeout(effectiveTimeout, () => {
|
|
329782
|
+
req.destroy(providerTimeoutError(effectiveTimeout));
|
|
329783
|
+
});
|
|
329784
|
+
}
|
|
329770
329785
|
req.on("error", reject);
|
|
329771
329786
|
const onAbort = () => req.destroy(abortFailure(signal));
|
|
329772
329787
|
if (signal?.aborted) onAbort();
|
|
@@ -329815,7 +329830,7 @@ var LLMProvider = class _LLMProvider {
|
|
|
329815
329830
|
" $raw = $headerJson | ConvertFrom-Json",
|
|
329816
329831
|
" foreach ($p in $raw.PSObject.Properties) { $headers[$p.Name] = [string]$p.Value }",
|
|
329817
329832
|
"}",
|
|
329818
|
-
|
|
329833
|
+
effectiveTimeout > 0 ? `$params = @{ Uri = $uri; Method = $method; Headers = $headers; UseBasicParsing = $true; TimeoutSec = ${Math.ceil(effectiveTimeout / 1e3)} }` : "$params = @{ Uri = $uri; Method = $method; Headers = $headers; UseBasicParsing = $true }",
|
|
329819
329834
|
'if ($method -eq "POST") { $params["Body"] = $bodyJson }',
|
|
329820
329835
|
'if ($method -eq "POST") { $params["ContentType"] = "application/json; charset=utf-8" }',
|
|
329821
329836
|
"$resp = Invoke-WebRequest @params",
|
|
@@ -329844,11 +329859,11 @@ var LLMProvider = class _LLMProvider {
|
|
|
329844
329859
|
};
|
|
329845
329860
|
if (signal?.aborted) onAbort();
|
|
329846
329861
|
else signal?.addEventListener("abort", onAbort, { once: true });
|
|
329847
|
-
const timer = setTimeout(() => {
|
|
329862
|
+
const timer = effectiveTimeout > 0 ? setTimeout(() => {
|
|
329848
329863
|
child.kill();
|
|
329849
329864
|
cleanup();
|
|
329850
329865
|
reject(providerTimeoutError(effectiveTimeout));
|
|
329851
|
-
}, effectiveTimeout + 5e3);
|
|
329866
|
+
}, effectiveTimeout + 5e3) : void 0;
|
|
329852
329867
|
child.stdout.setEncoding("utf8");
|
|
329853
329868
|
child.stderr.setEncoding("utf8");
|
|
329854
329869
|
child.stdout.on("data", (chunk) => {
|
|
@@ -330317,7 +330332,7 @@ ${responsePath}
|
|
|
330317
330332
|
if (signal?.aborted) forwardAbort();
|
|
330318
330333
|
else signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
330319
330334
|
const effectiveTimeout = this.effectiveRequestTimeout(12e4);
|
|
330320
|
-
const timer = setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout);
|
|
330335
|
+
const timer = effectiveTimeout > 0 ? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout) : void 0;
|
|
330321
330336
|
try {
|
|
330322
330337
|
try {
|
|
330323
330338
|
let response2 = await fetch(request.url, {
|
|
@@ -330462,7 +330477,7 @@ ${responsePath}
|
|
|
330462
330477
|
if (signal?.aborted) forwardAbort();
|
|
330463
330478
|
else signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
330464
330479
|
const effectiveTimeout = this.effectiveRequestTimeout(12e4);
|
|
330465
|
-
const timeout = setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout);
|
|
330480
|
+
const timeout = effectiveTimeout > 0 ? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout) : void 0;
|
|
330466
330481
|
let reader = null;
|
|
330467
330482
|
try {
|
|
330468
330483
|
let response;
|
|
@@ -333767,11 +333782,8 @@ function jpegCaptureScript(outPath, boundsScript, requestedMaxWidth, requestedMa
|
|
|
333767
333782
|
'Write-Output (@{ ok=$true; left=$x; top=$y; width=$w; height=$h; capture_max_width=$maxWidth; capture_max_height=$maxHeight; image_width=$imageWidth; image_height=$imageHeight; image_bytes=$fileSize; image_quality=$qualityUsed; image_available=$imageAvailable; image_mime="image/jpeg" } | ConvertTo-Json -Compress)'
|
|
333768
333783
|
].join("\r\n");
|
|
333769
333784
|
}
|
|
333770
|
-
function gradientPalette(
|
|
333771
|
-
|
|
333772
|
-
const configured = Array.isArray(input) ? input : [];
|
|
333773
|
-
const raw = configured.length ? configured.map((v) => String(v || "").trim()).filter(Boolean) : String(process.env.NEWMARK_COMPUTER_USE_GRADIENT || "").split(",").map((v) => v.trim()).filter(Boolean);
|
|
333774
|
-
return raw.length >= 2 ? raw.slice(0, 6) : fallback;
|
|
333785
|
+
function gradientPalette(_input) {
|
|
333786
|
+
return ["#000000", "#ffffff", "#000000", "#ffffff"];
|
|
333775
333787
|
}
|
|
333776
333788
|
async function stopTakeoverOverlay() {
|
|
333777
333789
|
const pid = takeoverOverlayPid;
|
|
@@ -333793,11 +333805,11 @@ async function stopTakeoverOverlay() {
|
|
|
333793
333805
|
async function startTakeoverOverlay(durationMs = 0, input = {}) {
|
|
333794
333806
|
if (process.platform !== "win32") return { ok: false, action: "takeover_start", error: "Computer Use takeover overlay is Windows-only." };
|
|
333795
333807
|
await stopTakeoverOverlay();
|
|
333796
|
-
lastTakeoverOverlayStyle = { colors:
|
|
333808
|
+
lastTakeoverOverlayStyle = { colors: gradientPalette(), speed: 3, width: 2 };
|
|
333797
333809
|
const colors = gradientPalette(input.colors);
|
|
333798
333810
|
const lifetime = Math.max(0, Math.floor(Number(durationMs || 0)));
|
|
333799
|
-
const width =
|
|
333800
|
-
const speedSeconds =
|
|
333811
|
+
const width = 2;
|
|
333812
|
+
const speedSeconds = 3;
|
|
333801
333813
|
const ownerPid = Math.max(0, Math.floor(Number(input.ownerPid ?? process.pid) || 0));
|
|
333802
333814
|
const scriptPath = path9.join(tempScreenshotDir(), `takeover-overlay-${timestampName()}-${crypto6.randomBytes(4).toString("hex")}.ps1`);
|
|
333803
333815
|
const script = [
|
|
@@ -336921,6 +336933,10 @@ var ToolExecutor = class {
|
|
|
336921
336933
|
async webSearch(query) {
|
|
336922
336934
|
return this.wsearch(query);
|
|
336923
336935
|
}
|
|
336936
|
+
/** OCR entry point for the runtime's final visual fallback. */
|
|
336937
|
+
async finalVisualFallbackOcr(dataUrl, signal) {
|
|
336938
|
+
return await this.localOcr.recognizeDataUrl(dataUrl, signal, "sparse-ui");
|
|
336939
|
+
}
|
|
336924
336940
|
setHostProfile(profile) {
|
|
336925
336941
|
this.hostProfile = { ...profile };
|
|
336926
336942
|
}
|
|
@@ -337568,9 +337584,9 @@ var ToolExecutor = class {
|
|
|
337568
337584
|
allowEphemeralVisionImage: context.allowEphemeralVisionImage === true,
|
|
337569
337585
|
captureMaxWidth: Number(args.capture_max_width),
|
|
337570
337586
|
captureMaxHeight: Number(args.capture_max_height),
|
|
337571
|
-
gradientColors: Array.isArray(args.gradient_colors) ? args.gradient_colors :
|
|
337572
|
-
gradientSpeed: args.gradient_speed !== void 0 ? Number(args.gradient_speed) :
|
|
337573
|
-
gradientWidth: args.gradient_width !== void 0 ? Number(args.gradient_width) :
|
|
337587
|
+
gradientColors: Array.isArray(args.gradient_colors) ? args.gradient_colors : void 0,
|
|
337588
|
+
gradientSpeed: args.gradient_speed !== void 0 ? Number(args.gradient_speed) : void 0,
|
|
337589
|
+
gradientWidth: args.gradient_width !== void 0 ? Number(args.gradient_width) : void 0,
|
|
337574
337590
|
invocation: context.invocation,
|
|
337575
337591
|
ownerId: owner,
|
|
337576
337592
|
includeRawUi: args.include_raw_ui === true,
|
|
@@ -340834,7 +340850,12 @@ async function runAgentKernel(agent) {
|
|
|
340834
340850
|
try {
|
|
340835
340851
|
const linkedPlanRevisionBeforeRun = agent.getLinkedPlan().revision;
|
|
340836
340852
|
const modelBeforeKernelRun = agent.model;
|
|
340837
|
-
|
|
340853
|
+
const preflightVisualFallback = !agent.activeModelConfig()?.vision ? await agent.finalVisualFallback("vision input not supported by the selected model", processSignal) : null;
|
|
340854
|
+
let lastTurn = preflightVisualFallback ? { text: preflightVisualFallback, stopReason: "stop", errorMessage: "" } : await runWithCompressionResume([], false);
|
|
340855
|
+
if (preflightVisualFallback) {
|
|
340856
|
+
tokens.push({ type: "text", text: preflightVisualFallback });
|
|
340857
|
+
agent.recordWorkStatus("Final visual fallback used: local mini OCR plus conservative text correction.");
|
|
340858
|
+
}
|
|
340838
340859
|
if (modelBeforeKernelRun && modelBeforeKernelRun !== agent.model && !tokens.some((t3) => t3.text?.includes("[Model fallback]"))) {
|
|
340839
340860
|
tokens.unshift({ type: "text", text: `[Model fallback] ${modelBeforeKernelRun} unavailable; switched to ${agent.model}.` });
|
|
340840
340861
|
}
|
|
@@ -340864,6 +340885,14 @@ async function runAgentKernel(agent) {
|
|
|
340864
340885
|
await agent.waitForPlannedRouteRetry();
|
|
340865
340886
|
lastTurn = await runWithCompressionResume([], false);
|
|
340866
340887
|
}
|
|
340888
|
+
if (kernelTurnFailed(agent, lastTurn)) {
|
|
340889
|
+
const visualFallback = await agent.finalVisualFallback(lastTurn.errorMessage || lastTurn.text, processSignal);
|
|
340890
|
+
if (visualFallback) {
|
|
340891
|
+
tokens.push({ type: "text", text: visualFallback });
|
|
340892
|
+
agent.recordWorkStatus("Final visual fallback used: local mini OCR plus conservative text correction.");
|
|
340893
|
+
lastTurn = { ...lastTurn, text: visualFallback, errorMessage: "", stopReason: "stop" };
|
|
340894
|
+
}
|
|
340895
|
+
}
|
|
340867
340896
|
if (kernelTurnFailed(agent, lastTurn)) {
|
|
340868
340897
|
throw new ProviderRunError(normalizePublicProviderError(lastTurn.errorMessage || lastTurn.text, [agent.activeModelConfig()?.api_key]));
|
|
340869
340898
|
}
|
|
@@ -350808,6 +350837,60 @@ ${msg.content}
|
|
|
350808
350837
|
};
|
|
350809
350838
|
return results;
|
|
350810
350839
|
}
|
|
350840
|
+
/**
|
|
350841
|
+
* Final visual safety net: OCR each submitted image and ask a text-only
|
|
350842
|
+
* request to conservatively repair the OCR. This is intentionally callable
|
|
350843
|
+
* only after a visual-input refusal and after same-provider vision routing
|
|
350844
|
+
* has been exhausted; the original image is never sent again.
|
|
350845
|
+
*/
|
|
350846
|
+
async finalVisualFallback(errorText, signal) {
|
|
350847
|
+
if (!/(?:vision|image|multimodal|image_url|input_image).*(?:not supported|unsupported|拒绝|不支持|failed|failure|invalid)|(?:not supported|unsupported|拒绝|不支持).*(?:vision|image|multimodal|image_url|input_image)/i.test(String(errorText || ""))) return null;
|
|
350848
|
+
const current = this.activeModelConfig();
|
|
350849
|
+
if (!current) return null;
|
|
350850
|
+
const alternateVision = this.config.allModels().some(
|
|
350851
|
+
(model) => model.enabled !== false && model.provider_id === current.provider_id && model.name !== current.name && !!model.vision && !!model.api_key && !!model.provider_url && !["unavailable", "auth_error", "invalid_config"].includes(String(model.evaluation?.status || model.validation?.status || "").toLowerCase())
|
|
350852
|
+
);
|
|
350853
|
+
if (alternateVision) return null;
|
|
350854
|
+
const latest = [...this.history].reverse().find((item) => item?.role === "user");
|
|
350855
|
+
const parts = latest?.content && Array.isArray(latest.content) ? latest.content : [];
|
|
350856
|
+
const images = parts.map((part) => {
|
|
350857
|
+
const image = part.image_url;
|
|
350858
|
+
return image && typeof image === "object" ? String(image.url || "") : "";
|
|
350859
|
+
}).filter((value) => /^data:image\/(?:png|jpeg);base64,/i.test(value)).slice(0, 4);
|
|
350860
|
+
if (!images.length) return null;
|
|
350861
|
+
const ocr = [];
|
|
350862
|
+
for (const [index, image] of images.entries()) {
|
|
350863
|
+
try {
|
|
350864
|
+
const result = await this.tools.finalVisualFallbackOcr(image, signal);
|
|
350865
|
+
if (result.ok && result.text.trim()) ocr.push({ index: index + 1, text: result.text.slice(0, 5e4), confidence: result.confidence });
|
|
350866
|
+
} catch {
|
|
350867
|
+
}
|
|
350868
|
+
}
|
|
350869
|
+
if (!ocr.length) return JSON.stringify({ ok: false, fallback: "mini_ocr_llm", error: "Local OCR returned no readable text; no visual content was fabricated." });
|
|
350870
|
+
const task = typeof latest?.content === "string" ? latest.content : "";
|
|
350871
|
+
const evidence = ocr.map((item) => `Image ${item.index} (OCR confidence ${item.confidence.toFixed(1)}):
|
|
350872
|
+
${item.text}`).join("\n\n");
|
|
350873
|
+
const prompt = `The provider rejected image input. Answer the user's task using only this approximate OCR evidence. Correct obvious character, spacing, and line-break errors only when supported by context. Preserve [uncertain] markers for ambiguity and never invent missing visual content.
|
|
350874
|
+
User task:
|
|
350875
|
+
${task.slice(0, 12e3)}
|
|
350876
|
+
OCR evidence:
|
|
350877
|
+
${evidence}`;
|
|
350878
|
+
let corrected = "";
|
|
350879
|
+
try {
|
|
350880
|
+
const provider = this.engineModel();
|
|
350881
|
+
if (provider) corrected = String(await provider.chat(this.activeModelName(), [{ role: "user", content: prompt }], "You are a text-only OCR correction assistant. Be conservative and explicit about uncertainty.", 0.05, 3e3, signal) || "").trim();
|
|
350882
|
+
} catch {
|
|
350883
|
+
}
|
|
350884
|
+
return JSON.stringify({
|
|
350885
|
+
ok: !!(corrected || ocr.length),
|
|
350886
|
+
fallback: "mini_ocr_llm",
|
|
350887
|
+
approximate: true,
|
|
350888
|
+
warning: "\u89C6\u89C9\u8F93\u5165\u88AB\u62D2\u7EDD\uFF1B\u4EE5\u4E0B\u5185\u5BB9\u6765\u81EA\u672C\u5730 OCR\uFF0C\u5E76\u7ECF\u6587\u672C\u6A21\u578B\u4FDD\u5B88\u6821\u6B63\uFF0C\u53EF\u80FD\u4E0D\u5B8C\u6574\u3002",
|
|
350889
|
+
raw_ocr: ocr,
|
|
350890
|
+
corrected: corrected || ocr.map((item) => item.text).join("\n\n"),
|
|
350891
|
+
uncertainty: corrected ? "preserved" : "raw_ocr_only"
|
|
350892
|
+
}, null, 2);
|
|
350893
|
+
}
|
|
350811
350894
|
engineModel() {
|
|
350812
350895
|
if (this.forcedProvider) {
|
|
350813
350896
|
const active = this.activeDeployment();
|
|
@@ -351190,8 +351273,10 @@ ${persisted.map((guide, index) => `Guide ${index + 1}: ${guide.text}`).join("\n"
|
|
|
351190
351273
|
}
|
|
351191
351274
|
const selectedModel = this.activeModelConfig();
|
|
351192
351275
|
if (images.length && !selectedModel?.vision) {
|
|
351193
|
-
|
|
351194
|
-
|
|
351276
|
+
const hasSameProviderVision = selectedModel && this.config.allModels().some(
|
|
351277
|
+
(model) => model.enabled !== false && model.provider_id === selectedModel.provider_id && model.name !== selectedModel.name && !!model.vision
|
|
351278
|
+
);
|
|
351279
|
+
if (hasSameProviderVision) this.switchToFallbackModel("vision input not supported by the selected model");
|
|
351195
351280
|
}
|
|
351196
351281
|
const now2 = this.nowLabel();
|
|
351197
351282
|
const visibleUserInput = inputEnvelope?.visibleUserInput === void 0 ? text : String(inputEnvelope.visibleUserInput || "");
|
package/dist/core/agent.d.ts
CHANGED
|
@@ -990,6 +990,13 @@ export declare class Agent {
|
|
|
990
990
|
isModelValidationRunning(): boolean;
|
|
991
991
|
modelValidationStatus(): ModelValidationProgress;
|
|
992
992
|
private runModelValidation;
|
|
993
|
+
/**
|
|
994
|
+
* Final visual safety net: OCR each submitted image and ask a text-only
|
|
995
|
+
* request to conservatively repair the OCR. This is intentionally callable
|
|
996
|
+
* only after a visual-input refusal and after same-provider vision routing
|
|
997
|
+
* has been exhausted; the original image is never sent again.
|
|
998
|
+
*/
|
|
999
|
+
finalVisualFallback(errorText: string, signal?: AbortSignal): Promise<string | null>;
|
|
993
1000
|
engineModel(): LLMProvider | null;
|
|
994
1001
|
/**
|
|
995
1002
|
* dev-0.4.3 模型原生思考强度档位映射表(模型名 → thinking_tier_map)。
|
package/dist/core/agent.js
CHANGED
|
@@ -6913,6 +6913,62 @@ class Agent {
|
|
|
6913
6913
|
};
|
|
6914
6914
|
return results;
|
|
6915
6915
|
}
|
|
6916
|
+
/**
|
|
6917
|
+
* Final visual safety net: OCR each submitted image and ask a text-only
|
|
6918
|
+
* request to conservatively repair the OCR. This is intentionally callable
|
|
6919
|
+
* only after a visual-input refusal and after same-provider vision routing
|
|
6920
|
+
* has been exhausted; the original image is never sent again.
|
|
6921
|
+
*/
|
|
6922
|
+
async finalVisualFallback(errorText, signal) {
|
|
6923
|
+
if (!/(?:vision|image|multimodal|image_url|input_image).*(?:not supported|unsupported|拒绝|不支持|failed|failure|invalid)|(?:not supported|unsupported|拒绝|不支持).*(?:vision|image|multimodal|image_url|input_image)/i.test(String(errorText || '')))
|
|
6924
|
+
return null;
|
|
6925
|
+
const current = this.activeModelConfig();
|
|
6926
|
+
if (!current)
|
|
6927
|
+
return null;
|
|
6928
|
+
const alternateVision = this.config.allModels().some(model => model.enabled !== false && model.provider_id === current.provider_id &&
|
|
6929
|
+
model.name !== current.name && !!model.vision && !!model.api_key && !!model.provider_url &&
|
|
6930
|
+
!['unavailable', 'auth_error', 'invalid_config'].includes(String(model.evaluation?.status || model.validation?.status || '').toLowerCase()));
|
|
6931
|
+
if (alternateVision)
|
|
6932
|
+
return null;
|
|
6933
|
+
const latest = [...this.history].reverse().find(item => item?.role === 'user');
|
|
6934
|
+
const parts = latest?.content && Array.isArray(latest.content) ? latest.content : [];
|
|
6935
|
+
const images = parts.map(part => {
|
|
6936
|
+
const image = part.image_url;
|
|
6937
|
+
return image && typeof image === 'object' ? String(image.url || '') : '';
|
|
6938
|
+
}).filter(value => /^data:image\/(?:png|jpeg);base64,/i.test(value)).slice(0, 4);
|
|
6939
|
+
if (!images.length)
|
|
6940
|
+
return null;
|
|
6941
|
+
const ocr = [];
|
|
6942
|
+
for (const [index, image] of images.entries()) {
|
|
6943
|
+
try {
|
|
6944
|
+
const result = await this.tools.finalVisualFallbackOcr(image, signal);
|
|
6945
|
+
if (result.ok && result.text.trim())
|
|
6946
|
+
ocr.push({ index: index + 1, text: result.text.slice(0, 50_000), confidence: result.confidence });
|
|
6947
|
+
}
|
|
6948
|
+
catch { }
|
|
6949
|
+
}
|
|
6950
|
+
if (!ocr.length)
|
|
6951
|
+
return JSON.stringify({ ok: false, fallback: 'mini_ocr_llm', error: 'Local OCR returned no readable text; no visual content was fabricated.' });
|
|
6952
|
+
const task = typeof latest?.content === 'string' ? latest.content : '';
|
|
6953
|
+
const evidence = ocr.map(item => `Image ${item.index} (OCR confidence ${item.confidence.toFixed(1)}):\n${item.text}`).join('\n\n');
|
|
6954
|
+
const prompt = `The provider rejected image input. Answer the user's task using only this approximate OCR evidence. Correct obvious character, spacing, and line-break errors only when supported by context. Preserve [uncertain] markers for ambiguity and never invent missing visual content.\nUser task:\n${task.slice(0, 12_000)}\nOCR evidence:\n${evidence}`;
|
|
6955
|
+
let corrected = '';
|
|
6956
|
+
try {
|
|
6957
|
+
const provider = this.engineModel();
|
|
6958
|
+
if (provider)
|
|
6959
|
+
corrected = String(await provider.chat(this.activeModelName(), [{ role: 'user', content: prompt }], 'You are a text-only OCR correction assistant. Be conservative and explicit about uncertainty.', 0.05, 3000, signal) || '').trim();
|
|
6960
|
+
}
|
|
6961
|
+
catch { }
|
|
6962
|
+
return JSON.stringify({
|
|
6963
|
+
ok: !!(corrected || ocr.length),
|
|
6964
|
+
fallback: 'mini_ocr_llm',
|
|
6965
|
+
approximate: true,
|
|
6966
|
+
warning: '视觉输入被拒绝;以下内容来自本地 OCR,并经文本模型保守校正,可能不完整。',
|
|
6967
|
+
raw_ocr: ocr,
|
|
6968
|
+
corrected: corrected || ocr.map(item => item.text).join('\n\n'),
|
|
6969
|
+
uncertainty: corrected ? 'preserved' : 'raw_ocr_only',
|
|
6970
|
+
}, null, 2);
|
|
6971
|
+
}
|
|
6916
6972
|
engineModel() {
|
|
6917
6973
|
if (this.forcedProvider) {
|
|
6918
6974
|
const active = this.activeDeployment();
|
|
@@ -7353,8 +7409,13 @@ class Agent {
|
|
|
7353
7409
|
}
|
|
7354
7410
|
const selectedModel = this.activeModelConfig();
|
|
7355
7411
|
if (images.length && !selectedModel?.vision) {
|
|
7356
|
-
|
|
7357
|
-
|
|
7412
|
+
// Give the normal route planner first chance to select another
|
|
7413
|
+
// same-provider vision deployment. If none is available, the kernel
|
|
7414
|
+
// preflight invokes the final mini-OCR + text-only correction path.
|
|
7415
|
+
const hasSameProviderVision = selectedModel && this.config.allModels().some(model => model.enabled !== false && model.provider_id === selectedModel.provider_id &&
|
|
7416
|
+
model.name !== selectedModel.name && !!model.vision);
|
|
7417
|
+
if (hasSameProviderVision)
|
|
7418
|
+
this.switchToFallbackModel('vision input not supported by the selected model');
|
|
7358
7419
|
}
|
|
7359
7420
|
const now = this.nowLabel();
|
|
7360
7421
|
const visibleUserInput = inputEnvelope?.visibleUserInput === undefined
|
|
@@ -429,7 +429,16 @@ async function runAgentKernel(agent) {
|
|
|
429
429
|
try {
|
|
430
430
|
const linkedPlanRevisionBeforeRun = agent.getLinkedPlan().revision;
|
|
431
431
|
const modelBeforeKernelRun = agent.model;
|
|
432
|
-
|
|
432
|
+
const preflightVisualFallback = !agent.activeModelConfig()?.vision
|
|
433
|
+
? await agent.finalVisualFallback('vision input not supported by the selected model', processSignal)
|
|
434
|
+
: null;
|
|
435
|
+
let lastTurn = preflightVisualFallback
|
|
436
|
+
? { text: preflightVisualFallback, stopReason: 'stop', errorMessage: '' }
|
|
437
|
+
: await runWithCompressionResume([], false);
|
|
438
|
+
if (preflightVisualFallback) {
|
|
439
|
+
tokens.push({ type: 'text', text: preflightVisualFallback });
|
|
440
|
+
agent.recordWorkStatus('Final visual fallback used: local mini OCR plus conservative text correction.');
|
|
441
|
+
}
|
|
433
442
|
if (modelBeforeKernelRun && modelBeforeKernelRun !== agent.model && !tokens.some(t => t.text?.includes('[Model fallback]'))) {
|
|
434
443
|
tokens.unshift({ type: 'text', text: `[Model fallback] ${modelBeforeKernelRun} unavailable; switched to ${agent.model}.` });
|
|
435
444
|
}
|
|
@@ -460,6 +469,14 @@ async function runAgentKernel(agent) {
|
|
|
460
469
|
await agent.waitForPlannedRouteRetry();
|
|
461
470
|
lastTurn = await runWithCompressionResume([], false);
|
|
462
471
|
}
|
|
472
|
+
if (kernelTurnFailed(agent, lastTurn)) {
|
|
473
|
+
const visualFallback = await agent.finalVisualFallback(lastTurn.errorMessage || lastTurn.text, processSignal);
|
|
474
|
+
if (visualFallback) {
|
|
475
|
+
tokens.push({ type: 'text', text: visualFallback });
|
|
476
|
+
agent.recordWorkStatus('Final visual fallback used: local mini OCR plus conservative text correction.');
|
|
477
|
+
lastTurn = { ...lastTurn, text: visualFallback, errorMessage: '', stopReason: 'stop' };
|
|
478
|
+
}
|
|
479
|
+
}
|
|
463
480
|
if (kernelTurnFailed(agent, lastTurn)) {
|
|
464
481
|
throw new ProviderRunError(normalizePublicProviderError(lastTurn.errorMessage || lastTurn.text, [agent.activeModelConfig()?.api_key]));
|
|
465
482
|
}
|
package/dist/core/config.js
CHANGED
|
@@ -87,7 +87,9 @@ class ConfigManager {
|
|
|
87
87
|
this.backupConfig(cp, 'invalid-shape');
|
|
88
88
|
return this.writeRecoveredConfig(cp);
|
|
89
89
|
}
|
|
90
|
-
|
|
90
|
+
const providerIdsMigrated = migrateProviderIdsInConfig(normalized);
|
|
91
|
+
const marqueeConfigRemoved = removeDeprecatedMarqueeConfig(normalized);
|
|
92
|
+
if (providerIdsMigrated || marqueeConfigRemoved) {
|
|
91
93
|
// Provider ids are routing identities, so legacy/malformed catalogs must
|
|
92
94
|
// not wait for an unrelated settings save before becoming collision-safe.
|
|
93
95
|
try {
|
|
@@ -457,6 +459,19 @@ function normalizeConfigShape(raw, withDefaults) {
|
|
|
457
459
|
}
|
|
458
460
|
return base;
|
|
459
461
|
}
|
|
462
|
+
function removeDeprecatedMarqueeConfig(config) {
|
|
463
|
+
const ui = config.ui;
|
|
464
|
+
if (!ui)
|
|
465
|
+
return false;
|
|
466
|
+
let changed = false;
|
|
467
|
+
for (const key of ['gradient_colors', 'gradient_speed', 'gradient_width']) {
|
|
468
|
+
if (Object.prototype.hasOwnProperty.call(ui, key)) {
|
|
469
|
+
delete ui[key];
|
|
470
|
+
changed = true;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
return changed;
|
|
474
|
+
}
|
|
460
475
|
function isConfigEntry(value) {
|
|
461
476
|
return !!value && typeof value === 'object' && !Array.isArray(value) && Object.prototype.hasOwnProperty.call(value, 'value');
|
|
462
477
|
}
|
|
@@ -892,9 +907,6 @@ function defaultConfig() {
|
|
|
892
907
|
auto_download: { _description: "Auto-download skills", _type: "choice", _values: ["aggressive", "conservative", "disabled"], value: "conservative" },
|
|
893
908
|
},
|
|
894
909
|
ui: {
|
|
895
|
-
gradient_colors: { _description: "Gradient colors (hex)", _type: "array", value: ["#00ff88", "#00ccff", "#aa44ff", "#ff4488"] },
|
|
896
|
-
gradient_speed: { _description: "Animation speed 1-5", _type: "range", _min: 1, _max: 5, value: 2 },
|
|
897
|
-
gradient_width: { _description: "Border width", _type: "integer", _min: 1, _max: 4, value: 2 },
|
|
898
910
|
glass_alpha: { _description: "Glass opacity", _type: "range", _min: 0, _max: 1, value: 0.85 },
|
|
899
911
|
show_mode_label: { _description: "Show mode on hover", _type: "boolean", value: true },
|
|
900
912
|
left_panel_collapsed: { _description: "Left panel collapsed", _type: "boolean", value: false },
|
package/dist/llm/provider.js
CHANGED
|
@@ -46,7 +46,9 @@ const providers_1 = require("../providers");
|
|
|
46
46
|
// Keep provider requests below the release-harness/user-visible command
|
|
47
47
|
// deadline. A provider that does not answer must produce one bounded error;
|
|
48
48
|
// it must not restart the same request through every Windows transport.
|
|
49
|
-
|
|
49
|
+
// Provider responses are intentionally unbounded. User cancellation, transport
|
|
50
|
+
// errors, and tool-specific limits remain the only automatic stop conditions.
|
|
51
|
+
const DEFAULT_PROVIDER_REQUEST_TIMEOUT_MS = 0;
|
|
50
52
|
const MIN_PROVIDER_REQUEST_TIMEOUT_MS = 50;
|
|
51
53
|
function providerTimeoutError(timeoutMs) {
|
|
52
54
|
const error = new Error(`Provider request timed out after ${timeoutMs}ms`);
|
|
@@ -115,13 +117,17 @@ class LLMProvider {
|
|
|
115
117
|
this.thinkingTierMaps = thinkingTierMaps;
|
|
116
118
|
}
|
|
117
119
|
effectiveRequestTimeout(timeoutMs) {
|
|
118
|
-
const requested = Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs :
|
|
120
|
+
const requested = Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 0;
|
|
119
121
|
const configured = Number.isFinite(this.requestTimeoutMs) && this.requestTimeoutMs > 0
|
|
120
122
|
? this.requestTimeoutMs
|
|
121
|
-
:
|
|
123
|
+
: 0;
|
|
124
|
+
if (requested <= 0 || configured <= 0)
|
|
125
|
+
return 0;
|
|
122
126
|
return Math.max(MIN_PROVIDER_REQUEST_TIMEOUT_MS, Math.min(requested, configured));
|
|
123
127
|
}
|
|
124
128
|
async withRequestTimeout(promise, timeoutMs, signal) {
|
|
129
|
+
if (timeoutMs <= 0)
|
|
130
|
+
return await abortable(promise, signal);
|
|
125
131
|
let timer;
|
|
126
132
|
const timeoutPromise = new Promise((_, reject) => {
|
|
127
133
|
timer = setTimeout(() => reject(providerTimeoutError(timeoutMs)), timeoutMs);
|
|
@@ -362,7 +368,9 @@ class LLMProvider {
|
|
|
362
368
|
forwardAbort();
|
|
363
369
|
else
|
|
364
370
|
signal?.addEventListener('abort', forwardAbort, { once: true });
|
|
365
|
-
const timer =
|
|
371
|
+
const timer = effectiveTimeout > 0
|
|
372
|
+
? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout)
|
|
373
|
+
: undefined;
|
|
366
374
|
try {
|
|
367
375
|
const response = await fetch(url, {
|
|
368
376
|
method: 'POST',
|
|
@@ -396,7 +404,9 @@ class LLMProvider {
|
|
|
396
404
|
async getJsonWithFetchFallback(url, headers, timeoutMs = 30000) {
|
|
397
405
|
const effectiveTimeout = this.effectiveRequestTimeout(timeoutMs);
|
|
398
406
|
const abort = new AbortController();
|
|
399
|
-
const timer =
|
|
407
|
+
const timer = effectiveTimeout > 0
|
|
408
|
+
? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout)
|
|
409
|
+
: undefined;
|
|
400
410
|
try {
|
|
401
411
|
const response = await fetch(url, { method: 'GET', headers, signal: abort.signal });
|
|
402
412
|
return response;
|
|
@@ -482,9 +492,11 @@ class LLMProvider {
|
|
|
482
492
|
fail(new Error('Node HTTP response closed before completion'));
|
|
483
493
|
});
|
|
484
494
|
});
|
|
485
|
-
|
|
486
|
-
req.
|
|
487
|
-
|
|
495
|
+
if (effectiveTimeout > 0) {
|
|
496
|
+
req.setTimeout(effectiveTimeout, () => {
|
|
497
|
+
req.destroy(providerTimeoutError(effectiveTimeout));
|
|
498
|
+
});
|
|
499
|
+
}
|
|
488
500
|
req.on('error', reject);
|
|
489
501
|
const onAbort = () => req.destroy(abortFailure(signal));
|
|
490
502
|
if (signal?.aborted)
|
|
@@ -538,7 +550,9 @@ class LLMProvider {
|
|
|
538
550
|
' $raw = $headerJson | ConvertFrom-Json',
|
|
539
551
|
' foreach ($p in $raw.PSObject.Properties) { $headers[$p.Name] = [string]$p.Value }',
|
|
540
552
|
'}',
|
|
541
|
-
|
|
553
|
+
effectiveTimeout > 0
|
|
554
|
+
? `$params = @{ Uri = $uri; Method = $method; Headers = $headers; UseBasicParsing = $true; TimeoutSec = ${Math.ceil(effectiveTimeout / 1000)} }`
|
|
555
|
+
: '$params = @{ Uri = $uri; Method = $method; Headers = $headers; UseBasicParsing = $true }',
|
|
542
556
|
'if ($method -eq "POST") { $params["Body"] = $bodyJson }',
|
|
543
557
|
'if ($method -eq "POST") { $params["ContentType"] = "application/json; charset=utf-8" }',
|
|
544
558
|
'$resp = Invoke-WebRequest @params',
|
|
@@ -569,11 +583,11 @@ class LLMProvider {
|
|
|
569
583
|
onAbort();
|
|
570
584
|
else
|
|
571
585
|
signal?.addEventListener('abort', onAbort, { once: true });
|
|
572
|
-
const timer = setTimeout(() => {
|
|
586
|
+
const timer = effectiveTimeout > 0 ? setTimeout(() => {
|
|
573
587
|
child.kill();
|
|
574
588
|
cleanup();
|
|
575
589
|
reject(providerTimeoutError(effectiveTimeout));
|
|
576
|
-
}, effectiveTimeout + 5000);
|
|
590
|
+
}, effectiveTimeout + 5000) : undefined;
|
|
577
591
|
child.stdout.setEncoding('utf8');
|
|
578
592
|
child.stderr.setEncoding('utf8');
|
|
579
593
|
child.stdout.on('data', chunk => { stdout += chunk; });
|
|
@@ -1083,7 +1097,9 @@ class LLMProvider {
|
|
|
1083
1097
|
else
|
|
1084
1098
|
signal?.addEventListener('abort', forwardAbort, { once: true });
|
|
1085
1099
|
const effectiveTimeout = this.effectiveRequestTimeout(120000);
|
|
1086
|
-
const timer =
|
|
1100
|
+
const timer = effectiveTimeout > 0
|
|
1101
|
+
? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout)
|
|
1102
|
+
: undefined;
|
|
1087
1103
|
try {
|
|
1088
1104
|
try {
|
|
1089
1105
|
let response = await fetch(request.url, {
|
|
@@ -1232,7 +1248,9 @@ class LLMProvider {
|
|
|
1232
1248
|
else
|
|
1233
1249
|
signal?.addEventListener('abort', forwardAbort, { once: true });
|
|
1234
1250
|
const effectiveTimeout = this.effectiveRequestTimeout(120000);
|
|
1235
|
-
const timeout =
|
|
1251
|
+
const timeout = effectiveTimeout > 0
|
|
1252
|
+
? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout)
|
|
1253
|
+
: undefined;
|
|
1236
1254
|
let reader = null;
|
|
1237
1255
|
try {
|
|
1238
1256
|
let response;
|
package/dist/main.js
CHANGED
|
@@ -3640,9 +3640,6 @@ else {
|
|
|
3640
3640
|
proxyEnabled: agent.config.getBool('proxy', 'enabled'),
|
|
3641
3641
|
proxyUrl: agent.config.getStr('proxy', 'url'),
|
|
3642
3642
|
proxyAuth: agent.config.getStr('proxy', 'auth'),
|
|
3643
|
-
gradientColors: agent.config.get('ui', 'gradient_colors') || [],
|
|
3644
|
-
gradientSpeed: agent.config.getNum('ui', 'gradient_speed'),
|
|
3645
|
-
gradientWidth: agent.config.getNum('ui', 'gradient_width'),
|
|
3646
3643
|
glassAlpha: agent.config.getNum('ui', 'glass_alpha') ?? 0.85,
|
|
3647
3644
|
leftPanelCollapsed: agent.config.getBool('ui', 'left_panel_collapsed'),
|
|
3648
3645
|
rightPanelCollapsed: agent.config.getBool('ui', 'right_panel_collapsed'),
|
|
@@ -3809,15 +3806,6 @@ else {
|
|
|
3809
3806
|
else {
|
|
3810
3807
|
for (const [key, value] of Object.entries(cfg || {})) {
|
|
3811
3808
|
switch (key) {
|
|
3812
|
-
case 'gradientColors':
|
|
3813
|
-
agent.config.set('ui', 'gradient_colors', value);
|
|
3814
|
-
break;
|
|
3815
|
-
case 'gradientSpeed':
|
|
3816
|
-
agent.config.set('ui', 'gradient_speed', value);
|
|
3817
|
-
break;
|
|
3818
|
-
case 'gradientWidth':
|
|
3819
|
-
agent.config.set('ui', 'gradient_width', value);
|
|
3820
|
-
break;
|
|
3821
3809
|
case 'glassAlpha':
|
|
3822
3810
|
agent.config.set('ui', 'glass_alpha', value);
|
|
3823
3811
|
break;
|
package/dist/server.js
CHANGED
|
@@ -459,15 +459,6 @@ function applyConfigPatch(cfg) {
|
|
|
459
459
|
return;
|
|
460
460
|
for (const [key, value] of Object.entries(cfg || {})) {
|
|
461
461
|
switch (key) {
|
|
462
|
-
case 'gradientColors':
|
|
463
|
-
agent.config.set('ui', 'gradient_colors', value);
|
|
464
|
-
break;
|
|
465
|
-
case 'gradientSpeed':
|
|
466
|
-
agent.config.set('ui', 'gradient_speed', value);
|
|
467
|
-
break;
|
|
468
|
-
case 'gradientWidth':
|
|
469
|
-
agent.config.set('ui', 'gradient_width', value);
|
|
470
|
-
break;
|
|
471
462
|
case 'glassAlpha':
|
|
472
463
|
agent.config.set('ui', 'glass_alpha', value);
|
|
473
464
|
break;
|
|
@@ -599,9 +590,6 @@ async function handleApi(req, res, body) {
|
|
|
599
590
|
conversationPlan: agent.getConversationPlan(),
|
|
600
591
|
historyMessages: agent.history.length,
|
|
601
592
|
conversationLocked: agent.isConversationLocked(),
|
|
602
|
-
gradientColors: agent.config.get('ui', 'gradient_colors') || [],
|
|
603
|
-
gradientSpeed: agent.config.getNum('ui', 'gradient_speed'),
|
|
604
|
-
gradientWidth: agent.config.getNum('ui', 'gradient_width'),
|
|
605
593
|
glassAlpha: agent.config.getNum('ui', 'glass_alpha'),
|
|
606
594
|
darkMode: agent.config.getStr('ui', 'dark_mode'),
|
|
607
595
|
backgroundColor: (0, uiPreferences_1.normalizeUiBackgroundColor)(agent.config.getStr('ui', 'background_color')),
|
|
@@ -210,13 +210,8 @@ function jpegCaptureScript(outPath, boundsScript, requestedMaxWidth, requestedMa
|
|
|
210
210
|
'Write-Output (@{ ok=$true; left=$x; top=$y; width=$w; height=$h; capture_max_width=$maxWidth; capture_max_height=$maxHeight; image_width=$imageWidth; image_height=$imageHeight; image_bytes=$fileSize; image_quality=$qualityUsed; image_available=$imageAvailable; image_mime="image/jpeg" } | ConvertTo-Json -Compress)',
|
|
211
211
|
].join('\r\n');
|
|
212
212
|
}
|
|
213
|
-
function gradientPalette(
|
|
214
|
-
|
|
215
|
-
const configured = Array.isArray(input) ? input : [];
|
|
216
|
-
const raw = configured.length
|
|
217
|
-
? configured.map(v => String(v || '').trim()).filter(Boolean)
|
|
218
|
-
: String(process.env.NEWMARK_COMPUTER_USE_GRADIENT || '').split(',').map(v => v.trim()).filter(Boolean);
|
|
219
|
-
return raw.length >= 2 ? raw.slice(0, 6) : fallback;
|
|
213
|
+
function gradientPalette(_input) {
|
|
214
|
+
return ['#000000', '#ffffff', '#000000', '#ffffff'];
|
|
220
215
|
}
|
|
221
216
|
async function stopTakeoverOverlay() {
|
|
222
217
|
const pid = takeoverOverlayPid;
|
|
@@ -239,11 +234,11 @@ async function startTakeoverOverlay(durationMs = 0, input = {}) {
|
|
|
239
234
|
if (process.platform !== 'win32')
|
|
240
235
|
return { ok: false, action: 'takeover_start', error: 'Computer Use takeover overlay is Windows-only.' };
|
|
241
236
|
await stopTakeoverOverlay();
|
|
242
|
-
lastTakeoverOverlayStyle = { colors:
|
|
237
|
+
lastTakeoverOverlayStyle = { colors: gradientPalette(), speed: 3, width: 2 };
|
|
243
238
|
const colors = gradientPalette(input.colors);
|
|
244
239
|
const lifetime = Math.max(0, Math.floor(Number(durationMs || 0)));
|
|
245
|
-
const width =
|
|
246
|
-
const speedSeconds =
|
|
240
|
+
const width = 2;
|
|
241
|
+
const speedSeconds = 3;
|
|
247
242
|
const ownerPid = Math.max(0, Math.floor(Number(input.ownerPid ?? process.pid) || 0));
|
|
248
243
|
const scriptPath = path.join(tempScreenshotDir(), `takeover-overlay-${timestampName()}-${crypto.randomBytes(4).toString('hex')}.ps1`);
|
|
249
244
|
const script = [
|
package/dist/tools/index.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { ConfigManager } from '../core/config';
|
|
|
2
2
|
import { NewmarkToolDefinition, NewmarkToolResult } from '../core/compat';
|
|
3
3
|
import { SshManager } from '../core/ssh';
|
|
4
4
|
import { WorkspaceManager } from '../core/workspace';
|
|
5
|
+
import { LocalOcrResult } from '../core/localOcr';
|
|
5
6
|
export interface ToolExecutionContext {
|
|
6
7
|
mode?: string;
|
|
7
8
|
workspacePath?: string;
|
|
@@ -30,6 +31,8 @@ export declare class ToolExecutor {
|
|
|
30
31
|
private hostProfile;
|
|
31
32
|
constructor(root: string, config: ConfigManager, ssh?: SshManager | undefined, workspace?: WorkspaceManager | undefined);
|
|
32
33
|
webSearch(query: string): Promise<string>;
|
|
34
|
+
/** OCR entry point for the runtime's final visual fallback. */
|
|
35
|
+
finalVisualFallbackOcr(dataUrl: string, signal?: AbortSignal): Promise<LocalOcrResult>;
|
|
33
36
|
setHostProfile(profile: ToolHostProfile): void;
|
|
34
37
|
definitions(mode?: string): unknown[];
|
|
35
38
|
canonicalDefinitions(mode?: string): NewmarkToolDefinition[];
|
package/dist/tools/index.js
CHANGED
|
@@ -234,6 +234,10 @@ class ToolExecutor {
|
|
|
234
234
|
async webSearch(query) {
|
|
235
235
|
return this.wsearch(query);
|
|
236
236
|
}
|
|
237
|
+
/** OCR entry point for the runtime's final visual fallback. */
|
|
238
|
+
async finalVisualFallbackOcr(dataUrl, signal) {
|
|
239
|
+
return await this.localOcr.recognizeDataUrl(dataUrl, signal, 'sparse-ui');
|
|
240
|
+
}
|
|
237
241
|
setHostProfile(profile) {
|
|
238
242
|
this.hostProfile = { ...profile };
|
|
239
243
|
}
|
|
@@ -901,9 +905,9 @@ class ToolExecutor {
|
|
|
901
905
|
allowEphemeralVisionImage: context.allowEphemeralVisionImage === true,
|
|
902
906
|
captureMaxWidth: Number(args.capture_max_width),
|
|
903
907
|
captureMaxHeight: Number(args.capture_max_height),
|
|
904
|
-
gradientColors: Array.isArray(args.gradient_colors) ? args.gradient_colors :
|
|
905
|
-
gradientSpeed: args.gradient_speed !== undefined ? Number(args.gradient_speed) :
|
|
906
|
-
gradientWidth: args.gradient_width !== undefined ? Number(args.gradient_width) :
|
|
908
|
+
gradientColors: Array.isArray(args.gradient_colors) ? args.gradient_colors : undefined,
|
|
909
|
+
gradientSpeed: args.gradient_speed !== undefined ? Number(args.gradient_speed) : undefined,
|
|
910
|
+
gradientWidth: args.gradient_width !== undefined ? Number(args.gradient_width) : undefined,
|
|
907
911
|
invocation: context.invocation,
|
|
908
912
|
ownerId: owner,
|
|
909
913
|
includeRawUi: args.include_raw_ui === true,
|
package/dist/ui/index.html
CHANGED
|
@@ -108,10 +108,10 @@ try {
|
|
|
108
108
|
--radius-full: 9999px;
|
|
109
109
|
|
|
110
110
|
/* Marquee gradient */
|
|
111
|
-
--g1: #
|
|
112
|
-
--g2: #
|
|
113
|
-
--g3: #
|
|
114
|
-
--g4: #
|
|
111
|
+
--g1: #000000;
|
|
112
|
+
--g2: #ffffff;
|
|
113
|
+
--g3: #000000;
|
|
114
|
+
--g4: #ffffff;
|
|
115
115
|
--marquee-speed: 3s;
|
|
116
116
|
--marquee-width: 2px;
|
|
117
117
|
|
|
@@ -6182,9 +6182,6 @@ var state = {
|
|
|
6182
6182
|
conversationArchiveRefreshTimer: null,
|
|
6183
6183
|
conversationArchiveActiveSyncTimer: null,
|
|
6184
6184
|
nextConversationSequence: 0,
|
|
6185
|
-
configGradientColors: ['#00ff88', '#00ccff', '#aa44ff', '#ff4488'],
|
|
6186
|
-
configGradientSpeed: 3,
|
|
6187
|
-
configGradientWidth: 2,
|
|
6188
6185
|
inputMode: 'guide',
|
|
6189
6186
|
theme: 'dark',
|
|
6190
6187
|
backgroundColor: '',
|
|
@@ -15679,7 +15676,6 @@ function renderGeneralSettings() {
|
|
|
15679
15676
|
for (var i = 0; i < themeOpts.length; i++) {
|
|
15680
15677
|
themeHtml += '<option value="' + themeOpts[i].v + '"' + (state.theme === themeOpts[i].v ? ' selected' : '') + '>' + themeOpts[i].l + '</option>';
|
|
15681
15678
|
}
|
|
15682
|
-
var gradColors = state.configGradientColors || ['#ff6b6b','#ffd93d','#6bcb77','#4d96ff'];
|
|
15683
15679
|
var backgroundColor = /^#[0-9a-f]{6}$/i.test(String(state.backgroundColor || ''))
|
|
15684
15680
|
? state.backgroundColor
|
|
15685
15681
|
: (state.theme === 'light' ? '#F0F2F8' : '#0A0A1A');
|
|
@@ -15748,16 +15744,6 @@ function renderGeneralSettings() {
|
|
|
15748
15744
|
'<div class="setting-desc" id="glass-desc" aria-live="polite">' + esc(glassDescription) + '</div></div>' +
|
|
15749
15745
|
'</div>' +
|
|
15750
15746
|
'<div class="setting-row">' +
|
|
15751
|
-
'<span class="setting-label">' + esc(t('settings.gradient')) + '</span>' +
|
|
15752
|
-
'<div class="setting-control" style="flex-wrap:wrap;gap:4px;">' +
|
|
15753
|
-
'<input type="color" value="' + gradColors[0] + '" onchange="window.setGradientColor(0,this.value)" style="width:36px;height:28px;padding:0;border:1px solid var(--border);border-radius:var(--radius-sm);background:none;cursor:pointer;">' +
|
|
15754
|
-
'<input type="color" value="' + gradColors[1] + '" onchange="window.setGradientColor(1,this.value)" style="width:36px;height:28px;padding:0;border:1px solid var(--border);border-radius:var(--radius-sm);background:none;cursor:pointer;">' +
|
|
15755
|
-
'<input type="color" value="' + gradColors[2] + '" onchange="window.setGradientColor(2,this.value)" style="width:36px;height:28px;padding:0;border:1px solid var(--border);border-radius:var(--radius-sm);background:none;cursor:pointer;">' +
|
|
15756
|
-
'<input type="color" value="' + gradColors[3] + '" onchange="window.setGradientColor(3,this.value)" style="width:36px;height:28px;padding:0;border:1px solid var(--border);border-radius:var(--radius-sm);background:none;cursor:pointer;">' +
|
|
15757
|
-
'<div class="setting-desc" style="width:100%;">' + esc(t('settings.speed')) + ': <input type="range" min="1" max="10" value="' + (state.configGradientSpeed || 3) + '" oninput="window.setGradientSpeed(this.value)" style="width:60px;vertical-align:middle;">' +
|
|
15758
|
-
' ' + esc(t('settings.width')) + ': <input type="range" min="1" max="6" value="' + (state.configGradientWidth || 2) + '" oninput="window.setGradientWidth(this.value)" style="width:60px;vertical-align:middle;"></div></div>' +
|
|
15759
|
-
'</div>' +
|
|
15760
|
-
'<div class="setting-row">' +
|
|
15761
15747
|
'<span class="setting-label">' + esc(t('settings.inputMode')) + '</span>' +
|
|
15762
15748
|
'<div class="setting-control"><select onchange="window.setInputMode(this.value)">' +
|
|
15763
15749
|
'<option value="guide"' + (state.inputMode === 'guide' ? ' selected' : '') + '>' + esc(t('input.guide')) + '</option>' +
|
|
@@ -16311,25 +16297,6 @@ window.commitGlassOpacity = function(value) {
|
|
|
16311
16297
|
if (api.saveConfig) api.saveConfig({ glassAlpha: presentation.alpha }).catch(function(){});
|
|
16312
16298
|
};
|
|
16313
16299
|
|
|
16314
|
-
window.setGradientColor = function(idx, color) {
|
|
16315
|
-
if (!state.configGradientColors) state.configGradientColors = ['#ff6b6b','#ffd93d','#6bcb77','#4d96ff'];
|
|
16316
|
-
state.configGradientColors[idx] = color;
|
|
16317
|
-
updateMarqueeFromConfig();
|
|
16318
|
-
api.saveConfig({gradientColors: state.configGradientColors});
|
|
16319
|
-
};
|
|
16320
|
-
|
|
16321
|
-
window.setGradientSpeed = function(v) {
|
|
16322
|
-
state.configGradientSpeed = parseInt(v);
|
|
16323
|
-
updateMarqueeFromConfig();
|
|
16324
|
-
api.saveConfig({gradientSpeed: state.configGradientSpeed});
|
|
16325
|
-
};
|
|
16326
|
-
|
|
16327
|
-
window.setGradientWidth = function(v) {
|
|
16328
|
-
state.configGradientWidth = parseInt(v);
|
|
16329
|
-
updateMarqueeFromConfig();
|
|
16330
|
-
api.saveConfig({gradientWidth: state.configGradientWidth});
|
|
16331
|
-
};
|
|
16332
|
-
|
|
16333
16300
|
window.setDialogStyle = function(v) {
|
|
16334
16301
|
state.dialogStyle = v;
|
|
16335
16302
|
api.saveConfig({ dialogStyle: v });
|
|
@@ -22415,27 +22382,12 @@ window.ensureFlowsLoaded = function(options) {
|
|
|
22415
22382
|
return state._flowLoadPromise;
|
|
22416
22383
|
};
|
|
22417
22384
|
|
|
22418
|
-
// === Update Marquee ===
|
|
22419
22385
|
var marqueeRAF = null;
|
|
22420
|
-
function updateMarqueeFromConfig() {
|
|
22421
|
-
var root = document.documentElement;
|
|
22422
|
-
var colors = state.configGradientColors;
|
|
22423
|
-
if (colors && colors.length >= 4) {
|
|
22424
|
-
root.style.setProperty('--g1', colors[0]);
|
|
22425
|
-
root.style.setProperty('--g2', colors[1]);
|
|
22426
|
-
root.style.setProperty('--g3', colors[2]);
|
|
22427
|
-
root.style.setProperty('--g4', colors[3]);
|
|
22428
|
-
}
|
|
22429
|
-
root.style.setProperty('--marquee-speed', (state.configGradientSpeed || 2) + 's');
|
|
22430
|
-
root.style.setProperty('--marquee-width', (state.configGradientWidth || 2) + 'px');
|
|
22431
|
-
}
|
|
22432
|
-
|
|
22433
22386
|
// JS-driven marquee fallback for browsers without @property support
|
|
22434
22387
|
function startMarqueeJS() {
|
|
22435
22388
|
if (marqueeRAF) return;
|
|
22436
22389
|
var root = document.documentElement;
|
|
22437
22390
|
var angle = 0;
|
|
22438
|
-
var speed = (state.configGradientSpeed || 3) * 60;
|
|
22439
22391
|
function tick() {
|
|
22440
22392
|
angle = (angle + 1) % 360;
|
|
22441
22393
|
root.style.setProperty('--marquee-angle', angle + 'deg');
|
|
@@ -23270,9 +23222,6 @@ function schedulePostStartupUiRendering() {
|
|
|
23270
23222
|
state.fontFamily = normalizeUiFontFamilyClient(s.fontFamily || '');
|
|
23271
23223
|
state.glassLevel = glassPresentationForOpacity((s.glassAlpha ?? 0.85) * 100).opacityPercent;
|
|
23272
23224
|
state.models = s.models || [];
|
|
23273
|
-
if (s.gradientColors && s.gradientColors.length) state.configGradientColors = s.gradientColors;
|
|
23274
|
-
if (s.gradientSpeed) state.configGradientSpeed = s.gradientSpeed;
|
|
23275
|
-
if (s.gradientWidth) state.configGradientWidth = s.gradientWidth;
|
|
23276
23225
|
applySavedLayoutState({
|
|
23277
23226
|
leftCollapsed: s.leftPanelCollapsed,
|
|
23278
23227
|
rightCollapsed: s.rightPanelCollapsed,
|
|
@@ -23356,7 +23305,6 @@ function schedulePostStartupUiRendering() {
|
|
|
23356
23305
|
});
|
|
23357
23306
|
}
|
|
23358
23307
|
|
|
23359
|
-
updateMarqueeFromConfig();
|
|
23360
23308
|
window.startRemoteTouchStatusPolling();
|
|
23361
23309
|
|
|
23362
23310
|
// Populate selects
|
|
@@ -327676,7 +327676,9 @@ var ConfigManager = class {
|
|
|
327676
327676
|
this.backupConfig(cp, "invalid-shape");
|
|
327677
327677
|
return this.writeRecoveredConfig(cp);
|
|
327678
327678
|
}
|
|
327679
|
-
|
|
327679
|
+
const providerIdsMigrated = migrateProviderIdsInConfig(normalized);
|
|
327680
|
+
const marqueeConfigRemoved = removeDeprecatedMarqueeConfig(normalized);
|
|
327681
|
+
if (providerIdsMigrated || marqueeConfigRemoved) {
|
|
327680
327682
|
try {
|
|
327681
327683
|
if (!this.readOnly) fs3.writeFileSync(cp, JSON.stringify(normalized, null, 2), "utf-8");
|
|
327682
327684
|
} catch {
|
|
@@ -327995,6 +327997,18 @@ function normalizeConfigShape(raw, withDefaults) {
|
|
|
327995
327997
|
}
|
|
327996
327998
|
return base2;
|
|
327997
327999
|
}
|
|
328000
|
+
function removeDeprecatedMarqueeConfig(config) {
|
|
328001
|
+
const ui = config.ui;
|
|
328002
|
+
if (!ui) return false;
|
|
328003
|
+
let changed = false;
|
|
328004
|
+
for (const key3 of ["gradient_colors", "gradient_speed", "gradient_width"]) {
|
|
328005
|
+
if (Object.prototype.hasOwnProperty.call(ui, key3)) {
|
|
328006
|
+
delete ui[key3];
|
|
328007
|
+
changed = true;
|
|
328008
|
+
}
|
|
328009
|
+
}
|
|
328010
|
+
return changed;
|
|
328011
|
+
}
|
|
327998
328012
|
function isConfigEntry(value) {
|
|
327999
328013
|
return !!value && typeof value === "object" && !Array.isArray(value) && Object.prototype.hasOwnProperty.call(value, "value");
|
|
328000
328014
|
}
|
|
@@ -328362,9 +328376,6 @@ function defaultConfig() {
|
|
|
328362
328376
|
auto_download: { _description: "Auto-download skills", _type: "choice", _values: ["aggressive", "conservative", "disabled"], value: "conservative" }
|
|
328363
328377
|
},
|
|
328364
328378
|
ui: {
|
|
328365
|
-
gradient_colors: { _description: "Gradient colors (hex)", _type: "array", value: ["#00ff88", "#00ccff", "#aa44ff", "#ff4488"] },
|
|
328366
|
-
gradient_speed: { _description: "Animation speed 1-5", _type: "range", _min: 1, _max: 5, value: 2 },
|
|
328367
|
-
gradient_width: { _description: "Border width", _type: "integer", _min: 1, _max: 4, value: 2 },
|
|
328368
328379
|
glass_alpha: { _description: "Glass opacity", _type: "range", _min: 0, _max: 1, value: 0.85 },
|
|
328369
328380
|
show_mode_label: { _description: "Show mode on hover", _type: "boolean", value: true },
|
|
328370
328381
|
left_panel_collapsed: { _description: "Left panel collapsed", _type: "boolean", value: false },
|
|
@@ -329400,7 +329411,7 @@ function createProviderAdapter(providerId, apiMode) {
|
|
|
329400
329411
|
}
|
|
329401
329412
|
|
|
329402
329413
|
// src/llm/provider.ts
|
|
329403
|
-
var DEFAULT_PROVIDER_REQUEST_TIMEOUT_MS =
|
|
329414
|
+
var DEFAULT_PROVIDER_REQUEST_TIMEOUT_MS = 0;
|
|
329404
329415
|
var MIN_PROVIDER_REQUEST_TIMEOUT_MS = 50;
|
|
329405
329416
|
function providerTimeoutError(timeoutMs) {
|
|
329406
329417
|
const error = new Error(`Provider request timed out after ${timeoutMs}ms`);
|
|
@@ -329461,11 +329472,13 @@ var LLMProvider = class _LLMProvider {
|
|
|
329461
329472
|
static powershellTransport = null;
|
|
329462
329473
|
temperatureUnsupported = /* @__PURE__ */ new Set();
|
|
329463
329474
|
effectiveRequestTimeout(timeoutMs) {
|
|
329464
|
-
const requested = Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs :
|
|
329465
|
-
const configured = Number.isFinite(this.requestTimeoutMs) && this.requestTimeoutMs > 0 ? this.requestTimeoutMs :
|
|
329475
|
+
const requested = Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 0;
|
|
329476
|
+
const configured = Number.isFinite(this.requestTimeoutMs) && this.requestTimeoutMs > 0 ? this.requestTimeoutMs : 0;
|
|
329477
|
+
if (requested <= 0 || configured <= 0) return 0;
|
|
329466
329478
|
return Math.max(MIN_PROVIDER_REQUEST_TIMEOUT_MS, Math.min(requested, configured));
|
|
329467
329479
|
}
|
|
329468
329480
|
async withRequestTimeout(promise, timeoutMs, signal) {
|
|
329481
|
+
if (timeoutMs <= 0) return await abortable(promise, signal);
|
|
329469
329482
|
let timer;
|
|
329470
329483
|
const timeoutPromise = new Promise((_3, reject) => {
|
|
329471
329484
|
timer = setTimeout(() => reject(providerTimeoutError(timeoutMs)), timeoutMs);
|
|
@@ -329665,7 +329678,7 @@ var LLMProvider = class _LLMProvider {
|
|
|
329665
329678
|
const forwardAbort = () => abort.abort(signal?.reason);
|
|
329666
329679
|
if (signal?.aborted) forwardAbort();
|
|
329667
329680
|
else signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
329668
|
-
const timer = setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout);
|
|
329681
|
+
const timer = effectiveTimeout > 0 ? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout) : void 0;
|
|
329669
329682
|
try {
|
|
329670
329683
|
const response = await fetch(url, {
|
|
329671
329684
|
method: "POST",
|
|
@@ -329694,7 +329707,7 @@ var LLMProvider = class _LLMProvider {
|
|
|
329694
329707
|
async getJsonWithFetchFallback(url, headers, timeoutMs = 3e4) {
|
|
329695
329708
|
const effectiveTimeout = this.effectiveRequestTimeout(timeoutMs);
|
|
329696
329709
|
const abort = new AbortController();
|
|
329697
|
-
const timer = setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout);
|
|
329710
|
+
const timer = effectiveTimeout > 0 ? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout) : void 0;
|
|
329698
329711
|
try {
|
|
329699
329712
|
const response = await fetch(url, { method: "GET", headers, signal: abort.signal });
|
|
329700
329713
|
return response;
|
|
@@ -329768,9 +329781,11 @@ var LLMProvider = class _LLMProvider {
|
|
|
329768
329781
|
else fail(new Error("Node HTTP response closed before completion"));
|
|
329769
329782
|
});
|
|
329770
329783
|
});
|
|
329771
|
-
|
|
329772
|
-
req.
|
|
329773
|
-
|
|
329784
|
+
if (effectiveTimeout > 0) {
|
|
329785
|
+
req.setTimeout(effectiveTimeout, () => {
|
|
329786
|
+
req.destroy(providerTimeoutError(effectiveTimeout));
|
|
329787
|
+
});
|
|
329788
|
+
}
|
|
329774
329789
|
req.on("error", reject);
|
|
329775
329790
|
const onAbort = () => req.destroy(abortFailure(signal));
|
|
329776
329791
|
if (signal?.aborted) onAbort();
|
|
@@ -329819,7 +329834,7 @@ var LLMProvider = class _LLMProvider {
|
|
|
329819
329834
|
" $raw = $headerJson | ConvertFrom-Json",
|
|
329820
329835
|
" foreach ($p in $raw.PSObject.Properties) { $headers[$p.Name] = [string]$p.Value }",
|
|
329821
329836
|
"}",
|
|
329822
|
-
|
|
329837
|
+
effectiveTimeout > 0 ? `$params = @{ Uri = $uri; Method = $method; Headers = $headers; UseBasicParsing = $true; TimeoutSec = ${Math.ceil(effectiveTimeout / 1e3)} }` : "$params = @{ Uri = $uri; Method = $method; Headers = $headers; UseBasicParsing = $true }",
|
|
329823
329838
|
'if ($method -eq "POST") { $params["Body"] = $bodyJson }',
|
|
329824
329839
|
'if ($method -eq "POST") { $params["ContentType"] = "application/json; charset=utf-8" }',
|
|
329825
329840
|
"$resp = Invoke-WebRequest @params",
|
|
@@ -329848,11 +329863,11 @@ var LLMProvider = class _LLMProvider {
|
|
|
329848
329863
|
};
|
|
329849
329864
|
if (signal?.aborted) onAbort();
|
|
329850
329865
|
else signal?.addEventListener("abort", onAbort, { once: true });
|
|
329851
|
-
const timer = setTimeout(() => {
|
|
329866
|
+
const timer = effectiveTimeout > 0 ? setTimeout(() => {
|
|
329852
329867
|
child.kill();
|
|
329853
329868
|
cleanup();
|
|
329854
329869
|
reject(providerTimeoutError(effectiveTimeout));
|
|
329855
|
-
}, effectiveTimeout + 5e3);
|
|
329870
|
+
}, effectiveTimeout + 5e3) : void 0;
|
|
329856
329871
|
child.stdout.setEncoding("utf8");
|
|
329857
329872
|
child.stderr.setEncoding("utf8");
|
|
329858
329873
|
child.stdout.on("data", (chunk) => {
|
|
@@ -330321,7 +330336,7 @@ ${responsePath}
|
|
|
330321
330336
|
if (signal?.aborted) forwardAbort();
|
|
330322
330337
|
else signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
330323
330338
|
const effectiveTimeout = this.effectiveRequestTimeout(12e4);
|
|
330324
|
-
const timer = setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout);
|
|
330339
|
+
const timer = effectiveTimeout > 0 ? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout) : void 0;
|
|
330325
330340
|
try {
|
|
330326
330341
|
try {
|
|
330327
330342
|
let response2 = await fetch(request.url, {
|
|
@@ -330466,7 +330481,7 @@ ${responsePath}
|
|
|
330466
330481
|
if (signal?.aborted) forwardAbort();
|
|
330467
330482
|
else signal?.addEventListener("abort", forwardAbort, { once: true });
|
|
330468
330483
|
const effectiveTimeout = this.effectiveRequestTimeout(12e4);
|
|
330469
|
-
const timeout = setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout);
|
|
330484
|
+
const timeout = effectiveTimeout > 0 ? setTimeout(() => abort.abort(providerTimeoutError(effectiveTimeout)), effectiveTimeout) : void 0;
|
|
330470
330485
|
let reader = null;
|
|
330471
330486
|
try {
|
|
330472
330487
|
let response;
|
|
@@ -333775,11 +333790,8 @@ function jpegCaptureScript(outPath, boundsScript, requestedMaxWidth, requestedMa
|
|
|
333775
333790
|
'Write-Output (@{ ok=$true; left=$x; top=$y; width=$w; height=$h; capture_max_width=$maxWidth; capture_max_height=$maxHeight; image_width=$imageWidth; image_height=$imageHeight; image_bytes=$fileSize; image_quality=$qualityUsed; image_available=$imageAvailable; image_mime="image/jpeg" } | ConvertTo-Json -Compress)'
|
|
333776
333791
|
].join("\r\n");
|
|
333777
333792
|
}
|
|
333778
|
-
function gradientPalette(
|
|
333779
|
-
|
|
333780
|
-
const configured = Array.isArray(input2) ? input2 : [];
|
|
333781
|
-
const raw = configured.length ? configured.map((v) => String(v || "").trim()).filter(Boolean) : String(process.env.NEWMARK_COMPUTER_USE_GRADIENT || "").split(",").map((v) => v.trim()).filter(Boolean);
|
|
333782
|
-
return raw.length >= 2 ? raw.slice(0, 6) : fallback;
|
|
333793
|
+
function gradientPalette(_input) {
|
|
333794
|
+
return ["#000000", "#ffffff", "#000000", "#ffffff"];
|
|
333783
333795
|
}
|
|
333784
333796
|
async function stopTakeoverOverlay() {
|
|
333785
333797
|
const pid = takeoverOverlayPid;
|
|
@@ -333801,11 +333813,11 @@ async function stopTakeoverOverlay() {
|
|
|
333801
333813
|
async function startTakeoverOverlay(durationMs = 0, input2 = {}) {
|
|
333802
333814
|
if (process.platform !== "win32") return { ok: false, action: "takeover_start", error: "Computer Use takeover overlay is Windows-only." };
|
|
333803
333815
|
await stopTakeoverOverlay();
|
|
333804
|
-
lastTakeoverOverlayStyle = { colors:
|
|
333816
|
+
lastTakeoverOverlayStyle = { colors: gradientPalette(), speed: 3, width: 2 };
|
|
333805
333817
|
const colors = gradientPalette(input2.colors);
|
|
333806
333818
|
const lifetime = Math.max(0, Math.floor(Number(durationMs || 0)));
|
|
333807
|
-
const width =
|
|
333808
|
-
const speedSeconds =
|
|
333819
|
+
const width = 2;
|
|
333820
|
+
const speedSeconds = 3;
|
|
333809
333821
|
const ownerPid = Math.max(0, Math.floor(Number(input2.ownerPid ?? process.pid) || 0));
|
|
333810
333822
|
const scriptPath = path9.join(tempScreenshotDir(), `takeover-overlay-${timestampName()}-${crypto6.randomBytes(4).toString("hex")}.ps1`);
|
|
333811
333823
|
const script = [
|
|
@@ -336925,6 +336937,10 @@ var ToolExecutor = class {
|
|
|
336925
336937
|
async webSearch(query) {
|
|
336926
336938
|
return this.wsearch(query);
|
|
336927
336939
|
}
|
|
336940
|
+
/** OCR entry point for the runtime's final visual fallback. */
|
|
336941
|
+
async finalVisualFallbackOcr(dataUrl, signal) {
|
|
336942
|
+
return await this.localOcr.recognizeDataUrl(dataUrl, signal, "sparse-ui");
|
|
336943
|
+
}
|
|
336928
336944
|
setHostProfile(profile) {
|
|
336929
336945
|
this.hostProfile = { ...profile };
|
|
336930
336946
|
}
|
|
@@ -337572,9 +337588,9 @@ var ToolExecutor = class {
|
|
|
337572
337588
|
allowEphemeralVisionImage: context.allowEphemeralVisionImage === true,
|
|
337573
337589
|
captureMaxWidth: Number(args.capture_max_width),
|
|
337574
337590
|
captureMaxHeight: Number(args.capture_max_height),
|
|
337575
|
-
gradientColors: Array.isArray(args.gradient_colors) ? args.gradient_colors :
|
|
337576
|
-
gradientSpeed: args.gradient_speed !== void 0 ? Number(args.gradient_speed) :
|
|
337577
|
-
gradientWidth: args.gradient_width !== void 0 ? Number(args.gradient_width) :
|
|
337591
|
+
gradientColors: Array.isArray(args.gradient_colors) ? args.gradient_colors : void 0,
|
|
337592
|
+
gradientSpeed: args.gradient_speed !== void 0 ? Number(args.gradient_speed) : void 0,
|
|
337593
|
+
gradientWidth: args.gradient_width !== void 0 ? Number(args.gradient_width) : void 0,
|
|
337578
337594
|
invocation: context.invocation,
|
|
337579
337595
|
ownerId: owner,
|
|
337580
337596
|
includeRawUi: args.include_raw_ui === true,
|
|
@@ -340838,7 +340854,12 @@ async function runAgentKernel(agent) {
|
|
|
340838
340854
|
try {
|
|
340839
340855
|
const linkedPlanRevisionBeforeRun = agent.getLinkedPlan().revision;
|
|
340840
340856
|
const modelBeforeKernelRun = agent.model;
|
|
340841
|
-
|
|
340857
|
+
const preflightVisualFallback = !agent.activeModelConfig()?.vision ? await agent.finalVisualFallback("vision input not supported by the selected model", processSignal) : null;
|
|
340858
|
+
let lastTurn = preflightVisualFallback ? { text: preflightVisualFallback, stopReason: "stop", errorMessage: "" } : await runWithCompressionResume([], false);
|
|
340859
|
+
if (preflightVisualFallback) {
|
|
340860
|
+
tokens.push({ type: "text", text: preflightVisualFallback });
|
|
340861
|
+
agent.recordWorkStatus("Final visual fallback used: local mini OCR plus conservative text correction.");
|
|
340862
|
+
}
|
|
340842
340863
|
if (modelBeforeKernelRun && modelBeforeKernelRun !== agent.model && !tokens.some((t3) => t3.text?.includes("[Model fallback]"))) {
|
|
340843
340864
|
tokens.unshift({ type: "text", text: `[Model fallback] ${modelBeforeKernelRun} unavailable; switched to ${agent.model}.` });
|
|
340844
340865
|
}
|
|
@@ -340868,6 +340889,14 @@ async function runAgentKernel(agent) {
|
|
|
340868
340889
|
await agent.waitForPlannedRouteRetry();
|
|
340869
340890
|
lastTurn = await runWithCompressionResume([], false);
|
|
340870
340891
|
}
|
|
340892
|
+
if (kernelTurnFailed(agent, lastTurn)) {
|
|
340893
|
+
const visualFallback = await agent.finalVisualFallback(lastTurn.errorMessage || lastTurn.text, processSignal);
|
|
340894
|
+
if (visualFallback) {
|
|
340895
|
+
tokens.push({ type: "text", text: visualFallback });
|
|
340896
|
+
agent.recordWorkStatus("Final visual fallback used: local mini OCR plus conservative text correction.");
|
|
340897
|
+
lastTurn = { ...lastTurn, text: visualFallback, errorMessage: "", stopReason: "stop" };
|
|
340898
|
+
}
|
|
340899
|
+
}
|
|
340871
340900
|
if (kernelTurnFailed(agent, lastTurn)) {
|
|
340872
340901
|
throw new ProviderRunError(normalizePublicProviderError(lastTurn.errorMessage || lastTurn.text, [agent.activeModelConfig()?.api_key]));
|
|
340873
340902
|
}
|
|
@@ -350812,6 +350841,60 @@ ${msg.content}
|
|
|
350812
350841
|
};
|
|
350813
350842
|
return results;
|
|
350814
350843
|
}
|
|
350844
|
+
/**
|
|
350845
|
+
* Final visual safety net: OCR each submitted image and ask a text-only
|
|
350846
|
+
* request to conservatively repair the OCR. This is intentionally callable
|
|
350847
|
+
* only after a visual-input refusal and after same-provider vision routing
|
|
350848
|
+
* has been exhausted; the original image is never sent again.
|
|
350849
|
+
*/
|
|
350850
|
+
async finalVisualFallback(errorText, signal) {
|
|
350851
|
+
if (!/(?:vision|image|multimodal|image_url|input_image).*(?:not supported|unsupported|拒绝|不支持|failed|failure|invalid)|(?:not supported|unsupported|拒绝|不支持).*(?:vision|image|multimodal|image_url|input_image)/i.test(String(errorText || ""))) return null;
|
|
350852
|
+
const current = this.activeModelConfig();
|
|
350853
|
+
if (!current) return null;
|
|
350854
|
+
const alternateVision = this.config.allModels().some(
|
|
350855
|
+
(model) => model.enabled !== false && model.provider_id === current.provider_id && model.name !== current.name && !!model.vision && !!model.api_key && !!model.provider_url && !["unavailable", "auth_error", "invalid_config"].includes(String(model.evaluation?.status || model.validation?.status || "").toLowerCase())
|
|
350856
|
+
);
|
|
350857
|
+
if (alternateVision) return null;
|
|
350858
|
+
const latest = [...this.history].reverse().find((item) => item?.role === "user");
|
|
350859
|
+
const parts = latest?.content && Array.isArray(latest.content) ? latest.content : [];
|
|
350860
|
+
const images = parts.map((part) => {
|
|
350861
|
+
const image = part.image_url;
|
|
350862
|
+
return image && typeof image === "object" ? String(image.url || "") : "";
|
|
350863
|
+
}).filter((value) => /^data:image\/(?:png|jpeg);base64,/i.test(value)).slice(0, 4);
|
|
350864
|
+
if (!images.length) return null;
|
|
350865
|
+
const ocr = [];
|
|
350866
|
+
for (const [index, image] of images.entries()) {
|
|
350867
|
+
try {
|
|
350868
|
+
const result = await this.tools.finalVisualFallbackOcr(image, signal);
|
|
350869
|
+
if (result.ok && result.text.trim()) ocr.push({ index: index + 1, text: result.text.slice(0, 5e4), confidence: result.confidence });
|
|
350870
|
+
} catch {
|
|
350871
|
+
}
|
|
350872
|
+
}
|
|
350873
|
+
if (!ocr.length) return JSON.stringify({ ok: false, fallback: "mini_ocr_llm", error: "Local OCR returned no readable text; no visual content was fabricated." });
|
|
350874
|
+
const task = typeof latest?.content === "string" ? latest.content : "";
|
|
350875
|
+
const evidence = ocr.map((item) => `Image ${item.index} (OCR confidence ${item.confidence.toFixed(1)}):
|
|
350876
|
+
${item.text}`).join("\n\n");
|
|
350877
|
+
const prompt = `The provider rejected image input. Answer the user's task using only this approximate OCR evidence. Correct obvious character, spacing, and line-break errors only when supported by context. Preserve [uncertain] markers for ambiguity and never invent missing visual content.
|
|
350878
|
+
User task:
|
|
350879
|
+
${task.slice(0, 12e3)}
|
|
350880
|
+
OCR evidence:
|
|
350881
|
+
${evidence}`;
|
|
350882
|
+
let corrected = "";
|
|
350883
|
+
try {
|
|
350884
|
+
const provider = this.engineModel();
|
|
350885
|
+
if (provider) corrected = String(await provider.chat(this.activeModelName(), [{ role: "user", content: prompt }], "You are a text-only OCR correction assistant. Be conservative and explicit about uncertainty.", 0.05, 3e3, signal) || "").trim();
|
|
350886
|
+
} catch {
|
|
350887
|
+
}
|
|
350888
|
+
return JSON.stringify({
|
|
350889
|
+
ok: !!(corrected || ocr.length),
|
|
350890
|
+
fallback: "mini_ocr_llm",
|
|
350891
|
+
approximate: true,
|
|
350892
|
+
warning: "\u89C6\u89C9\u8F93\u5165\u88AB\u62D2\u7EDD\uFF1B\u4EE5\u4E0B\u5185\u5BB9\u6765\u81EA\u672C\u5730 OCR\uFF0C\u5E76\u7ECF\u6587\u672C\u6A21\u578B\u4FDD\u5B88\u6821\u6B63\uFF0C\u53EF\u80FD\u4E0D\u5B8C\u6574\u3002",
|
|
350893
|
+
raw_ocr: ocr,
|
|
350894
|
+
corrected: corrected || ocr.map((item) => item.text).join("\n\n"),
|
|
350895
|
+
uncertainty: corrected ? "preserved" : "raw_ocr_only"
|
|
350896
|
+
}, null, 2);
|
|
350897
|
+
}
|
|
350815
350898
|
engineModel() {
|
|
350816
350899
|
if (this.forcedProvider) {
|
|
350817
350900
|
const active = this.activeDeployment();
|
|
@@ -351194,8 +351277,10 @@ ${persisted.map((guide, index) => `Guide ${index + 1}: ${guide.text}`).join("\n"
|
|
|
351194
351277
|
}
|
|
351195
351278
|
const selectedModel = this.activeModelConfig();
|
|
351196
351279
|
if (images.length && !selectedModel?.vision) {
|
|
351197
|
-
|
|
351198
|
-
|
|
351280
|
+
const hasSameProviderVision = selectedModel && this.config.allModels().some(
|
|
351281
|
+
(model) => model.enabled !== false && model.provider_id === selectedModel.provider_id && model.name !== selectedModel.name && !!model.vision
|
|
351282
|
+
);
|
|
351283
|
+
if (hasSameProviderVision) this.switchToFallbackModel("vision input not supported by the selected model");
|
|
351199
351284
|
}
|
|
351200
351285
|
const now2 = this.nowLabel();
|
|
351201
351286
|
const visibleUserInput = inputEnvelope?.visibleUserInput === void 0 ? text : String(inputEnvelope.visibleUserInput || "");
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "newmark-agent",
|
|
3
3
|
"productName": "Newmark Agent",
|
|
4
|
-
"version": "0.5.
|
|
4
|
+
"version": "0.5.4",
|
|
5
5
|
"description": "Newmark Agent — Portable AI coding agent with rich GUI and CLI (TypeScript)",
|
|
6
6
|
"homepage": "https://github.com/positer/Newmark-Agent",
|
|
7
7
|
"repository": {
|