newmark-agent 0.4.7 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/conversation-utility-host.bundle.cjs +407 -36
- package/dist/conversation-utility-host.js +3 -0
- package/dist/core/agent.d.ts +3 -0
- package/dist/core/agent.js +122 -10
- package/dist/core/agentKernelRunner.js +28 -11
- package/dist/core/autoRouter.d.ts +1 -1
- package/dist/core/autoRouter.js +16 -7
- package/dist/core/conversationKernel.d.ts +43 -0
- package/dist/core/conversationKernel.js +262 -8
- package/dist/core/electronUtilityAgentClient.d.ts +2 -0
- package/dist/core/electronUtilityAgentClient.js +5 -1
- package/dist/core/electronUtilityRuntimePool.d.ts +3 -0
- package/dist/core/electronUtilityRuntimePool.js +11 -0
- package/dist/core/toolPolicy.js +3 -0
- package/dist/core/utilityAgentProtocol.d.ts +23 -1
- package/dist/core/utilityHostToolRouter.js +18 -0
- package/dist/core/wslAgentClient.d.ts +2 -0
- package/dist/core/wslAgentClient.js +5 -1
- package/dist/core/wslAgentProtocol.d.ts +12 -1
- package/dist/core/wslAgentRuntimePool.d.ts +3 -0
- package/dist/core/wslAgentRuntimePool.js +11 -0
- package/dist/main.js +226 -13
- package/dist/preload.js +2 -0
- package/dist/server.d.ts +47 -1
- package/dist/server.js +304 -38
- package/dist/tools/index.js +46 -1
- package/dist/tools/nativeTools.js +1 -0
- package/dist/ui/index.html +150 -14
- package/dist/wsl-agent-host.bundle.cjs +407 -36
- package/dist/wsl-agent-host.js +3 -0
- package/package.json +5 -2
|
@@ -79,6 +79,9 @@ async function handle(request) {
|
|
|
79
79
|
target,
|
|
80
80
|
});
|
|
81
81
|
}
|
|
82
|
+
if (request.method === 'queue_action') {
|
|
83
|
+
return kernel.queueAction(checkedTarget(request.params.target), request.params.action, request.params.input);
|
|
84
|
+
}
|
|
82
85
|
if (request.method === 'checkpoint')
|
|
83
86
|
return kernel.checkpoint(checkedTarget(request.params.target));
|
|
84
87
|
if (request.method === 'context_compress') {
|
package/dist/core/agent.d.ts
CHANGED
|
@@ -346,6 +346,7 @@ export declare class Agent {
|
|
|
346
346
|
private memoryLabRebuildState;
|
|
347
347
|
private memoryLabRebuildError;
|
|
348
348
|
private displayImageDescriptionCache;
|
|
349
|
+
private latestCapturedImageInputs;
|
|
349
350
|
private activeProcessAbortController;
|
|
350
351
|
private automationManager;
|
|
351
352
|
private activeAgentKernelRuntime;
|
|
@@ -622,6 +623,7 @@ export declare class Agent {
|
|
|
622
623
|
order: number;
|
|
623
624
|
branchCommunication: boolean;
|
|
624
625
|
}>;
|
|
626
|
+
registerCapturedImageInput(dataUrl: string, name?: string): ConversationImageAttachment | null;
|
|
625
627
|
private subagentConcurrencyLimit;
|
|
626
628
|
/** 当前工作区使用内存中的前台选择;后台工作区从各自持久化状态读取 active id。 */
|
|
627
629
|
activeConversationIdForWorkspace(ws: WorkspaceInfo | null): string;
|
|
@@ -668,6 +670,7 @@ export declare class Agent {
|
|
|
668
670
|
isBranchCommunicationEnabled(): boolean;
|
|
669
671
|
switchConversationBranch(conversationId: string, branchId: string, branchGroupId?: string): ConversationSnapshot;
|
|
670
672
|
setConversationPinned(id: string, pinned: boolean, ws?: WorkspaceInfo | null): boolean;
|
|
673
|
+
reorderConversationContinuations(orderedIds: string[]): ConversationContinuation[];
|
|
671
674
|
renameConversation(id: string, title: string, ws?: WorkspaceInfo | null): boolean;
|
|
672
675
|
/** 为指定工作区创建空白对话,不临时切换全局前台工作区。 */
|
|
673
676
|
createConversationInWorkspace(ws: WorkspaceInfo, title?: string): {
|
package/dist/core/agent.js
CHANGED
|
@@ -241,6 +241,7 @@ class Agent {
|
|
|
241
241
|
memoryLabRebuildState = 'idle';
|
|
242
242
|
memoryLabRebuildError = '';
|
|
243
243
|
displayImageDescriptionCache = new Map();
|
|
244
|
+
latestCapturedImageInputs = [];
|
|
244
245
|
activeProcessAbortController = null;
|
|
245
246
|
automationManager = null;
|
|
246
247
|
activeAgentKernelRuntime = null;
|
|
@@ -1050,7 +1051,9 @@ class Agent {
|
|
|
1050
1051
|
}
|
|
1051
1052
|
updateProviders(value) {
|
|
1052
1053
|
const before = this.config.providers();
|
|
1053
|
-
|
|
1054
|
+
const merged = (0, config_1.mergeProviderSecrets)(value, before);
|
|
1055
|
+
resetEditedModelValidationEvidence(merged, before);
|
|
1056
|
+
this.config.set('models', 'providers', merged);
|
|
1054
1057
|
const after = this.config.providers();
|
|
1055
1058
|
const beforeById = new Map(before.map(provider => [provider.id, provider]));
|
|
1056
1059
|
const afterById = new Map(after.map(provider => [provider.id, provider]));
|
|
@@ -2859,6 +2862,14 @@ class Agent {
|
|
|
2859
2862
|
listConversationStates() {
|
|
2860
2863
|
return this.listWorkspaceConversationStates(this.workspace.current);
|
|
2861
2864
|
}
|
|
2865
|
+
registerCapturedImageInput(dataUrl, name = 'active-screenshot.jpg') {
|
|
2866
|
+
if (!String(dataUrl || '').startsWith('data:image/'))
|
|
2867
|
+
return null;
|
|
2868
|
+
const prepared = this.prepareSubmittedConversationImages([{ dataUrl, name, type: dataUrl.slice(5, dataUrl.indexOf(';')) }]);
|
|
2869
|
+
const attachment = prepared.attachments[0] || null;
|
|
2870
|
+
this.latestCapturedImageInputs = attachment ? [attachment] : [];
|
|
2871
|
+
return attachment;
|
|
2872
|
+
}
|
|
2862
2873
|
subagentConcurrencyLimit() {
|
|
2863
2874
|
return this.intelligence === 'ultra' ? 16 : 4;
|
|
2864
2875
|
}
|
|
@@ -3470,6 +3481,27 @@ class Agent {
|
|
|
3470
3481
|
this.writeStoredConversationState(stored, targetWs);
|
|
3471
3482
|
return true;
|
|
3472
3483
|
}
|
|
3484
|
+
reorderConversationContinuations(orderedIds) {
|
|
3485
|
+
const currentIds = this.continuations
|
|
3486
|
+
.filter(item => item.queueMode === 'followUp' && !!item.clientMessageId)
|
|
3487
|
+
.map(item => String(item.clientMessageId));
|
|
3488
|
+
const completeOrder = orderedIds.length === currentIds.length
|
|
3489
|
+
&& new Set(orderedIds).size === orderedIds.length
|
|
3490
|
+
&& orderedIds.every(id => currentIds.includes(id));
|
|
3491
|
+
if (!completeOrder)
|
|
3492
|
+
throw new Error('A complete queue order with unique current item ids is required');
|
|
3493
|
+
const byId = new Map(this.continuations.flatMap(item => item.queueMode === 'followUp' && item.clientMessageId
|
|
3494
|
+
? [[String(item.clientMessageId), item]]
|
|
3495
|
+
: []));
|
|
3496
|
+
let nextIndex = 0;
|
|
3497
|
+
this.continuations = this.continuations.map(item => {
|
|
3498
|
+
if (item.queueMode !== 'followUp' || !item.clientMessageId)
|
|
3499
|
+
return item;
|
|
3500
|
+
return byId.get(orderedIds[nextIndex++]);
|
|
3501
|
+
});
|
|
3502
|
+
this.saveWorkspaceConversationState(true);
|
|
3503
|
+
return this.conversationContinuations();
|
|
3504
|
+
}
|
|
3473
3505
|
renameConversation(id, title, ws = this.workspace.current) {
|
|
3474
3506
|
const targetWs = ws || this.workspace.current;
|
|
3475
3507
|
if (!targetWs)
|
|
@@ -6584,6 +6616,13 @@ class Agent {
|
|
|
6584
6616
|
const fallbackEnabled = this.config.getBool('models', 'fallback_on_unavailable');
|
|
6585
6617
|
const observedFailure = (0, autoRouter_1.classifyRouteFailure)(errorText);
|
|
6586
6618
|
const observedDeployment = this.activeDeployment();
|
|
6619
|
+
// Keep balance exhaustion scoped to the deployment that actually failed.
|
|
6620
|
+
// Provider adapters normally record this before returning an error, but
|
|
6621
|
+
// fallback callers are also a public recovery boundary and must not rely
|
|
6622
|
+
// on every adapter/error path having performed that side effect first.
|
|
6623
|
+
if (observedFailure.type === 'balance_exhausted' && observedDeployment) {
|
|
6624
|
+
this.providerBalanceBlockedUntilByDeployment.set(deploymentIdentity(observedDeployment), Date.now() + 5 * 60_000);
|
|
6625
|
+
}
|
|
6587
6626
|
const previousAttempt = observedDeployment && this.lastRouteDecision
|
|
6588
6627
|
? [...this.lastRouteDecision.attempts].reverse().find(attempt => deploymentIdentity(attempt.deployment) === deploymentIdentity(observedDeployment))
|
|
6589
6628
|
: undefined;
|
|
@@ -6645,16 +6684,15 @@ class Agent {
|
|
|
6645
6684
|
}
|
|
6646
6685
|
if (!fallbackEnabled)
|
|
6647
6686
|
return null;
|
|
6648
|
-
if (!observedFailure.
|
|
6687
|
+
if (!observedFailure.switchAllowed || this.routeStreamCommitted || this.routeSideEffectCommitted)
|
|
6649
6688
|
return null;
|
|
6650
6689
|
const current = this.model;
|
|
6651
|
-
const
|
|
6690
|
+
const currentDeployment = observedDeployment;
|
|
6691
|
+
const all = this.scopedSwitchModels(current).filter(m => !currentDeployment
|
|
6692
|
+
|| deploymentIdentity(this.deploymentRef(m)) !== deploymentIdentity(currentDeployment));
|
|
6652
6693
|
if (!all.length)
|
|
6653
6694
|
return null;
|
|
6654
|
-
const usable = all.filter(m =>
|
|
6655
|
-
const status = String(m.evaluation?.status || 'unknown').toLowerCase();
|
|
6656
|
-
return status !== 'unavailable' && !status.startsWith('error');
|
|
6657
|
-
});
|
|
6695
|
+
const usable = all.filter(m => !this.isBalanceBlockedDeployment(this.deploymentRef(m)) && !modelConfigIsUnavailable(m));
|
|
6658
6696
|
if (!usable.length)
|
|
6659
6697
|
return null;
|
|
6660
6698
|
const pref = this.config.autoSwitchPreference();
|
|
@@ -7427,7 +7465,7 @@ class Agent {
|
|
|
7427
7465
|
throw e;
|
|
7428
7466
|
}
|
|
7429
7467
|
const msg = e instanceof Error ? e.message : String(e);
|
|
7430
|
-
if (/\b402\b|insufficient balance|insufficient funds|payment required
|
|
7468
|
+
if (/\b402\b|insufficient balance|insufficient funds|payment required|insufficient[_ -]?quota|quota (?:exceeded|exhausted)|credit(?:s| balance)? (?:exhausted|depleted)|billing hard limit|budget exhausted|余额不足|额度不足|配额(?:不足|耗尽|超限)/i.test(msg)) {
|
|
7431
7469
|
this.noteProviderBalanceFailure();
|
|
7432
7470
|
}
|
|
7433
7471
|
this.status = 'error';
|
|
@@ -8382,6 +8420,17 @@ class Agent {
|
|
|
8382
8420
|
}
|
|
8383
8421
|
latestSubmittedImages(attachmentId = '') {
|
|
8384
8422
|
const normalizedId = String(attachmentId || '').trim();
|
|
8423
|
+
const captured = (0, conversationAttachments_1.hydrateConversationImageAttachments)(this.rootPath, this.latestCapturedImageInputs).flatMap(attachment => attachment.dataUrl
|
|
8424
|
+
? [{ id: attachment.id, dataUrl: attachment.dataUrl }]
|
|
8425
|
+
: []);
|
|
8426
|
+
if (normalizedId) {
|
|
8427
|
+
const capturedMatch = captured.find(item => item.id === normalizedId);
|
|
8428
|
+
if (capturedMatch)
|
|
8429
|
+
return [capturedMatch];
|
|
8430
|
+
}
|
|
8431
|
+
else if (captured.length) {
|
|
8432
|
+
return captured;
|
|
8433
|
+
}
|
|
8385
8434
|
for (let index = this.chatMessages.length - 1; index >= 0; index -= 1) {
|
|
8386
8435
|
const message = this.chatMessages[index];
|
|
8387
8436
|
if (message?.role !== 'user')
|
|
@@ -9519,11 +9568,74 @@ function routeProviderFingerprint(provider) {
|
|
|
9519
9568
|
enabled: provider.enabled,
|
|
9520
9569
|
models: (provider.models || []).map(model => ({
|
|
9521
9570
|
name: model.name,
|
|
9571
|
+
display: model.display,
|
|
9572
|
+
description: model.description,
|
|
9573
|
+
maxTokens: model.max_tokens,
|
|
9574
|
+
vision: model.vision,
|
|
9575
|
+
thinking: !!model.thinking,
|
|
9576
|
+
imageOutput: !!model.image_output,
|
|
9522
9577
|
enabled: model.enabled !== false,
|
|
9578
|
+
preview: !!model.preview,
|
|
9523
9579
|
logicalModelGroupId: model.logical_model_group_id || '',
|
|
9580
|
+
privacy: model.privacy || [],
|
|
9581
|
+
capabilities: model.capabilities || [],
|
|
9582
|
+
supportedParameters: model.supported_parameters || [],
|
|
9583
|
+
routePreference: model.route_preference,
|
|
9584
|
+
fallbackOnly: !!model.fallback_only,
|
|
9585
|
+
thinkingTierMap: model.thinking_tier_map || {},
|
|
9524
9586
|
})),
|
|
9525
9587
|
});
|
|
9526
9588
|
}
|
|
9589
|
+
function modelConfigurationFingerprint(model) {
|
|
9590
|
+
const { validation, evaluation, _previous_name, previous_name, ...configuration } = model;
|
|
9591
|
+
void validation;
|
|
9592
|
+
void evaluation;
|
|
9593
|
+
void _previous_name;
|
|
9594
|
+
void previous_name;
|
|
9595
|
+
return JSON.stringify(configuration);
|
|
9596
|
+
}
|
|
9597
|
+
function resetEditedModelValidationEvidence(incomingProviders, existingProviders) {
|
|
9598
|
+
const existingById = new Map(existingProviders.map(provider => [provider.id, provider]));
|
|
9599
|
+
const existingByName = new Map(existingProviders.map(provider => [provider.name, provider]));
|
|
9600
|
+
for (const rawProvider of incomingProviders) {
|
|
9601
|
+
if (!rawProvider || typeof rawProvider !== 'object' || Array.isArray(rawProvider))
|
|
9602
|
+
continue;
|
|
9603
|
+
const provider = rawProvider;
|
|
9604
|
+
const previousProvider = existingById.get(String(provider.id || ''))
|
|
9605
|
+
|| existingByName.get(String(provider._previous_name || provider.previous_name || provider.name || ''));
|
|
9606
|
+
const models = Array.isArray(provider.models) ? provider.models : [];
|
|
9607
|
+
// Endpoint/protocol edits invalidate capability evidence. A display-name
|
|
9608
|
+
// change or credential rotation only resets runtime health/circuits via the
|
|
9609
|
+
// provider fingerprint and must not discard still-valid model capabilities.
|
|
9610
|
+
const providerConnectionChanged = !!previousProvider && (String(provider.base_url || provider.endpoint || '') !== previousProvider.base_url
|
|
9611
|
+
|| String(provider.protocol || '') !== previousProvider.protocol);
|
|
9612
|
+
for (const rawModel of models) {
|
|
9613
|
+
if (!rawModel || typeof rawModel !== 'object' || Array.isArray(rawModel))
|
|
9614
|
+
continue;
|
|
9615
|
+
const model = rawModel;
|
|
9616
|
+
const previousName = String(model._previous_name || model.previous_name || model.name || '');
|
|
9617
|
+
const previousModel = previousProvider?.models.find(candidate => candidate.name === previousName);
|
|
9618
|
+
const edited = providerConnectionChanged || (!!previousModel
|
|
9619
|
+
&& modelConfigurationFingerprint(model) !== modelConfigurationFingerprint(previousModel));
|
|
9620
|
+
delete model._previous_name;
|
|
9621
|
+
delete model.previous_name;
|
|
9622
|
+
if (!edited)
|
|
9623
|
+
continue;
|
|
9624
|
+
model.validation = { level: 'discovered', status: 'degraded', checked_at: '', capabilities: {} };
|
|
9625
|
+
delete model.evaluation;
|
|
9626
|
+
model.speed_rating = 'unknown';
|
|
9627
|
+
model.capability_rating = 'unknown';
|
|
9628
|
+
}
|
|
9629
|
+
}
|
|
9630
|
+
}
|
|
9631
|
+
function modelConfigIsUnavailable(model) {
|
|
9632
|
+
const validationStatus = effectiveModelValidationStatus(model);
|
|
9633
|
+
if (model.validation?.level !== 'discovered'
|
|
9634
|
+
&& (validationStatus === 'unavailable' || validationStatus === 'auth_error' || validationStatus === 'invalid_config'))
|
|
9635
|
+
return true;
|
|
9636
|
+
const evaluationStatus = validationStatus === 'degraded' ? 'degraded' : String(model.evaluation?.status || '').toLowerCase();
|
|
9637
|
+
return evaluationStatus === 'unavailable' || evaluationStatus.startsWith('error');
|
|
9638
|
+
}
|
|
9527
9639
|
function parseDeploymentSelectionValue(value) {
|
|
9528
9640
|
const marker = String(value || '').trim();
|
|
9529
9641
|
if (!marker.startsWith('deployment:'))
|
|
@@ -9544,6 +9656,8 @@ function effectiveModelValidationStatus(model) {
|
|
|
9544
9656
|
const raw = String(model.validation?.status || '').toLowerCase();
|
|
9545
9657
|
if (raw === 'auth_error')
|
|
9546
9658
|
return raw;
|
|
9659
|
+
if (String(model.validation?.level || '').toLowerCase() === 'discovered')
|
|
9660
|
+
return 'degraded';
|
|
9547
9661
|
const textEvidence = model.validation?.capabilities?.text === true
|
|
9548
9662
|
|| model.validation?.capabilities?.text_input === true
|
|
9549
9663
|
|| model.validation?.capabilities?.text_output === true
|
|
@@ -9551,8 +9665,6 @@ function effectiveModelValidationStatus(model) {
|
|
|
9551
9665
|
|| model.evaluation?.text_output === true;
|
|
9552
9666
|
if (textEvidence && raw === 'unavailable')
|
|
9553
9667
|
return 'degraded';
|
|
9554
|
-
if (!raw && String(model.validation?.level || '').toLowerCase() === 'discovered')
|
|
9555
|
-
return 'degraded';
|
|
9556
9668
|
return (['verified', 'degraded', 'unavailable', 'auth_error', 'rate_limited', 'invalid_config'].includes(raw)
|
|
9557
9669
|
? raw
|
|
9558
9670
|
: 'unavailable');
|
|
@@ -662,7 +662,7 @@ async function runAgentKernel(agent) {
|
|
|
662
662
|
return;
|
|
663
663
|
}
|
|
664
664
|
const publicError = normalizePublicProviderError(error, [currentAgent.activeModelConfig()?.api_key]);
|
|
665
|
-
if (/\b402\b|insufficient balance|insufficient funds|payment required
|
|
665
|
+
if (/\b402\b|insufficient balance|insufficient funds|payment required|insufficient[_ -]?quota|quota (?:exceeded|exhausted)|credit(?:s| balance)? (?:exhausted|depleted)|billing hard limit|budget exhausted|余额不足|额度不足|配额(?:不足|耗尽|超限)/i.test(publicError)) {
|
|
666
666
|
currentAgent.noteProviderBalanceFailure();
|
|
667
667
|
}
|
|
668
668
|
const final = assistantMessage(model, [{ type: 'text', text: `[Error] ${publicError}` }], 'error');
|
|
@@ -1387,10 +1387,13 @@ function toKernelTools(agent, definitions, provisioning) {
|
|
|
1387
1387
|
throw new Error(rawText);
|
|
1388
1388
|
}
|
|
1389
1389
|
const visionImage = visualFallbackImageInput(agent, name, rawText);
|
|
1390
|
+
const capturedInput = name === 'screen_capture' ? agent.registerCapturedImageInput(visionImage.image || '', 'active-screenshot.jpg') : null;
|
|
1390
1391
|
const directImage = imageInspectDataUrl(name, rawText);
|
|
1391
|
-
const text = spillOversizedToolResult(agent, name, sanitizeVisualToolText(name, rawText));
|
|
1392
|
+
const text = spillOversizedToolResult(agent, name, capturedImageToolText(name, sanitizeVisualToolText(name, rawText), capturedInput?.id));
|
|
1392
1393
|
const content = [{ type: 'text', text }];
|
|
1393
|
-
if (
|
|
1394
|
+
if (name === 'screen_capture' && capturedInput?.dataUrl)
|
|
1395
|
+
content.push({ type: 'image', image: capturedInput.dataUrl, mimeType: capturedInput.mimeType });
|
|
1396
|
+
else if (visionImage.imagePath)
|
|
1394
1397
|
content.push({ type: 'image', imagePath: visionImage.imagePath, mimeType: imageMimeForPath(visionImage.imagePath) });
|
|
1395
1398
|
else if (visionImage.image)
|
|
1396
1399
|
content.push({ type: 'image', image: visionImage.image, mimeType: visionImage.mimeType });
|
|
@@ -1407,7 +1410,7 @@ function toKernelTools(agent, definitions, provisioning) {
|
|
|
1407
1410
|
}
|
|
1408
1411
|
catch { }
|
|
1409
1412
|
}
|
|
1410
|
-
return { content, details: { tool: name, ok: true, terminate, ...(launchReceipt ? { launchReceipt } : {}), visionImagePath: visionImage.imagePath || undefined, ephemeralVisionImage: !!visionImage.image, displayImage }, terminate };
|
|
1413
|
+
return { content, details: { tool: name, ok: true, terminate, ...(launchReceipt ? { launchReceipt } : {}), visionImagePath: visionImage.imagePath || undefined, ephemeralVisionImage: !!visionImage.image, capturedAttachmentId: capturedInput?.id, displayImage }, terminate };
|
|
1411
1414
|
},
|
|
1412
1415
|
};
|
|
1413
1416
|
}).filter((tool) => !!tool.name);
|
|
@@ -1440,7 +1443,7 @@ function boundInlineToolResult(name, text) {
|
|
|
1440
1443
|
if (value.length <= INLINE_TOOL_RESULT_MAX_CHARS)
|
|
1441
1444
|
return value;
|
|
1442
1445
|
// 结构化结果(JSON/视觉/浏览器/子代理/计划等)不可安全截断,保持原样。
|
|
1443
|
-
if (['computer_use', 'browser_use', 'pdf_read', 'image_inspect', 'image_display', 'task', 'subagent_create', 'SubAgent', 'subagent_send', 'subagent_result', 'subagent_read', 'linked_plan', 'question'].includes(name)) {
|
|
1446
|
+
if (['screen_capture', 'computer_use', 'browser_use', 'pdf_read', 'image_inspect', 'image_display', 'task', 'subagent_create', 'SubAgent', 'subagent_send', 'subagent_result', 'subagent_read', 'linked_plan', 'question'].includes(name)) {
|
|
1444
1447
|
return value;
|
|
1445
1448
|
}
|
|
1446
1449
|
const headChars = Math.floor(INLINE_TOOL_RESULT_MAX_CHARS * 0.6);
|
|
@@ -1460,7 +1463,7 @@ function spillOversizedToolResult(agent, name, text) {
|
|
|
1460
1463
|
if (value.length <= INLINE_TOOL_RESULT_MAX_CHARS)
|
|
1461
1464
|
return value;
|
|
1462
1465
|
// 结构化结果不可安全落盘引用(破坏 JSON 结构),保持原样。
|
|
1463
|
-
if (['computer_use', 'browser_use', 'pdf_read', 'image_inspect', 'image_display', 'task', 'subagent_create', 'SubAgent', 'subagent_send', 'subagent_result', 'subagent_read', 'linked_plan', 'question'].includes(name)) {
|
|
1466
|
+
if (['screen_capture', 'computer_use', 'browser_use', 'pdf_read', 'image_inspect', 'image_display', 'task', 'subagent_create', 'SubAgent', 'subagent_send', 'subagent_result', 'subagent_read', 'linked_plan', 'question'].includes(name)) {
|
|
1464
1467
|
return value;
|
|
1465
1468
|
}
|
|
1466
1469
|
const artifactId = agent.storeToolResultArtifact(name, value);
|
|
@@ -1475,11 +1478,11 @@ function spillOversizedToolResult(agent, name, text) {
|
|
|
1475
1478
|
].join('\n');
|
|
1476
1479
|
}
|
|
1477
1480
|
function sanitizeVisualToolText(name, text) {
|
|
1478
|
-
if (name !== 'computer_use' && name !== 'browser_use' && name !== 'pdf_read' && name !== 'image_inspect')
|
|
1481
|
+
if (name !== 'screen_capture' && name !== 'computer_use' && name !== 'browser_use' && name !== 'pdf_read' && name !== 'image_inspect')
|
|
1479
1482
|
return text;
|
|
1480
1483
|
try {
|
|
1481
1484
|
const parsed = JSON.parse(text);
|
|
1482
|
-
if (name === 'computer_use' || name === 'browser_use' || name === 'pdf_read') {
|
|
1485
|
+
if (name === 'screen_capture' || name === 'computer_use' || name === 'browser_use' || name === 'pdf_read') {
|
|
1483
1486
|
delete parsed.vision_image_path;
|
|
1484
1487
|
delete parsed.vision_image_data_url;
|
|
1485
1488
|
if (name === 'pdf_read' && parsed.result && typeof parsed.result === 'object') {
|
|
@@ -1497,7 +1500,7 @@ function sanitizeVisualToolText(name, text) {
|
|
|
1497
1500
|
}
|
|
1498
1501
|
}
|
|
1499
1502
|
function discardComputerUseVisionImage(name, text) {
|
|
1500
|
-
if (name !== 'computer_use')
|
|
1503
|
+
if (name !== 'screen_capture' && name !== 'computer_use')
|
|
1501
1504
|
return;
|
|
1502
1505
|
try {
|
|
1503
1506
|
const parsed = JSON.parse(text);
|
|
@@ -1519,8 +1522,22 @@ function imageInspectDataUrl(name, text) {
|
|
|
1519
1522
|
return '';
|
|
1520
1523
|
}
|
|
1521
1524
|
}
|
|
1525
|
+
function capturedImageToolText(name, text, attachmentId) {
|
|
1526
|
+
if (name !== 'screen_capture' || !attachmentId)
|
|
1527
|
+
return text;
|
|
1528
|
+
try {
|
|
1529
|
+
const parsed = JSON.parse(text);
|
|
1530
|
+
parsed.attachment_id = attachmentId;
|
|
1531
|
+
parsed.image_input_channel = 'user-image';
|
|
1532
|
+
parsed.inspect_next = { tool: 'image_inspect', actions: ['source_info', 'crop'], max_scale: 4 };
|
|
1533
|
+
return JSON.stringify(parsed, null, 2);
|
|
1534
|
+
}
|
|
1535
|
+
catch {
|
|
1536
|
+
return text;
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1522
1539
|
function visualFallbackImageInput(agent, name, text) {
|
|
1523
|
-
if (name !== 'computer_use' && name !== 'browser_use' && name !== 'pdf_read')
|
|
1540
|
+
if (name !== 'screen_capture' && name !== 'computer_use' && name !== 'browser_use' && name !== 'pdf_read')
|
|
1524
1541
|
return {};
|
|
1525
1542
|
const model = agent.activeModelConfig();
|
|
1526
1543
|
if (!model?.vision)
|
|
@@ -1685,7 +1702,7 @@ async function executeNewmarkTool(agent, name, args, inputSchema, signal) {
|
|
|
1685
1702
|
actorId: agent.runtimeActorId,
|
|
1686
1703
|
workspaceId: (0, terminalTakeover_1.terminalTakeoverWorkspaceId)(wsDir),
|
|
1687
1704
|
backend: process.env.NEWMARK_WSL_DISTRO ? 'wsl' : (process.platform === 'win32' ? 'windows' : process.platform),
|
|
1688
|
-
allowEphemeralVisionImage: (name === 'computer_use' || name === 'browser_use' || name === 'pdf_read' || name === 'ocr_read')
|
|
1705
|
+
allowEphemeralVisionImage: (name === 'screen_capture' || name === 'computer_use' || name === 'browser_use' || name === 'pdf_read' || name === 'ocr_read')
|
|
1689
1706
|
&& !!agent.activeModelConfig()?.vision,
|
|
1690
1707
|
signal,
|
|
1691
1708
|
});
|
|
@@ -85,7 +85,7 @@ export interface RankedRouteCandidate {
|
|
|
85
85
|
export type RouteAttemptStatus = 'planned' | 'success' | 'failed' | 'blocked';
|
|
86
86
|
export interface RouteAttempt {
|
|
87
87
|
deployment: DeploymentRef;
|
|
88
|
-
kind: 'initial' | 'retry_same_deployment' | 'equivalent_deployment' | 'fallback_model';
|
|
88
|
+
kind: 'initial' | 'retry_same_deployment' | 'equivalent_deployment' | 'fallback_model' | 'alternate_model';
|
|
89
89
|
status: RouteAttemptStatus;
|
|
90
90
|
errorType?: RouteFailureType;
|
|
91
91
|
durationMs?: number;
|
package/dist/core/autoRouter.js
CHANGED
|
@@ -115,8 +115,8 @@ function classifyRouteFailure(error) {
|
|
|
115
115
|
if (statusCode === 400 || /bad request|invalid parameter|invalid request/i.test(text)) {
|
|
116
116
|
return { type: 'invalid_request', retryable: false, switchAllowed: false, statusCode };
|
|
117
117
|
}
|
|
118
|
-
if (statusCode === 402 || /insufficient balance|insufficient funds|payment required
|
|
119
|
-
return { type: 'balance_exhausted', retryable: false, switchAllowed:
|
|
118
|
+
if (statusCode === 402 || /insufficient balance|insufficient funds|payment required|insufficient[_ -]?quota|quota (?:exceeded|exhausted)|credit(?:s| balance)? (?:exhausted|depleted)|billing hard limit|budget exhausted|余额不足|额度不足|配额(?:不足|耗尽|超限)/i.test(text)) {
|
|
119
|
+
return { type: 'balance_exhausted', retryable: false, switchAllowed: true, statusCode: statusCode || 402 };
|
|
120
120
|
}
|
|
121
121
|
if (statusCode === 429 || /rate limit|too many requests/i.test(text)) {
|
|
122
122
|
return { type: 'rate_limited', retryable: true, switchAllowed: true, statusCode: 429, retryAfterMs };
|
|
@@ -270,7 +270,7 @@ class AutoRouter {
|
|
|
270
270
|
}
|
|
271
271
|
planAttempts(decision, candidates, failure) {
|
|
272
272
|
const current = decision.resolvedDeployment;
|
|
273
|
-
if (!current || !failure.error.
|
|
273
|
+
if (!current || !failure.error.switchAllowed || failure.streamCommitted || failure.sideEffectCommitted)
|
|
274
274
|
return [];
|
|
275
275
|
const remainingAttempts = Math.max(0, 3 - decision.attempts.length);
|
|
276
276
|
if (!remainingAttempts)
|
|
@@ -279,7 +279,7 @@ class AutoRouter {
|
|
|
279
279
|
const retryDelayMs = failure.error.retryAfterMs ?? 250;
|
|
280
280
|
const alreadyRetriedCurrent = decision.attempts.some(attempt => attempt.kind === 'retry_same_deployment'
|
|
281
281
|
&& sameDeployment(attempt.deployment, current));
|
|
282
|
-
if (!alreadyRetriedCurrent && retryDelayMs <= (decision.retryBudgetMs ?? 5_000)) {
|
|
282
|
+
if (failure.error.retryable && !alreadyRetriedCurrent && retryDelayMs <= (decision.retryBudgetMs ?? 5_000)) {
|
|
283
283
|
attempts.push({
|
|
284
284
|
deployment: { ...current },
|
|
285
285
|
kind: 'retry_same_deployment',
|
|
@@ -310,12 +310,21 @@ class AutoRouter {
|
|
|
310
310
|
? eligible.find(candidate => candidate.deployment.logicalModelGroupId === currentGroup && !candidate.fallbackOnly)
|
|
311
311
|
: undefined;
|
|
312
312
|
const fallback = eligible.find(candidate => candidate.fallbackOnly);
|
|
313
|
-
|
|
313
|
+
const rankedAlternates = decision.rankedCandidates
|
|
314
|
+
.map(ranked => eligible.find(candidate => sameDeployment(candidate.deployment, ranked.deployment)))
|
|
315
|
+
.filter((candidate) => !!candidate && candidate !== equivalent && candidate !== fallback && !candidate.fallbackOnly);
|
|
316
|
+
for (const candidate of eligible) {
|
|
317
|
+
if (candidate === equivalent || candidate === fallback || candidate.fallbackOnly
|
|
318
|
+
|| rankedAlternates.some(existing => sameDeployment(existing.deployment, candidate.deployment)))
|
|
319
|
+
continue;
|
|
320
|
+
rankedAlternates.push(candidate);
|
|
321
|
+
}
|
|
322
|
+
for (const next of [equivalent, fallback, ...rankedAlternates]) {
|
|
314
323
|
if (!next || attempts.length >= 2)
|
|
315
324
|
continue;
|
|
316
325
|
attempts.push({
|
|
317
326
|
deployment: { ...next.deployment },
|
|
318
|
-
kind: next === equivalent ? 'equivalent_deployment' : 'fallback_model',
|
|
327
|
+
kind: next === equivalent ? 'equivalent_deployment' : next === fallback ? 'fallback_model' : 'alternate_model',
|
|
319
328
|
status: 'planned',
|
|
320
329
|
errorType: failure.error.type,
|
|
321
330
|
streamCommitted: false,
|
|
@@ -578,6 +587,6 @@ function percentile(values, fraction) {
|
|
|
578
587
|
}
|
|
579
588
|
function failureFromType(type) {
|
|
580
589
|
const retryable = type === 'timeout' || type === 'rate_limited' || type === 'transport' || type === 'server_error' || type === 'empty_response';
|
|
581
|
-
return { type, retryable, switchAllowed: retryable };
|
|
590
|
+
return { type, retryable, switchAllowed: retryable || type === 'balance_exhausted' };
|
|
582
591
|
}
|
|
583
592
|
//# sourceMappingURL=autoRouter.js.map
|
|
@@ -3,6 +3,24 @@ import { AgentMode, AgentWorkEvent, ConversationInputEnvelope, ConversationImage
|
|
|
3
3
|
import { AutomationManager } from './automation';
|
|
4
4
|
import { ConversationRuntimeTarget, NormalizedConversationTarget } from './conversationTarget';
|
|
5
5
|
export type ConversationQueueMode = 'steer' | 'followUp';
|
|
6
|
+
export interface ConversationQueueItemSnapshot {
|
|
7
|
+
id: string;
|
|
8
|
+
text: string;
|
|
9
|
+
queueMode: ConversationQueueMode;
|
|
10
|
+
requestedMode?: string;
|
|
11
|
+
goalObjective?: string;
|
|
12
|
+
runId?: string;
|
|
13
|
+
createdAt: string;
|
|
14
|
+
}
|
|
15
|
+
export type ConversationQueueAction = 'enqueue' | 'update' | 'delete' | 'reorder' | 'toggle_pause' | 'guide';
|
|
16
|
+
export interface ConversationQueueActionInput {
|
|
17
|
+
id?: string;
|
|
18
|
+
text?: string;
|
|
19
|
+
requestedMode?: string;
|
|
20
|
+
goalObjective?: string;
|
|
21
|
+
createdAt?: string;
|
|
22
|
+
orderedIds?: string[];
|
|
23
|
+
}
|
|
6
24
|
export interface AgentPromptMessage {
|
|
7
25
|
text: string;
|
|
8
26
|
/** Public transcript text when the execution prompt contains hidden orchestration instructions. */
|
|
@@ -176,6 +194,28 @@ export declare class ConversationKernel {
|
|
|
176
194
|
steering: string[];
|
|
177
195
|
followUp: string[];
|
|
178
196
|
};
|
|
197
|
+
queueItems(target: ConversationTargetInput): ConversationQueueItemSnapshot[];
|
|
198
|
+
enqueueNext(target: ConversationTargetInput, input: {
|
|
199
|
+
id: string;
|
|
200
|
+
text: string;
|
|
201
|
+
requestedMode?: string;
|
|
202
|
+
goalObjective?: string;
|
|
203
|
+
createdAt?: string;
|
|
204
|
+
}): ConversationQueueItemSnapshot;
|
|
205
|
+
updateQueueItem(target: ConversationTargetInput, idInput: string, textInput: string): ConversationQueueItemSnapshot;
|
|
206
|
+
deleteQueueItem(target: ConversationTargetInput, idInput: string): boolean;
|
|
207
|
+
reorderQueueItems(target: ConversationTargetInput, orderedIdsInput: string[]): ConversationQueueItemSnapshot[];
|
|
208
|
+
setQueuePaused(target: ConversationTargetInput, paused: boolean): boolean;
|
|
209
|
+
queueAction(target: ConversationTargetInput, action: ConversationQueueAction, input?: ConversationQueueActionInput): {
|
|
210
|
+
ok: boolean;
|
|
211
|
+
queueItems: ConversationQueueItemSnapshot[];
|
|
212
|
+
queuePaused: boolean;
|
|
213
|
+
queued: {
|
|
214
|
+
steering: string[];
|
|
215
|
+
followUp: string[];
|
|
216
|
+
};
|
|
217
|
+
receipt?: GuideReceipt;
|
|
218
|
+
};
|
|
179
219
|
events(target: ConversationTargetInput): AgentWorkEvent[];
|
|
180
220
|
waitForIdle(target: ConversationTargetInput): Promise<void>;
|
|
181
221
|
pendingOptions(target: ConversationTargetInput): OptionQuestion[] | undefined;
|
|
@@ -185,6 +225,8 @@ export declare class ConversationKernel {
|
|
|
185
225
|
steering: string[];
|
|
186
226
|
followUp: string[];
|
|
187
227
|
};
|
|
228
|
+
queueItems: ConversationQueueItemSnapshot[];
|
|
229
|
+
queuePaused: boolean;
|
|
188
230
|
workEvents: AgentWorkEvent[];
|
|
189
231
|
runtime: ConversationRuntimeState | null;
|
|
190
232
|
mode: Agent['mode'];
|
|
@@ -258,6 +300,7 @@ export declare class ConversationKernel {
|
|
|
258
300
|
private enqueueSameSession;
|
|
259
301
|
private trackQueuedMessage;
|
|
260
302
|
private consumeQueuedMessage;
|
|
303
|
+
private replaceTrackedQueuedMessage;
|
|
261
304
|
private emitQueueUpdate;
|
|
262
305
|
private clearQueued;
|
|
263
306
|
private queueState;
|