newmark-agent 0.6.3 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.example.json +13 -3
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +53 -2
- package/dist/cli-discovery.js +3 -0
- package/dist/conversation-utility-host.bundle.cjs +3046 -1038
- package/dist/conversation-utility-host.js +94 -7
- package/dist/core/agent.d.ts +149 -12
- package/dist/core/agent.js +843 -176
- package/dist/core/agentKernelRunner.d.ts +7 -1
- package/dist/core/agentKernelRunner.js +171 -22
- package/dist/core/autoRouter.d.ts +49 -44
- package/dist/core/autoRouter.js +117 -302
- package/dist/core/branchIdentity.d.ts +29 -0
- package/dist/core/branchIdentity.js +54 -0
- package/dist/core/config.js +27 -11
- package/dist/core/continuation/contracts.d.ts +1 -1
- package/dist/core/continuation/store.d.ts +119 -1
- package/dist/core/continuation/store.js +337 -19
- package/dist/core/conversationKernel.d.ts +62 -8
- package/dist/core/conversationKernel.js +361 -47
- package/dist/core/conversationStateDocument.d.ts +18 -0
- package/dist/core/conversationStateDocument.js +88 -0
- package/dist/core/conversationVisualBudget.d.ts +74 -0
- package/dist/core/conversationVisualBudget.js +153 -0
- package/dist/core/electronUtilityAgentClient.d.ts +3 -2
- package/dist/core/electronUtilityAgentClient.js +32 -8
- package/dist/core/electronUtilityRuntimePool.d.ts +28 -3
- package/dist/core/electronUtilityRuntimePool.js +93 -24
- package/dist/core/hostRuntimeHooks.d.ts +23 -0
- package/dist/core/hostRuntimeHooks.js +17 -0
- package/dist/core/jevDecision.d.ts +144 -0
- package/dist/core/jevDecision.js +226 -0
- package/dist/core/memoryProbe.d.ts +24 -0
- package/dist/core/memoryProbe.js +104 -0
- package/dist/core/mobilePairing.d.ts +7 -1
- package/dist/core/mobilePairing.js +9 -1
- package/dist/core/performanceDiagnostics.d.ts +1 -1
- package/dist/core/routeDecisionValidator.d.ts +86 -0
- package/dist/core/routeDecisionValidator.js +249 -0
- package/dist/core/routeEligibility.d.ts +52 -0
- package/dist/core/routeEligibility.js +108 -0
- package/dist/core/runtimeMemoryBudget.d.ts +6 -0
- package/dist/core/runtimeMemoryBudget.js +12 -0
- package/dist/core/utilityAgentProtocol.d.ts +8 -10
- package/dist/core/utilityHostToolRouter.js +2 -0
- package/dist/core/visualDownscale.d.ts +6 -0
- package/dist/core/visualDownscale.js +137 -0
- package/dist/core/wslAgentClient.d.ts +1 -2
- package/dist/core/wslAgentClient.js +0 -8
- package/dist/core/wslAgentProtocol.d.ts +1 -10
- package/dist/core/wslAgentRuntimePool.d.ts +1 -3
- package/dist/core/wslAgentRuntimePool.js +17 -22
- package/dist/llm/provider.d.ts +3 -1
- package/dist/llm/provider.js +53 -9
- package/dist/main.js +108 -31
- package/dist/preload.js +3 -1
- package/dist/server.js +30 -18
- package/dist/tools/computerUse.d.ts +4 -0
- package/dist/tools/computerUse.js +202 -4
- package/dist/tools/computerUsePowerShellHost.d.ts +6 -1
- package/dist/tools/computerUsePowerShellHost.js +105 -31
- package/dist/tools/index.js +4 -1
- package/dist/tui/src/adapters/core-runtime-adapter.js +13 -1
- package/dist/tui/src/i18n.js +12 -1
- package/dist/tui/src/render.js +9 -2
- package/dist/tui/src/settings-schema.js +19 -0
- package/dist/tui/src/state.js +69 -1
- package/dist/ui/index.html +994 -277
- package/dist/ui/lucide-sprite.svg +0 -8
- package/dist/wsl-agent-host.bundle.cjs +2851 -1013
- package/dist/wsl-agent-host.js +0 -3
- package/package.json +38 -9
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Agent } from './agent';
|
|
2
2
|
import { StreamToken } from './types';
|
|
3
3
|
import { type ToolchainCore } from '../toolchain';
|
|
4
|
+
import { type ConversationVisualBudget } from './conversationVisualBudget';
|
|
4
5
|
export declare function filterPublicAssistantDelta(agent: Agent, delta: string): string;
|
|
5
6
|
export declare function resetPublicAssistantDeltaFilter(agent: Agent): void;
|
|
6
7
|
interface KernelTextContent {
|
|
@@ -143,7 +144,12 @@ declare function visualFallbackImageInput(agent: Agent, name: string, text: stri
|
|
|
143
144
|
export declare function toolResultObjectiveOutcome(result: string): boolean | undefined;
|
|
144
145
|
declare function toKernelMessagesFromHistory(history: Array<Record<string, unknown>>, agent: Agent): KernelMessage[];
|
|
145
146
|
declare function toProviderToolDefinitions(tools: KernelTool[]): unknown[];
|
|
146
|
-
declare function fromKernelMessages(messages: KernelMessage[], includeEphemeralImages?: boolean
|
|
147
|
+
declare function fromKernelMessages(messages: KernelMessage[], includeEphemeralImages?: boolean, visualBudget?: ConversationVisualBudget, onVisualRelease?: (info: {
|
|
148
|
+
releasedImages: number;
|
|
149
|
+
releasedBytes: number;
|
|
150
|
+
retainedBytes: number;
|
|
151
|
+
budget: ConversationVisualBudget;
|
|
152
|
+
}) => void): Array<Record<string, unknown>>;
|
|
147
153
|
declare function imagePathToOpenAIContentPart(imagePath: string): Record<string, unknown> | null;
|
|
148
154
|
export declare const agentKernelRunnerInternals: {
|
|
149
155
|
buildRequestTaskFocus: typeof buildRequestTaskFocus;
|
|
@@ -39,6 +39,7 @@ exports.resetPublicAssistantDeltaFilter = resetPublicAssistantDeltaFilter;
|
|
|
39
39
|
exports.runAgentKernel = runAgentKernel;
|
|
40
40
|
exports.routeToolSurfaceV2 = routeToolSurfaceV2;
|
|
41
41
|
exports.toolResultObjectiveOutcome = toolResultObjectiveOutcome;
|
|
42
|
+
const branchIdentity_1 = require("./branchIdentity");
|
|
42
43
|
const fs = __importStar(require("fs"));
|
|
43
44
|
const path = __importStar(require("path"));
|
|
44
45
|
const terminalTakeover_1 = require("../tools/terminalTakeover");
|
|
@@ -47,8 +48,13 @@ const toolPolicy_1 = require("./toolPolicy");
|
|
|
47
48
|
const performanceDiagnostics_1 = require("./performanceDiagnostics");
|
|
48
49
|
const agentKernelDiagnostics_1 = require("./agentKernelDiagnostics");
|
|
49
50
|
const toolchain_1 = require("../toolchain");
|
|
51
|
+
const conversationVisualBudget_1 = require("./conversationVisualBudget");
|
|
52
|
+
const visualDownscale_1 = require("./visualDownscale");
|
|
50
53
|
const emptyResponseRetry_1 = require("./emptyResponseRetry");
|
|
51
54
|
const publicStreamFilters = new WeakMap();
|
|
55
|
+
/** dev-0.6.6: throttles the "screenshots released" work-status line. */
|
|
56
|
+
let visualReleaseReportedAt = 0;
|
|
57
|
+
const VISUAL_MIB = 1024 ** 2;
|
|
52
58
|
const brokerOnlyAssistantBuffers = new WeakMap();
|
|
53
59
|
const BROKER_PREFACE_BUFFER_CHARS = 96;
|
|
54
60
|
const HIDDEN_LINE_PREFIXES = [
|
|
@@ -75,13 +81,25 @@ const HIDDEN_LINE_PREFIXES = [
|
|
|
75
81
|
* a fork that reuses the root conversation id can read the other branch's
|
|
76
82
|
* history. Keep the id stable within one branch (normal sequential turns still
|
|
77
83
|
* cache-hit) and change it the moment the runtime branch changes.
|
|
84
|
+
*
|
|
85
|
+
* dev-0.6.4: 分支身份固定在 Build 启动时创建的那条分支节点上(用户分页分支与
|
|
86
|
+
* 实验性分支交流分支共用同一规则)。这里绝不读取「当前正在查看/激活的分支」,
|
|
87
|
+
* 否则正在并行运行的兄弟分支或中途切换的页面会把请求绑到别的分支的远程会话。
|
|
78
88
|
*/
|
|
79
89
|
function providerSessionIdentity(agent) {
|
|
80
90
|
const conversationId = String(agent.activeConversationId || 'default');
|
|
91
|
+
let runBranchId = '';
|
|
92
|
+
try {
|
|
93
|
+
runBranchId = String(agent.activeWorkRunBranchId() || '');
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
runBranchId = '';
|
|
97
|
+
}
|
|
98
|
+
if (runBranchId)
|
|
99
|
+
return (0, branchIdentity_1.branchConversationIdentity)(conversationId, runBranchId);
|
|
81
100
|
try {
|
|
82
101
|
const snapshot = agent.getConversationSnapshot(conversationId);
|
|
83
|
-
|
|
84
|
-
return branchId ? `${conversationId}::branch:${branchId}` : conversationId;
|
|
102
|
+
return (0, branchIdentity_1.branchConversationIdentity)(conversationId, String(snapshot.runtimeBranchId || snapshot.activeBranchId || ''));
|
|
85
103
|
}
|
|
86
104
|
catch {
|
|
87
105
|
return conversationId;
|
|
@@ -447,7 +465,11 @@ async function runAgentKernel(agent) {
|
|
|
447
465
|
agent.chatMessages.push({ role: 'user', content: KernelMessageText(msg), mode: agent.modeName(), model: agent.model, timestamp: agent.nowLabel() });
|
|
448
466
|
agent.history.push(toHistoryMessage(msg));
|
|
449
467
|
}
|
|
450
|
-
|
|
468
|
+
// Mid-turn persistence is coalesced (see Agent.scheduleStoredConversationState):
|
|
469
|
+
// the terminal save at the end of the run still writes synchronously, so
|
|
470
|
+
// durability at turn boundaries is unchanged while per-turn full-document
|
|
471
|
+
// rewrites drop from one-per-event to at most one per throttle window.
|
|
472
|
+
agent.saveWorkspaceConversationState(false);
|
|
451
473
|
}
|
|
452
474
|
await kernel.prompt(promptMessages);
|
|
453
475
|
const assistant = lastAssistant;
|
|
@@ -506,7 +528,7 @@ async function runAgentKernel(agent) {
|
|
|
506
528
|
agent.emitWorkEvent({ type: 'final_response', content: visualFallback });
|
|
507
529
|
agent.chatMessages.push({ role: 'assistant', content: visualFallback, mode: agent.modeName(), model: agent.model, timestamp: agent.nowLabel(), runId: agent.currentWorkRunId() || undefined });
|
|
508
530
|
agent.history.push({ role: 'assistant', content: visualFallback, run_id: agent.currentWorkRunId() || undefined });
|
|
509
|
-
agent.saveWorkspaceConversationState();
|
|
531
|
+
agent.saveWorkspaceConversationState(false);
|
|
510
532
|
agent.recordWorkStatus('Final visual fallback used: local mini OCR plus conservative text correction.');
|
|
511
533
|
lastTurn = { ...lastTurn, text: visualFallback, errorMessage: '', stopReason: 'stop' };
|
|
512
534
|
}
|
|
@@ -645,7 +667,9 @@ async function runAgentKernel(agent) {
|
|
|
645
667
|
agent.attachAgentKernelRuntime(null);
|
|
646
668
|
}
|
|
647
669
|
agent.status = 'idle';
|
|
648
|
-
|
|
670
|
+
// Coalesced: the caller (Agent.process) flushes synchronously right after,
|
|
671
|
+
// and every in-memory reader already sees this status.
|
|
672
|
+
agent.saveWorkspaceConversationState(false);
|
|
649
673
|
return agent.sanitizeVisibleTokens(tokens);
|
|
650
674
|
function streamWithNewmarkProvider(currentAgent, compat) {
|
|
651
675
|
return async (model, context, options) => {
|
|
@@ -683,7 +707,15 @@ async function runAgentKernel(agent) {
|
|
|
683
707
|
// endpoint owns its real limit; protocols that require a positive
|
|
684
708
|
// max_tokens resolve one internally.
|
|
685
709
|
const maxTokens = 0;
|
|
686
|
-
const
|
|
710
|
+
const requestVisualBudget = visualBudgetForAgent(currentAgent);
|
|
711
|
+
const newmarkMessages = fromKernelMessages(context.messages, true, requestVisualBudget, info => {
|
|
712
|
+
// Observable, throttled: one status line per run so the user learns
|
|
713
|
+
// that screenshots were reclaimed instead of silently losing them.
|
|
714
|
+
if (visualReleaseReportedAt && Date.now() - visualReleaseReportedAt < 60_000)
|
|
715
|
+
return;
|
|
716
|
+
visualReleaseReportedAt = Date.now();
|
|
717
|
+
currentAgent.recordWorkStatus(`Released ${info.releasedImages} older ComputerUse/BrowserControl screenshot(s) (~${Math.round(info.releasedBytes / VISUAL_MIB)} MiB) to stay within the ${(0, conversationVisualBudget_1.describeVisualBudget)(info.budget)} budget.`);
|
|
718
|
+
}).map(message => message.role === 'assistant'
|
|
687
719
|
// Use the same public-content boundary on its first provider
|
|
688
720
|
// submission and after persistence. Trimming only the durable
|
|
689
721
|
// copy changes earlier tool-call envelopes on mailbox recovery.
|
|
@@ -1038,7 +1070,7 @@ async function handleKernelEvent(agent, event, tokens) {
|
|
|
1038
1070
|
// Otherwise a cold continuation loses an already-read instruction
|
|
1039
1071
|
// and deletes that message from the provider's retained prefix.
|
|
1040
1072
|
agent.history.push({ ...toHistoryMessage(event.message), run_id: agent.currentWorkRunId() || undefined });
|
|
1041
|
-
agent.saveWorkspaceConversationState(
|
|
1073
|
+
agent.saveWorkspaceConversationState(false);
|
|
1042
1074
|
}
|
|
1043
1075
|
agent.notifyAgentKernelUserMessageStart(text, event.message.clientMessageId);
|
|
1044
1076
|
}
|
|
@@ -1145,7 +1177,7 @@ async function handleKernelEvent(agent, event, tokens) {
|
|
|
1145
1177
|
historyMessage.content = text;
|
|
1146
1178
|
historyMessage.run_id = agent.currentWorkRunId() || undefined;
|
|
1147
1179
|
agent.history.push(historyMessage);
|
|
1148
|
-
agent.saveWorkspaceConversationState();
|
|
1180
|
+
agent.saveWorkspaceConversationState(false);
|
|
1149
1181
|
}
|
|
1150
1182
|
resetPublicAssistantDeltaFilter(agent);
|
|
1151
1183
|
resetAssistantToolVisibility(agent);
|
|
@@ -1156,6 +1188,9 @@ async function handleKernelEvent(agent, event, tokens) {
|
|
|
1156
1188
|
...(!event.message.isError && receipt ? { subagent_settlement_receipt: { ...receipt } } : {}),
|
|
1157
1189
|
};
|
|
1158
1190
|
agent.history.push(historyMessage);
|
|
1191
|
+
// Kept synchronous on purpose: the catch below clears the settlement
|
|
1192
|
+
// receipt so a later notification cannot mistake an unsaved result for a
|
|
1193
|
+
// committed one, which requires the write failure to surface here.
|
|
1159
1194
|
try {
|
|
1160
1195
|
agent.saveWorkspaceConversationState();
|
|
1161
1196
|
}
|
|
@@ -1777,8 +1812,17 @@ function visualFallbackImageInput(agent, name, text) {
|
|
|
1777
1812
|
if (name !== 'pdf_read' && nested.action !== 'observe' && nested.action !== 'app_observe')
|
|
1778
1813
|
return {};
|
|
1779
1814
|
const directImage = String(nested.vision_image_data_url || '');
|
|
1780
|
-
if (/^data:image\/(?:png|jpeg);base64,[A-Za-z0-9+/]+={0,2}$/i.test(directImage)
|
|
1781
|
-
|
|
1815
|
+
if (/^data:image\/(?:png|jpeg);base64,[A-Za-z0-9+/]+={0,2}$/i.test(directImage)) {
|
|
1816
|
+
// dev-0.6.6: ComputerUse/BrowserControl screenshots are the dominant
|
|
1817
|
+
// memory consumer of a long conversation, so the accepted single-image
|
|
1818
|
+
// size follows the machine's available memory instead of a fixed 2 MiB.
|
|
1819
|
+
const budget = visualBudgetForAgent(agent);
|
|
1820
|
+
const fitted = (0, conversationVisualBudget_1.visualDataUrlBytes)(directImage) <= budget.maxSingleImageBytes
|
|
1821
|
+
? directImage
|
|
1822
|
+
: (0, visualDownscale_1.downscaleDataUrlToBytes)(directImage, budget.maxSingleImageBytes);
|
|
1823
|
+
if (fitted)
|
|
1824
|
+
return { image: fitted, mimeType: fitted.slice(5, fitted.indexOf(';')).toLowerCase() };
|
|
1825
|
+
return {};
|
|
1782
1826
|
}
|
|
1783
1827
|
const screenshotPath = String(nested.vision_image_path || '');
|
|
1784
1828
|
if (!screenshotPath)
|
|
@@ -1790,6 +1834,47 @@ function visualFallbackImageInput(agent, name, text) {
|
|
|
1790
1834
|
return {};
|
|
1791
1835
|
}
|
|
1792
1836
|
}
|
|
1837
|
+
const visualBudgetCache = new WeakMap();
|
|
1838
|
+
/**
|
|
1839
|
+
* The conversation's visual budget for right now. Sampled at most once per
|
|
1840
|
+
* second per agent so a tool-heavy round does not re-read OS memory state on
|
|
1841
|
+
* every screenshot.
|
|
1842
|
+
*/
|
|
1843
|
+
function visualBudgetForAgent(agent) {
|
|
1844
|
+
const cached = visualBudgetCache.get(agent);
|
|
1845
|
+
const now = Date.now();
|
|
1846
|
+
if (cached && now - cached.at < 1000)
|
|
1847
|
+
return cached.budget;
|
|
1848
|
+
const usage = (() => {
|
|
1849
|
+
try {
|
|
1850
|
+
return process.memoryUsage();
|
|
1851
|
+
}
|
|
1852
|
+
catch {
|
|
1853
|
+
return { rss: 0 };
|
|
1854
|
+
}
|
|
1855
|
+
})();
|
|
1856
|
+
const budget = (0, conversationVisualBudget_1.conversationVisualBudget)({ rssBytes: Number(usage?.rss || 0) });
|
|
1857
|
+
visualBudgetCache.set(agent, { at: now, budget });
|
|
1858
|
+
return budget;
|
|
1859
|
+
}
|
|
1860
|
+
/** Bytes an image part costs when materialized into the provider request. */
|
|
1861
|
+
function imagePartBytes(part) {
|
|
1862
|
+
const inline = String(part.image || '');
|
|
1863
|
+
if (inline.startsWith('data:'))
|
|
1864
|
+
return (0, conversationVisualBudget_1.visualDataUrlBytes)(inline);
|
|
1865
|
+
const imagePath = String(part.imagePath || '');
|
|
1866
|
+
if (!imagePath)
|
|
1867
|
+
return 0;
|
|
1868
|
+
try {
|
|
1869
|
+
// Materializing a path-based screenshot produces `data:<mime>;base64,<...>`;
|
|
1870
|
+
// budget the base64 payload length the request will actually hold.
|
|
1871
|
+
const size = fs.statSync(imagePath).size;
|
|
1872
|
+
return 4 * Math.ceil(size / 3);
|
|
1873
|
+
}
|
|
1874
|
+
catch {
|
|
1875
|
+
return 0;
|
|
1876
|
+
}
|
|
1877
|
+
}
|
|
1793
1878
|
async function executeNewmarkTool(agent, name, args, inputSchema, signal, onSettlementReceipt) {
|
|
1794
1879
|
const stopToolTimer = (0, performanceDiagnostics_1.performanceTimer)('tool_execution', { conversationId: agent.activeConversationId, detail: { tool: name } });
|
|
1795
1880
|
try {
|
|
@@ -1940,8 +2025,11 @@ async function executeNewmarkTool(agent, name, args, inputSchema, signal, onSett
|
|
|
1940
2025
|
throw abortError();
|
|
1941
2026
|
trackFileDiff(agent, name, args);
|
|
1942
2027
|
const objectiveResult = toolResultObjectiveOutcome(result);
|
|
2028
|
+
// Objective postconditions are operational health evidence only. Auto Router
|
|
2029
|
+
// v2 keeps tool outcomes out of model preference: the observation can demote
|
|
2030
|
+
// an endpoint's live health, and it can never teach a future route.
|
|
1943
2031
|
if (objectiveResult !== undefined)
|
|
1944
|
-
agent.
|
|
2032
|
+
agent.recordRouteToolOutcome(objectiveResult);
|
|
1945
2033
|
return result;
|
|
1946
2034
|
}
|
|
1947
2035
|
finally {
|
|
@@ -2068,9 +2156,53 @@ function toProviderToolDefinitions(tools) {
|
|
|
2068
2156
|
},
|
|
2069
2157
|
}));
|
|
2070
2158
|
}
|
|
2071
|
-
function fromKernelMessages(messages, includeEphemeralImages = true) {
|
|
2072
|
-
|
|
2073
|
-
|
|
2159
|
+
function fromKernelMessages(messages, includeEphemeralImages = true, visualBudget, onVisualRelease) {
|
|
2160
|
+
const budget = visualBudget || (0, conversationVisualBudget_1.conversationVisualBudget)();
|
|
2161
|
+
// dev-0.6.6: a late-stage ComputerUse/BrowserControl conversation stores one
|
|
2162
|
+
// screenshot per observation. Materializing all of them per request is what
|
|
2163
|
+
// pushed the utility host to a 3 GB V8 abort, so the newest screenshots are
|
|
2164
|
+
// kept within this machine's visual budget and older ones are released
|
|
2165
|
+
// (text/tool results stay untouched).
|
|
2166
|
+
const candidates = [];
|
|
2167
|
+
messages.forEach((message, index) => {
|
|
2168
|
+
if (message.role !== 'toolResult')
|
|
2169
|
+
return;
|
|
2170
|
+
const image = message.content.find((part) => part.type === 'image');
|
|
2171
|
+
if (!image)
|
|
2172
|
+
return;
|
|
2173
|
+
candidates.push({ key: String(index), bytes: imagePartBytes(image) });
|
|
2174
|
+
});
|
|
2175
|
+
const plan = (0, conversationVisualBudget_1.planVisualRetention)(candidates, budget);
|
|
2176
|
+
if (plan.released.length && onVisualRelease) {
|
|
2177
|
+
onVisualRelease({
|
|
2178
|
+
releasedImages: plan.released.length,
|
|
2179
|
+
releasedBytes: plan.releasedBytes,
|
|
2180
|
+
retainedBytes: plan.retainedBytes,
|
|
2181
|
+
budget,
|
|
2182
|
+
});
|
|
2183
|
+
}
|
|
2184
|
+
return messages.flatMap((message, index) => {
|
|
2185
|
+
const key = String(index);
|
|
2186
|
+
const hadImage = message.role === 'toolResult'
|
|
2187
|
+
&& message.content.some(part => part.type === 'image');
|
|
2188
|
+
const imageAllowed = !hadImage || plan.keep.has(key);
|
|
2189
|
+
const projected = toHistoryMessage(message, includeEphemeralImages && imageAllowed, budget);
|
|
2190
|
+
if (hadImage && !imageAllowed) {
|
|
2191
|
+
// Tell the model the screenshot existed but was released, so it can
|
|
2192
|
+
// re-observe instead of assuming the picture was empty.
|
|
2193
|
+
// Kept to one short line: a long conversation can release hundreds of
|
|
2194
|
+
// historic screenshots, and the note must not become its own payload.
|
|
2195
|
+
const releaseNote = `[visual_memory_budget] screenshot released (${budget.pressure} memory budget); re-run observe if you need it again.`;
|
|
2196
|
+
if (typeof projected.content === 'string')
|
|
2197
|
+
projected.content = `${projected.content}\n${releaseNote}`;
|
|
2198
|
+
else if (Array.isArray(projected.content)) {
|
|
2199
|
+
const textPart = projected.content.find(part => part?.type === 'text');
|
|
2200
|
+
if (textPart)
|
|
2201
|
+
textPart.text = `${String(textPart.text || '')}\n${releaseNote}`;
|
|
2202
|
+
else
|
|
2203
|
+
projected.content.unshift({ type: 'text', text: releaseNote });
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2074
2206
|
if (message.role !== 'toolResult' || !Array.isArray(projected.content))
|
|
2075
2207
|
return [projected];
|
|
2076
2208
|
const parts = projected.content;
|
|
@@ -2109,7 +2241,7 @@ function publicHistoryFromKernelMessages(messages) {
|
|
|
2109
2241
|
return [toHistoryMessage({ ...message, content: publicContent }, false)];
|
|
2110
2242
|
});
|
|
2111
2243
|
}
|
|
2112
|
-
function toHistoryMessage(message, includeEphemeralImages = false) {
|
|
2244
|
+
function toHistoryMessage(message, includeEphemeralImages = false, visualBudget) {
|
|
2113
2245
|
if (message.role === 'user') {
|
|
2114
2246
|
if (typeof message.content === 'string')
|
|
2115
2247
|
return {
|
|
@@ -2141,13 +2273,7 @@ function toHistoryMessage(message, includeEphemeralImages = false) {
|
|
|
2141
2273
|
const ephemeralImage = message.content.find((c) => c.type === 'image');
|
|
2142
2274
|
const imagePath = ephemeralImage?.imagePath || '';
|
|
2143
2275
|
const directImage = ephemeralImage?.image || '';
|
|
2144
|
-
const imagePart = includeEphemeralImages
|
|
2145
|
-
? (imagePath
|
|
2146
|
-
? imagePathToOpenAIContentPart(imagePath)
|
|
2147
|
-
: directImage.startsWith('data:image/')
|
|
2148
|
-
? { type: 'image_url', image_url: { url: directImage } }
|
|
2149
|
-
: null)
|
|
2150
|
-
: null;
|
|
2276
|
+
const imagePart = includeEphemeralImages ? materializeVisualImagePart(imagePath, directImage, visualBudget) : null;
|
|
2151
2277
|
if (imagePart && ephemeralImage) {
|
|
2152
2278
|
// Tool-result images are one provider-input capability, not history. Once
|
|
2153
2279
|
// this request has materialized the image part, consume it from the live
|
|
@@ -2173,6 +2299,29 @@ function imagePathToOpenAIContentPart(imagePath) {
|
|
|
2173
2299
|
return null;
|
|
2174
2300
|
}
|
|
2175
2301
|
}
|
|
2302
|
+
/**
|
|
2303
|
+
* dev-0.6.6: materialize one screenshot into a provider image part while
|
|
2304
|
+
* respecting the conversation's per-image cap. Oversized screenshots are
|
|
2305
|
+
* downscaled (PNG, then JPEG) instead of being dropped, so a memory-tight
|
|
2306
|
+
* machine keeps vision and the agent keeps working.
|
|
2307
|
+
*/
|
|
2308
|
+
function materializeVisualImagePart(imagePath, directImage, visualBudget) {
|
|
2309
|
+
const budget = visualBudget || (0, conversationVisualBudget_1.conversationVisualBudget)();
|
|
2310
|
+
let dataUrl = '';
|
|
2311
|
+
if (String(directImage || '').startsWith('data:image/'))
|
|
2312
|
+
dataUrl = String(directImage);
|
|
2313
|
+
else if (imagePath)
|
|
2314
|
+
dataUrl = imagePathToDataUrl(imagePath) || '';
|
|
2315
|
+
if (!dataUrl)
|
|
2316
|
+
return null;
|
|
2317
|
+
if ((0, conversationVisualBudget_1.visualDataUrlBytes)(dataUrl) > budget.maxSingleImageBytes) {
|
|
2318
|
+
const fitted = (0, visualDownscale_1.downscaleDataUrlToBytes)(dataUrl, budget.maxSingleImageBytes);
|
|
2319
|
+
if (!fitted)
|
|
2320
|
+
return null;
|
|
2321
|
+
dataUrl = fitted;
|
|
2322
|
+
}
|
|
2323
|
+
return { type: 'image_url', image_url: { url: dataUrl } };
|
|
2324
|
+
}
|
|
2176
2325
|
exports.agentKernelRunnerInternals = {
|
|
2177
2326
|
buildRequestTaskFocus,
|
|
2178
2327
|
buildBuildContextBootstrap,
|
|
@@ -18,24 +18,29 @@ export type ModelSelection = {
|
|
|
18
18
|
policyId: string;
|
|
19
19
|
subset?: DeploymentRef[];
|
|
20
20
|
};
|
|
21
|
-
export type RouteMode = '
|
|
21
|
+
export type RouteMode = 'conservative' | 'balanced' | 'aggressive';
|
|
22
22
|
export type RoutePrivacy = 'default' | 'no_training' | 'zdr';
|
|
23
23
|
export type TaskClass = 'chat' | 'coding' | 'reasoning' | 'long_context' | 'vision' | 'image_generation' | 'tool_use' | 'computer_use';
|
|
24
24
|
export type ValidationLevel = 'discovered' | 'legacy_basic' | 'basic' | 'standard' | 'extended';
|
|
25
25
|
export type ValidationStatus = 'verified' | 'degraded' | 'unavailable' | 'auth_error' | 'rate_limited' | 'invalid_config';
|
|
26
26
|
export interface RoutePolicy {
|
|
27
27
|
mode: RouteMode;
|
|
28
|
-
maxQualityLoss: number;
|
|
29
|
-
maxExpectedCostUsd?: number;
|
|
30
28
|
allowPreview: boolean;
|
|
31
29
|
privacy: RoutePrivacy;
|
|
32
30
|
requiredCapabilities: string[];
|
|
33
31
|
dataRegion?: string;
|
|
34
32
|
requiredProtocolParameters?: string[];
|
|
33
|
+
maxExpectedCostUsd?: number;
|
|
35
34
|
}
|
|
36
35
|
export interface AutoRouteCandidate {
|
|
37
36
|
deployment: DeploymentRef;
|
|
38
37
|
enabled: boolean;
|
|
38
|
+
/**
|
|
39
|
+
* Set by the Agent when a deployment is explicitly unusable right now
|
|
40
|
+
* (for example a provider-reported balance exhaustion window). It carries a
|
|
41
|
+
* guard reason instead of silently mutating `enabled`.
|
|
42
|
+
*/
|
|
43
|
+
unavailableReason?: string;
|
|
39
44
|
validation: {
|
|
40
45
|
level: ValidationLevel;
|
|
41
46
|
status: ValidationStatus;
|
|
@@ -49,39 +54,24 @@ export interface AutoRouteCandidate {
|
|
|
49
54
|
supportedProtocolParameters?: string[];
|
|
50
55
|
expectedInputCostUsdPerM?: number;
|
|
51
56
|
expectedOutputCostUsdPerM?: number;
|
|
57
|
+
/** Current operational measurements. Never persisted as a preference. */
|
|
52
58
|
latencyMs?: number;
|
|
53
59
|
reliability?: number;
|
|
54
60
|
toolValidity?: number;
|
|
55
61
|
throughput?: number;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
attempts: number;
|
|
59
|
-
}>>;
|
|
62
|
+
circuit?: 'closed' | 'open' | 'half_open';
|
|
63
|
+
/** Explicit user configuration (`route_preference`), not learned history. */
|
|
60
64
|
preference?: number;
|
|
61
65
|
fallbackOnly?: boolean;
|
|
62
66
|
}
|
|
63
67
|
export interface RouteRequest {
|
|
64
68
|
transactionId: string;
|
|
65
|
-
affinityKey: string;
|
|
66
69
|
taskText: string;
|
|
67
70
|
estimatedInputTokens: number;
|
|
68
71
|
expectedOutputTokens: number;
|
|
69
72
|
requiredCapabilities: string[];
|
|
70
73
|
batch?: boolean;
|
|
71
74
|
}
|
|
72
|
-
export interface RankedRouteCandidate {
|
|
73
|
-
deployment: DeploymentRef;
|
|
74
|
-
quality: number;
|
|
75
|
-
utility: number;
|
|
76
|
-
expectedCostUsd?: number;
|
|
77
|
-
components: {
|
|
78
|
-
cost: number;
|
|
79
|
-
reliability: number;
|
|
80
|
-
speed: number;
|
|
81
|
-
cache: number;
|
|
82
|
-
preference: number;
|
|
83
|
-
};
|
|
84
|
-
}
|
|
85
75
|
export type RouteAttemptStatus = 'planned' | 'success' | 'failed' | 'blocked';
|
|
86
76
|
export interface RouteAttempt {
|
|
87
77
|
deployment: DeploymentRef;
|
|
@@ -97,15 +87,24 @@ export interface RouteDecision {
|
|
|
97
87
|
routeId: string;
|
|
98
88
|
requestedSelection: ModelSelection;
|
|
99
89
|
policyVersion: string;
|
|
90
|
+
decisionSource: 'jev';
|
|
100
91
|
catalogSnapshotHash: string;
|
|
101
92
|
taskClasses: TaskClass[];
|
|
102
93
|
excludedCandidates: Array<{
|
|
103
94
|
deployment: DeploymentRef;
|
|
104
95
|
reasons: string[];
|
|
105
96
|
}>;
|
|
106
|
-
rankedCandidates: RankedRouteCandidate[];
|
|
107
97
|
resolvedDeployment?: DeploymentRef;
|
|
108
|
-
|
|
98
|
+
/** Ordered recovery hints produced by the decision engine, already validated. */
|
|
99
|
+
alternatives: DeploymentRef[];
|
|
100
|
+
jev: {
|
|
101
|
+
modelVersion: string;
|
|
102
|
+
confidence?: number;
|
|
103
|
+
reasonCodes: string[];
|
|
104
|
+
latencyMs: number;
|
|
105
|
+
};
|
|
106
|
+
decisionError?: string;
|
|
107
|
+
pinReason?: 'transaction';
|
|
109
108
|
attempts: RouteAttempt[];
|
|
110
109
|
finalStatus: 'resolved' | 'no_candidate' | 'fixed_unavailable' | 'retrying' | 'succeeded' | 'failed' | 'blocked';
|
|
111
110
|
retryBudgetMs?: number;
|
|
@@ -121,34 +120,45 @@ export interface RouteFailure {
|
|
|
121
120
|
export interface PlannedRouteAttempt extends RouteAttempt {
|
|
122
121
|
status: 'planned';
|
|
123
122
|
}
|
|
124
|
-
export
|
|
125
|
-
deployment: DeploymentRef;
|
|
126
|
-
taskClass: TaskClass;
|
|
127
|
-
score: number;
|
|
128
|
-
source: 'manual_switch' | 'explicit_rating' | 'objective_success';
|
|
129
|
-
at?: number;
|
|
130
|
-
}
|
|
123
|
+
export declare const ROUTE_POLICY_VERSION = "newmark-auto-v2";
|
|
131
124
|
export declare function normalizeAutoPreference(value: string): RouteMode;
|
|
132
125
|
export declare function defaultRoutePolicy(mode?: RouteMode): RoutePolicy;
|
|
133
126
|
export declare function classifyTaskClasses(taskText: string, requiredCapabilities: string[], estimatedInputTokens?: number): TaskClass[];
|
|
134
127
|
export declare function classifyRouteFailure(error: unknown): RouteFailure;
|
|
135
|
-
export declare
|
|
128
|
+
export declare function createRouteId(): string;
|
|
129
|
+
export declare function catalogSnapshotHash(candidates: AutoRouteCandidate[]): string;
|
|
130
|
+
/**
|
|
131
|
+
* Operational execution controller.
|
|
132
|
+
*
|
|
133
|
+
* It owns endpoint health, the circuit breaker, build-scoped transaction pins
|
|
134
|
+
* and the provider-local recovery ladder. It never ranks models and never
|
|
135
|
+
* learns a preference; health observations only decide whether an endpoint is
|
|
136
|
+
* attempted right now.
|
|
137
|
+
*/
|
|
138
|
+
export declare class RouteExecutionController {
|
|
136
139
|
private readonly now;
|
|
137
140
|
private readonly policyVersion;
|
|
138
|
-
private readonly affinityTtlMs;
|
|
139
|
-
private readonly switchThreshold;
|
|
140
141
|
private readonly transactionPins;
|
|
141
|
-
private readonly affinities;
|
|
142
142
|
private readonly endpointHealth;
|
|
143
|
-
private readonly feedback;
|
|
144
143
|
constructor(options?: {
|
|
145
144
|
now?: () => number;
|
|
146
145
|
policyVersion?: string;
|
|
147
|
-
affinityTtlMs?: number;
|
|
148
|
-
switchThreshold?: number;
|
|
149
146
|
});
|
|
150
|
-
|
|
147
|
+
version(): string;
|
|
148
|
+
/** Execution-consistency pin for one Build / route transaction. */
|
|
149
|
+
pinTransaction(transactionId: string, deployment: DeploymentRef): void;
|
|
150
|
+
transactionPin(transactionId: string): DeploymentRef | undefined;
|
|
151
151
|
endTransaction(transactionId: string): void;
|
|
152
|
+
/**
|
|
153
|
+
* Provider-local recovery ladder.
|
|
154
|
+
*
|
|
155
|
+
* Order of preference: retry the same deployment (when the failure is
|
|
156
|
+
* retryable inside the retry budget), then an equivalent deployment of the
|
|
157
|
+
* same logical model group, then the alternates the decision engine already
|
|
158
|
+
* validated (in the order it produced them), then explicit fallback-only
|
|
159
|
+
* models. Recovery never crosses the failed provider boundary and never
|
|
160
|
+
* re-runs model selection.
|
|
161
|
+
*/
|
|
152
162
|
planAttempts(decision: RouteDecision, candidates: AutoRouteCandidate[], failure: {
|
|
153
163
|
error: RouteFailure;
|
|
154
164
|
streamCommitted: boolean;
|
|
@@ -170,14 +180,9 @@ export declare class AutoRouter {
|
|
|
170
180
|
toolValidity: number;
|
|
171
181
|
circuit: 'closed' | 'open' | 'half_open';
|
|
172
182
|
};
|
|
173
|
-
recordFeedback(event: RouteFeedbackEvent): boolean;
|
|
174
|
-
clearLearnedPreferences(): void;
|
|
175
|
-
learnedPreference(deployment: DeploymentRef, taskClasses: TaskClass[]): number;
|
|
176
|
-
private rankEligible;
|
|
177
|
-
private rankCandidate;
|
|
178
|
-
private affinityKey;
|
|
179
183
|
private healthFor;
|
|
180
184
|
private passedInitialHardFilters;
|
|
181
185
|
private circuitState;
|
|
182
186
|
}
|
|
187
|
+
export declare function validationEvidenceIsStale(checkedAt: string | undefined, now?: number): boolean;
|
|
183
188
|
//# sourceMappingURL=autoRouter.d.ts.map
|