newmark-agent 0.6.4 → 0.6.7

package/dist/core/agentKernelRunner.d.ts

@@ -1,6 +1,7 @@
 import { Agent } from './agent';
 import { StreamToken } from './types';
 import { type ToolchainCore } from '../toolchain';
+import { type ConversationVisualBudget } from './conversationVisualBudget';
 export declare function filterPublicAssistantDelta(agent: Agent, delta: string): string;
 export declare function resetPublicAssistantDeltaFilter(agent: Agent): void;
 interface KernelTextContent {
@@ -143,7 +144,12 @@ declare function visualFallbackImageInput(agent: Agent, name: string, text: stri
 export declare function toolResultObjectiveOutcome(result: string): boolean | undefined;
 declare function toKernelMessagesFromHistory(history: Array<Record<string, unknown>>, agent: Agent): KernelMessage[];
 declare function toProviderToolDefinitions(tools: KernelTool[]): unknown[];
-declare function fromKernelMessages(messages: KernelMessage[], includeEphemeralImages?: boolean): Array<Record<string, unknown>>;
+declare function fromKernelMessages(messages: KernelMessage[], includeEphemeralImages?: boolean, visualBudget?: ConversationVisualBudget, onVisualRelease?: (info: {
+    releasedImages: number;
+    releasedBytes: number;
+    retainedBytes: number;
+    budget: ConversationVisualBudget;
+}) => void): Array<Record<string, unknown>>;
 declare function imagePathToOpenAIContentPart(imagePath: string): Record<string, unknown> | null;
 export declare const agentKernelRunnerInternals: {
     buildRequestTaskFocus: typeof buildRequestTaskFocus;

package/dist/core/agentKernelRunner.js

@@ -48,8 +48,13 @@ const toolPolicy_1 = require("./toolPolicy");
 const performanceDiagnostics_1 = require("./performanceDiagnostics");
 const agentKernelDiagnostics_1 = require("./agentKernelDiagnostics");
 const toolchain_1 = require("../toolchain");
+const conversationVisualBudget_1 = require("./conversationVisualBudget");
+const visualDownscale_1 = require("./visualDownscale");
 const emptyResponseRetry_1 = require("./emptyResponseRetry");
 const publicStreamFilters = new WeakMap();
+/** dev-0.6.6: throttles the "screenshots released" work-status line. */
+let visualReleaseReportedAt = 0;
+const VISUAL_MIB = 1024 ** 2;
 const brokerOnlyAssistantBuffers = new WeakMap();
 const BROKER_PREFACE_BUFFER_CHARS = 96;
 const HIDDEN_LINE_PREFIXES = [
@@ -460,7 +465,11 @@ async function runAgentKernel(agent) {
                     agent.chatMessages.push({ role: 'user', content: KernelMessageText(msg), mode: agent.modeName(), model: agent.model, timestamp: agent.nowLabel() });
                     agent.history.push(toHistoryMessage(msg));
                 }
-                agent.saveWorkspaceConversationState();
+                // Mid-turn persistence is coalesced (see Agent.scheduleStoredConversationState):
+                // the terminal save at the end of the run still writes synchronously, so
+                // durability at turn boundaries is unchanged while per-turn full-document
+                // rewrites drop from one-per-event to at most one per throttle window.
+                agent.saveWorkspaceConversationState(false);
             }
             await kernel.prompt(promptMessages);
             const assistant = lastAssistant;
@@ -519,7 +528,7 @@ async function runAgentKernel(agent) {
                 agent.emitWorkEvent({ type: 'final_response', content: visualFallback });
                 agent.chatMessages.push({ role: 'assistant', content: visualFallback, mode: agent.modeName(), model: agent.model, timestamp: agent.nowLabel(), runId: agent.currentWorkRunId() || undefined });
                 agent.history.push({ role: 'assistant', content: visualFallback, run_id: agent.currentWorkRunId() || undefined });
-                agent.saveWorkspaceConversationState();
+                agent.saveWorkspaceConversationState(false);
                 agent.recordWorkStatus('Final visual fallback used: local mini OCR plus conservative text correction.');
                 lastTurn = { ...lastTurn, text: visualFallback, errorMessage: '', stopReason: 'stop' };
             }
@@ -658,7 +667,9 @@ async function runAgentKernel(agent) {
         agent.attachAgentKernelRuntime(null);
     }
     agent.status = 'idle';
-    agent.saveWorkspaceConversationState();
+    // Coalesced: the caller (Agent.process) flushes synchronously right after,
+    // and every in-memory reader already sees this status.
+    agent.saveWorkspaceConversationState(false);
     return agent.sanitizeVisibleTokens(tokens);
     function streamWithNewmarkProvider(currentAgent, compat) {
         return async (model, context, options) => {
@@ -696,7 +707,15 @@ async function runAgentKernel(agent) {
                     // endpoint owns its real limit; protocols that require a positive
                     // max_tokens resolve one internally.
                     const maxTokens = 0;
-                    const newmarkMessages = fromKernelMessages(context.messages).map(message => message.role === 'assistant'
+                    const requestVisualBudget = visualBudgetForAgent(currentAgent);
+                    const newmarkMessages = fromKernelMessages(context.messages, true, requestVisualBudget, info => {
+                        // Observable, throttled: one status line per run so the user learns
+                        // that screenshots were reclaimed instead of silently losing them.
+                        if (visualReleaseReportedAt && Date.now() - visualReleaseReportedAt < 60_000)
+                            return;
+                        visualReleaseReportedAt = Date.now();
+                        currentAgent.recordWorkStatus(`Released ${info.releasedImages} older ComputerUse/BrowserControl screenshot(s) (~${Math.round(info.releasedBytes / VISUAL_MIB)} MiB) to stay within the ${(0, conversationVisualBudget_1.describeVisualBudget)(info.budget)} budget.`);
+                    }).map(message => message.role === 'assistant'
                         // Use the same public-content boundary on its first provider
                         // submission and after persistence. Trimming only the durable
                         // copy changes earlier tool-call envelopes on mailbox recovery.
@@ -1051,7 +1070,7 @@ async function handleKernelEvent(agent, event, tokens) {
                     // Otherwise a cold continuation loses an already-read instruction
                     // and deletes that message from the provider's retained prefix.
                     agent.history.push({ ...toHistoryMessage(event.message), run_id: agent.currentWorkRunId() || undefined });
-                    agent.saveWorkspaceConversationState(true);
+                    agent.saveWorkspaceConversationState(false);
                 }
                 agent.notifyAgentKernelUserMessageStart(text, event.message.clientMessageId);
             }
@@ -1158,7 +1177,7 @@ async function handleKernelEvent(agent, event, tokens) {
                     historyMessage.content = text;
                     historyMessage.run_id = agent.currentWorkRunId() || undefined;
                     agent.history.push(historyMessage);
-                    agent.saveWorkspaceConversationState();
+                    agent.saveWorkspaceConversationState(false);
                 }
                 resetPublicAssistantDeltaFilter(agent);
                 resetAssistantToolVisibility(agent);
@@ -1169,6 +1188,9 @@ async function handleKernelEvent(agent, event, tokens) {
                     ...(!event.message.isError && receipt ? { subagent_settlement_receipt: { ...receipt } } : {}),
                 };
                 agent.history.push(historyMessage);
+                // Kept synchronous on purpose: the catch below clears the settlement
+                // receipt so a later notification cannot mistake an unsaved result for a
+                // committed one, which requires the write failure to surface here.
                 try {
                     agent.saveWorkspaceConversationState();
                 }
@@ -1790,8 +1812,17 @@ function visualFallbackImageInput(agent, name, text) {
         if (name !== 'pdf_read' && nested.action !== 'observe' && nested.action !== 'app_observe')
             return {};
         const directImage = String(nested.vision_image_data_url || '');
-        if (/^data:image\/(?:png|jpeg);base64,[A-Za-z0-9+/]+={0,2}$/i.test(directImage) && directImage.length <= 2 * 1024 * 1024) {
-            return { image: directImage, mimeType: directImage.slice(5, directImage.indexOf(';')).toLowerCase() };
+        if (/^data:image\/(?:png|jpeg);base64,[A-Za-z0-9+/]+={0,2}$/i.test(directImage)) {
+            // dev-0.6.6: ComputerUse/BrowserControl screenshots are the dominant
+            // memory consumer of a long conversation, so the accepted single-image
+            // size follows the machine's available memory instead of a fixed 2 MiB.
+            const budget = visualBudgetForAgent(agent);
+            const fitted = (0, conversationVisualBudget_1.visualDataUrlBytes)(directImage) <= budget.maxSingleImageBytes
+                ? directImage
+                : (0, visualDownscale_1.downscaleDataUrlToBytes)(directImage, budget.maxSingleImageBytes);
+            if (fitted)
+                return { image: fitted, mimeType: fitted.slice(5, fitted.indexOf(';')).toLowerCase() };
+            return {};
         }
         const screenshotPath = String(nested.vision_image_path || '');
         if (!screenshotPath)
@@ -1803,6 +1834,47 @@ function visualFallbackImageInput(agent, name, text) {
         return {};
     }
 }
+const visualBudgetCache = new WeakMap();
+/**
+ * The conversation's visual budget for right now. Sampled at most once per
+ * second per agent so a tool-heavy round does not re-read OS memory state on
+ * every screenshot.
+ */
+function visualBudgetForAgent(agent) {
+    const cached = visualBudgetCache.get(agent);
+    const now = Date.now();
+    if (cached && now - cached.at < 1000)
+        return cached.budget;
+    const usage = (() => {
+        try {
+            return process.memoryUsage();
+        }
+        catch {
+            return { rss: 0 };
+        }
+    })();
+    const budget = (0, conversationVisualBudget_1.conversationVisualBudget)({ rssBytes: Number(usage?.rss || 0) });
+    visualBudgetCache.set(agent, { at: now, budget });
+    return budget;
+}
+/** Bytes an image part costs when materialized into the provider request. */
+function imagePartBytes(part) {
+    const inline = String(part.image || '');
+    if (inline.startsWith('data:'))
+        return (0, conversationVisualBudget_1.visualDataUrlBytes)(inline);
+    const imagePath = String(part.imagePath || '');
+    if (!imagePath)
+        return 0;
+    try {
+        // Materializing a path-based screenshot produces `data:<mime>;base64,<...>`;
+        // budget the base64 payload length the request will actually hold.
+        const size = fs.statSync(imagePath).size;
+        return 4 * Math.ceil(size / 3);
+    }
+    catch {
+        return 0;
+    }
+}
 async function executeNewmarkTool(agent, name, args, inputSchema, signal, onSettlementReceipt) {
     const stopToolTimer = (0, performanceDiagnostics_1.performanceTimer)('tool_execution', { conversationId: agent.activeConversationId, detail: { tool: name } });
     try {
@@ -1953,8 +2025,11 @@ async function executeNewmarkTool(agent, name, args, inputSchema, signal, onSett
             throw abortError();
         trackFileDiff(agent, name, args);
         const objectiveResult = toolResultObjectiveOutcome(result);
+        // Objective postconditions are operational health evidence only. Auto Router
+        // v2 keeps tool outcomes out of model preference: the observation can demote
+        // an endpoint's live health, and it can never teach a future route.
         if (objectiveResult !== undefined)
-            agent.recordObjectiveRouteResult(objectiveResult);
+            agent.recordRouteToolOutcome(objectiveResult);
         return result;
     }
     finally {
@@ -2081,9 +2156,53 @@ function toProviderToolDefinitions(tools) {
         },
     }));
 }
-function fromKernelMessages(messages, includeEphemeralImages = true) {
-    return messages.flatMap(message => {
-        const projected = toHistoryMessage(message, includeEphemeralImages);
+function fromKernelMessages(messages, includeEphemeralImages = true, visualBudget, onVisualRelease) {
+    const budget = visualBudget || (0, conversationVisualBudget_1.conversationVisualBudget)();
+    // dev-0.6.6: a late-stage ComputerUse/BrowserControl conversation stores one
+    // screenshot per observation. Materializing all of them per request is what
+    // pushed the utility host to a 3 GB V8 abort, so the newest screenshots are
+    // kept within this machine's visual budget and older ones are released
+    // (text/tool results stay untouched).
+    const candidates = [];
+    messages.forEach((message, index) => {
+        if (message.role !== 'toolResult')
+            return;
+        const image = message.content.find((part) => part.type === 'image');
+        if (!image)
+            return;
+        candidates.push({ key: String(index), bytes: imagePartBytes(image) });
+    });
+    const plan = (0, conversationVisualBudget_1.planVisualRetention)(candidates, budget);
+    if (plan.released.length && onVisualRelease) {
+        onVisualRelease({
+            releasedImages: plan.released.length,
+            releasedBytes: plan.releasedBytes,
+            retainedBytes: plan.retainedBytes,
+            budget,
+        });
+    }
+    return messages.flatMap((message, index) => {
+        const key = String(index);
+        const hadImage = message.role === 'toolResult'
+            && message.content.some(part => part.type === 'image');
+        const imageAllowed = !hadImage || plan.keep.has(key);
+        const projected = toHistoryMessage(message, includeEphemeralImages && imageAllowed, budget);
+        if (hadImage && !imageAllowed) {
+            // Tell the model the screenshot existed but was released, so it can
+            // re-observe instead of assuming the picture was empty.
+            // Kept to one short line: a long conversation can release hundreds of
+            // historic screenshots, and the note must not become its own payload.
+            const releaseNote = `[visual_memory_budget] screenshot released (${budget.pressure} memory budget); re-run observe if you need it again.`;
+            if (typeof projected.content === 'string')
+                projected.content = `${projected.content}\n${releaseNote}`;
+            else if (Array.isArray(projected.content)) {
+                const textPart = projected.content.find(part => part?.type === 'text');
+                if (textPart)
+                    textPart.text = `${String(textPart.text || '')}\n${releaseNote}`;
+                else
+                    projected.content.unshift({ type: 'text', text: releaseNote });
+            }
+        }
         if (message.role !== 'toolResult' || !Array.isArray(projected.content))
             return [projected];
         const parts = projected.content;
@@ -2122,7 +2241,7 @@ function publicHistoryFromKernelMessages(messages) {
         return [toHistoryMessage({ ...message, content: publicContent }, false)];
     });
 }
-function toHistoryMessage(message, includeEphemeralImages = false) {
+function toHistoryMessage(message, includeEphemeralImages = false, visualBudget) {
     if (message.role === 'user') {
         if (typeof message.content === 'string')
             return {
@@ -2154,13 +2273,7 @@ function toHistoryMessage(message, includeEphemeralImages = false) {
     const ephemeralImage = message.content.find((c) => c.type === 'image');
     const imagePath = ephemeralImage?.imagePath || '';
     const directImage = ephemeralImage?.image || '';
-    const imagePart = includeEphemeralImages
-        ? (imagePath
-            ? imagePathToOpenAIContentPart(imagePath)
-            : directImage.startsWith('data:image/')
-                ? { type: 'image_url', image_url: { url: directImage } }
-                : null)
-        : null;
+    const imagePart = includeEphemeralImages ? materializeVisualImagePart(imagePath, directImage, visualBudget) : null;
     if (imagePart && ephemeralImage) {
         // Tool-result images are one provider-input capability, not history. Once
         // this request has materialized the image part, consume it from the live
@@ -2186,6 +2299,29 @@ function imagePathToOpenAIContentPart(imagePath) {
         return null;
     }
 }
+/**
+ * dev-0.6.6: materialize one screenshot into a provider image part while
+ * respecting the conversation's per-image cap. Oversized screenshots are
+ * downscaled (PNG, then JPEG) instead of being dropped, so a memory-tight
+ * machine keeps vision and the agent keeps working.
+ */
+function materializeVisualImagePart(imagePath, directImage, visualBudget) {
+    const budget = visualBudget || (0, conversationVisualBudget_1.conversationVisualBudget)();
+    let dataUrl = '';
+    if (String(directImage || '').startsWith('data:image/'))
+        dataUrl = String(directImage);
+    else if (imagePath)
+        dataUrl = imagePathToDataUrl(imagePath) || '';
+    if (!dataUrl)
+        return null;
+    if ((0, conversationVisualBudget_1.visualDataUrlBytes)(dataUrl) > budget.maxSingleImageBytes) {
+        const fitted = (0, visualDownscale_1.downscaleDataUrlToBytes)(dataUrl, budget.maxSingleImageBytes);
+        if (!fitted)
+            return null;
+        dataUrl = fitted;
+    }
+    return { type: 'image_url', image_url: { url: dataUrl } };
+}
 exports.agentKernelRunnerInternals = {
     buildRequestTaskFocus,
     buildBuildContextBootstrap,

package/dist/core/autoRouter.d.ts

@@ -18,24 +18,29 @@ export type ModelSelection = {
     policyId: string;
     subset?: DeploymentRef[];
 };
-export type RouteMode = 'quality' | 'balanced' | 'cost' | 'speed';
+export type RouteMode = 'conservative' | 'balanced' | 'aggressive';
 export type RoutePrivacy = 'default' | 'no_training' | 'zdr';
 export type TaskClass = 'chat' | 'coding' | 'reasoning' | 'long_context' | 'vision' | 'image_generation' | 'tool_use' | 'computer_use';
 export type ValidationLevel = 'discovered' | 'legacy_basic' | 'basic' | 'standard' | 'extended';
 export type ValidationStatus = 'verified' | 'degraded' | 'unavailable' | 'auth_error' | 'rate_limited' | 'invalid_config';
 export interface RoutePolicy {
     mode: RouteMode;
-    maxQualityLoss: number;
-    maxExpectedCostUsd?: number;
     allowPreview: boolean;
     privacy: RoutePrivacy;
     requiredCapabilities: string[];
     dataRegion?: string;
     requiredProtocolParameters?: string[];
+    maxExpectedCostUsd?: number;
 }
 export interface AutoRouteCandidate {
     deployment: DeploymentRef;
     enabled: boolean;
+    /**
+     * Set by the Agent when a deployment is explicitly unusable right now
+     * (for example a provider-reported balance exhaustion window). It carries a
+     * guard reason instead of silently mutating `enabled`.
+     */
+    unavailableReason?: string;
     validation: {
         level: ValidationLevel;
         status: ValidationStatus;
@@ -49,39 +54,24 @@ export interface AutoRouteCandidate {
     supportedProtocolParameters?: string[];
     expectedInputCostUsdPerM?: number;
     expectedOutputCostUsdPerM?: number;
+    /** Current operational measurements. Never persisted as a preference. */
     latencyMs?: number;
     reliability?: number;
     toolValidity?: number;
     throughput?: number;
-    qualityByTask?: Partial<Record<TaskClass, {
-        successes: number;
-        attempts: number;
-    }>>;
+    circuit?: 'closed' | 'open' | 'half_open';
+    /** Explicit user configuration (`route_preference`), not learned history. */
     preference?: number;
     fallbackOnly?: boolean;
 }
 export interface RouteRequest {
     transactionId: string;
-    affinityKey: string;
     taskText: string;
     estimatedInputTokens: number;
     expectedOutputTokens: number;
     requiredCapabilities: string[];
     batch?: boolean;
 }
-export interface RankedRouteCandidate {
-    deployment: DeploymentRef;
-    quality: number;
-    utility: number;
-    expectedCostUsd?: number;
-    components: {
-        cost: number;
-        reliability: number;
-        speed: number;
-        cache: number;
-        preference: number;
-    };
-}
 export type RouteAttemptStatus = 'planned' | 'success' | 'failed' | 'blocked';
 export interface RouteAttempt {
     deployment: DeploymentRef;
@@ -97,15 +87,24 @@ export interface RouteDecision {
     routeId: string;
     requestedSelection: ModelSelection;
     policyVersion: string;
+    decisionSource: 'jev';
     catalogSnapshotHash: string;
     taskClasses: TaskClass[];
     excludedCandidates: Array<{
         deployment: DeploymentRef;
         reasons: string[];
     }>;
-    rankedCandidates: RankedRouteCandidate[];
     resolvedDeployment?: DeploymentRef;
-    pinReason?: 'transaction' | 'cache_affinity';
+    /** Ordered recovery hints produced by the decision engine, already validated. */
+    alternatives: DeploymentRef[];
+    jev: {
+        modelVersion: string;
+        confidence?: number;
+        reasonCodes: string[];
+        latencyMs: number;
+    };
+    decisionError?: string;
+    pinReason?: 'transaction';
     attempts: RouteAttempt[];
     finalStatus: 'resolved' | 'no_candidate' | 'fixed_unavailable' | 'retrying' | 'succeeded' | 'failed' | 'blocked';
     retryBudgetMs?: number;
@@ -121,34 +120,45 @@ export interface RouteFailure {
 export interface PlannedRouteAttempt extends RouteAttempt {
     status: 'planned';
 }
-export interface RouteFeedbackEvent {
-    deployment: DeploymentRef;
-    taskClass: TaskClass;
-    score: number;
-    source: 'manual_switch' | 'explicit_rating' | 'objective_success';
-    at?: number;
-}
+export declare const ROUTE_POLICY_VERSION = "newmark-auto-v2";
 export declare function normalizeAutoPreference(value: string): RouteMode;
 export declare function defaultRoutePolicy(mode?: RouteMode): RoutePolicy;
 export declare function classifyTaskClasses(taskText: string, requiredCapabilities: string[], estimatedInputTokens?: number): TaskClass[];
 export declare function classifyRouteFailure(error: unknown): RouteFailure;
-export declare class AutoRouter {
+export declare function createRouteId(): string;
+export declare function catalogSnapshotHash(candidates: AutoRouteCandidate[]): string;
+/**
+ * Operational execution controller.
+ *
+ * It owns endpoint health, the circuit breaker, build-scoped transaction pins
+ * and the provider-local recovery ladder. It never ranks models and never
+ * learns a preference; health observations only decide whether an endpoint is
+ * attempted right now.
+ */
+export declare class RouteExecutionController {
     private readonly now;
     private readonly policyVersion;
-    private readonly affinityTtlMs;
-    private readonly switchThreshold;
     private readonly transactionPins;
-    private readonly affinities;
     private readonly endpointHealth;
-    private readonly feedback;
     constructor(options?: {
         now?: () => number;
         policyVersion?: string;
-        affinityTtlMs?: number;
-        switchThreshold?: number;
     });
-    route(selection: ModelSelection, policy: RoutePolicy, candidates: AutoRouteCandidate[], request: RouteRequest): RouteDecision;
+    version(): string;
+    /** Execution-consistency pin for one Build / route transaction. */
+    pinTransaction(transactionId: string, deployment: DeploymentRef): void;
+    transactionPin(transactionId: string): DeploymentRef | undefined;
     endTransaction(transactionId: string): void;
+    /**
+     * Provider-local recovery ladder.
+     *
+     * Order of preference: retry the same deployment (when the failure is
+     * retryable inside the retry budget), then an equivalent deployment of the
+     * same logical model group, then the alternates the decision engine already
+     * validated (in the order it produced them), then explicit fallback-only
+     * models. Recovery never crosses the failed provider boundary and never
+     * re-runs model selection.
+     */
     planAttempts(decision: RouteDecision, candidates: AutoRouteCandidate[], failure: {
         error: RouteFailure;
         streamCommitted: boolean;
@@ -170,14 +180,9 @@ export declare class AutoRouter {
         toolValidity: number;
         circuit: 'closed' | 'open' | 'half_open';
     };
-    recordFeedback(event: RouteFeedbackEvent): boolean;
-    clearLearnedPreferences(): void;
-    learnedPreference(deployment: DeploymentRef, taskClasses: TaskClass[]): number;
-    private rankEligible;
-    private rankCandidate;
-    private affinityKey;
     private healthFor;
     private passedInitialHardFilters;
     private circuitState;
 }
+export declare function validationEvidenceIsStale(checkedAt: string | undefined, now?: number): boolean;
 //# sourceMappingURL=autoRouter.d.ts.map