@myagentroam/agent 0.11.21 → 0.11.23

package/dist/agent-core/runtime/contracts.d.ts

@@ -1,11 +1,10 @@
 import type { MarAgentReasoningEffort } from '../model/configuration.js';
+import type { ModelImage } from '../model/image.js';
 /** @deprecated MAR Agent always executes tools directly through the Host. */
 export type AccessMode = 'full-access';
 export type ExecutionMode = 'run' | 'plan' | 'compact';
-export interface RunInputImage {
+export interface RunInputImage extends ModelImage {
     name: string;
-    mimeType: string;
-    dataBase64: string;
 }
 export interface MarAgentImageMetadata {
     name: string;

package/dist/agent-core/runtime/execution.d.ts

@@ -27,7 +27,7 @@ export interface ExecutionLoopInput {
     readonly compactionFallback?: (() => boolean) | undefined;
     readonly observeUsage: (usage: Omit<AuthoritativeModelUsage, 'historyMessages'> | undefined) => void;
     readonly observeContextTokens: (tokens: number) => void;
-    readonly afterModel?: (closedPrefixLength: number) => Promise<void>;
+    readonly afterModel?: (closedPrefixLength: number, acceptedRequest?: ModelRequest) => Promise<void>;
     readonly checkpoint: (round: PreparedExecutionRound, usage: ModelUsage | undefined) => Promise<void>;
     readonly mailbox: {
         hasPending(): boolean;

package/dist/agent-core/runtime/execution.js

@@ -323,11 +323,14 @@ export class ExecutionLoop {
             }
             recoveringFromContextOverflow = false;
             input.observeUsage(roundServerUsage);
-            await input.afterModel?.(pendingTools.length ? roundMessageCount : messages.length);
+            const emptyCompletion = stop === 'completed' && !sawTool && !this.finalAnswer.trim();
+            await input.afterModel?.(pendingTools.length ? roundMessageCount : messages.length, emptyCompletion || stop === 'cancelled' ? undefined : round.request);
             const toolResults = await executeRoundTools(pendingTools, round.toolScope);
             for (const result of toolResults) {
-                if ('images' in result && result.images?.length)
-                    input.toolImages.push(...result.images.map((image) => ({
+                if ('sourceImages' in result && result.sourceImages?.length)
+                    input.toolImages.push(...result.sourceImages
+                        .filter((image) => image.dataBase64 !== undefined)
+                        .map((image) => ({
                         mimeType: image.mimeType,
                         data: Buffer.from(image.dataBase64, 'base64')
                     })));
@@ -354,6 +357,9 @@ export class ExecutionLoop {
                     content: message.content,
                     ...(message.calls ? { calls: message.calls } : {}),
                     ...(message.observations ? { observations: message.observations } : {}),
+                    ...(message.images?.length
+                        ? { images: message.images.map(({ dataBase64: _bytes, ...image }) => image) }
+                        : {}),
                     ...(message.toolSearchTools === undefined
                         ? {}
                         : { toolSearchTools: message.toolSearchTools })
@@ -401,7 +407,7 @@ export class ExecutionLoop {
                 const compactSummary = mode === 'compact' ? this.finalAnswer.trim() : '';
                 if (mode === 'compact' && !compactSummary)
                     throw new MarAgentError('MAR_AGENT_COMPACT_FAILED', 'Compaction returned an empty handoff.');
-                if (mode !== 'compact' && !this.finalAnswer.trim()) {
+                if (mode !== 'compact' && emptyCompletion) {
                     const error = new MarAgentError(input.sourceType === 'subagent'
                         ? 'MAR_AGENT_SUBAGENT_FAILED'
                         : 'MAR_AGENT_MODEL_OUTPUT_INCOMPLETE', input.sourceType === 'subagent'

package/dist/agent-core/runtime/resource-publication.d.ts

@@ -0,0 +1,8 @@
+import type { MarAgentHost } from '../host/contracts.js';
+import type { ModelImage } from '../model/image.js';
+import type { SessionStore } from '../session/store/contracts.js';
+/** Persist immutable bytes first; publication never changes an existing resource URL. */
+export declare function publishModelResources<T extends ModelImage>(host: MarAgentHost, images: readonly T[] | undefined, context: {
+    sessionId: string;
+    executionId: string;
+}, store?: SessionStore): Promise<readonly T[] | undefined>;

package/dist/agent-core/runtime/resource-publication.js

@@ -0,0 +1,60 @@
+import { isModelImageUrl } from '../model/image.js';
+import { MarAgentError } from '../error.js';
+const pending = new WeakMap();
+/** Persist immutable bytes first; publication never changes an existing resource URL. */
+export async function publishModelResources(host, images, context, store) {
+    if (!images?.length)
+        return images;
+    if (store === undefined) {
+        if (host.publishModelResource === undefined)
+            return images;
+        throw new MarAgentError('MAR_AGENT_IMAGE_PUBLICATION_FAILED', 'Image publication requires a SessionStore.');
+    }
+    let publications = pending.get(store);
+    if (publications === undefined)
+        pending.set(store, (publications = new Map()));
+    return Promise.all(images.map(async (image) => {
+        if (image.dataBase64 === undefined &&
+            image.resourceId === undefined &&
+            isModelImageUrl(image.url))
+            return image;
+        let resource = await store.persistResource(context.sessionId, image);
+        if (resource.url === undefined && image.url !== undefined && image.urlExpiresAt !== undefined)
+            resource = await store.setResourceUrl(context.sessionId, resource.resourceId, {
+                url: image.url,
+                urlExpiresAt: image.urlExpiresAt
+            });
+        const key = `${context.sessionId}\0${resource.resourceId}`;
+        let publication = publications.get(key);
+        if (publication === undefined) {
+            publication = (async () => {
+                if (resource.url === undefined && host.publishModelResource !== undefined) {
+                    const published = await host.publishModelResource(resource, {
+                        ...context,
+                        homeDirectory: (await host.describe()).homeDirectory
+                    });
+                    if (published === undefined ||
+                        !isModelImageUrl(published.url) ||
+                        !Number.isSafeInteger(published.urlExpiresAt) ||
+                        published.urlExpiresAt <= 0)
+                        throw new MarAgentError('MAR_AGENT_IMAGE_PUBLICATION_FAILED', 'Image publication did not return a valid fixed URL.');
+                    resource = await store.setResourceUrl(context.sessionId, resource.resourceId, published);
+                }
+                const { sha256: _hash, ...reference } = resource;
+                return resource.url === undefined && image.dataBase64 !== undefined
+                    ? { ...reference, dataBase64: image.dataBase64 }
+                    : reference;
+            })();
+            publications.set(key, publication);
+        }
+        try {
+            const published = await publication;
+            const { dataBase64: _bytes, ...metadata } = image;
+            return { ...metadata, ...published, resourceInputVersion: 1 };
+        }
+        finally {
+            if (publications.get(key) === publication)
+                publications.delete(key);
+        }
+    }));
+}

package/dist/agent-core/sdk/agent.js

@@ -20,6 +20,7 @@ export async function createMarAgentWithAdapter(options) {
     let { models, defaultModelId } = resolvedModels;
     let modelGeneration = 0;
     const { credentialProviders } = resolvedModels;
+    let retainedExtensionProviders;
     const description = negotiateHost(await options.host.describe());
     if (description.mode !== options.mode)
         throw new MarAgentError('MAR_AGENT_HOST_INCOMPATIBLE', 'Host mode does not match Agent mode.');
@@ -318,6 +319,19 @@ export async function createMarAgentWithAdapter(options) {
                 }
                 for (const commit of reconfigurationCommits.values())
                     commit();
+                if (extensions.hasReferences() || activeSessions.size > 0) {
+                    if (!retainedExtensionProviders ||
+                        retainedExtensionProviders.revision !== extensions.revision) {
+                        const providers = new Set();
+                        for (const configuration of models.values())
+                            addModelCredentialProviders(providers, configuration);
+                        // A pending composition retains its original providers, not every intermediate catalog.
+                        retainedExtensionProviders = { revision: extensions.revision, providers };
+                    }
+                }
+                else {
+                    retainedExtensionProviders = undefined;
+                }
                 models = next.models;
                 defaultModelId = next.defaultModelId;
                 modelGeneration++;
@@ -327,6 +341,8 @@ export async function createMarAgentWithAdapter(options) {
                     runtime.adapterGeneration = modelGeneration;
                 }
                 const liveProviders = new Set(next.credentialProviders);
+                for (const provider of retainedExtensionProviders?.providers ?? [])
+                    liveProviders.add(provider);
                 for (const adapter of activeExecutionAdapters)
                     addModelCredentialProviders(liveProviders, adapter.configuration);
                 for (const runtime of sessionRuntimes.values())

package/dist/agent-core/sdk/model-runtime.js

@@ -48,6 +48,8 @@ export function createModelTurnSession(adapter) {
             }
         },
         resetContinuation: () => turn.resetContinuation?.(),
+        continuationInputCount: () => turn.continuationInputCount?.() ?? 0,
+        commitContinuationInput: (count) => turn.commitContinuationInput?.(count),
         close: () => turn.close()
     };
 }

package/dist/agent-core/sdk/resource-input.d.ts

@@ -0,0 +1,31 @@
+import type { ModelMessage, ModelRequest } from '../model/contracts.js';
+import { type ModelImage } from '../model/image.js';
+import type { SessionStore } from '../session/store/contracts.js';
+import type { SessionRecord } from '../session/store/types.js';
+import { type InlineMediaGcCandidate, type InlineMediaState } from '../session/inline-media.js';
+interface Options {
+    records?: readonly SessionRecord[];
+    persist?: (state: InlineMediaState) => Promise<void>;
+}
+/** SDK-owned ordered visual window; durable image references are never rewritten. */
+export declare class ResourceInputProjection {
+    #private;
+    private readonly useResourceUrl;
+    private readonly store;
+    private readonly sessionId;
+    private readonly options;
+    constructor(useResourceUrl: boolean, store: Pick<SessionStore, 'readResource'>, sessionId: string, history: readonly ModelMessage[], options?: Options);
+    observe(messages: readonly ModelMessage[]): void;
+    /** Called only at an explicit context replacement, never as a history diff detector. */
+    replace(messages: readonly ModelMessage[]): void;
+    hasPendingImages(messages: readonly ModelMessage[]): boolean;
+    /** Reference-only view shared by GC; no byte loading or new eviction decisions. */
+    historyView(messages: readonly ModelMessage[]): ModelMessage[];
+    checkpoint(messages?: readonly ModelMessage[]): InlineMediaState;
+    prepare(request: ModelRequest, mirroredImages?: readonly ModelImage[], canonicalMessages?: readonly ModelMessage[]): Promise<ModelRequest>;
+    complete(request: ModelRequest): Promise<void>;
+    /** Propose a cursor; only the unified GC checkpoint may adopt it. */
+    gcCandidate(request?: ModelRequest, mirroredImages?: readonly ModelImage[]): Promise<InlineMediaGcCandidate | undefined>;
+    adoptGc(state: InlineMediaState | undefined): void;
+}
+export {};

package/dist/agent-core/sdk/resource-input.js

@@ -0,0 +1,342 @@
+import { isDeepStrictEqual } from 'node:util';
+import { MarAgentError } from '../error.js';
+import { isModelImageUrl } from '../model/image.js';
+import { observationImages, renderObservationResult, selectObservationImages } from '../tools/code-mode/observations.js';
+import { INLINE_MEDIA_POLICY as policy, assertInlineMediaBudget, inlineMediaBytes, inlineMediaHistory, inlineMediaLimit, inlineMediaSource } from '../session/inline-media.js';
+function releasedImageNotice(message, indices) {
+    return indices
+        .map((index) => {
+        const resourceId = message.images?.[index]?.resourceId;
+        const hint = resourceId
+            ? ` Reopen with view_image(${JSON.stringify({ resourceId })}) if needed.`
+            : '';
+        return `<mar_context_gc kind="image">Image released.${hint}</mar_context_gc>`;
+    })
+        .join('\n');
+}
+function projectRetainedImages(message, images, removed) {
+    if (!removed.size)
+        return { ...message, images };
+    if (message.observations) {
+        const observations = message.observations.map((observation) => {
+            const removedIndices = observationImages(observation).filter((index) => removed.has(index));
+            if (!removedIndices.length)
+                return observation;
+            const notice = releasedImageNotice(message, removedIndices);
+            return observation.type === 'tool_result'
+                ? {
+                    ...observation,
+                    result: { ...observation.result, content: `${observation.result.content}\n${notice}` }
+                }
+                : { ...observation, value: `${observation.value}\n${notice}` };
+        });
+        const selected = selectObservationImages(observations, images);
+        return {
+            ...message,
+            ...selected,
+            content: renderObservationResult(selected.observations, message.content)
+        };
+    }
+    const notice = releasedImageNotice(message, [...removed]);
+    return {
+        ...message,
+        images: images.filter(Boolean),
+        content: `${message.content}\n${notice}`,
+        ...(message.textBlocks ? { textBlocks: [...message.textBlocks, { text: notice }] } : {})
+    };
+}
+/** SDK-owned ordered visual window; durable image references are never rewritten. */
+export class ResourceInputProjection {
+    useResourceUrl;
+    store;
+    sessionId;
+    options;
+    #entries = [];
+    #sources = new Map();
+    #candidates = new WeakMap();
+    #considered = -1;
+    #rollout = 0;
+    #evicted = -1;
+    constructor(useResourceUrl, store, sessionId, history, options = {}) {
+        this.useResourceUrl = useResourceUrl;
+        this.store = store;
+        this.sessionId = sessionId;
+        this.options = options;
+        const restored = inlineMediaHistory(options.records ?? []);
+        this.#rollout = restored.rollouts;
+        for (const cursor of restored.cursors)
+            this.#entries[this.#entry(cursor.sourceId, cursor.imageIndex)].rollout = cursor.rollout;
+        this.observe(history);
+        if (restored.state) {
+            this.#considered = this.#position(restored.state.consideredThrough);
+            this.#evicted = this.#position(restored.state.evictedThrough);
+        }
+        else if (options.records === undefined)
+            this.#considered = this.#entries.length - 1;
+    }
+    #entry(source, imageIndex) {
+        let positions = this.#sources.get(source);
+        if (!positions)
+            this.#sources.set(source, (positions = []));
+        let position = positions[imageIndex];
+        if (position === undefined) {
+            position = this.#entries.length;
+            positions[imageIndex] = position;
+            this.#entries.push({ source, imageIndex, active: false, rollout: this.#rollout });
+        }
+        return position;
+    }
+    #position(cursor) {
+        if (!cursor)
+            return -1;
+        const position = this.#sources.get(cursor.sourceId)?.[cursor.imageIndex];
+        if (position === undefined)
+            throw new MarAgentError('MAR_AGENT_SESSION_CORRUPT', 'Inline media boundary is missing.');
+        return position;
+    }
+    observe(messages) {
+        for (const message of messages)
+            for (const [index, image] of (message.images ?? []).entries()) {
+                const entry = this.#entries[this.#entry(inlineMediaSource(message), index)];
+                entry.image = image;
+                entry.active = true;
+            }
+    }
+    /** Called only at an explicit context replacement, never as a history diff detector. */
+    replace(messages) {
+        for (const entry of this.#entries) {
+            entry.active = false;
+            delete entry.image;
+        }
+        this.observe(messages);
+    }
+    hasPendingImages(messages) {
+        return messages.some((message) => (message.images ?? []).some((_, index) => {
+            const position = this.#sources.get(inlineMediaSource(message))?.[index];
+            return position === undefined || position > this.#considered;
+        }));
+    }
+    /** Reference-only view shared by GC; no byte loading or new eviction decisions. */
+    historyView(messages) {
+        return this.#historyView(messages, this.#evicted);
+    }
+    #historyView(messages, evicted, fromEvicted = -1) {
+        return messages.map((message) => {
+            const positions = this.#sources
+                .get(inlineMediaSource(message))
+                ?.filter((position) => position > fromEvicted);
+            const images = message.images ??
+                positions?.flatMap((position) => {
+                    const entry = this.#entries[position];
+                    return entry.active && entry.image ? [entry.image] : [];
+                });
+            if (!images?.length)
+                return message;
+            const removed = new Set();
+            const retained = [];
+            images.forEach((image, index) => {
+                if (!this.useResourceUrl &&
+                    positions?.[index] !== undefined &&
+                    positions[index] <= evicted)
+                    removed.add(index);
+                else
+                    retained[index] = image;
+            });
+            return projectRetainedImages({ ...message, images }, retained, removed);
+        });
+    }
+    checkpoint(messages) {
+        const retained = messages && new Set(messages.map(inlineMediaSource));
+        const cursor = (through) => {
+            for (let index = through; index >= 0; index--) {
+                const entry = this.#entries[index];
+                if (typeof entry.source === 'string' && (!retained || retained.has(entry.source)))
+                    return { sourceId: entry.source, imageIndex: entry.imageIndex };
+            }
+            return undefined;
+        };
+        const consideredThrough = cursor(this.#considered);
+        const evictedThrough = cursor(this.#evicted);
+        return {
+            version: 1,
+            ...(consideredThrough ? { consideredThrough } : {}),
+            ...(evictedThrough ? { evictedThrough } : {})
+        };
+    }
+    async prepare(request, mirroredImages = [], canonicalMessages = request.messages) {
+        this.observe(canonicalMessages);
+        const through = this.#entries.length - 1;
+        const cached = new Map();
+        const project = (image) => {
+            let value = cached.get(image);
+            if (!value)
+                cached.set(image, (value = this.#project(image)));
+            return value;
+        };
+        const sized = async (image) => image.dataBase64 !== undefined || image.byteSize !== undefined ? image : project(image);
+        const top = (request.images ?? []).filter((image) => !mirroredImages.some((mirror) => isDeepStrictEqual(image, mirror)));
+        const evicted = this.#evicted;
+        if (!this.useResourceUrl) {
+            let total = 0;
+            for (const message of request.messages)
+                for (const [index, image] of (message.images ?? []).entries()) {
+                    const position = this.#sources.get(inlineMediaSource(message))?.[index];
+                    const original = position === undefined ? undefined : this.#entries[position]?.image;
+                    if (position !== undefined && position <= evicted && isDeepStrictEqual(original, image))
+                        continue;
+                    total += inlineMediaBytes(await sized(image));
+                }
+            for (const image of top)
+                total += inlineMediaBytes(await sized(image));
+            if (total > policy.requestInlineMediaHardLimit)
+                inlineMediaLimit();
+        }
+        const messages = await Promise.all(request.messages.map(async (message) => {
+            if (!message.images?.length)
+                return message;
+            const removed = new Set();
+            const images = [];
+            await Promise.all(message.images.map(async (image, index) => {
+                const position = this.#sources.get(inlineMediaSource(message))?.[index];
+                const original = position === undefined ? undefined : this.#entries[position]?.image;
+                if (!this.useResourceUrl &&
+                    position !== undefined &&
+                    position <= evicted &&
+                    isDeepStrictEqual(original, image))
+                    removed.add(index);
+                else
+                    images[index] = await project(image);
+            }));
+            return projectRetainedImages(message, images, removed);
+        }));
+        const { images: _topImages, ...retained } = request;
+        const images = await Promise.all(top.map(project));
+        const result = { ...retained, messages, ...(images.length ? { images } : {}) };
+        if (!this.useResourceUrl)
+            assertInlineMediaBudget(result);
+        this.#candidates.set(result.messages, through);
+        return result;
+    }
+    async complete(request) {
+        const candidate = this.#candidates.get(request.messages);
+        if (candidate === undefined)
+            return;
+        this.#candidates.delete(request.messages);
+        this.#rollout++;
+        const considered = Math.max(this.#considered, candidate);
+        if (considered === this.#considered)
+            return;
+        const before = this.#considered;
+        this.#considered = considered;
+        try {
+            await this.options.persist?.(this.checkpoint());
+        }
+        catch (error) {
+            this.#considered = before;
+            this.#rollout--;
+            throw error;
+        }
+    }
+    /** Propose a cursor; only the unified GC checkpoint may adopt it. */
+    async gcCandidate(request, mirroredImages = []) {
+        if (this.useResourceUrl)
+            return undefined;
+        const size = async (image) => inlineMediaBytes(image.byteSize !== undefined || image.dataBase64 !== undefined
+            ? image
+            : await this.#project(image));
+        const history = await Promise.all(this.#entries.map(async (entry, index) => ({
+            index,
+            bytes: entry.active && entry.image && index > this.#evicted && index <= this.#considered
+                ? await size(entry.image)
+                : 0
+        })));
+        const beforeBytes = history.reduce((sum, entry) => sum + entry.bytes, 0);
+        let freshBytes = 0;
+        let requestBytes = 0;
+        const requestHistory = new Map();
+        if (request) {
+            for (const message of request.messages)
+                for (const [index, image] of (message.images ?? []).entries()) {
+                    const position = this.#sources.get(inlineMediaSource(message))?.[index];
+                    const entry = position === undefined ? undefined : this.#entries[position];
+                    const original = entry && isDeepStrictEqual(entry.image, image);
+                    if (original && position <= this.#evicted)
+                        continue;
+                    const bytes = await size(image);
+                    requestBytes += bytes;
+                    if (original && position <= this.#considered)
+                        requestHistory.set(position, (requestHistory.get(position) ?? 0) + bytes);
+                    else
+                        freshBytes += bytes;
+                }
+            for (const image of request.images ?? []) {
+                if (mirroredImages.some((mirror) => isDeepStrictEqual(image, mirror)))
+                    continue;
+                const bytes = await size(image);
+                freshBytes += bytes;
+                requestBytes += bytes;
+            }
+            if (freshBytes > policy.requestInlineMediaHardLimit)
+                inlineMediaLimit();
+        }
+        const hard = requestBytes > policy.requestInlineMediaHardLimit;
+        if (!hard && beforeBytes <= policy.historicalInlineMediaTarget)
+            return undefined;
+        const target = hard
+            ? Math.max(0, policy.historicalInlineMediaTarget - freshBytes)
+            : policy.historicalInlineMediaTarget;
+        let afterBytes = beforeBytes;
+        let evicted = this.#evicted;
+        for (const item of history) {
+            if (afterBytes <= target && (!hard || requestBytes <= policy.requestInlineMediaHardLimit))
+                break;
+            if (!item.bytes)
+                continue;
+            if (!hard && this.#rollout - this.#entries[item.index].rollout < 2)
+                break;
+            afterBytes -= item.bytes;
+            requestBytes -= requestHistory.get(item.index) ?? 0;
+            evicted = item.index;
+        }
+        if (afterBytes > target || (hard && requestBytes > policy.requestInlineMediaHardLimit))
+            return undefined;
+        const entry = this.#entries[evicted];
+        if (!entry || typeof entry.source !== 'string' || evicted <= this.#evicted)
+            return undefined;
+        return {
+            reason: hard
+                ? 'HARD_LIMIT'
+                : beforeBytes > policy.historicalInlineMediaTrigger
+                    ? 'SOFT_LIMIT'
+                    : 'TEXT_GC',
+            state: {
+                ...this.checkpoint(),
+                evictedThrough: { sourceId: entry.source, imageIndex: entry.imageIndex }
+            },
+            beforeBytes,
+            afterBytes,
+            project: (messages) => this.#historyView(messages, evicted, this.#evicted)
+        };
+    }
+    adoptGc(state) {
+        if (state)
+            this.#evicted = this.#position(state.evictedThrough);
+    }
+    async #project(image) {
+        if (this.useResourceUrl) {
+            if (!isModelImageUrl(image.url))
+                throw new MarAgentError('MAR_AGENT_RESOURCE_URL_UNAVAILABLE', 'Resource URL is unavailable.');
+            const { dataBase64: _bytes, ...reference } = image;
+            return reference;
+        }
+        const bytes = image.dataBase64 !== undefined
+            ? image
+            : image.resourceId === undefined
+                ? undefined
+                : await this.store.readResource(this.sessionId, image.resourceId);
+        if (!bytes?.dataBase64)
+            throw new MarAgentError('MAR_AGENT_SESSION_CORRUPT', 'Resource bytes are unavailable.');
+        const { url: _url, urlExpiresAt: _expires, ...reference } = image;
+        return { ...reference, dataBase64: bytes.dataBase64 };
+    }
+}