@retinue/agentkit 0.1.0

package/dist/models/streaming.d.ts

@@ -0,0 +1,145 @@
+/**
+ * Neutral model-turn streaming — `docs/03-intelligence-runtime.md`.
+ *
+ * The one place the Vercel AI SDK's `streamText` tool loop is used (boundary rule R3 keeps the SDK
+ * inside `models/`). It runs a multi-step turn — the model calls tools, their results feed back, it
+ * continues — and yields provider-neutral chunks. The agent engine consumes these chunks without
+ * ever importing the SDK, so switching providers changes nothing above this layer.
+ */
+import { type LanguageModel } from "ai";
+import type { InputModality } from "./index.js";
+/** Opaque handle to a provider model. Aliased here so layers above `models/` never import the SDK. */
+export type ResolvedModel = LanguageModel;
+/** A conversation turn message the engine hands down (text-only for v1; multimodal is additive). */
+/**
+ * One piece of a turn's content — #185.
+ *
+ * `image` and `file` exist because the platform has always accepted them: `ImagePart` and `FilePart` are in the
+ * message contract, `InputModality` covers image, audio, video and pdf, and `resolveModel` filters on
+ * `requiredModalities`. What was missing was the last hop — the bridge to the provider took a string, so an
+ * attachment was stored, authorized, rendered and billed for, and then not mentioned to the model.
+ *
+ * A URL or a data payload rather than a platform file id, deliberately: this layer is the SDK boundary and knows
+ * nothing about the file store. The caller resolves a `FilePart` into bytes or a URL **through the mediated read
+ * path**, which is what keeps the modality bridge from becoming a way around file authorization.
+ */
+export type TurnContentPart = {
+    readonly kind: "text";
+    readonly text: string;
+} | {
+    readonly kind: "image";
+    readonly image: string | URL;
+    readonly mediaType?: string;
+} | {
+    readonly kind: "file";
+    readonly data: string | URL;
+    readonly mediaType: string;
+    readonly filename?: string;
+};
+export type TurnMessage = {
+    readonly role: "system" | "user" | "assistant";
+    /**
+     * A string for a text turn, parts when the turn carries an attachment.
+     *
+     * A union rather than `text` plus an optional `parts`, which was the other option and is worse: two fields
+     * that can both carry text leave the relationship between them undefined, and somebody eventually sets one and
+     * reads the other.
+     */
+    readonly content: string | readonly TurnContentPart[];
+};
+/** The text of a turn, for callers that count tokens or log. Non-text parts contribute nothing. */
+export declare const turnText: (message: TurnMessage) => string;
+/** The modalities a turn actually needs, so a model can be checked against it rather than assumed. */
+/**
+ * What non-text input a turn is carrying — REQ-036 (#185), AC-4.
+ *
+ * Counted, not inferred: one image part is one image. Audio is not counted yet and the field is deliberately
+ * absent rather than zero — a `TurnContentPart` has no duration, so the only honest answer is "this layer does
+ * not know", and a zero would be indistinguishable from a silent audio turn.
+ */
+export declare const nonTextCounts: (messages: readonly TurnMessage[]) => {
+    readonly imageCount?: number;
+};
+export declare const modalitiesOf: (messages: readonly TurnMessage[]) => readonly InputModality[];
+/** A tool the model may call this turn. `execute` is the platform's guarded execution path. */
+export type ModelTurnTool = {
+    readonly name: string;
+    readonly description?: string;
+    /** Zod schema or JSON-schema object; a permissive object schema is used when absent. */
+    readonly inputSchema?: unknown;
+    execute(input: unknown): Promise<unknown>;
+};
+export type ModelTurnRequest = {
+    readonly model: ResolvedModel;
+    readonly system?: string;
+    readonly messages: readonly TurnMessage[];
+    /**
+     * What the resolved model accepts — #185. Optional, and its absence means "do not check".
+     *
+     * Absent rather than defaulting to `["text"]`, because a caller that has not said what the model takes has not
+     * said the model takes text only. Defaulting would refuse every image turn from every caller that has not been
+     * updated, which is an outage dressed as a safety check.
+     */
+    readonly modelModalities?: readonly InputModality[];
+    readonly tools?: readonly ModelTurnTool[];
+    readonly maxSteps?: number;
+    readonly abortSignal?: AbortSignal;
+    /**
+     * Generation parameters — #160.
+     *
+     * None of these existed, and none was sent: `streamText` was called with model, system, messages, tools and
+     * `stopWhen` only. So `ModelDefinition.limits.maxOutputTokens` was **decorative** in the text path — a
+     * definition declaring a 4,096-token cap capped nothing, and a run got whatever the provider's default
+     * happened to be. `models/vision.ts` was the only place that applied it.
+     *
+     * `temperature` matters for a second reason: the evaluation harness (#141) rests its reproducibility argument
+     * partly on temperature zero, and there was no way to ask for it on a real run.
+     */
+    readonly maxOutputTokens?: number;
+    readonly temperature?: number;
+    readonly topP?: number;
+    readonly stopSequences?: readonly string[];
+};
+export type NeutralUsage = {
+    readonly inputTokens: number;
+    readonly outputTokens: number;
+    readonly cachedInputTokens: number;
+    readonly reasoningTokens?: number;
+    /**
+     * Non-text input, counted from what **we sent** rather than from what the provider reported — #185 AC-4.
+     *
+     * Deliberately not read out of the provider's usage object. The AI SDK's neutral usage has no modality
+     * breakdown, and the providers that expose one put it in provider-specific metadata under provider-specific
+     * names — so a platform reading it would work for one vendor and silently report zero for the rest, which is
+     * worse than not reporting at all because it looks like data.
+     *
+     * We know exactly what went out: the turn's parts. Counting there is provider-independent, always available,
+     * and auditable against the transcript — and it is the number a per-image price is charged against anyway.
+     */
+    readonly imageCount?: number;
+    readonly audioSeconds?: number;
+};
+/** Provider-neutral stream chunk — the engine maps these to `RunEvent`s. */
+export type NeutralStreamChunk = {
+    readonly type: "text-delta";
+    readonly id: string;
+    readonly text: string;
+} | {
+    readonly type: "tool-call";
+    readonly toolCallId: string;
+    readonly toolName: string;
+    readonly input: unknown;
+} | {
+    readonly type: "tool-result";
+    readonly toolCallId: string;
+    readonly toolName: string;
+    readonly output: unknown;
+} | {
+    readonly type: "finish";
+    readonly usage: NeutralUsage;
+} | {
+    readonly type: "error";
+    readonly error: unknown;
+};
+export declare function streamModelTurn(req: ModelTurnRequest): AsyncIterable<NeutralStreamChunk>;
+//# sourceMappingURL=streaming.d.ts.map

package/dist/models/streaming.js

@@ -0,0 +1,272 @@
+/**
+ * Neutral model-turn streaming — `docs/03-intelligence-runtime.md`.
+ *
+ * The one place the Vercel AI SDK's `streamText` tool loop is used (boundary rule R3 keeps the SDK
+ * inside `models/`). It runs a multi-step turn — the model calls tools, their results feed back, it
+ * continues — and yields provider-neutral chunks. The agent engine consumes these chunks without
+ * ever importing the SDK, so switching providers changes nothing above this layer.
+ */
+import { jsonSchema, stepCountIs, streamText, tool as aiTool } from "ai";
+import { AgentPlatformError } from "../core/errors.js";
+/** The text of a turn, for callers that count tokens or log. Non-text parts contribute nothing. */
+export const turnText = (message) => typeof message.content === "string"
+    ? message.content
+    : message.content
+        .filter((part) => part.kind === "text")
+        .map((part) => part.text)
+        .join("\n");
+/** The modalities a turn actually needs, so a model can be checked against it rather than assumed. */
+/**
+ * What non-text input a turn is carrying — REQ-036 (#185), AC-4.
+ *
+ * Counted, not inferred: one image part is one image. Audio is not counted yet and the field is deliberately
+ * absent rather than zero — a `TurnContentPart` has no duration, so the only honest answer is "this layer does
+ * not know", and a zero would be indistinguishable from a silent audio turn.
+ */
+export const nonTextCounts = (messages) => {
+    let images = 0;
+    for (const message of messages) {
+        if (typeof message.content === "string")
+            continue;
+        for (const part of message.content)
+            if (part.kind === "image")
+                images += 1;
+    }
+    return images === 0 ? {} : { imageCount: images };
+};
+export const modalitiesOf = (messages) => {
+    const found = new Set();
+    for (const message of messages) {
+        if (typeof message.content === "string")
+            continue;
+        for (const part of message.content) {
+            if (part.kind === "image")
+                found.add("image");
+            // A file's modality is its media type: a PDF needs `pdf`, an audio file needs `audio`. Anything else is
+            // sent as a file and left to the provider — guessing a modality we cannot name would defeat the check.
+            if (part.kind === "file") {
+                if (part.mediaType === "application/pdf")
+                    found.add("pdf");
+                else if (part.mediaType.startsWith("audio/"))
+                    found.add("audio");
+                else if (part.mediaType.startsWith("video/"))
+                    found.add("video");
+                else if (part.mediaType.startsWith("image/"))
+                    found.add("image");
+            }
+        }
+    }
+    return [...found];
+};
+const isZodSchema = (s) => typeof s === "object" && s !== null && typeof s.safeParse === "function";
+/** A JSON-schema-shaped object: `{ type: "object", … }`. Enough to tell one from a Zod schema or from absence. */
+const isJsonSchema = (s) => typeof s === "object" && s !== null && typeof s.type === "string";
+const toToolSet = (tools) => {
+    const set = {};
+    for (const t of tools) {
+        set[t.name] = aiTool({
+            ...(t.description ? { description: t.description } : {}),
+            /**
+             * A **JSON schema is honoured**, not discarded — #159.
+             *
+             * `ModelTurnTool.inputSchema` documents itself as "Zod schema or JSON-schema object", and every tool the
+             * platform builds carries a JSON schema: that is what `defineDelegatingTool` takes and what
+             * `ToolDescriptor.inputSchema` holds. The previous version kept only a Zod schema and replaced everything
+             * else with a permissive `{ type: "object", additionalProperties: true }`.
+             *
+             * A permissive schema tells the model the tool takes *any* object — so it has no parameter names to fill in
+             * and emits calls with **empty arguments**. Every JSON-schema tool in the platform was reaching the model
+             * effectively undocumented, and the work it was asked to do silently did not happen: the assistant was
+             * asked to remember a fact, called `remember` with `{}`, and two turns later said it knew nothing.
+             *
+             * Wrapping with `jsonSchema()` is all that was missing. The permissive fallback stays for a tool that
+             * genuinely declares nothing, which is the only case it was meant for.
+             */
+            inputSchema: isZodSchema(t.inputSchema)
+                ? t.inputSchema
+                : isJsonSchema(t.inputSchema)
+                    ? jsonSchema(t.inputSchema)
+                    : jsonSchema({ type: "object", additionalProperties: true }),
+            execute: (input) => t.execute(input),
+        });
+    }
+    return set;
+};
+const num = (v) => (typeof v === "number" && Number.isFinite(v) ? v : 0);
+/**
+ * Run one turn and yield neutral chunks. `streamText` drives the model↔tool loop up to `maxSteps`;
+ * this reads its `fullStream` and normalizes the pieces the engine cares about (text deltas, tool
+ * calls/results, final usage, errors).
+ */
+/**
+ * A short, safe message for a thrown tool error.
+ *
+ * Bounded and message-only. The model needs enough to decide whether to retry or explain, and a validation
+ * message ("expected string, received undefined") is exactly that — but the value lands in the durable event log,
+ * so it is truncated and the object itself never travels.
+ */
+const errorCodeOf = (error) => {
+    if (typeof error === "object" && error !== null) {
+        const code = error.code;
+        if (typeof code === "string" && code.length > 0 && code.length <= 64)
+            return code;
+        const name = error.constructor?.name;
+        if (typeof name === "string" && name.length > 0)
+            return name;
+    }
+    return "tool-failed";
+};
+const errorMessageOf = (error) => {
+    const message = typeof error === "object" && error !== null && typeof error.message === "string"
+        ? error.message
+        : String(error);
+    return message.slice(0, 200);
+};
+/**
+ * A turn's parts, in the SDK's shape.
+ *
+ * A message whose content is a plain string stays a plain string rather than being wrapped in a single text
+ * part. Providers treat the two identically, but the wire form differs, and a change that rewrites every
+ * existing text turn is a change whose blast radius is every conversation rather than the ones with an
+ * attachment in them.
+ */
+const toModelContent = (content) => {
+    if (typeof content === "string")
+        return content;
+    return content.map((part) => part.kind === "text"
+        ? { type: "text", text: part.text }
+        : part.kind === "image"
+            ? { type: "image", image: part.image, ...(part.mediaType === undefined ? {} : { mediaType: part.mediaType }) }
+            : {
+                type: "file",
+                data: part.data,
+                mediaType: part.mediaType,
+                ...(part.filename === undefined ? {} : { filename: part.filename }),
+            });
+};
+export async function* streamModelTurn(req) {
+    /**
+     * Refuse a modality the model cannot take — #185.
+     *
+     * Fail closed, and loudly. The alternatives are both worse: dropping the attachment sends the model a turn
+     * that reads as if the user attached nothing, and it answers confidently about a message it never saw; and
+     * substituting a text description silently makes the transcript a record of something that did not happen.
+     *
+     * `resolveModel` already refuses to *hand out* a model that lacks a required modality, so in the normal path
+     * this never fires. It fires when a caller resolved a model for a text turn and an attachment arrived later in
+     * the conversation — which is exactly the case the resolution-time check cannot see.
+     */
+    if (req.modelModalities !== undefined) {
+        const needed = modalitiesOf(req.messages);
+        const missing = needed.filter((m) => !req.modelModalities.includes(m));
+        if (missing.length > 0)
+            throw new AgentPlatformError({
+                code: "capability_unavailable",
+                message: `this turn carries ${missing.join(", ")} and the resolved model accepts only ` +
+                    `${req.modelModalities.join(", ")}. Resolve a model with the required modalities, or remove the ` +
+                    `attachment from the turn.`,
+                retryable: false,
+            });
+    }
+    const messages = req.messages.map((m) => ({ role: m.role, content: toModelContent(m.content) }));
+    const result = streamText({
+        model: req.model,
+        ...(req.system ? { system: req.system } : {}),
+        messages,
+        ...(req.tools && req.tools.length > 0 ? { tools: toToolSet(req.tools) } : {}),
+        stopWhen: stepCountIs(req.maxSteps ?? 8),
+        ...(req.abortSignal ? { abortSignal: req.abortSignal } : {}),
+        // Spread conditionally so an unset parameter leaves the provider's own default alone, rather than pinning it
+        // to a value this layer invented. #160: none of these was sent at all before.
+        ...(req.maxOutputTokens === undefined ? {} : { maxOutputTokens: req.maxOutputTokens }),
+        ...(req.temperature === undefined ? {} : { temperature: req.temperature }),
+        ...(req.topP === undefined ? {} : { topP: req.topP }),
+        ...(req.stopSequences === undefined ? {} : { stopSequences: [...req.stopSequences] }),
+    });
+    for await (const chunk of result.fullStream) {
+        switch (chunk.type) {
+            case "text-delta":
+                yield { type: "text-delta", id: chunk.id ?? "text", text: chunk.text ?? "" };
+                break;
+            case "tool-call":
+                yield {
+                    type: "tool-call",
+                    toolCallId: chunk.toolCallId,
+                    toolName: chunk.toolName,
+                    input: chunk.input,
+                };
+                break;
+            case "tool-result":
+                yield {
+                    type: "tool-result",
+                    toolCallId: chunk.toolCallId,
+                    toolName: chunk.toolName,
+                    output: chunk.output,
+                };
+                break;
+            /**
+             * A tool whose `execute` threw — #159.
+             *
+             * The SDK runs the tools itself and emits `tool-error` when one throws. This case did not exist, so the
+             * chunk fell into `default: break` and the failure produced **no neutral chunk at all**: the model never
+             * learned the call failed, the projection was left with a `tool-call` and no matching `tool-result`, and
+             * the dangling call was only finalised as `tool.failed` at the end of the run — after the answer had
+             * already been written. A user asked the assistant to remember something and was told, two turns later,
+             * that it knew nothing about them.
+             *
+             * Mapped to a `tool-result` carrying an error payload rather than to a bare `error`, and that choice is the
+             * point: a bare error is a *run* failure, but a tool failing is a normal thing the model should see and
+             * respond to. Emitting it as the call's result both resolves the dangling call and puts the failure in the
+             * model's own history, so it can retry or say what went wrong.
+             */
+            case "tool-error":
+                yield {
+                    type: "tool-result",
+                    toolCallId: chunk.toolCallId,
+                    toolName: chunk.toolName,
+                    output: {
+                        ok: false,
+                        // The message only, never the thrown object: a stack or a cause chain routinely carries a URL with a
+                        // token in it, and this value goes into the run's durable event log. Same rule as #143's `recordError`.
+                        error: {
+                            code: errorCodeOf(chunk.error),
+                            message: errorMessageOf(chunk.error),
+                        },
+                    },
+                };
+                break;
+            case "error":
+                yield { type: "error", error: chunk.error };
+                break;
+            case "finish": {
+                const usage = chunk.totalUsage ?? {};
+                yield {
+                    type: "finish",
+                    usage: {
+                        inputTokens: num(usage.inputTokens),
+                        outputTokens: num(usage.outputTokens),
+                        cachedInputTokens: num(usage.cachedInputTokens),
+                        ...(usage.reasoningTokens !== undefined ? { reasoningTokens: num(usage.reasoningTokens) } : {}),
+                        /**
+                         * Non-text input, counted from the request rather than read from the response — #185 AC-4.
+                         *
+                         * This is the only place that knows both things at once: what went out, and that the turn has
+                         * finished. The provider's usage object has no modality breakdown in the SDK's neutral shape, and
+                         * the vendors that expose one bury it under vendor-specific metadata — so reading it there would
+                         * work for one provider and silently report zero for the others, which is worse than reporting
+                         * nothing because it looks like data.
+                         *
+                         * Only emitted when there is something to say. A text-only turn carries no `imageCount: 0`, so a
+                         * pricing record that charges per image cannot be handed a zero it might treat as "unknown".
+                         */
+                        ...nonTextCounts(req.messages),
+                    },
+                };
+                break;
+            }
+            default:
+                break;
+        }
+    }
+}
+//# sourceMappingURL=streaming.js.map

package/dist/models/vision.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Image description through a vision-capable model (#132).
+ *
+ * Lives in `models/` because boundary rule R3 keeps the Vercel AI SDK here: the layers above call
+ * `describeImage` and never see the SDK, exactly as they call `streamModelTurn` and never see `streamText`.
+ *
+ * **The capability check is not in this file, and that is the design.** `ModelRegistry.resolve` already
+ * refuses when no model satisfies `requiredModalities: ["image"]` — it throws `capability_unavailable`. So
+ * AC-3 ("a model without vision capability is never used for an image request") holds because the caller
+ * cannot *obtain* a model to pass here, not because this function checks one. A check here would be a second
+ * gate to keep in step with the first, and the weaker of the two would be the one that mattered.
+ */
+import { type LanguageModel } from "ai";
+import type { NeutralUsage } from "./streaming.js";
+/** The prompt a description is produced against. Injectable, because what matters differs by caller. */
+export declare const DEFAULT_VISION_PROMPT: string;
+export type DescribeImageRequest = {
+    readonly model: LanguageModel;
+    readonly bytes: Uint8Array;
+    readonly mediaType: string;
+    readonly prompt?: string;
+    readonly maxOutputTokens?: number;
+    readonly abortSignal?: AbortSignal;
+};
+export type DescribeImageResult = {
+    readonly text: string;
+    readonly usage: NeutralUsage;
+    readonly modelId: string;
+};
+/**
+ * One vision call, returning the description and its token usage.
+ *
+ * The usage comes back rather than being recorded here: this module has no `ExecutionContext` and no
+ * `UsageRecorder`, and reaching for either would put a billing decision inside a provider call. The caller
+ * records it, which is also what makes the record idempotent on the *extraction*, not on the HTTP request.
+ */
+export declare const describeImage: (req: DescribeImageRequest) => Promise<DescribeImageResult>;
+//# sourceMappingURL=vision.d.ts.map

package/dist/models/vision.js

@@ -0,0 +1,62 @@
+/**
+ * Image description through a vision-capable model (#132).
+ *
+ * Lives in `models/` because boundary rule R3 keeps the Vercel AI SDK here: the layers above call
+ * `describeImage` and never see the SDK, exactly as they call `streamModelTurn` and never see `streamText`.
+ *
+ * **The capability check is not in this file, and that is the design.** `ModelRegistry.resolve` already
+ * refuses when no model satisfies `requiredModalities: ["image"]` — it throws `capability_unavailable`. So
+ * AC-3 ("a model without vision capability is never used for an image request") holds because the caller
+ * cannot *obtain* a model to pass here, not because this function checks one. A check here would be a second
+ * gate to keep in step with the first, and the weaker of the two would be the one that mattered.
+ */
+import { generateText } from "ai";
+import { AgentPlatformError } from "../core/errors.js";
+/** The prompt a description is produced against. Injectable, because what matters differs by caller. */
+export const DEFAULT_VISION_PROMPT = "Describe this image for someone who cannot see it. State what it shows, transcribe any text exactly as " +
+    "it appears, and describe any chart's axes and values. Do not speculate about anything not visible.";
+const num = (v) => (typeof v === "number" && Number.isFinite(v) ? v : 0);
+/**
+ * One vision call, returning the description and its token usage.
+ *
+ * The usage comes back rather than being recorded here: this module has no `ExecutionContext` and no
+ * `UsageRecorder`, and reaching for either would put a billing decision inside a provider call. The caller
+ * records it, which is also what makes the record idempotent on the *extraction*, not on the HTTP request.
+ */
+export const describeImage = async (req) => {
+    const content = [
+        { type: "text", text: req.prompt ?? DEFAULT_VISION_PROMPT },
+        // The SDK takes the bytes and the media type; it decides per provider whether that becomes a base64
+        // data part or an upload. Passing a URL instead would mean a signed URL in a provider request, which is
+        // the thing #129 clamps to fifteen minutes precisely to avoid.
+        { type: "image", image: req.bytes, mediaType: req.mediaType },
+    ];
+    try {
+        const result = await generateText({
+            model: req.model,
+            messages: [{ role: "user", content }],
+            ...(req.maxOutputTokens === undefined ? {} : { maxOutputTokens: req.maxOutputTokens }),
+            ...(req.abortSignal ? { abortSignal: req.abortSignal } : {}),
+        });
+        const usage = result.usage;
+        return {
+            text: result.text,
+            usage: {
+                inputTokens: num(usage?.inputTokens),
+                outputTokens: num(usage?.outputTokens),
+                cachedInputTokens: num(usage?.cachedInputTokens),
+            },
+            modelId: typeof req.model === "string" ? req.model : (req.model.modelId ?? "unknown"),
+        };
+    }
+    catch (error) {
+        // Wrapped so a provider's error shape does not reach the extraction pipeline, which would then have to
+        // know about three SDKs to decide whether a failure is retryable.
+        throw new AgentPlatformError({
+            code: "provider_unavailable",
+            message: "The vision model could not describe that image.",
+            retryable: true,
+        }, { cause: error });
+    }
+};
+//# sourceMappingURL=vision.js.map