@kuralle-agents/core 0.7.2 → 0.8.0

package/dist/flow/collectUntilComplete.js

@@ -1,5 +1,6 @@
 import { runCollectDigression } from './collectDigression.js';
 import { hasPendingUserInput } from '../runtime/channels/inputBuffer.js';
+import { userInputToText } from '../runtime/userInput.js';
 import { resolveCollectExtractionNode } from './nodeBuilders.js';
 import { computeMissingFields, createExtractionSubmitTool, getCollectData, incrementCollectTurns, emitExtractionTelemetry, mergeTurnExtraction, projectCollectData, schemaSatisfied, } from './extraction.js';
 import { normalizeTransition } from './normalizeTransition.js';
@@ -127,8 +128,10 @@ function mergeExtractionFromTurn(node, run, turn, ctx) {
 function peekLatestUserMessage(run) {
     for (let i = run.messages.length - 1; i >= 0; i -= 1) {
         const message = run.messages[i];
-        if (message?.role === 'user' && typeof message.content === 'string') {
-            return message.content;
+        if (message?.role === 'user') {
+            const text = userInputToText(message.content);
+            if (text)
+                return text;
         }
     }
     return undefined;

package/dist/flow/runFlow.js

@@ -1,6 +1,7 @@
 import { getFlowPark } from './collectDigression.js';
 import { parseConfirmation } from './confirmParse.js';
 import { hasPendingUserInput } from '../runtime/channels/inputBuffer.js';
+import { userInputToText } from '../runtime/userInput.js';
 import { collectUntilComplete } from './collectUntilComplete.js';
 import { isActionNode, isCollectNode, isDecideNode, isReplyNode, } from './nodeKinds.js';
 import { normalizeTransition, resolveNodeRef } from './normalizeTransition.js';
@@ -53,8 +54,10 @@ function appendUserMessage(run, input) {
 function latestUserText(run) {
     for (let i = run.messages.length - 1; i >= 0; i -= 1) {
         const message = run.messages[i];
-        if (message?.role === 'user' && typeof message.content === 'string') {
-            return message.content;
+        if (message?.role === 'user') {
+            const text = userInputToText(message.content);
+            if (text)
+                return text;
         }
     }
     return '';
@@ -67,7 +70,7 @@ async function dispatchConfirmGate(node, run, driver, ctx) {
     let input = '';
     if (hasPendingUserInput(ctx.session)) {
         const signal = await driver.awaitUser(ctx);
-        input = signal.input;
+        input = userInputToText(signal.input);
         appendUserMessage(run, signal.input);
     }
     else {

package/dist/index.d.ts

@@ -90,10 +90,12 @@ export type { HarnessStreamPart } from './types/stream.js';
 export { harnessToUIMessageStream } from './ai-sdk/uiMessageStream.js';
 export type { KuralleMetadata, KuralleDataParts, KuralleUIMessage, } from './ai-sdk/uiMessageStream.js';
 export type { ChoiceOption, ResolvedSelection } from './types/selection.js';
-export type { RunState, StepRecord } from './runtime/durable/types.js';
+export type { RunState, StepRecord, SignalDelivery, SessionDurableRuns, PersistedRun, } from './runtime/durable/types.js';
+export { DURABLE_RUNS_KEY } from './runtime/durable/types.js';
 export type { RunStore } from './runtime/durable/RunStore.js';
 export type { ChannelDriver, TextDriver } from './runtime/channels/index.js';
 export type { TurnResult } from './types/channel.js';
 export type { RunContext } from './types/run-context.js';
 export { createRuntime, Runtime, type HarnessConfig, type RunOptions, } from './runtime/Runtime.js';
 export type { RuntimeLike } from './runtime/RuntimeLike.js';
+export { userInputToText, hasMediaParts, transcribeAudioParts, type UserInputContent, } from './runtime/userInput.js';

package/dist/index.js

@@ -47,4 +47,6 @@ export { SkillsCapability, wireAgentSkills, collectRegisteredNames, validateSkil
 export { buildToolSet, toolToAiSdk, wrapAiSdkTool, ToolApprovalDeniedError, ToolTimeoutError, } from './tools/effect/index.js';
 export { parseConfirmation } from './flow/confirmParse.js';
 export { harnessToUIMessageStream } from './ai-sdk/uiMessageStream.js';
+export { DURABLE_RUNS_KEY } from './runtime/durable/types.js';
 export { createRuntime, Runtime, } from './runtime/Runtime.js';
+export { userInputToText, hasMediaParts, transcribeAudioParts, } from './runtime/userInput.js';

package/dist/runtime/Runtime.d.ts

@@ -1,4 +1,5 @@
-import type { LanguageModel, ModelMessage } from 'ai';
+import type { LanguageModel, ModelMessage, TranscriptionModel } from 'ai';
+import type { UserInputContent } from './userInput.js';
 import type { Session } from '../types/session.js';
 import type { SessionStore } from '../session/SessionStore.js';
 import type { AuditListOptions, ConversationAuditEntry } from '../audit/types.js';
@@ -31,10 +32,17 @@ export interface HarnessConfig {
     memoryService?: V1MemoryService;
     /** Default store for `agent.memory.workingMemory` when `workingMemory.store` is omitted. */
     defaultWorkingMemoryStore?: PersistentMemoryStore;
+    /**
+     * Optional AI SDK transcription model. When set, inbound audio file parts (voice
+     * notes) are transcribed to text before the model turn — so voice input works on
+     * text-only models. When omitted, audio parts pass through to audio-capable models.
+     */
+    transcriptionModel?: TranscriptionModel;
 }
 export interface RunOptions {
     sessionId?: string;
-    input?: string;
+    /** The user turn: plain text, or AI SDK multimodal content (text + file/image/audio parts). */
+    input?: UserInputContent;
     selection?: ResolvedSelection;
     userId?: string;
     agentId?: string;

package/dist/runtime/Runtime.js

@@ -60,6 +60,7 @@ export class Runtime {
                 seedMessages: opts.seedMessages,
                 historyDelta: opts.historyDelta,
                 signalDelivery: opts.signalDelivery,
+                transcriptionModel: this.config.transcriptionModel,
                 defaultAgentId: this.config.defaultAgentId,
                 sessionStore: this.sessionStore,
             });

package/dist/runtime/channels/inputBuffer.d.ts

@@ -1,5 +1,6 @@
 import type { Session } from '../../types/session.js';
-export declare function setPendingUserInput(session: Session, input: string): void;
-export declare function consumePendingUserInput(session: Session): string;
-export declare function peekPendingUserInput(session: Session): string | undefined;
+import type { UserInputContent } from '../userInput.js';
+export declare function setPendingUserInput(session: Session, input: UserInputContent): void;
+export declare function consumePendingUserInput(session: Session): UserInputContent;
+export declare function peekPendingUserInput(session: Session): UserInputContent | undefined;
 export declare function hasPendingUserInput(session: Session): boolean;

package/dist/runtime/ctx.js

@@ -112,6 +112,15 @@ function makeCtx(deps) {
         bargeIn: deps.bargeIn,
         abortSignal: deps.abortSignal,
         turnInputConsumed: false,
+        // Rebase durable effect callsites to 0. Called at flow entry so a flow's
+        // effects (and any suspend/resume callsite) are anchored to the flow itself —
+        // identical whether the flow was entered fresh after an answering turn (which
+        // may have consumed callsites via enter_flow / tool calls) or re-entered on
+        // resume (where that answering turn does not re-run). Without this, a suspend's
+        // recorded callsite would not match on resume and the run would re-suspend.
+        resetCallsites: () => {
+            effectOrdinal = 0;
+        },
         tool: async (name, args, options) => {
             // needsApproval gate: a tool flagged `needsApproval` must be approved by a human
             // before it runs. Approval is a durable pause (the `__approval` signal); on resume

package/dist/runtime/hostLoop.js

@@ -59,6 +59,12 @@ async function runAnsweringAgent(agent, run, driver, ctx, classify) {
 async function runActiveFlow(flow, run, driver, ctx, agent) {
     incrementTurnCount(run);
     assertWithinTurnLimit(run, ctx.limits);
+    // Anchor durable effect callsites to the flow. On a fresh entry the answering
+    // turn may have consumed callsites (enter_flow / tool calls); on resume that
+    // turn does not re-run. Rebasing here makes the flow's callsites (and any
+    // suspend/resume key) identical across both paths, so a resumed run does not
+    // re-suspend on a callsite mismatch.
+    ctx.resetCallsites();
     const result = await runFlow(flow, run, driver, ctx, agent);
     if (result.kind === 'handoff') {
         return { kind: 'handoff', to: result.to, reason: result.reason };

package/dist/runtime/index.d.ts

@@ -4,4 +4,5 @@ export { SessionWorkingMemory } from './WorkingMemory.js';
 export { TextDriver, VoiceDriver } from './channels/index.js';
 export type { VoiceDriverConfig } from './channels/index.js';
 export { setPendingUserInput, consumePendingUserInput, peekPendingUserInput, hasPendingUserInput, } from './channels/index.js';
+export { userInputToText, hasMediaParts, transcribeAudioParts, type UserInputContent, } from './userInput.js';
 export { isAbortSignal, type InterruptionEvent, type AbortOptions, type CancellationReason, } from '../types/index.js';

package/dist/runtime/index.js

@@ -5,4 +5,8 @@ export { TextDriver, VoiceDriver } from './channels/index.js';
 // implement awaitUser the same FIFO-aware way the built-in drivers do (the
 // buffer is an ordered queue since 0.3.13/H3, not a single slot).
 export { setPendingUserInput, consumePendingUserInput, peekPendingUserInput, hasPendingUserInput, } from './channels/index.js';
+// Multimodal user input — `UserInputContent` is the runtime's accepted user-turn
+// shape (text or AI SDK file/image/audio parts). Helpers project to text and
+// transcribe audio so ingress adapters (web/messaging) can build it uniformly.
+export { userInputToText, hasMediaParts, transcribeAudioParts, } from './userInput.js';
 export { isAbortSignal, } from '../types/index.js';

package/dist/runtime/openRun.d.ts

@@ -1,4 +1,5 @@
-import type { ModelMessage } from 'ai';
+import type { ModelMessage, TranscriptionModel } from 'ai';
+import { type UserInputContent } from './userInput.js';
 import type { Session } from '../types/session.js';
 import type { SessionStore } from '../session/SessionStore.js';
 import type { AgentConfig } from '../types/agentConfig.js';
@@ -9,12 +10,13 @@ import type { ResolvedSelection } from '../types/selection.js';
 export interface OpenRunOptions {
     sessionId: string;
     userId?: string;
-    input?: string;
+    input?: UserInputContent;
     selection?: ResolvedSelection;
     agentId?: string;
     seedMessages?: ModelMessage[];
     historyDelta?: ModelMessage[];
     signalDelivery?: SignalDelivery;
+    transcriptionModel?: TranscriptionModel;
     defaultAgentId: string;
     sessionStore: SessionStore;
 }

package/dist/runtime/openRun.js

@@ -1,4 +1,5 @@
 import { randomUUID } from 'node:crypto';
+import { transcribeAudioParts } from './userInput.js';
 import { setPendingUserInput } from './channels/inputBuffer.js';
 import { SessionRunStore } from './durable/SessionRunStore.js';
 import { recordSignalDelivery } from './durable/replay.js';
@@ -45,8 +46,14 @@ export async function openRun(agentsById, options) {
         runState.updatedAt = Date.now();
         await runStore.putRunState(runState);
     }
-    const effectiveInput = options.selection?.id ?? options.input;
-    if (effectiveInput) {
+    const rawInput = options.selection?.id ?? options.input;
+    const effectiveInput = rawInput === undefined
+        ? undefined
+        : await transcribeAudioParts(rawInput, options.transcriptionModel);
+    const hasInput = typeof effectiveInput === 'string'
+        ? effectiveInput.length > 0
+        : Array.isArray(effectiveInput) && effectiveInput.length > 0;
+    if (hasInput && effectiveInput !== undefined) {
         runState.updatedAt = Date.now();
         if (runState.activeFlow) {
             await runStore.putRunState(runState);

package/dist/runtime/userInput.d.ts

@@ -0,0 +1,24 @@
+import { type UserContent, type TranscriptionModel } from 'ai';
+/**
+ * User-turn content the runtime accepts: plain text, or AI SDK multimodal parts
+ * (text + file/image/audio). This is exactly the `content` of a user `ModelMessage`,
+ * so it threads into the model with no translation.
+ *
+ * Durability invariant: any `FilePart.data` flowing through the runtime must be
+ * JSON-serializable (a base64 string, data URL, or https URL) — never a raw
+ * `Buffer`/`Uint8Array`. `RunState.messages`, `session.messages`, and the pending
+ * input buffer are all persisted through the `SessionStore` (JSON/Redis/Postgres).
+ */
+export type UserInputContent = UserContent;
+/** Text projection of user input — for confirm-gate parsing, choice matching, and
+ *  extraction hints. Non-text parts are dropped. A plain string returns as-is. */
+export declare function userInputToText(input: UserInputContent): string;
+/** Whether the input carries any non-text (file/image/audio) parts. */
+export declare function hasMediaParts(input: UserInputContent): boolean;
+/**
+ * Replace audio file parts with their transcript (a text part) using an AI SDK
+ * transcription model. With no model configured, audio parts pass through
+ * unchanged — audio-capable models (e.g. Gemini) accept them directly. Non-audio
+ * parts (images, documents) are always left untouched.
+ */
+export declare function transcribeAudioParts(input: UserInputContent, transcriptionModel: TranscriptionModel | undefined): Promise<UserInputContent>;

package/dist/runtime/userInput.js

@@ -0,0 +1,67 @@
+import { experimental_transcribe as transcribe, } from 'ai';
+function isFilePart(part) {
+    return part.type === 'file';
+}
+function isAudioPart(part) {
+    return typeof part.mediaType === 'string' && part.mediaType.startsWith('audio/');
+}
+/**
+ * Normalize a FilePart's `data` into something AI SDK `transcribe` accepts as
+ * `audio`. A bare string is treated by `transcribe` as base64, so http(s) URLs
+ * must become a `URL` (fetched via the built-in download) and `data:` URLs must
+ * be reduced to their base64 payload. Bytes and `URL`s pass through unchanged.
+ */
+function audioSource(data) {
+    if (typeof data !== 'string')
+        return data;
+    if (data.startsWith('data:')) {
+        const comma = data.indexOf(',');
+        return comma === -1 ? data : data.slice(comma + 1);
+    }
+    if (data.startsWith('http://') || data.startsWith('https://')) {
+        return new URL(data);
+    }
+    return data;
+}
+/** Text projection of user input — for confirm-gate parsing, choice matching, and
+ *  extraction hints. Non-text parts are dropped. A plain string returns as-is. */
+export function userInputToText(input) {
+    if (typeof input === 'string')
+        return input;
+    return input
+        .filter((p) => p.type === 'text')
+        .map((p) => p.text)
+        .join(' ')
+        .trim();
+}
+/** Whether the input carries any non-text (file/image/audio) parts. */
+export function hasMediaParts(input) {
+    return typeof input !== 'string' && input.some((p) => p.type !== 'text');
+}
+/**
+ * Replace audio file parts with their transcript (a text part) using an AI SDK
+ * transcription model. With no model configured, audio parts pass through
+ * unchanged — audio-capable models (e.g. Gemini) accept them directly. Non-audio
+ * parts (images, documents) are always left untouched.
+ */
+export async function transcribeAudioParts(input, transcriptionModel) {
+    if (!transcriptionModel || typeof input === 'string')
+        return input;
+    if (!input.some((p) => isFilePart(p) && isAudioPart(p)))
+        return input;
+    const out = [];
+    for (const part of input) {
+        if (isFilePart(part) && isAudioPart(part)) {
+            const { text } = await transcribe({
+                model: transcriptionModel,
+                audio: audioSource(part.data),
+            });
+            if (text.trim())
+                out.push({ type: 'text', text });
+        }
+        else {
+            out.push(part);
+        }
+    }
+    return out;
+}

package/dist/testing/mocks.d.ts

@@ -1,10 +1,11 @@
 import type { HarnessStreamPart, TurnHandle } from '../types/stream.js';
 import type { TurnResult } from '../types/channel.js';
 import type { Session } from '../types/session.js';
+import type { UserInputContent } from '../runtime/userInput.js';
 import type { RuntimeLike } from '../runtime/RuntimeLike.js';
 export interface MockRuntimeRunCall {
     sessionId?: string;
-    input?: string;
+    input?: UserInputContent;
     agentId?: string;
     seedMessages?: unknown[];
 }

package/dist/types/channel.d.ts

@@ -1,4 +1,5 @@
 import type { ToolSet } from 'ai';
+import type { UserInputContent } from '../runtime/userInput.js';
 import type { RunContext } from './run-context.js';
 import type { FlowNode } from './flow.js';
 import type { AnyTool } from './effectTool.js';
@@ -44,7 +45,7 @@ export interface TurnResult {
 }
 export type UserSignal = {
     type: 'message';
-    input: string;
+    input: UserInputContent;
 };
 interface ToolResultRecord {
     name: string;

package/dist/types/run-context.d.ts

@@ -112,6 +112,10 @@ export interface RunContext {
     }): Promise<unknown>;
     now(): Promise<number>;
     uuid(): Promise<string>;
+    /** Rebase durable effect callsites to 0. The runtime calls this at flow entry so
+     *  a flow's durable callsites are anchored to the flow — identical on fresh entry
+     *  (after an answering turn) and on resume (where that turn does not re-run). */
+    resetCallsites(): void;
 }
 export type ActionContext = Pick<RunContext, 'tool' | 'approve' | 'signal' | 'now' | 'uuid' | 'emit' | 'fs'>;
 export type ToolContext = Pick<RunContext, 'session' | 'runState' | 'tool' | 'now' | 'uuid' | 'emit' | 'fs'>;

package/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/kuralle/kuralle-agents.git",
     "directory": "packages/kuralle-core"
   },
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "A framework for structured conversational AI agents",
   "publishConfig": {
     "access": "public"
@@ -103,7 +103,7 @@
     "typescript": "^5.3.0",
     "vitest": "^3.2.4",
     "zod": "^4.0.0",
-    "@kuralle-agents/realtime-audio": "0.7.2"
+    "@kuralle-agents/realtime-audio": "0.8.0"
   },
   "dependencies": {
     "chrono-node": "^2.6.0"