@miraiclip/assistant 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,358 @@
+import { Project, Command, ProjectDocument } from '@miraiclip/core';
+
+/**
+ * The vendor-neutral model contract. An adapter turns these neutral messages
+ * and tools into one vendor's API and back; nothing else in the package knows
+ * which vendor is on the other end.
+ */
+/** A tool call the model made. `arguments` is already parsed JSON. */
+interface ToolCall {
+    id: string;
+    name: string;
+    arguments: Record<string, unknown>;
+}
+type ChatMessage = {
+    role: "system";
+    content: string;
+} | {
+    role: "user";
+    content: string;
+} | {
+    role: "assistant";
+    content: string;
+    toolCalls?: ToolCall[];
+} | {
+    role: "tool";
+    toolCallId: string;
+    name: string;
+    content: string;
+};
+/** A tool as the model sees it: name, description and a JSON Schema for its input. */
+interface ToolSpec {
+    name: string;
+    description: string;
+    inputSchema: Record<string, unknown>;
+}
+interface ChatRequest {
+    messages: ChatMessage[];
+    tools?: ToolSpec[];
+    /** Model id, when the adapter serves several. */
+    model?: string;
+    maxOutputTokens?: number;
+    /** Vendor-specific request fields, passed through untouched (e.g. `{ reasoning_effort: "low" }`). */
+    params?: Record<string, unknown>;
+}
+interface ChatUsage {
+    inputTokens?: number;
+    outputTokens?: number;
+}
+interface ChatResponse {
+    content: string;
+    toolCalls: ToolCall[];
+    /** Why the model stopped. */
+    finish: "stop" | "tool_calls" | "length" | "content_filter" | "other";
+    model?: string;
+    usage?: ChatUsage;
+}
+interface ChatOptions {
+    signal?: AbortSignal;
+    /** Streaming text, as it arrives (adapters that can't stream call it once). */
+    onText?: (delta: string) => void;
+}
+interface ChatModelInfo {
+    /** Adapter id, e.g. "openai". Alphanumeric with - or _. */
+    id: string;
+    label: string;
+    /** Models this adapter serves; the first is the default. */
+    models?: {
+        id: string;
+        label?: string;
+    }[];
+}
+interface ChatModel extends ChatModelInfo {
+    complete(request: ChatRequest, options?: ChatOptions): Promise<ChatResponse>;
+}
+/** An error from a model vendor, carrying its HTTP status when there was one. */
+declare class ChatModelError extends Error {
+    readonly provider: string;
+    readonly status?: number | undefined;
+    constructor(provider: string, message: string, status?: number | undefined);
+}
+type FetchLike = (url: string, init?: RequestInit) => Promise<Response>;
+/** Check an adapter's shape once, at registration. */
+declare function defineChatModel<M extends ChatModel>(model: M): M;
+
+/**
+ * OpenAI adapter, over the Chat Completions API with function calling.
+ *
+ * Chat Completions is also what most other servers speak (Azure OpenAI,
+ * OpenRouter, Groq, Together, vLLM, Ollama, LM Studio…), so `baseUrl` points
+ * this one adapter at any of them.
+ */
+
+interface OpenAIChatModelOptions {
+    /** Model id, e.g. "gpt-5.4-mini". Required: model names change faster than this package. */
+    model: string;
+    /** Other models the app may pick per request (`ChatRequest.model`). */
+    models?: {
+        id: string;
+        label?: string;
+    }[];
+    /** API key. Omit for local servers that don't need one. Keep it server-side (see `createChatHandler`). */
+    apiKey?: string;
+    /** Default https://api.openai.com/v1. Any OpenAI-compatible base URL works. */
+    baseUrl?: string;
+    organization?: string;
+    project?: string;
+    headers?: Record<string, string>;
+    /** Default request fields merged into every call (e.g. `{ reasoning_effort: "low" }`). */
+    params?: Record<string, unknown>;
+    /** Stream responses (default true). */
+    stream?: boolean;
+    /** Adapter id / label (default "openai" / "OpenAI"), e.g. "ollama" when pointing elsewhere. */
+    id?: string;
+    label?: string;
+    fetch?: FetchLike;
+}
+type OpenAIMessage = {
+    role: "system" | "user";
+    content: string;
+} | {
+    role: "assistant";
+    content: string | null;
+    tool_calls?: {
+        id: string;
+        type: "function";
+        function: {
+            name: string;
+            arguments: string;
+        };
+    }[];
+} | {
+    role: "tool";
+    tool_call_id: string;
+    content: string;
+};
+/** Neutral messages → Chat Completions messages. */
+declare function toOpenAIMessages(messages: readonly ChatMessage[]): OpenAIMessage[];
+/** Parse a tool call's JSON arguments; unparseable JSON is reported to the model as a tool error. */
+declare function parseToolArguments(raw: string): Record<string, unknown>;
+declare function openAIChatModel(options: OpenAIChatModelOptions): ChatModel;
+
+/**
+ * Keys stay on the server: run the real model adapter in your backend behind
+ * `createChatHandler`, and give the browser `remoteChatModel(endpoint)` — a
+ * proxy with the same `ChatModel` shape. The agent loop and the tools run in
+ * the browser, next to the project; only model calls cross the network.
+ *
+ *   GET  {base}/       → { id, label, models }
+ *   POST {base}/chat   { request: ChatRequest } →
+ *        application/x-ndjson, one JSON object per line:
+ *        { "type": "text", "delta": "…" }   (as it streams)
+ *        { "type": "done", "response": ChatResponse }
+ *        { "type": "error", "error": "…", "status": 401 }
+ *   errors before streaming → { error } with 400 / 404 / 413, or the vendor's status
+ *
+ * Fetch-API handler (`Request` → `Response`): mounts in Node 18+, edge
+ * runtimes, Next, Hono, Express (via an adapter).
+ */
+
+interface ChatHandlerOptions {
+    /** Path prefix the handler is mounted at, e.g. "/api/assistant". */
+    basePath?: string;
+    /** Reject request bodies larger than this (default 2 MB). */
+    maxBodyBytes?: number;
+    /** Return a Response to refuse a request (auth, rate limits); undefined to allow. */
+    authorize?: (request: Request) => Promise<Response | undefined> | Response | undefined;
+    /** Adjust each request server-side (pin a model, cap tokens, add instructions). */
+    prepare?: (request: ChatRequest) => ChatRequest;
+}
+declare function createChatHandler(model: ChatModel, options?: ChatHandlerOptions): (request: Request) => Promise<Response | null>;
+interface RemoteChatModelOptions {
+    fetch?: FetchLike;
+    headers?: Record<string, string>;
+    /** Skip the GET for model info (id/label/models default to "remote"). */
+    info?: {
+        id: string;
+        label: string;
+        models?: {
+            id: string;
+            label?: string;
+        }[];
+    };
+}
+/** The model a backend serves, as a browser-side `ChatModel`. */
+declare function remoteChatModel(endpoint: string, options?: RemoteChatModelOptions): Promise<ChatModel>;
+
+/** What a tool's `run` gets. `project` is the turn's working copy. */
+interface ToolContext {
+    project: Project;
+    signal?: AbortSignal;
+}
+/** A tool's answer. `changes` are human-readable lines for the app's "what changed" list. */
+type ToolOutput = {
+    ok: true;
+    result?: unknown;
+    changes?: string[];
+} | {
+    ok: false;
+    error: unknown;
+};
+interface AssistantTool extends ToolSpec {
+    run(input: Record<string, unknown>, ctx: ToolContext): Promise<ToolOutput> | ToolOutput;
+}
+declare function defineTool(tool: AssistantTool): AssistantTool;
+/**
+ * Adopt tools that come as definitions plus one runner — e.g.
+ * `@miraiclip/audio-sources`' `audioToolDefinitions` + `runAudioTool`.
+ * Accepts `{ name, description, input_schema }` or OpenAI function tools.
+ */
+declare function toolsFromDefinitions(definitions: readonly unknown[], run: (name: string, input: Record<string, unknown>, ctx: ToolContext) => Promise<{
+    ok: boolean;
+    result?: unknown;
+    error?: unknown;
+}>, options?: {
+    changes?: (name: string, input: Record<string, unknown>, result: unknown) => string[] | undefined;
+}): AssistantTool[];
+/** One line per command, for a "what changed" list (keyframe edits fold into one line per clip). */
+declare function describeCommands(commands: readonly Command[]): string[];
+interface EditorToolsOptions {
+    /** Transition kinds to offer (default: the built-ins). Add your registered custom kinds. */
+    transitionKinds?: string[];
+    /** The command catalog to offer in apply_commands (default: the built-ins). Pass `project.commandCatalog()` when you register custom commands. */
+    commandCatalog?: Record<string, unknown>;
+}
+/**
+ * The tools that edit a Miraiclip project: read state, look up a command's
+ * schema, apply command batches, and two high-level tools (transitions,
+ * animation presets) that use the same logic as the editor's panels.
+ */
+declare function editorTools(options?: EditorToolsOptions): AssistantTool[];
+/** apply_commands' input schema: `type` is an enum of the catalog's commands. */
+declare function applyCommandsSchema(catalog: Record<string, unknown>): Record<string, unknown>;
+
+/**
+ * A working copy of a project that records every command that sticks.
+ *
+ * The assistant edits the copy while the model thinks (async, many steps);
+ * the real project only changes when the turn commits, by replaying the
+ * recorded commands in one transaction — one undo step, and nothing half-done
+ * if the request fails. Commands inside a transaction that rolls back are
+ * dropped, exactly as the copy drops their effects.
+ *
+ * Replay must produce the same ids the model saw, so commands whose ids are
+ * optional get explicit ones before they run.
+ */
+interface RecordingProject extends Project {
+    /** Commands that changed the copy, in order. */
+    readonly recorded: readonly Command[];
+}
+interface ForkOptions {
+    /** Build the copy (default: `createProject(doc)`). Register custom commands here. */
+    createFork?: (doc: ReturnType<Project["toJSON"]>) => Project;
+    /** Give commands explicit ids (default handles the built-ins with optional ids). */
+    ensureIds?: (command: Command) => Command;
+}
+/** Built-in commands that would otherwise get a random id inside the engine. */
+declare function withExplicitIds(command: Command): Command;
+declare function forkProject(project: Project, options?: ForkOptions): RecordingProject;
+type CommitResult = {
+    ok: true;
+    applied: number;
+} | {
+    ok: false;
+    error: string;
+};
+/** Replay a fork's commands onto the real project as one undoable step. */
+declare function commitFork(project: Project, fork: RecordingProject, label?: string): CommitResult;
+
+interface AssistantOptions {
+    model: ChatModel;
+    /** Default `editorTools()`. Add your own (audio, captions, stock media…). */
+    tools?: AssistantTool[];
+    /** Extra system instructions (brand voice, house rules). */
+    instructions?: string;
+    /** Model round-trips per request (default 12). */
+    maxSteps?: number;
+    /** Model id when the adapter serves several. */
+    modelId?: string;
+    maxOutputTokens?: number;
+    /** Vendor request fields, passed through. */
+    params?: Record<string, unknown>;
+    fork?: ForkOptions;
+}
+type AssistantEvent = {
+    type: "text";
+    delta: string;
+} | {
+    type: "tool-start";
+    call: ToolCall;
+} | {
+    type: "tool-end";
+    call: ToolCall;
+    ok: boolean;
+    changes?: string[];
+    error?: string;
+} | {
+    type: "step";
+    index: number;
+};
+interface TurnOptions {
+    /** Earlier messages of this conversation (`turn.messages` from the last turn). */
+    history?: readonly ChatMessage[];
+    /** What the user is looking at, e.g. "Selected: clip t-1. Playhead: 4.2s." */
+    context?: string;
+    /** "auto" (default) commits when the request finishes; "review" waits for `turn.apply()`. */
+    apply?: "auto" | "review";
+    signal?: AbortSignal;
+    onEvent?: (event: AssistantEvent) => void;
+    /** History label for the undo step (default "Assistant: <request>"). */
+    label?: string;
+}
+interface AssistantTurn {
+    /** The model's final answer to the user. */
+    reply: string;
+    /** Human-readable list of what the turn changed (empty when nothing). */
+    changes: string[];
+    /** The commands the turn makes, in order (replayed onto the project on apply). */
+    commands: readonly Command[];
+    /**
+     * applied: committed as one undo step. review: waiting for apply()/discard().
+     * no-changes: nothing to apply. failed / canceled: the project wasn't touched.
+     */
+    status: "applied" | "review" | "no-changes" | "failed" | "canceled";
+    error?: string;
+    /** The project as the turn left it (for previews in review mode). */
+    preview: ProjectDocument;
+    /** The conversation so far, without the system prompt: pass back as `history`. */
+    messages: ChatMessage[];
+    usage: Required<ChatUsage>;
+    /** Commit a reviewed turn (no-op once applied). */
+    apply(): CommitResult;
+    discard(): void;
+}
+interface Assistant {
+    readonly tools: readonly AssistantTool[];
+    run(project: Project, request: string, options?: TurnOptions): Promise<AssistantTurn>;
+}
+declare const DEFAULT_INSTRUCTIONS = "You are the editing assistant inside a video editor. You change the user's project by calling tools; every change you make lands as ONE undo step when you finish, so act confidently.\n\nHow to work:\n- The current project is summarized below. Clip, track and asset ids there are what tools expect. Call get_state to re-read it after big changes.\n- Prefer the high-level tools when they fit (add_transition, animate_clip, and any audio or media tools). Use apply_commands for everything else, batching related commands; look up a command's payload with get_command_schema the first time you use it.\n- Commands use integer microseconds (1 second = 1000000). High-level tools take seconds.\n- Positions (transform x, y) are fractions of the frame: 0.5, 0.5 is the center.\n- Only reference media that exists in the project or that a tool returned. Never invent file URLs.\n- If a tool fails, read the error, fix the input and retry. If something can't be done, say why.\n- If the request is ambiguous in a way that matters, ask one short question instead of guessing.\n\nWhen you're done, reply in one to three short sentences: what you changed, in plain words (times in seconds, names over ids). No markdown headings.";
+declare function createAssistant(options: AssistantOptions): Assistant;
+
+type ScriptStep = string | {
+    text?: string;
+    toolCalls?: (Omit<ToolCall, "id"> & {
+        id?: string;
+    })[];
+} | ((request: ChatRequest, index: number) => ScriptStep);
+/**
+ * A model that answers from a script: for tests, demos and offline
+ * development. Each `complete` call takes the next step; a string is a final
+ * text answer, an object can call tools. Requests are kept in `requests`.
+ */
+declare function scriptedChatModel(steps: ScriptStep[], options?: {
+    id?: string;
+}): ChatModel & {
+    requests: ChatRequest[];
+};
+
+export { type Assistant, type AssistantEvent, type AssistantOptions, type AssistantTool, type AssistantTurn, type ChatHandlerOptions, type ChatMessage, type ChatModel, ChatModelError, type ChatModelInfo, type ChatOptions, type ChatRequest, type ChatResponse, type ChatUsage, type CommitResult, DEFAULT_INSTRUCTIONS, type EditorToolsOptions, type FetchLike, type ForkOptions, type OpenAIChatModelOptions, type RecordingProject, type RemoteChatModelOptions, type ScriptStep, type ToolCall, type ToolContext, type ToolOutput, type ToolSpec, type TurnOptions, applyCommandsSchema, commitFork, createAssistant, createChatHandler, defineChatModel, defineTool, describeCommands, editorTools, forkProject, openAIChatModel, parseToolArguments, remoteChatModel, scriptedChatModel, toOpenAIMessages, toolsFromDefinitions, withExplicitIds };