@miraiclip/assistant 0.1.0 → 0.1.2

package/dist/index.d.cts

@@ -1,4 +1,4 @@
-import { Project, Command, ProjectDocument } from '@miraiclip/core';
+import { ProjectDocument, Project, Command } from '@miraiclip/core';
 
 /**
  * The vendor-neutral model contract. An adapter turns these neutral messages
@@ -109,6 +109,12 @@ interface OpenAIChatModelOptions {
     params?: Record<string, unknown>;
     /** Stream responses (default true). */
     stream?: boolean;
+    /**
+     * Retries on rate limits (429) and server errors (500/502/503/504), waiting
+     * as long as the server asks (Retry-After, or "try again in 1.2s"), else
+     * backing off 1 s, 2 s, 4 s… (capped at 30 s). Default 4; 0 disables.
+     */
+    maxRetries?: number;
     /** Adapter id / label (default "openai" / "OpenAI"), e.g. "ollama" when pointing elsewhere. */
     id?: string;
     label?: string;
@@ -137,6 +143,8 @@ type OpenAIMessage = {
 declare function toOpenAIMessages(messages: readonly ChatMessage[]): OpenAIMessage[];
 /** Parse a tool call's JSON arguments; unparseable JSON is reported to the model as a tool error. */
 declare function parseToolArguments(raw: string): Record<string, unknown>;
+/** How long the server asked us to wait (Retry-After header, or "try again in 1.2s / 350ms"), else exponential backoff. */
+declare function retryDelayMs(res: Pick<Response, "headers">, message: string, attempt: number): number;
 declare function openAIChatModel(options: OpenAIChatModelOptions): ChatModel;
 
 /**
@@ -184,6 +192,38 @@ interface RemoteChatModelOptions {
 /** The model a backend serves, as a browser-side `ChatModel`. */
 declare function remoteChatModel(endpoint: string, options?: RemoteChatModelOptions): Promise<ChatModel>;
 
+interface RateLimitOptions {
+    /** Tokens per rolling minute across every caller of this model (input + output). */
+    tokensPerMinute?: number;
+    /** Requests per rolling minute. */
+    requestsPerMinute?: number;
+    /** Token estimate for a request before it's sent (default: JSON length ÷ 4, plus room for the reply). */
+    estimate?: (request: ChatRequest) => number;
+}
+/**
+ * Wrap a model so calls wait for room under your provider's per-minute
+ * limits instead of failing with 429s — for batch jobs (evals, bulk edits)
+ * and for a server sharing one key. Each call reserves an estimate, which is
+ * replaced by the real usage when the response reports it.
+ */
+declare function rateLimitedChatModel(model: ChatModel, options: RateLimitOptions): ChatModel;
+
+/**
+ * The project as the model sees it: core's `describeProject`, plus what a
+ * model needs to make RELATIVE edits ("bigger", "louder", "keep the
+ * entrance") — track names, every clip's position, size and opacity, text
+ * styling, volume, effects and its animation recipe.
+ */
+declare function describeForAssistant(doc: ProjectDocument): string;
+
+interface UnknownFields {
+    unknown: string[];
+    /** The fields the command does take (for the error message). */
+    allowed: string[];
+}
+/** Unknown payload fields for a command, judged against the best-matching schema branch. */
+declare function unknownFields(schema: unknown, payload: unknown): UnknownFields | null;
+
 /** What a tool's `run` gets. `project` is the turn's working copy. */
 interface ToolContext {
     project: Project;
@@ -214,6 +254,8 @@ declare function toolsFromDefinitions(definitions: readonly unknown[], run: (nam
 }>, options?: {
     changes?: (name: string, input: Record<string, unknown>, result: unknown) => string[] | undefined;
 }): AssistantTool[];
+/** The html template set_background uses: one `{{color}}` fill. */
+declare const BACKGROUND_TEMPLATE = "<div style=\"width:100%;height:100%;background:{{color}}\"></div>";
 /** One line per command, for a "what changed" list (keyframe edits fold into one line per clip). */
 declare function describeCommands(commands: readonly Command[]): string[];
 interface EditorToolsOptions {
@@ -228,6 +270,13 @@ interface EditorToolsOptions {
  * animation presets) that use the same logic as the editor's panels.
  */
 declare function editorTools(options?: EditorToolsOptions): AssistantTool[];
+/**
+ * Apply a batch as one all-or-nothing step, with two checks core doesn't do:
+ * fields a command would silently ignore are errors, and commands that
+ * changed nothing are reported, so the model never claims an edit that didn't
+ * happen.
+ */
+declare function applyChecked(project: Project, commands: Command[]): ToolOutput;
 /** apply_commands' input schema: `type` is an enum of the catalog's commands. */
 declare function applyCommandsSchema(catalog: Record<string, unknown>): Record<string, unknown>;
 
@@ -335,7 +384,7 @@ interface Assistant {
     readonly tools: readonly AssistantTool[];
     run(project: Project, request: string, options?: TurnOptions): Promise<AssistantTurn>;
 }
-declare const DEFAULT_INSTRUCTIONS = "You are the editing assistant inside a video editor. You change the user's project by calling tools; every change you make lands as ONE undo step when you finish, so act confidently.\n\nHow to work:\n- The current project is summarized below. Clip, track and asset ids there are what tools expect. Call get_state to re-read it after big changes.\n- Prefer the high-level tools when they fit (add_transition, animate_clip, and any audio or media tools). Use apply_commands for everything else, batching related commands; look up a command's payload with get_command_schema the first time you use it.\n- Commands use integer microseconds (1 second = 1000000). High-level tools take seconds.\n- Positions (transform x, y) are fractions of the frame: 0.5, 0.5 is the center.\n- Only reference media that exists in the project or that a tool returned. Never invent file URLs.\n- If a tool fails, read the error, fix the input and retry. If something can't be done, say why.\n- If the request is ambiguous in a way that matters, ask one short question instead of guessing.\n\nWhen you're done, reply in one to three short sentences: what you changed, in plain words (times in seconds, names over ids). No markdown headings.";
+declare const DEFAULT_INSTRUCTIONS = "You are the editing assistant inside a video editor. You change the user's project by calling tools; every change you make lands as ONE undo step when you finish, so act confidently.\n\nHow to work:\n- The current project is summarized below. Clip, track and asset ids there are what tools expect. Call get_state to re-read it after big changes.\n- Prefer the high-level tools when they fit (add_transition, animate_clip, add_effect, set_effects_enabled, set_background, trim_clip, set_keyframes, close_gaps, and any audio or media tools). Call tools directly; apply_commands takes only editing commands like \"clip/add\", never tool names. Use apply_commands for everything else, batching related commands; look up a command's payload with get_command_schema the first time you use it, and use only the fields it lists.\n- Fading a picture (text, image, logo, video) in or out is animate_clip (in / out: fade). fadeInUs / fadeOutUs are audio fades. \"Fade the video in at the start / out at the end\" means the first clip's entrance and the last clip's exit.\n- Constant values are set directly with clip/set-property: rotation (\"rotate 45\u00B0\", \"tilt\"), volume (\"lower to 30%\", \"louder\"), position, size, opacity. Only values that CHANGE over a time range (\"duck the music between 10 and 15 s\", \"fade to silence between 18 and 20 s\", \"spin one full turn\", \"pan across\") are keyframes: set_keyframes, with times in timeline seconds. Remove an animation with keyframe/clear.\n- \"Cuts\" are the ones listed in the project summary; use their times (e.g. one sound per cut).\n- \"Add N more seconds of footage\" adds a new clip (clip/add with trimStartUs for the source point) after the last one; it does not stretch an existing clip.\n- Caption looks (box behind the text, outline, shadow, colors, word preset) are the caption clip's style (clip/set-property { style }).\n- \"The whole video\" means from 0 to the composition length (trim_clip startSeconds / endSeconds).\n- A solid color background behind everything is set_background (add, recolor, remove). Other html clips are recolored through the params their template uses (shown in the clip details). A new track goes on top unless track/add gets index 0 (the bottom).\n- \"Replace X with Y\" removes X and adds Y in the same turn, whichever tool adds Y (audio tools take replaceClipId, which does both).\n- Solo / mute / lock / hide a track with track/set-property (solo, muted, locked, hidden). \"Solo X\" is solo: true on X, not muting the others.\n- Relative requests (\"bigger\", \"louder\", \"a bit brighter\", \"twice as big\") start from the current values in the clip details below: make a visible change in the asked direction (about 25% for \"bigger\"/\"smaller\", \u00D72 for \"twice\").\n- When something is selected, \"this\", \"it\", \"the clip\" and any request that names no target mean ONLY the selection, even with words like \"at the end\" (the end of that clip). Edit nothing else. \"Here\" / \"now\" / \"this point\" mean the playhead. \"Clips\" without a kind mean the video clips on the main video track.\n- \"Again\" / \"also\" / \"another\" mean keep the original and add a copy (clip/duplicate, or clip/add with the same asset); never move the original. \"Instead\" means move or replace.\n- Removing a clip leaves a gap; when asked to close it, also run close_gaps (or trim_clip with ripple).\n- Commands use integer microseconds (1 second = 1000000). High-level tools take seconds.\n- Positions (transform x, y) are fractions of the frame: 0.5, 0.5 is the center.\n- Only reference media that exists in the project or that a tool returned. Never invent file URLs.\n- If a tool fails, read the error, fix the input and retry. A failed apply_commands batch applied NONE of its commands: resend all of them, corrected. If something can't be done, say why.\n- If the request is ambiguous in a way that matters, ask one short question instead of guessing.\n\nWhen you're done, reply in one to three short sentences: what you changed, in plain words (times in seconds, names over ids). Describe only edits your tools reported as applied; if a tool says something changed nothing or failed, fix it or say so. No markdown headings.";
 declare function createAssistant(options: AssistantOptions): Assistant;
 
 type ScriptStep = string | {
@@ -355,4 +404,4 @@ declare function scriptedChatModel(steps: ScriptStep[], options?: {
     requests: ChatRequest[];
 };
 
-export { type Assistant, type AssistantEvent, type AssistantOptions, type AssistantTool, type AssistantTurn, type ChatHandlerOptions, type ChatMessage, type ChatModel, ChatModelError, type ChatModelInfo, type ChatOptions, type ChatRequest, type ChatResponse, type ChatUsage, type CommitResult, DEFAULT_INSTRUCTIONS, type EditorToolsOptions, type FetchLike, type ForkOptions, type OpenAIChatModelOptions, type RecordingProject, type RemoteChatModelOptions, type ScriptStep, type ToolCall, type ToolContext, type ToolOutput, type ToolSpec, type TurnOptions, applyCommandsSchema, commitFork, createAssistant, createChatHandler, defineChatModel, defineTool, describeCommands, editorTools, forkProject, openAIChatModel, parseToolArguments, remoteChatModel, scriptedChatModel, toOpenAIMessages, toolsFromDefinitions, withExplicitIds };
+export { type Assistant, type AssistantEvent, type AssistantOptions, type AssistantTool, type AssistantTurn, BACKGROUND_TEMPLATE, type ChatHandlerOptions, type ChatMessage, type ChatModel, ChatModelError, type ChatModelInfo, type ChatOptions, type ChatRequest, type ChatResponse, type ChatUsage, type CommitResult, DEFAULT_INSTRUCTIONS, type EditorToolsOptions, type FetchLike, type ForkOptions, type OpenAIChatModelOptions, type RateLimitOptions, type RecordingProject, type RemoteChatModelOptions, type ScriptStep, type ToolCall, type ToolContext, type ToolOutput, type ToolSpec, type TurnOptions, type UnknownFields, applyChecked, applyCommandsSchema, commitFork, createAssistant, createChatHandler, defineChatModel, defineTool, describeCommands, describeForAssistant, editorTools, forkProject, openAIChatModel, parseToolArguments, rateLimitedChatModel, remoteChatModel, retryDelayMs, scriptedChatModel, toOpenAIMessages, toolsFromDefinitions, unknownFields, withExplicitIds };

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { Project, Command, ProjectDocument } from '@miraiclip/core';
+import { ProjectDocument, Project, Command } from '@miraiclip/core';
 
 /**
  * The vendor-neutral model contract. An adapter turns these neutral messages
@@ -109,6 +109,12 @@ interface OpenAIChatModelOptions {
     params?: Record<string, unknown>;
     /** Stream responses (default true). */
     stream?: boolean;
+    /**
+     * Retries on rate limits (429) and server errors (500/502/503/504), waiting
+     * as long as the server asks (Retry-After, or "try again in 1.2s"), else
+     * backing off 1 s, 2 s, 4 s… (capped at 30 s). Default 4; 0 disables.
+     */
+    maxRetries?: number;
     /** Adapter id / label (default "openai" / "OpenAI"), e.g. "ollama" when pointing elsewhere. */
     id?: string;
     label?: string;
@@ -137,6 +143,8 @@ type OpenAIMessage = {
 declare function toOpenAIMessages(messages: readonly ChatMessage[]): OpenAIMessage[];
 /** Parse a tool call's JSON arguments; unparseable JSON is reported to the model as a tool error. */
 declare function parseToolArguments(raw: string): Record<string, unknown>;
+/** How long the server asked us to wait (Retry-After header, or "try again in 1.2s / 350ms"), else exponential backoff. */
+declare function retryDelayMs(res: Pick<Response, "headers">, message: string, attempt: number): number;
 declare function openAIChatModel(options: OpenAIChatModelOptions): ChatModel;
 
 /**
@@ -184,6 +192,38 @@ interface RemoteChatModelOptions {
 /** The model a backend serves, as a browser-side `ChatModel`. */
 declare function remoteChatModel(endpoint: string, options?: RemoteChatModelOptions): Promise<ChatModel>;
 
+interface RateLimitOptions {
+    /** Tokens per rolling minute across every caller of this model (input + output). */
+    tokensPerMinute?: number;
+    /** Requests per rolling minute. */
+    requestsPerMinute?: number;
+    /** Token estimate for a request before it's sent (default: JSON length ÷ 4, plus room for the reply). */
+    estimate?: (request: ChatRequest) => number;
+}
+/**
+ * Wrap a model so calls wait for room under your provider's per-minute
+ * limits instead of failing with 429s — for batch jobs (evals, bulk edits)
+ * and for a server sharing one key. Each call reserves an estimate, which is
+ * replaced by the real usage when the response reports it.
+ */
+declare function rateLimitedChatModel(model: ChatModel, options: RateLimitOptions): ChatModel;
+
+/**
+ * The project as the model sees it: core's `describeProject`, plus what a
+ * model needs to make RELATIVE edits ("bigger", "louder", "keep the
+ * entrance") — track names, every clip's position, size and opacity, text
+ * styling, volume, effects and its animation recipe.
+ */
+declare function describeForAssistant(doc: ProjectDocument): string;
+
+interface UnknownFields {
+    unknown: string[];
+    /** The fields the command does take (for the error message). */
+    allowed: string[];
+}
+/** Unknown payload fields for a command, judged against the best-matching schema branch. */
+declare function unknownFields(schema: unknown, payload: unknown): UnknownFields | null;
+
 /** What a tool's `run` gets. `project` is the turn's working copy. */
 interface ToolContext {
     project: Project;
@@ -214,6 +254,8 @@ declare function toolsFromDefinitions(definitions: readonly unknown[], run: (nam
 }>, options?: {
     changes?: (name: string, input: Record<string, unknown>, result: unknown) => string[] | undefined;
 }): AssistantTool[];
+/** The html template set_background uses: one `{{color}}` fill. */
+declare const BACKGROUND_TEMPLATE = "<div style=\"width:100%;height:100%;background:{{color}}\"></div>";
 /** One line per command, for a "what changed" list (keyframe edits fold into one line per clip). */
 declare function describeCommands(commands: readonly Command[]): string[];
 interface EditorToolsOptions {
@@ -228,6 +270,13 @@ interface EditorToolsOptions {
  * animation presets) that use the same logic as the editor's panels.
  */
 declare function editorTools(options?: EditorToolsOptions): AssistantTool[];
+/**
+ * Apply a batch as one all-or-nothing step, with two checks core doesn't do:
+ * fields a command would silently ignore are errors, and commands that
+ * changed nothing are reported, so the model never claims an edit that didn't
+ * happen.
+ */
+declare function applyChecked(project: Project, commands: Command[]): ToolOutput;
 /** apply_commands' input schema: `type` is an enum of the catalog's commands. */
 declare function applyCommandsSchema(catalog: Record<string, unknown>): Record<string, unknown>;
 
@@ -335,7 +384,7 @@ interface Assistant {
     readonly tools: readonly AssistantTool[];
     run(project: Project, request: string, options?: TurnOptions): Promise<AssistantTurn>;
 }
-declare const DEFAULT_INSTRUCTIONS = "You are the editing assistant inside a video editor. You change the user's project by calling tools; every change you make lands as ONE undo step when you finish, so act confidently.\n\nHow to work:\n- The current project is summarized below. Clip, track and asset ids there are what tools expect. Call get_state to re-read it after big changes.\n- Prefer the high-level tools when they fit (add_transition, animate_clip, and any audio or media tools). Use apply_commands for everything else, batching related commands; look up a command's payload with get_command_schema the first time you use it.\n- Commands use integer microseconds (1 second = 1000000). High-level tools take seconds.\n- Positions (transform x, y) are fractions of the frame: 0.5, 0.5 is the center.\n- Only reference media that exists in the project or that a tool returned. Never invent file URLs.\n- If a tool fails, read the error, fix the input and retry. If something can't be done, say why.\n- If the request is ambiguous in a way that matters, ask one short question instead of guessing.\n\nWhen you're done, reply in one to three short sentences: what you changed, in plain words (times in seconds, names over ids). No markdown headings.";
+declare const DEFAULT_INSTRUCTIONS = "You are the editing assistant inside a video editor. You change the user's project by calling tools; every change you make lands as ONE undo step when you finish, so act confidently.\n\nHow to work:\n- The current project is summarized below. Clip, track and asset ids there are what tools expect. Call get_state to re-read it after big changes.\n- Prefer the high-level tools when they fit (add_transition, animate_clip, add_effect, set_effects_enabled, set_background, trim_clip, set_keyframes, close_gaps, and any audio or media tools). Call tools directly; apply_commands takes only editing commands like \"clip/add\", never tool names. Use apply_commands for everything else, batching related commands; look up a command's payload with get_command_schema the first time you use it, and use only the fields it lists.\n- Fading a picture (text, image, logo, video) in or out is animate_clip (in / out: fade). fadeInUs / fadeOutUs are audio fades. \"Fade the video in at the start / out at the end\" means the first clip's entrance and the last clip's exit.\n- Constant values are set directly with clip/set-property: rotation (\"rotate 45\u00B0\", \"tilt\"), volume (\"lower to 30%\", \"louder\"), position, size, opacity. Only values that CHANGE over a time range (\"duck the music between 10 and 15 s\", \"fade to silence between 18 and 20 s\", \"spin one full turn\", \"pan across\") are keyframes: set_keyframes, with times in timeline seconds. Remove an animation with keyframe/clear.\n- \"Cuts\" are the ones listed in the project summary; use their times (e.g. one sound per cut).\n- \"Add N more seconds of footage\" adds a new clip (clip/add with trimStartUs for the source point) after the last one; it does not stretch an existing clip.\n- Caption looks (box behind the text, outline, shadow, colors, word preset) are the caption clip's style (clip/set-property { style }).\n- \"The whole video\" means from 0 to the composition length (trim_clip startSeconds / endSeconds).\n- A solid color background behind everything is set_background (add, recolor, remove). Other html clips are recolored through the params their template uses (shown in the clip details). A new track goes on top unless track/add gets index 0 (the bottom).\n- \"Replace X with Y\" removes X and adds Y in the same turn, whichever tool adds Y (audio tools take replaceClipId, which does both).\n- Solo / mute / lock / hide a track with track/set-property (solo, muted, locked, hidden). \"Solo X\" is solo: true on X, not muting the others.\n- Relative requests (\"bigger\", \"louder\", \"a bit brighter\", \"twice as big\") start from the current values in the clip details below: make a visible change in the asked direction (about 25% for \"bigger\"/\"smaller\", \u00D72 for \"twice\").\n- When something is selected, \"this\", \"it\", \"the clip\" and any request that names no target mean ONLY the selection, even with words like \"at the end\" (the end of that clip). Edit nothing else. \"Here\" / \"now\" / \"this point\" mean the playhead. \"Clips\" without a kind mean the video clips on the main video track.\n- \"Again\" / \"also\" / \"another\" mean keep the original and add a copy (clip/duplicate, or clip/add with the same asset); never move the original. \"Instead\" means move or replace.\n- Removing a clip leaves a gap; when asked to close it, also run close_gaps (or trim_clip with ripple).\n- Commands use integer microseconds (1 second = 1000000). High-level tools take seconds.\n- Positions (transform x, y) are fractions of the frame: 0.5, 0.5 is the center.\n- Only reference media that exists in the project or that a tool returned. Never invent file URLs.\n- If a tool fails, read the error, fix the input and retry. A failed apply_commands batch applied NONE of its commands: resend all of them, corrected. If something can't be done, say why.\n- If the request is ambiguous in a way that matters, ask one short question instead of guessing.\n\nWhen you're done, reply in one to three short sentences: what you changed, in plain words (times in seconds, names over ids). Describe only edits your tools reported as applied; if a tool says something changed nothing or failed, fix it or say so. No markdown headings.";
 declare function createAssistant(options: AssistantOptions): Assistant;
 
 type ScriptStep = string | {
@@ -355,4 +404,4 @@ declare function scriptedChatModel(steps: ScriptStep[], options?: {
     requests: ChatRequest[];
 };
 
-export { type Assistant, type AssistantEvent, type AssistantOptions, type AssistantTool, type AssistantTurn, type ChatHandlerOptions, type ChatMessage, type ChatModel, ChatModelError, type ChatModelInfo, type ChatOptions, type ChatRequest, type ChatResponse, type ChatUsage, type CommitResult, DEFAULT_INSTRUCTIONS, type EditorToolsOptions, type FetchLike, type ForkOptions, type OpenAIChatModelOptions, type RecordingProject, type RemoteChatModelOptions, type ScriptStep, type ToolCall, type ToolContext, type ToolOutput, type ToolSpec, type TurnOptions, applyCommandsSchema, commitFork, createAssistant, createChatHandler, defineChatModel, defineTool, describeCommands, editorTools, forkProject, openAIChatModel, parseToolArguments, remoteChatModel, scriptedChatModel, toOpenAIMessages, toolsFromDefinitions, withExplicitIds };
+export { type Assistant, type AssistantEvent, type AssistantOptions, type AssistantTool, type AssistantTurn, BACKGROUND_TEMPLATE, type ChatHandlerOptions, type ChatMessage, type ChatModel, ChatModelError, type ChatModelInfo, type ChatOptions, type ChatRequest, type ChatResponse, type ChatUsage, type CommitResult, DEFAULT_INSTRUCTIONS, type EditorToolsOptions, type FetchLike, type ForkOptions, type OpenAIChatModelOptions, type RateLimitOptions, type RecordingProject, type RemoteChatModelOptions, type ScriptStep, type ToolCall, type ToolContext, type ToolOutput, type ToolSpec, type TurnOptions, type UnknownFields, applyChecked, applyCommandsSchema, commitFork, createAssistant, createChatHandler, defineChatModel, defineTool, describeCommands, describeForAssistant, editorTools, forkProject, openAIChatModel, parseToolArguments, rateLimitedChatModel, remoteChatModel, retryDelayMs, scriptedChatModel, toOpenAIMessages, toolsFromDefinitions, unknownFields, withExplicitIds };