@chai-ui/core 0.7.1 → 0.9.0

package/src/composer-view.ts

@@ -9,15 +9,29 @@ import { readMediaFiles } from "./media-analyzer.js";
 import type { EditRegion } from "./region-edit.js";
 import { deriveGenerationUseCase } from "./run.js";
 import { suggestedModels } from "./suggested-models.js";
-import type { MediaKind, ModelOption, ParameterOption } from "./types.js";
+import type { ModelOption, OutputKind, ParameterOption } from "./types.js";
 
-export type ComposerAttachMenuAction = "add-media" | "create-image" | "create-video" | "edit-media";
+/** Which prompt field an Enhance click is for: the bar's prompt, or a world's environment or character. */
+export type ComposerPromptField = "prompt" | "environment" | "character";
 
-/** The `+` menu, in order. "Add media" is the only item with a divider after it. */
-export const COMPOSER_ATTACH_MENU_ITEMS: { action: ComposerAttachMenuAction; label: string; divider?: boolean }[] = [
+export type ComposerAttachMenuAction = "add-media" | "create-image" | "create-video" | "create-world" | "edit-media";
+
+/**
+ * The `+` menu, in order. "Add media" is the only item with a divider after
+ * it. An `optIn` item shows only when `attachMenuActions` lists it.
+ */
+export const COMPOSER_ATTACH_MENU_ITEMS: {
+  action: ComposerAttachMenuAction;
+  label: string;
+  divider?: boolean;
+  optIn?: boolean;
+}[] = [
   { action: "add-media", label: "Add media", divider: true },
   { action: "create-image", label: "Create image" },
   { action: "create-video", label: "Create video" },
+  // No suggested world models or shipped world engine yet: a builder adds
+  // this with their own models and `engineByKind.world`.
+  { action: "create-world", label: "Create world", optIn: true },
   { action: "edit-media", label: "Edit image" },
 ];
 
@@ -41,7 +55,7 @@ export interface ComposerViewInput {
   defaultUseCase?: ComposerUseCase;
   multiSelectUseCases?: boolean;
   useCases?: ComposerUseCase[];
-  modelsByKind?: Partial<Record<MediaKind, ModelOption[]>>;
+  modelsByKind?: Partial<Record<OutputKind, ModelOption[]>>;
   multiSelectModels?: boolean;
   modelIds?: string[];
   models?: ModelOption[];
@@ -51,6 +65,9 @@ export interface ComposerViewInput {
   autoSelectModel?: boolean;
   aspectRatios?: ParameterOption<string>[];
   aspectRatio?: string | null;
+  /** A world's character field. */
+  character?: string;
+  frameRate?: number | null;
   submitError?: string | null;
   submitDisabled?: boolean;
   submitting?: boolean;
@@ -75,6 +92,8 @@ export function composerView(props: ComposerViewInput) {
     autoSelectModel = false,
     aspectRatios,
     aspectRatio = null,
+    character = "",
+    frameRate = null,
     submitError = null,
     submitting = false,
     regions = [],
@@ -88,17 +107,27 @@ export function composerView(props: ComposerViewInput) {
   // An edit needs an image, then a change: a prompt for the whole image, a
   // region with its own instruction, or both.
   const editing = activeUseCases.some((u) => u.edit);
+  // A world takes two prompts: the environment (`value`, required) and the
+  // character the person plays (optional). An image can start the world,
+  // but the environment is still needed.
+  const world = activeUseCases.some((u) => u.kind === "world");
   const hasImage = attachments.some((a) => a.kind === "image");
   const hasContent = editing
     ? hasImage && (value.trim().length > 0 || regions.length > 0)
-    : value.trim().length > 0 || attachments.length > 0;
+    : world
+      ? value.trim().length > 0
+      : value.trim().length > 0 || attachments.length > 0;
   const placeholder =
     props.placeholder ??
     (editing
       ? hasImage
         ? "Describe a change across the whole image, or leave this empty"
         : "Attach an image to edit"
-      : "Describe media to create");
+      : world
+        ? attachments.length > 0
+          ? "Describe the world, and how to use the attached media"
+          : "Describe the world and how you move through it"
+        : "Describe media to create");
   // The stop icon only appears with somewhere for its click to go.
   const showStop = submitting && Boolean(props.onAbort);
 
@@ -153,6 +182,13 @@ export function composerView(props: ComposerViewInput) {
   const offeredAspect = showAspectRatio && aspectOptions.some((o) => o.value === aspectRatio) ? aspectRatio : null;
   const selectedAspect = aspectOptions.find((o) => o.value === offeredAspect) ?? null;
 
+  // A world's frame rate works like aspect ratio: only rates every model
+  // that might run takes, and a pick outside them is never sent.
+  const frameRateOptions = world
+    ? (ratioModels.reduce<number[] | null>((shared, m) => (shared ?? m.frameRates ?? []).filter((r) => m.frameRates?.includes(r)), null) ?? [])
+    : [];
+  const offeredFrameRate = frameRate != null && frameRateOptions.includes(frameRate) ? frameRate : null;
+
   const selections = (): ComposerSelection[] =>
     activeUseCases.map((u) => ({
       useCase: u,
@@ -183,6 +219,24 @@ export function composerView(props: ComposerViewInput) {
     aspectOptions,
     offeredAspect,
     selectedAspect,
+    world,
+    /**
+     * What an Enhance click rewrites for: the active use case, derived like
+     * a submit's `mode` ("text-to-text" with no use case picked, "vision"
+     * with media attached, "image-edit" while editing). Undefined for a world.
+     */
+    enhanceMode: editing ? ("image-edit" as const) : deriveGenerationUseCase(activeUseCases[0], attachments),
+    /**
+     * Why submit can't go yet, shown under the environment: there's
+     * something to send (a character or an attachment) but no environment.
+     */
+    environmentHint:
+      world && !value.trim() && (character.trim().length > 0 || attachments.length > 0)
+        ? "Add an environment to create a world"
+        : null,
+    characterPlaceholder: "Describe who you play as, or leave this empty",
+    frameRateOptions,
+    offeredFrameRate,
     /** The model ids after a row click: the one picked, or with multi-select that one toggled. */
     toggledModelIds: (id: string) => (modelIds.includes(id) ? modelIds.filter((m) => m !== id) : [...modelIds, id]),
     /** Everything a submit sends. */
@@ -195,20 +249,25 @@ export function composerView(props: ComposerViewInput) {
       selections: selections(),
       autoSelectModel,
       regions: editing ? regions : [],
+      ...(world && { character, frameRate: offeredFrameRate }),
     }),
   };
 }
 
 export type ComposerView = ReturnType<typeof composerView>;
 
-/** The `+` menu items that can work: "Add media" needs somewhere to put files, the use-case items a handler. */
+/**
+ * The `+` menu items that can work: "Add media" needs somewhere to put
+ * files, the use-case items a handler. With no `actions`, every item that
+ * isn't opt-in.
+ */
 export function composerAttachMenuItems(
   actions: ComposerAttachMenuAction[] | undefined,
   { canAddMedia, canSelect }: { canAddMedia: boolean; canSelect: boolean }
 ) {
   return COMPOSER_ATTACH_MENU_ITEMS.filter(
     (item) =>
-      (actions ?? COMPOSER_ATTACH_MENU_ITEMS.map((i) => i.action)).includes(item.action) &&
+      (actions ? actions.includes(item.action) : !item.optIn) &&
       (item.action === "add-media" ? canAddMedia : canSelect)
   );
 }

package/src/composer.ts

@@ -28,7 +28,8 @@ import {
   type ResultStatus,
   type Run,
 } from "./run.js";
-import type { DroppedMedia, GenerationUseCase, MediaKind, ModelOption } from "./types.js";
+import type { ComposerPromptField } from "./composer-view.js";
+import type { DroppedMedia, GenerationUseCase, MediaKind, ModelOption, OutputKind } from "./types.js";
 
 export interface ComposerAttachment extends DroppedMedia {
   id: string;
@@ -43,7 +44,7 @@ export interface ComposerAttachment extends DroppedMedia {
  * the design). With none picked, `defaultUseCase` applies.
  */
 export interface ComposerUseCase {
-  kind: MediaKind;
+  kind: OutputKind;
   label: string;
   /**
    * An edit of the attached image rather than a new one: no aspect ratio,
@@ -87,6 +88,10 @@ export interface ComposerSubmitPayload {
   autoSelectModel: boolean;
   /** An edit's marked regions, each with its own instruction. Empty unless the use case is an edit. */
   regions?: EditRegion[];
+  /** A world's character, the person's own figure in it. With a world use case, `value` is the environment. Undefined for other use cases. */
+  character?: string;
+  /** The picked frame rate for a world, or null for the model's default. Undefined for other use cases. */
+  frameRate?: number | null;
 }
 
 /**
@@ -116,7 +121,7 @@ export interface ComposerOptions {
    * for multi-use-case select, since one Fal engine outputs one kind, e.g.
    * `{ image: createFalEngine({ outputKind: "image" }), video: createFalEngine({ outputKind: "video" }) }`.
    */
-  engineByKind?: Partial<Record<MediaKind, GenerationEngine>>;
+  engineByKind?: Partial<Record<OutputKind, GenerationEngine>>;
   /**
    * The engine for text requests (a Composer with no use case picked, or a
    * text use case). Defaults to `reasoningEngine`, and to a mock text engine
@@ -137,8 +142,12 @@ export interface ComposerOptions {
   reasoningEngine?: GenerationEngine;
   /** Set with `reasoningEngine` to override the provider's pair; used for every kind (§5.5). */
   reasoningModel?: string;
-  /** Overrides the instruction wrapping a raw prompt for Enhance. Defaults to a generic still/video-aware rewrite instruction. */
-  enhancePrompt?: (value: string, mode?: GenerationUseCase) => string;
+  /**
+   * Overrides the instruction wrapping a raw prompt for Enhance. Defaults to
+   * a rewrite instruction for the kind of prompt: still, video, text, or a
+   * world's environment or character (`field`).
+   */
+  enhancePrompt?: (value: string, mode?: GenerationUseCase, field?: ComposerPromptField) => string;
   /**
    * Auto-select: with `autoSelectModel` on in the payload, the reasoning
    * model picks one model per use case at submit, then these fire with the
@@ -195,7 +204,14 @@ export interface ComposerEdit {
 
 const mockTextEngine = createMockEngine({ outputKind: "text" });
 
-function defaultEnhancePrompt(value: string, mode?: GenerationUseCase): string {
+function defaultEnhancePrompt(value: string, mode?: GenerationUseCase, field: ComposerPromptField = "prompt"): string {
+  const reply = "Reply with only the rewritten description, nothing else.";
+  if (field === "environment") {
+    return `Rewrite the following description of an interactive world for a world model to be more detailed and specific, keeping the same intent: the setting, its look and lighting, and how the person moves through it. Don't add a character. ${reply}\n\n${value}`;
+  }
+  if (field === "character") {
+    return `Rewrite the following description of the character a person plays in an interactive world to be more detailed and specific, keeping the same intent: who or what they are, how they look, and how they move. Don't describe the world around them. ${reply}\n\n${value}`;
+  }
   const subject = mode?.includes("video")
     ? "a video-generation"
     : mode === "text-to-text" || mode === "vision"
@@ -293,6 +309,11 @@ export function createComposerStore(getOptions: () => ComposerOptions, getProvid
     const fallback = isText ? (opts.textModel ?? (r && reasoningModelFor(r, attachments))) : undefined;
     const params: Record<string, unknown> = payload.aspectRatio && !isEdit ? { aspectRatio: payload.aspectRatio } : {};
     if (isEdit) params.regions = payload.regions ?? [];
+    // A world's extra inputs, for a world engine to map to its model.
+    if (selection.useCase.kind === "world") {
+      if (payload.character?.trim()) params.character = payload.character;
+      if (payload.frameRate != null) params.frameRate = payload.frameRate;
+    }
     const request: Request = {
       mode: deriveGenerationUseCase(selection.useCase, attachments),
       prompt: payload.value,
@@ -359,6 +380,8 @@ export function createComposerStore(getOptions: () => ComposerOptions, getProvid
             prompt,
             attachments: newRun.request.attachments,
             aspectRatio: newRun.request.params.aspectRatio as string | undefined,
+            character: newRun.request.params.character as string | undefined,
+            frameRate: newRun.request.params.frameRate as number | undefined,
             // Text engines that stream fill `output` in while still "running".
             onText: (text) => update(newRun.id, (r) => streamResult(r, result.id, text)),
             signal,
@@ -462,13 +485,17 @@ export function createComposerStore(getOptions: () => ComposerOptions, getProvid
     set((s) => ({ routing: false, runs: cancelAll(s.runs), editRuns: cancelAll(s.editRuns) }));
   }
 
-  /** Rewrites a prompt with the reasoning model. Only offer it when `reasoning()` is set. */
-  async function enhance(value: string): Promise<string> {
+  /**
+   * Rewrites a prompt with the reasoning model, worded for its `field` and
+   * `mode` (the Composer passes both to `onEnhance`, so `onEnhance={enhance}`
+   * rewrites for the use case on screen). Only offer it when `reasoning()` is set.
+   */
+  async function enhance(value: string, field?: ComposerPromptField, mode?: GenerationUseCase): Promise<string> {
     const r = reasoning();
     if (!r) throw new Error("[chai-ui] Enhance needs a reasoning engine and model.");
     set(() => ({ enhancing: true }));
     try {
-      const prompt = (getOptions().enhancePrompt ?? defaultEnhancePrompt)(value, state.runs[0]?.request.mode);
+      const prompt = (getOptions().enhancePrompt ?? defaultEnhancePrompt)(value, mode ?? state.runs[0]?.request.mode, field);
       const { src } = await r.engine.generate({ modelId: r.model, prompt, attachments: [] });
       return src;
     } finally {

package/src/engine.ts

@@ -23,6 +23,10 @@ export interface GenerationEngine {
     attachments: DroppedMedia[];
     /** The output aspect ratio the person picked, e.g. `"16:9"`. Only set for a model that lists it in `ModelOption.aspectRatios`. */
     aspectRatio?: string;
+    /** A world's character (Create world), when the person gave one. `prompt` is the environment. */
+    character?: string;
+    /** A world's frame rate, e.g. `16`. Only set for a model that lists it in `ModelOption.frameRates`. */
+    frameRate?: number;
     /**
      * Optional streaming hook for text output. An engine that can stream
      * calls this with the full text received so far (not a delta) each time

package/src/engines/fal.ts

@@ -24,6 +24,7 @@
 import { abortError, abortableSleep, isAbortError, readEvaluations, type GenerationEngine } from "../engine.js";
 import type { DroppedMedia, MediaKind } from "../types.js";
 import { SUGGESTED_FAL_MODELS } from "../suggested-models.js";
+import type { Transcribe } from "../voice-input.js";
 
 /** Output kinds this engine can produce — text goes through a reasoning engine instead (see file doc comment). */
 type FalOutputKind = Exclude<MediaKind, "text">;
@@ -77,6 +78,9 @@ interface FalResultResponse {
   video?: { url: string };
   audio?: { url: string };
   audio_url?: string;
+  /** Speech-to-text models answer with the transcript in `text` or `output`. */
+  text?: string;
+  output?: string;
   error?: unknown;
   detail?: unknown;
 }
@@ -205,6 +209,12 @@ async function awaitFalJob(args: {
   return json;
 }
 
+/** The transcript in a Fal speech-to-text result: `text` (Wizper, Scribe, Cohere) or `output` (Fal's own, Nemotron). */
+export function extractFalTranscript(json: { text?: unknown; output?: unknown }): string | undefined {
+  const text = typeof json.text === "string" ? json.text : typeof json.output === "string" ? json.output : undefined;
+  return text?.trim();
+}
+
 /**
  * Asks Fal to drop a job (`PUT …/requests/<id>/cancel`). A queued job is
  * removed and never runs; a running one gets a cancel signal and may still
@@ -318,3 +328,48 @@ export async function pingFal(
     return { ok: false, error: err instanceof Error ? err.message : String(err) };
   }
 }
+
+export interface FalTranscriberOptions {
+  /** A Fal speech-to-text model id, e.g. `"fal-ai/wizper"`. Required: Chai doesn't pick one for you. */
+  model: string;
+  /** Same proxy route `createFalEngine` uses. Defaults to `/api/chai/fal`. */
+  endpoint?: string;
+  fetchImpl?: typeof fetch;
+  pollIntervalMs?: number;
+  timeoutMs?: number;
+}
+
+/**
+ * A `Transcribe` function on a Fal speech-to-text model, through the same
+ * server route as `createFalEngine`, so the key stays on the server. The
+ * recording goes to the model as `audio_url`, a data URL, like images do.
+ * With `allowedModels` on the handler, list the transcription model too.
+ */
+export function createFalTranscriber(options: FalTranscriberOptions): Transcribe {
+  const endpoint = options.endpoint ?? "/api/chai/fal";
+  // Called through a wrapper: browsers throw "Illegal invocation" when fetch runs with an object as `this`.
+  const fetchImpl: typeof fetch = (input, init) => (options.fetchImpl ?? fetch)(input, init);
+  const { model } = options;
+  return async (audio, { signal } = {}) => {
+    if (signal?.aborted) throw abortError();
+    const { requestId, status } = await submitFalJob({ endpoint, model, body: { audio_url: audio.src }, fetchImpl });
+    if (signal?.aborted) {
+      await cancelFalJob({ endpoint, model, requestId, fetchImpl });
+      throw abortError();
+    }
+    const result = await awaitFalJob({
+      endpoint,
+      model,
+      requestId,
+      status,
+      fetchImpl,
+      pollIntervalMs: options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS,
+      timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+      signal,
+    }).catch(async (err) => {
+      if (isAbortError(err)) await cancelFalJob({ endpoint, model, requestId, fetchImpl });
+      throw err;
+    });
+    return extractFalTranscript(result) ?? "";
+  };
+}

package/src/index.ts

@@ -15,3 +15,4 @@ export * from "./media-actions.js";
 export * from "./result-format.js";
 export * from "./composer-view.js";
 export * from "./edit-card.js";
+export * from "./voice-input.js";

package/src/model-routing.ts

@@ -6,13 +6,13 @@
  */
 
 import { normalizeGenerationError, type GenerationEngine } from "./engine.js";
-import type { DroppedMedia, MediaKind, ModelOption } from "./types.js";
+import type { DroppedMedia, MediaKind, ModelOption, OutputKind } from "./types.js";
 
 export interface ModelRoutingInput {
   /** What the person typed. Empty for media-only requests (e.g. Media Analyzer). */
   prompt: string;
   /** What the request produces, e.g. "image" for Create image, "text" for analysis. */
-  outputKind: MediaKind;
+  outputKind: OutputKind;
   /** Kinds of media attached, if any. */
   attachmentKinds: MediaKind[];
   /** The candidates: only models the builder offers for this request. */

package/src/run.ts

@@ -4,7 +4,7 @@
  * (`@chai-ui/react`) is the thin React binding on top.
  */
 
-import type { DroppedMedia, Evaluation, GenerationUseCase, MediaKind, Usage } from "./types.js";
+import type { DroppedMedia, Evaluation, GenerationUseCase, MediaKind, OutputKind, Usage } from "./types.js";
 
 export interface Request {
   /** The specific generation use case, when one could be derived — see `deriveGenerationUseCase`. Undefined when the use case (or its kind) has no `GenerationUseCase` match yet, e.g. audio. */
@@ -135,12 +135,12 @@ export function cancelResult(run: Run, resultId: string): Run {
  * case plus whether anything is attached — this is real derivation logic,
  * not just plumbing: the same `kind: "video"` means `"text-to-video"` with
  * nothing attached but `"image-to-video"` once an image is. Returns
- * `undefined` for "audio" (no `GenerationUseCase` member exists yet;
- * audio generation is planned). `"text"` is a text request: a reply to
+ * `undefined` for "audio" and "world" (no `GenerationUseCase` member exists
+ * yet; audio generation is planned, world models await provider specs). `"text"` is a text request: a reply to
  * the prompt, or a description of the attached media ("vision").
  */
 export function deriveGenerationUseCase(
-  useCase: { kind: MediaKind } | null | undefined,
+  useCase: { kind: OutputKind } | null | undefined,
   attachments: DroppedMedia[]
 ): GenerationUseCase | undefined {
   if (!useCase) return undefined;

package/src/suggested-models.ts

@@ -4,7 +4,8 @@ import { DEFAULT_MAX_REGIONS } from "./region-edit.js";
 /**
  * Suggested models, so builders don't have to write model lists. Composer
  * and MediaAnalyzer use them when no `models` is passed; passing `models`
- * still replaces them. The first model in each list is the default pick.
+ * still replaces them. Nothing is picked for the person: with a Model menu
+ * they pick or turn on auto-select; with no menu, the list's first runs.
  *
  * Generation runs on Fal (the default engine), analysis on OpenRouter (the
  * reasoning engine). Every entry was checked against the provider's own

package/src/types.ts

@@ -5,6 +5,13 @@
 /** The kinds of media a component can take in or produce. */
 export type MediaKind = "image" | "video" | "audio" | "text";
 
+/**
+ * What a generation produces: any `MediaKind`, or an interactive world from
+ * a world model (Create world). A world is never an attachment, so it lives
+ * here rather than in `MediaKind`.
+ */
+export type OutputKind = MediaKind | "world";
+
 /**
  * A piece of media a person attached (or an engine received) — just enough
  * to know its kind and where it lives.
@@ -106,6 +113,12 @@ export interface ModelOption {
    * the input image): no menu shows.
    */
   aspectRatios?: string[];
+  /**
+   * The frame rates a world model runs at, e.g. `[16]`. Composer's frame
+   * rate menu offers only rates the selected models share. Leave it out for
+   * a model with one fixed rate it doesn't take as an input: no menu shows.
+   */
+  frameRates?: number[];
 }
 
 /** See `ModelOption.falInput`. */