@tanstack/ai 0.52.3 → 0.54.0

package/src/activities/generateWorld/index.ts

@@ -0,0 +1,339 @@
+/**
+ * World Activity (Experimental)
+ *
+ * Mints a session token for a live, prompt-steerable world. Unlike
+ * generateVideo (a job that finishes with a URL), the browser then connects
+ * with the token, sets the prompt, and streams until pause/reset/close.
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+
+import { aiEventClient } from '@tanstack/ai-event-client'
+import { streamGenerationResult } from '../stream-generation-result.js'
+import { resolveDebugOption } from '../../logger/resolve'
+import {
+  applyGenerationResultTransforms,
+  createGenerationContext,
+  runGenerationAbort,
+  runGenerationError,
+  runGenerationFinish,
+  runGenerationStart,
+  runGenerationUsage,
+} from '../middleware/run'
+import {
+  abortReasonMessage,
+  createActivityAbortControls,
+  isActivityAbortError,
+  raceWithAbort,
+} from '../../utilities/activity-abort'
+import type { InternalLogger } from '../../logger/internal-logger'
+import type { DebugOption } from '../../logger/types'
+import type { GenerationMiddleware } from '../middleware/types'
+import type { WorldAdapter } from './adapter'
+import type { StreamChunk, WorldGenerationResult } from '../../types'
+
+// ===========================
+// Activity Kind
+// ===========================
+
+/** The adapter kind this activity handles */
+export const kind = 'world' as const
+
+// ===========================
+// Type Extraction Helpers
+// ===========================
+
+/**
+ * Extract provider options from a WorldAdapter via ~types.
+ */
+export type WorldProviderOptions<TAdapter> = TAdapter extends {
+  '~types': { providerOptions: infer P extends object }
+}
+  ? P
+  : object
+
+// ===========================
+// Activity Options Type
+// ===========================
+
+/**
+ * Options for the world generation activity.
+ * The model is extracted from the adapter's model property.
+ *
+ * @template TAdapter - The world adapter type
+ * @template TStream - Whether to stream the output
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+export interface WorldActivityOptions<
+  TAdapter extends WorldAdapter<string, WorldProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+> {
+  /** The world adapter to use (must be created with a model) */
+  adapter: TAdapter & { kind: typeof kind }
+  /** Natural-language description of the world or scene */
+  prompt: string
+  /** Provider-specific options for world generation */
+  modelOptions?: WorldProviderOptions<TAdapter>
+  /**
+   * Whether to wrap the token result as StreamChunks for SSE transport.
+   * This is not the live video. When false or omitted, returns
+   * Promise<WorldGenerationResult>.
+   *
+   * @default false
+   */
+  stream?: TStream
+  /**
+   * Enable debug logging. Pass `true` to enable all categories, `false` to
+   * silence everything including errors, or a `DebugConfig` object for granular
+   * control and/or a custom `Logger`.
+   */
+  debug?: DebugOption
+  /**
+   * Observe-only middleware notified on start, usage, success, and error. Pass
+   * `otelMiddleware()` to emit OpenTelemetry spans, or implement the
+   * `GenerationMiddleware` contract for a custom backend.
+   */
+  middleware?: Array<GenerationMiddleware>
+  /** Stable conversation/thread id for correlating this run when persisted. */
+  threadId?: string
+  /** Stable run id for correlating this run when persisted. */
+  runId?: string
+  /**
+   * Maximum duration of the token mint in milliseconds.
+   * No SDK-wide default. Composed with {@link abortSignal}; the first abort wins.
+   */
+  timeout?: number
+  /**
+   * Caller cancellation signal (request disconnects, job/runtime cancellation).
+   * Composed with {@link timeout} into an effective signal forwarded to the
+   * adapter. Request-specific — not stored on global provider client config.
+   */
+  abortSignal?: AbortSignal
+}
+
+// ===========================
+// Activity Result Type
+// ===========================
+
+/**
+ * Result type for the world generation activity.
+ * - If stream is true: AsyncIterable<StreamChunk>
+ * - Otherwise: Promise<WorldGenerationResult>
+ */
+export type WorldActivityResult<TStream extends boolean = false> =
+  TStream extends true
+    ? AsyncIterable<StreamChunk>
+    : Promise<WorldGenerationResult>
+
+function createId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
+}
+
+// ===========================
+// Activity Implementation
+// ===========================
+
+/**
+ * World generation activity - opens a live, prompt-steerable world session.
+ *
+ * @example Mint a session token on the server
+ * ```ts
+ * import { generateWorld } from '@tanstack/ai'
+ * import { reactorWorld } from '@tanstack/ai-reactor'
+ *
+ * const world = await generateWorld({
+ *   adapter: reactorWorld('visko-orbis-stable'),
+ *   prompt: 'A neon cyberpunk city at night, slow aerial drift',
+ * })
+ *
+ * // Hand world.token, world.model, and world.prompt to the browser.
+ * ```
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+export function generateWorld<
+  TAdapter extends WorldAdapter<string, WorldProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: WorldActivityOptions<TAdapter, TStream>,
+): WorldActivityResult<TStream> {
+  if (options.stream) {
+    return streamGenerationResult(
+      (resolved) => runGenerateWorld({ ...options, runId: resolved.runId }),
+      options,
+    ) as WorldActivityResult<TStream>
+  }
+  return runGenerateWorld(options) as WorldActivityResult<TStream>
+}
+
+/**
+ * Run the core world generation logic (non-streaming).
+ */
+async function runGenerateWorld<
+  TAdapter extends WorldAdapter<string, WorldProviderOptions<TAdapter>>,
+>(
+  options: WorldActivityOptions<TAdapter, boolean>,
+): Promise<WorldGenerationResult> {
+  const {
+    adapter,
+    stream: _stream,
+    debug: _debug,
+    middleware,
+    threadId,
+    runId,
+    timeout,
+    abortSignal: callerAbortSignal,
+    ...rest
+  } = options
+  const model = adapter.model
+  const requestId = createId('world')
+  const startTime = Date.now()
+  const logger: InternalLogger = resolveDebugOption(options.debug)
+  const abortControls = createActivityAbortControls({
+    timeout,
+    abortSignal: callerAbortSignal,
+  })
+  const providerName =
+    (adapter as { name?: string; provider?: string }).provider ??
+    (adapter as { name?: string }).name ??
+    'unknown'
+
+  const mwCtx = createGenerationContext({
+    requestId,
+    activity: 'world',
+    provider: adapter.name,
+    model,
+    modelOptions: rest.modelOptions,
+    threadId,
+    runId,
+    artifactInputs: { prompt: rest.prompt },
+    createId,
+  })
+
+  await runGenerationStart(middleware, mwCtx)
+
+  aiEventClient.emit('world:request:started', {
+    requestId,
+    provider: adapter.name,
+    model,
+    prompt: rest.prompt,
+    timestamp: startTime,
+    ...(rest.modelOptions !== undefined && {
+      modelOptions: rest.modelOptions as Record<string, unknown>,
+    }),
+  })
+
+  logger.request(`activity=generateWorld provider=${providerName}`, {
+    provider: providerName,
+    model,
+  })
+
+  try {
+    const rawResult = await raceWithAbort(
+      adapter.createWorld({
+        ...rest,
+        model,
+        logger,
+        ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),
+      }),
+      abortControls.signal,
+    )
+    abortControls.clear()
+    const result = await applyGenerationResultTransforms(mwCtx, rawResult)
+    const elapsedMs = Date.now() - startTime
+
+    aiEventClient.emit('world:request:completed', {
+      requestId,
+      provider: adapter.name,
+      model: result.model,
+      prompt: result.prompt,
+      status: result.status,
+      duration: elapsedMs,
+      timestamp: Date.now(),
+      ...(rest.modelOptions !== undefined && {
+        modelOptions: rest.modelOptions as Record<string, unknown>,
+      }),
+    })
+
+    if (result.usage) {
+      aiEventClient.emit('world:usage', {
+        requestId,
+        model: result.model,
+        usage: result.usage,
+        timestamp: Date.now(),
+        ...(rest.modelOptions !== undefined && {
+          modelOptions: rest.modelOptions as Record<string, unknown>,
+        }),
+      })
+    }
+
+    logger.output(`activity=generateWorld provider=${providerName}`, {
+      model: result.model,
+      status: result.status,
+    })
+
+    if (result.usage) await runGenerationUsage(middleware, mwCtx, result.usage)
+    await runGenerationFinish(middleware, mwCtx, {
+      duration: elapsedMs,
+      usage: result.usage,
+    })
+
+    return result
+  } catch (error) {
+    abortControls.clear()
+    const elapsedMs = Date.now() - startTime
+    const err = error as Error
+    aiEventClient.emit('world:request:error', {
+      requestId,
+      provider: adapter.name,
+      model,
+      error: { message: err.message, name: err.name },
+      duration: elapsedMs,
+      timestamp: Date.now(),
+      ...(rest.modelOptions !== undefined && {
+        modelOptions: rest.modelOptions as Record<string, unknown>,
+      }),
+    })
+    if (isActivityAbortError(error, abortControls.signal)) {
+      await runGenerationAbort(middleware, mwCtx, {
+        reason: abortReasonMessage(error, abortControls.signal),
+        duration: elapsedMs,
+      })
+    } else {
+      await runGenerationError(middleware, mwCtx, {
+        error,
+        duration: elapsedMs,
+      })
+    }
+    logger.errors('generateWorld activity failed', {
+      error,
+      source: 'generateWorld',
+    })
+    throw error
+  }
+}
+
+// ===========================
+// Options Factory
+// ===========================
+
+/**
+ * Create typed options for the generateWorld() function without executing.
+ */
+export function createWorldOptions<
+  TAdapter extends WorldAdapter<string, WorldProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: WorldActivityOptions<TAdapter, TStream>,
+): WorldActivityOptions<TAdapter, TStream> {
+  return options
+}
+
+// Re-export adapter types
+export type {
+  WorldAdapter,
+  WorldAdapterConfig,
+  AnyWorldAdapter,
+} from './adapter'
+export { BaseWorldAdapter } from './adapter'

package/src/activities/index.ts

@@ -23,6 +23,8 @@ import type { AnyTTSAdapter } from './generateSpeech/adapter'
 import type { AnyTranscriptionAdapter } from './generateTranscription/adapter'
 import type { AnyEmbeddingAdapter } from './embed/adapter'
 import type { AnyRerankAdapter } from './rerank/adapter'
+import type { AnyWorldAdapter } from './generateWorld/adapter'
+import type { AnyLiveVideoAdapter } from './generateLiveVideo/adapter'
 
 // ===========================
 // Chat Activity
@@ -210,6 +212,44 @@ export {
   type AnyEmbeddingAdapter,
 } from './embed/adapter'
 
+// ===========================
+// World Activity (Experimental)
+// ===========================
+
+export {
+  kind as worldKind,
+  generateWorld,
+  type WorldActivityOptions,
+  type WorldActivityResult,
+  type WorldProviderOptions,
+} from './generateWorld/index'
+
+export {
+  BaseWorldAdapter,
+  type WorldAdapter,
+  type WorldAdapterConfig,
+  type AnyWorldAdapter,
+} from './generateWorld/adapter'
+
+// ===========================
+// Live Activity (Experimental)
+// ===========================
+
+export {
+  kind as liveVideoKind,
+  generateLiveVideo,
+  type LiveVideoActivityOptions,
+  type LiveVideoActivityResult,
+  type LiveVideoProviderOptions,
+} from './generateLiveVideo/index'
+
+export {
+  BaseLiveVideoAdapter,
+  type LiveVideoAdapter,
+  type LiveVideoAdapterConfig,
+  type AnyLiveVideoAdapter,
+} from './generateLiveVideo/adapter'
+
 // ===========================
 // Adapter Union Types
 // ===========================
@@ -225,6 +265,8 @@ export type AIAdapter =
   | AnyTranscriptionAdapter
   | AnyEmbeddingAdapter
   | AnyRerankAdapter
+  | AnyWorldAdapter
+  | AnyLiveVideoAdapter
 
 /** Union type of all adapter kinds */
 export type AdapterKind =
@@ -237,3 +279,5 @@ export type AdapterKind =
   | 'transcription'
   | 'embedding'
   | 'rerank'
+  | 'world'
+  | 'liveVideo'

package/src/activities/middleware/types.ts

@@ -44,6 +44,8 @@ export type GenerationActivity =
   | 'embedding'
   | 'rerank'
   | 'summarize'
+  | 'world'
+  | 'liveVideo'
 
 /**
  * Stable context passed to every {@link GenerationMiddleware} hook. Created

package/src/activities/summarize/chat-stream-summarize.ts

@@ -95,6 +95,8 @@ const MAX_TOKENS_KEY_BY_ADAPTER: Record<string, string> = {
   // LLM Gateway exposes an OpenAI-compatible Chat Completions surface whose
   // only output cap is `max_tokens` — it does not read `max_completion_tokens`.
   llmgateway: 'max_tokens',
+  // Workers AI's OpenAI-compatible Chat Completions surface reads `max_tokens`.
+  cloudflare: 'max_tokens',
 }
 
 /**

package/src/byok/define-provider.ts

@@ -12,6 +12,11 @@ export interface ByokProvider<TId extends string = string> {
    * values here. This object is imported on the client.
    */
   readonly env?: ReadonlyArray<string>
+  /**
+   * Other descriptors this credential needs. A send for this provider also
+   * carries their headers and prompts for each one that is missing.
+   */
+  readonly with?: ReadonlyArray<ByokProvider>
 }
 
 /**
@@ -22,6 +27,7 @@ export type ByokProviderInit<TId extends string> = {
   readonly id: undefined extends TId ? never : TId
   readonly label: string
   readonly env?: string | ReadonlyArray<string>
+  readonly with?: ReadonlyArray<ByokProvider>
 }
 
 function normalizeEnv(
@@ -42,5 +48,6 @@ export function defineByokProvider<const TId extends string>(
     id: provider.id,
     label: provider.label,
     ...(env ? { env } : {}),
+    ...(provider.with ? { with: provider.with } : {}),
   }
 }

package/src/byok/get-key.ts

@@ -25,3 +25,21 @@ export function getByokKey(
   }
   return null
 }
+
+/**
+ * Read several keys at once, one per name. Same rules as {@link getByokKey}
+ * for each entry. Use it for a credential made of more than one value.
+ */
+export function getByokKeys<
+  const TProviders extends Record<string, ProviderId | ByokProvider>,
+>(
+  request: Request,
+  providers: TProviders,
+): { [K in keyof TProviders]: string | null } {
+  return Object.fromEntries(
+    Object.entries(providers).map(([name, provider]) => [
+      name,
+      getByokKey(request, provider),
+    ]),
+  ) as { [K in keyof TProviders]: string | null }
+}

package/src/byok/server.ts

@@ -1,3 +1,3 @@
-export { getByokKey } from './get-key'
+export { getByokKey, getByokKeys } from './get-key'
 export { byokMissing, isByokMissingBody } from './missing'
 export type { ByokMissingBody } from './missing'

package/src/client.ts

@@ -4,6 +4,8 @@ import type {
   TTSOptions,
   TranscriptionOptions,
   VideoGenerationOptions,
+  WorldGenerationOptions,
+  LiveVideoGenerationOptions,
 } from './types'
 
 export type GenerationKind =
@@ -12,6 +14,8 @@ export type GenerationKind =
   | 'tts'
   | 'video'
   | 'transcription'
+  | 'world'
+  | 'liveVideo'
 
 type GenerationInputByKind = {
   image: Omit<ImageGenerationOptions, 'logger' | 'model'>
@@ -19,6 +23,8 @@ type GenerationInputByKind = {
   tts: Omit<TTSOptions, 'logger' | 'model'>
   video: Omit<VideoGenerationOptions, 'logger' | 'model'>
   transcription: Omit<TranscriptionOptions, 'logger' | 'model'>
+  world: Omit<WorldGenerationOptions, 'logger' | 'model'>
+  liveVideo: Omit<LiveVideoGenerationOptions, 'logger' | 'model'>
 }
 
 export interface GenerationParams<TKind extends GenerationKind> {
@@ -34,6 +40,8 @@ const generationKinds = [
   'tts',
   'video',
   'transcription',
+  'world',
+  'liveVideo',
 ] as const satisfies ReadonlyArray<GenerationKind>
 
 function isRecord(value: unknown): value is Record<string, unknown> {

package/src/index.ts

@@ -10,6 +10,8 @@ export {
   generateSpeech,
   generateTranscription,
   embed,
+  generateWorld,
+  generateLiveVideo,
 } from './activities/index'
 
 // Create options functions - for pre-defining typed configurations
@@ -22,6 +24,8 @@ export { createVideoOptions } from './activities/generateVideo/index'
 export { createSpeechOptions } from './activities/generateSpeech/index'
 export { createTranscriptionOptions } from './activities/generateTranscription/index'
 export { createEmbedOptions } from './activities/embed/index'
+export { createWorldOptions } from './activities/generateWorld/index'
+export { createLiveVideoOptions } from './activities/generateLiveVideo/index'
 
 // Re-export types
 export type {
@@ -44,6 +48,10 @@ export type {
   EmbeddingAdapter,
   AnyRerankAdapter,
   RerankAdapter,
+  AnyWorldAdapter,
+  WorldAdapter,
+  AnyLiveVideoAdapter,
+  LiveVideoAdapter,
 } from './activities/index'
 
 // Rerank adapter base + types

package/src/middlewares/otel.ts

@@ -91,6 +91,8 @@ const OPERATION_NAME: Record<GenerationActivity, string> = {
   embedding: 'embeddings',
   rerank: 'rerank',
   summarize: 'summarize',
+  world: 'world_generation',
+  liveVideo: 'live_video_generation',
 }
 
 export interface OtelMiddlewareOptions {

package/src/types.ts

@@ -2218,6 +2218,134 @@ export interface VideoUrlResult {
   artifacts?: Array<PersistedArtifactRef>
 }
 
+// ============================================================================
+// World Generation Types (Experimental)
+// ============================================================================
+
+/**
+ * Options for world generation (live, prompt-steerable sessions).
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+export interface WorldGenerationOptions<
+  TProviderOptions extends object = object,
+> {
+  /** The model to use for world generation */
+  model: string
+  /** Natural-language description of the world or scene */
+  prompt: string
+  /**
+   * Provider mint options. Reactor resolution/seed/audio are browser
+   * `sendCommand` fields, not token-mint fields.
+   */
+  modelOptions?: TProviderOptions
+  /**
+   * Internal logger threaded from the generateWorld() entry point. Adapters
+   * must call logger.request() before the SDK call and logger.errors() in
+   * catch blocks.
+   */
+  logger: InternalLogger
+  /**
+   * Effective abort signal composed by the activity from caller `abortSignal`
+   * and/or `timeout`. Adapters should forward this to the provider SDK when
+   * supported. Request-specific — never store on a global client config.
+   */
+  abortSignal?: AbortSignal
+}
+
+/**
+ * Result of world generation. JSON-serializable so a server route can return
+ * it to a browser. The browser uses `token` + `model` to open the live
+ * session (set the prompt, start streaming, steer mid-run).
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+export interface WorldGenerationResult {
+  /** Unique identifier for this generation */
+  id: string
+  /** Model used for generation (provider connect slug) */
+  model: string
+  /** Short-lived session token for the client connection */
+  token: string
+  /** Token expiry as milliseconds since epoch */
+  expiresAt: number
+  /** Prompt the client should send when it starts the session */
+  prompt: string
+  /** Session status after the server half finishes */
+  status: 'ready' | 'waiting'
+  /** Provider session id, when the adapter created one */
+  sessionId?: string
+  /** Token usage / billing, when the adapter can report it */
+  usage?: TokenUsage
+}
+
+// ============================================================================
+// Live Generation Types (Experimental)
+// ============================================================================
+
+/**
+ * Options for live generation (prompt-steerable video sessions).
+ *
+ * @experimental Live generation is an experimental feature and may change.
+ */
+export interface LiveVideoGenerationOptions<
+  TProviderOptions extends object = object,
+> {
+  /** The model to use for live generation */
+  model: string
+  /** Natural-language description of the shot or scene */
+  prompt: string
+  /**
+   * Provider mint options. For fal live this is `tokenDuration`. Reactor
+   * resolution/seed/audio are browser `sendCommand` fields.
+   */
+  modelOptions?: TProviderOptions
+  /**
+   * Internal logger threaded from the generateLiveVideo() entry point. Adapters
+   * must call logger.request() before the SDK call and logger.errors() in
+   * catch blocks.
+   */
+  logger: InternalLogger
+  /**
+   * Effective abort signal composed by the activity from caller `abortSignal`
+   * and/or `timeout`. Adapters should forward this to the provider SDK when
+   * supported. Request-specific — never store on a global client config.
+   */
+  abortSignal?: AbortSignal
+}
+
+/**
+ * Result of live generation. JSON-serializable so a server route can return
+ * it to a browser.
+ *
+ * Reactor: connect with `token` and `model` (the connect slug).
+ * fal: `model` is the WMA app id. Open `wma(model)` through a server proxy
+ * that attaches `FAL_KEY`. Do not send `token` as Key credentials.
+ *
+ * @experimental Live generation is an experimental feature and may change.
+ */
+export interface LiveVideoGenerationResult {
+  /** Unique identifier for this generation */
+  id: string
+  /**
+   * Connect id for the browser client. Reactor: `reactor/helios`.
+   * fal: WMA app id `fal-ai/minimax-h3-max-director`.
+   */
+  model: string
+  /** Short-lived session token. Reactor uses this to connect. fal does not. */
+  token: string
+  /** Token expiry as milliseconds since epoch */
+  expiresAt: number
+  /** Prompt the client should send when it starts the session */
+  prompt: string
+  /** Session status after the server half finishes */
+  status: 'ready' | 'waiting'
+  /** Provider session id, when the adapter created one */
+  sessionId?: string
+  /** Token usage / billing, when the adapter can report it */
+  usage?: TokenUsage
+}
+
 // ============================================================================
 // Text-to-Speech (TTS) Types
 // ============================================================================