@tanstack/ai 0.53.0 → 0.55.0

package/src/activities/generateWorld/adapter.ts

@@ -0,0 +1,96 @@
+import type { WorldGenerationOptions, WorldGenerationResult } from '../../types'
+
+/**
+ * Configuration for world generation adapter instances.
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+export interface WorldAdapterConfig {
+  apiKey?: string
+  baseUrl?: string
+  timeout?: number
+  maxRetries?: number
+  headers?: Record<string, string>
+}
+
+/**
+ * World adapter interface with pre-resolved generics.
+ *
+ * An adapter is created by a provider function: `provider('model')` → `adapter`.
+ * All type resolution happens at the provider call site, not in this interface.
+ *
+ * Generic parameters:
+ * - TModel: The specific model name (e.g. 'visko-orbis-stable')
+ * - TProviderOptions: Provider-specific options (already resolved)
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+export interface WorldAdapter<
+  TModel extends string = string,
+  TProviderOptions extends object = Record<string, unknown>,
+> {
+  /** Discriminator for adapter kind - used to determine API shape */
+  readonly kind: 'world'
+  /** Adapter name identifier */
+  readonly name: string
+  /** The model this adapter is configured for */
+  readonly model: TModel
+
+  /**
+   * @internal Type-only properties for inference. Not assigned at runtime.
+   */
+  '~types': {
+    providerOptions: TProviderOptions
+  }
+
+  /**
+   * Open a world session from a prompt.
+   *
+   * Server adapters typically mint a short-lived token and return it with the
+   * prompt so a browser can connect, set the prompt, and start streaming.
+   */
+  createWorld: (
+    options: WorldGenerationOptions<TProviderOptions>,
+  ) => Promise<WorldGenerationResult>
+}
+
+/**
+ * A WorldAdapter with any/unknown type parameters.
+ * Useful as a constraint in generic functions and interfaces.
+ */
+export type AnyWorldAdapter = WorldAdapter<any, any>
+
+/**
+ * Abstract base class for world generation adapters.
+ * Extend this class to implement a world adapter for a specific provider.
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+export abstract class BaseWorldAdapter<
+  TModel extends string = string,
+  TProviderOptions extends object = Record<string, unknown>,
+> implements WorldAdapter<TModel, TProviderOptions> {
+  readonly kind = 'world' as const
+  abstract readonly name: string
+  readonly model: TModel
+
+  // Type-only property - never assigned at runtime
+  declare '~types': {
+    providerOptions: TProviderOptions
+  }
+
+  protected config: WorldAdapterConfig
+
+  constructor(model: TModel, config: WorldAdapterConfig = {}) {
+    this.config = config
+    this.model = model
+  }
+
+  abstract createWorld(
+    options: WorldGenerationOptions<TProviderOptions>,
+  ): Promise<WorldGenerationResult>
+
+  protected generateId(): string {
+    return `${this.name}-${Date.now()}-${Math.random().toString(36).substring(7)}`
+  }
+}

package/src/activities/generateWorld/index.ts

@@ -0,0 +1,339 @@
+/**
+ * World Activity (Experimental)
+ *
+ * Mints a session token for a live, prompt-steerable world. Unlike
+ * generateVideo (a job that finishes with a URL), the browser then connects
+ * with the token, sets the prompt, and streams until pause/reset/close.
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+
+import { aiEventClient } from '@tanstack/ai-event-client'
+import { streamGenerationResult } from '../stream-generation-result.js'
+import { resolveDebugOption } from '../../logger/resolve'
+import {
+  applyGenerationResultTransforms,
+  createGenerationContext,
+  runGenerationAbort,
+  runGenerationError,
+  runGenerationFinish,
+  runGenerationStart,
+  runGenerationUsage,
+} from '../middleware/run'
+import {
+  abortReasonMessage,
+  createActivityAbortControls,
+  isActivityAbortError,
+  raceWithAbort,
+} from '../../utilities/activity-abort'
+import type { InternalLogger } from '../../logger/internal-logger'
+import type { DebugOption } from '../../logger/types'
+import type { GenerationMiddleware } from '../middleware/types'
+import type { WorldAdapter } from './adapter'
+import type { StreamChunk, WorldGenerationResult } from '../../types'
+
+// ===========================
+// Activity Kind
+// ===========================
+
+/** The adapter kind this activity handles */
+export const kind = 'world' as const
+
+// ===========================
+// Type Extraction Helpers
+// ===========================
+
+/**
+ * Extract provider options from a WorldAdapter via ~types.
+ */
+export type WorldProviderOptions<TAdapter> = TAdapter extends {
+  '~types': { providerOptions: infer P extends object }
+}
+  ? P
+  : object
+
+// ===========================
+// Activity Options Type
+// ===========================
+
+/**
+ * Options for the world generation activity.
+ * The model is extracted from the adapter's model property.
+ *
+ * @template TAdapter - The world adapter type
+ * @template TStream - Whether to stream the output
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+export interface WorldActivityOptions<
+  TAdapter extends WorldAdapter<string, WorldProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+> {
+  /** The world adapter to use (must be created with a model) */
+  adapter: TAdapter & { kind: typeof kind }
+  /** Natural-language description of the world or scene */
+  prompt: string
+  /** Provider-specific options for world generation */
+  modelOptions?: WorldProviderOptions<TAdapter>
+  /**
+   * Whether to wrap the token result as StreamChunks for SSE transport.
+   * This is not the live video. When false or omitted, returns
+   * Promise<WorldGenerationResult>.
+   *
+   * @default false
+   */
+  stream?: TStream
+  /**
+   * Enable debug logging. Pass `true` to enable all categories, `false` to
+   * silence everything including errors, or a `DebugConfig` object for granular
+   * control and/or a custom `Logger`.
+   */
+  debug?: DebugOption
+  /**
+   * Observe-only middleware notified on start, usage, success, and error. Pass
+   * `otelMiddleware()` to emit OpenTelemetry spans, or implement the
+   * `GenerationMiddleware` contract for a custom backend.
+   */
+  middleware?: Array<GenerationMiddleware>
+  /** Stable conversation/thread id for correlating this run when persisted. */
+  threadId?: string
+  /** Stable run id for correlating this run when persisted. */
+  runId?: string
+  /**
+   * Maximum duration of the token mint in milliseconds.
+   * No SDK-wide default. Composed with {@link abortSignal}; the first abort wins.
+   */
+  timeout?: number
+  /**
+   * Caller cancellation signal (request disconnects, job/runtime cancellation).
+   * Composed with {@link timeout} into an effective signal forwarded to the
+   * adapter. Request-specific — not stored on global provider client config.
+   */
+  abortSignal?: AbortSignal
+}
+
+// ===========================
+// Activity Result Type
+// ===========================
+
+/**
+ * Result type for the world generation activity.
+ * - If stream is true: AsyncIterable<StreamChunk>
+ * - Otherwise: Promise<WorldGenerationResult>
+ */
+export type WorldActivityResult<TStream extends boolean = false> =
+  TStream extends true
+    ? AsyncIterable<StreamChunk>
+    : Promise<WorldGenerationResult>
+
+function createId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`
+}
+
+// ===========================
+// Activity Implementation
+// ===========================
+
+/**
+ * World generation activity - opens a live, prompt-steerable world session.
+ *
+ * @example Mint a session token on the server
+ * ```ts
+ * import { generateWorld } from '@tanstack/ai'
+ * import { reactorWorld } from '@tanstack/ai-reactor'
+ *
+ * const world = await generateWorld({
+ *   adapter: reactorWorld('visko-orbis-stable'),
+ *   prompt: 'A neon cyberpunk city at night, slow aerial drift',
+ * })
+ *
+ * // Hand world.token, world.model, and world.prompt to the browser.
+ * ```
+ *
+ * @experimental World generation is an experimental feature and may change.
+ */
+export function generateWorld<
+  TAdapter extends WorldAdapter<string, WorldProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: WorldActivityOptions<TAdapter, TStream>,
+): WorldActivityResult<TStream> {
+  if (options.stream) {
+    return streamGenerationResult(
+      (resolved) => runGenerateWorld({ ...options, runId: resolved.runId }),
+      options,
+    ) as WorldActivityResult<TStream>
+  }
+  return runGenerateWorld(options) as WorldActivityResult<TStream>
+}
+
+/**
+ * Run the core world generation logic (non-streaming).
+ */
+async function runGenerateWorld<
+  TAdapter extends WorldAdapter<string, WorldProviderOptions<TAdapter>>,
+>(
+  options: WorldActivityOptions<TAdapter, boolean>,
+): Promise<WorldGenerationResult> {
+  const {
+    adapter,
+    stream: _stream,
+    debug: _debug,
+    middleware,
+    threadId,
+    runId,
+    timeout,
+    abortSignal: callerAbortSignal,
+    ...rest
+  } = options
+  const model = adapter.model
+  const requestId = createId('world')
+  const startTime = Date.now()
+  const logger: InternalLogger = resolveDebugOption(options.debug)
+  const abortControls = createActivityAbortControls({
+    timeout,
+    abortSignal: callerAbortSignal,
+  })
+  const providerName =
+    (adapter as { name?: string; provider?: string }).provider ??
+    (adapter as { name?: string }).name ??
+    'unknown'
+
+  const mwCtx = createGenerationContext({
+    requestId,
+    activity: 'world',
+    provider: adapter.name,
+    model,
+    modelOptions: rest.modelOptions,
+    threadId,
+    runId,
+    artifactInputs: { prompt: rest.prompt },
+    createId,
+  })
+
+  await runGenerationStart(middleware, mwCtx)
+
+  aiEventClient.emit('world:request:started', {
+    requestId,
+    provider: adapter.name,
+    model,
+    prompt: rest.prompt,
+    timestamp: startTime,
+    ...(rest.modelOptions !== undefined && {
+      modelOptions: rest.modelOptions as Record<string, unknown>,
+    }),
+  })
+
+  logger.request(`activity=generateWorld provider=${providerName}`, {
+    provider: providerName,
+    model,
+  })
+
+  try {
+    const rawResult = await raceWithAbort(
+      adapter.createWorld({
+        ...rest,
+        model,
+        logger,
+        ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),
+      }),
+      abortControls.signal,
+    )
+    abortControls.clear()
+    const result = await applyGenerationResultTransforms(mwCtx, rawResult)
+    const elapsedMs = Date.now() - startTime
+
+    aiEventClient.emit('world:request:completed', {
+      requestId,
+      provider: adapter.name,
+      model: result.model,
+      prompt: result.prompt,
+      status: result.status,
+      duration: elapsedMs,
+      timestamp: Date.now(),
+      ...(rest.modelOptions !== undefined && {
+        modelOptions: rest.modelOptions as Record<string, unknown>,
+      }),
+    })
+
+    if (result.usage) {
+      aiEventClient.emit('world:usage', {
+        requestId,
+        model: result.model,
+        usage: result.usage,
+        timestamp: Date.now(),
+        ...(rest.modelOptions !== undefined && {
+          modelOptions: rest.modelOptions as Record<string, unknown>,
+        }),
+      })
+    }
+
+    logger.output(`activity=generateWorld provider=${providerName}`, {
+      model: result.model,
+      status: result.status,
+    })
+
+    if (result.usage) await runGenerationUsage(middleware, mwCtx, result.usage)
+    await runGenerationFinish(middleware, mwCtx, {
+      duration: elapsedMs,
+      usage: result.usage,
+    })
+
+    return result
+  } catch (error) {
+    abortControls.clear()
+    const elapsedMs = Date.now() - startTime
+    const err = error as Error
+    aiEventClient.emit('world:request:error', {
+      requestId,
+      provider: adapter.name,
+      model,
+      error: { message: err.message, name: err.name },
+      duration: elapsedMs,
+      timestamp: Date.now(),
+      ...(rest.modelOptions !== undefined && {
+        modelOptions: rest.modelOptions as Record<string, unknown>,
+      }),
+    })
+    if (isActivityAbortError(error, abortControls.signal)) {
+      await runGenerationAbort(middleware, mwCtx, {
+        reason: abortReasonMessage(error, abortControls.signal),
+        duration: elapsedMs,
+      })
+    } else {
+      await runGenerationError(middleware, mwCtx, {
+        error,
+        duration: elapsedMs,
+      })
+    }
+    logger.errors('generateWorld activity failed', {
+      error,
+      source: 'generateWorld',
+    })
+    throw error
+  }
+}
+
+// ===========================
+// Options Factory
+// ===========================
+
+/**
+ * Create typed options for the generateWorld() function without executing.
+ */
+export function createWorldOptions<
+  TAdapter extends WorldAdapter<string, WorldProviderOptions<TAdapter>>,
+  TStream extends boolean = false,
+>(
+  options: WorldActivityOptions<TAdapter, TStream>,
+): WorldActivityOptions<TAdapter, TStream> {
+  return options
+}
+
+// Re-export adapter types
+export type {
+  WorldAdapter,
+  WorldAdapterConfig,
+  AnyWorldAdapter,
+} from './adapter'
+export { BaseWorldAdapter } from './adapter'

package/src/activities/index.ts

@@ -23,6 +23,8 @@ import type { AnyTTSAdapter } from './generateSpeech/adapter'
 import type { AnyTranscriptionAdapter } from './generateTranscription/adapter'
 import type { AnyEmbeddingAdapter } from './embed/adapter'
 import type { AnyRerankAdapter } from './rerank/adapter'
+import type { AnyWorldAdapter } from './generateWorld/adapter'
+import type { AnyLiveVideoAdapter } from './generateLiveVideo/adapter'
 
 // ===========================
 // Chat Activity
@@ -210,6 +212,44 @@ export {
   type AnyEmbeddingAdapter,
 } from './embed/adapter'
 
+// ===========================
+// World Activity (Experimental)
+// ===========================
+
+export {
+  kind as worldKind,
+  generateWorld,
+  type WorldActivityOptions,
+  type WorldActivityResult,
+  type WorldProviderOptions,
+} from './generateWorld/index'
+
+export {
+  BaseWorldAdapter,
+  type WorldAdapter,
+  type WorldAdapterConfig,
+  type AnyWorldAdapter,
+} from './generateWorld/adapter'
+
+// ===========================
+// Live Activity (Experimental)
+// ===========================
+
+export {
+  kind as liveVideoKind,
+  generateLiveVideo,
+  type LiveVideoActivityOptions,
+  type LiveVideoActivityResult,
+  type LiveVideoProviderOptions,
+} from './generateLiveVideo/index'
+
+export {
+  BaseLiveVideoAdapter,
+  type LiveVideoAdapter,
+  type LiveVideoAdapterConfig,
+  type AnyLiveVideoAdapter,
+} from './generateLiveVideo/adapter'
+
 // ===========================
 // Adapter Union Types
 // ===========================
@@ -225,6 +265,8 @@ export type AIAdapter =
   | AnyTranscriptionAdapter
   | AnyEmbeddingAdapter
   | AnyRerankAdapter
+  | AnyWorldAdapter
+  | AnyLiveVideoAdapter
 
 /** Union type of all adapter kinds */
 export type AdapterKind =
@@ -237,3 +279,5 @@ export type AdapterKind =
   | 'transcription'
   | 'embedding'
   | 'rerank'
+  | 'world'
+  | 'liveVideo'

package/src/activities/middleware/types.ts

@@ -44,6 +44,8 @@ export type GenerationActivity =
   | 'embedding'
   | 'rerank'
   | 'summarize'
+  | 'world'
+  | 'liveVideo'
 
 /**
  * Stable context passed to every {@link GenerationMiddleware} hook. Created

package/src/client.ts

@@ -4,6 +4,8 @@ import type {
   TTSOptions,
   TranscriptionOptions,
   VideoGenerationOptions,
+  WorldGenerationOptions,
+  LiveVideoGenerationOptions,
 } from './types'
 
 export type GenerationKind =
@@ -12,6 +14,8 @@ export type GenerationKind =
   | 'tts'
   | 'video'
   | 'transcription'
+  | 'world'
+  | 'liveVideo'
 
 type GenerationInputByKind = {
   image: Omit<ImageGenerationOptions, 'logger' | 'model'>
@@ -19,6 +23,8 @@ type GenerationInputByKind = {
   tts: Omit<TTSOptions, 'logger' | 'model'>
   video: Omit<VideoGenerationOptions, 'logger' | 'model'>
   transcription: Omit<TranscriptionOptions, 'logger' | 'model'>
+  world: Omit<WorldGenerationOptions, 'logger' | 'model'>
+  liveVideo: Omit<LiveVideoGenerationOptions, 'logger' | 'model'>
 }
 
 export interface GenerationParams<TKind extends GenerationKind> {
@@ -34,6 +40,8 @@ const generationKinds = [
   'tts',
   'video',
   'transcription',
+  'world',
+  'liveVideo',
 ] as const satisfies ReadonlyArray<GenerationKind>
 
 function isRecord(value: unknown): value is Record<string, unknown> {

package/src/index.ts

@@ -10,6 +10,8 @@ export {
   generateSpeech,
   generateTranscription,
   embed,
+  generateWorld,
+  generateLiveVideo,
 } from './activities/index'
 
 // Create options functions - for pre-defining typed configurations
@@ -22,6 +24,8 @@ export { createVideoOptions } from './activities/generateVideo/index'
 export { createSpeechOptions } from './activities/generateSpeech/index'
 export { createTranscriptionOptions } from './activities/generateTranscription/index'
 export { createEmbedOptions } from './activities/embed/index'
+export { createWorldOptions } from './activities/generateWorld/index'
+export { createLiveVideoOptions } from './activities/generateLiveVideo/index'
 
 // Re-export types
 export type {
@@ -44,6 +48,10 @@ export type {
   EmbeddingAdapter,
   AnyRerankAdapter,
   RerankAdapter,
+  AnyWorldAdapter,
+  WorldAdapter,
+  AnyLiveVideoAdapter,
+  LiveVideoAdapter,
 } from './activities/index'
 
 // Rerank adapter base + types

package/src/middlewares/otel.ts

@@ -91,6 +91,8 @@ const OPERATION_NAME: Record<GenerationActivity, string> = {
   embedding: 'embeddings',
   rerank: 'rerank',
   summarize: 'summarize',
+  world: 'world_generation',
+  liveVideo: 'live_video_generation',
 }
 
 export interface OtelMiddlewareOptions {