lumiverse-spindle-types 0.5.28 → 0.5.30

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "lumiverse-spindle-types",
 
-  "version": "0.5.28",
+  "version": "0.5.30",
   "types": "./src/index.ts",
   "keywords": [
     "lumiverse",

package/src/api.ts

@@ -615,6 +615,7 @@ export interface ImageGenResultDTO {
 // ─── Image DTOs ─────────────────────────────────────────────────────────
 
 export type ImageSpecificityDTO = "full" | "sm" | "lg";
+export type ImageVideoCodecDTO = "h264" | "hevc";
 
 export interface ImageListOptionsDTO {
   limit?: number;
@@ -644,7 +645,7 @@ export interface ImageGetOptionsDTO {
   userId?: string;
 }
 
-/** Safe representation of an image exposed to extensions. */
+/** Safe representation of an image/video asset exposed to extensions. */
 export interface ImageDTO {
   id: string;
   original_filename: string;
@@ -663,7 +664,7 @@ export interface ImageDTO {
 
 /** Upload payload for `spindle.images.upload()` */
 export interface ImageUploadDTO {
-  /** Raw image bytes. */
+  /** Raw image or video bytes. */
   data: Uint8Array;
   /** Optional filename used to preserve the extension/MIME when storing the image. */
   filename?: string;
@@ -673,6 +674,12 @@ export interface ImageUploadDTO {
   owner_character_id?: string;
   /** Optional chat ownership tag for the persisted image. */
   owner_chat_id?: string;
+  /** For video uploads, strip any audio tracks from the stored output when possible. */
+  strip_audio?: boolean;
+  /** For video uploads, transcode the primary stored asset to this codec. */
+  transcode_video_codec?: ImageVideoCodecDTO;
+  /** Optional extra video variants to generate alongside the primary stored asset. */
+  sidecar_video_codecs?: ImageVideoCodecDTO[];
 }
 
 export interface ImageUploadFromDataUrlOptionsDTO {
@@ -685,6 +692,159 @@ export interface ImageUploadFromDataUrlOptionsDTO {
   userId?: string;
 }
 
+// ─── Media DTOs ─────────────────────────────────────────────────────────
+
+export type MediaSourceDTO =
+  | {
+      kind: "inline";
+      data: Uint8Array;
+      filename?: string;
+      mime_type?: string;
+    }
+  | {
+      kind: "upload";
+      upload_id: string;
+      filename?: string;
+      mime_type?: string;
+    }
+  | {
+      /**
+       * Image or video asset already stored in Lumiverse's images table.
+       * This can point at either a still image or a video upload.
+       */
+      kind: "image";
+      image_id: string;
+    }
+  | {
+      /** Audio asset already stored in Lumiverse's audio_files table. */
+      kind: "audio";
+      audio_id: string;
+    };
+
+export type MediaAudioFormatDTO =
+  | "mp3"
+  | "wav"
+  | "ogg"
+  | "aac"
+  | "flac"
+  | "m4a"
+  | "webm";
+
+export type MediaVideoFormatDTO =
+  | "mp4"
+  | "webm"
+  | "mov"
+  | "mkv";
+
+export type MediaVideoCodecDTO =
+  | "h264"
+  | "hevc"
+  | "vp9"
+  | "av1"
+  | "copy";
+
+export type MediaAudioCodecDTO =
+  | "aac"
+  | "mp3"
+  | "opus"
+  | "vorbis"
+  | "flac"
+  | "pcm_s16le"
+  | "copy";
+
+export type MediaFitModeDTO = "contain" | "cover" | "stretch";
+
+export interface MediaTransformResultDTO {
+  data: Uint8Array;
+  filename: string;
+  mime_type: string;
+  byte_size: number;
+  duration_ms?: number | null;
+  width?: number | null;
+  height?: number | null;
+}
+
+export interface MediaConvertAudioRequestDTO {
+  source: MediaSourceDTO;
+  output_format: MediaAudioFormatDTO;
+  audio_codec?: MediaAudioCodecDTO;
+  bitrate_kbps?: number;
+  sample_rate?: number;
+  channels?: number;
+  filename?: string;
+  /** For operator-scoped extensions. */
+  userId?: string;
+}
+
+export interface MediaConvertVideoRequestDTO {
+  source: MediaSourceDTO;
+  output_format: MediaVideoFormatDTO;
+  filename?: string;
+  /** For operator-scoped extensions. */
+  userId?: string;
+}
+
+export interface MediaTranscodeVideoRequestDTO {
+  source: MediaSourceDTO;
+  output_format?: MediaVideoFormatDTO;
+  video_codec?: MediaVideoCodecDTO;
+  audio_codec?: MediaAudioCodecDTO | "none";
+  video_bitrate_kbps?: number;
+  audio_bitrate_kbps?: number;
+  crf?: number;
+  preset?: string;
+  width?: number;
+  height?: number;
+  fps?: number;
+  pixel_format?: string;
+  faststart?: boolean;
+  filename?: string;
+  /** For operator-scoped extensions. */
+  userId?: string;
+}
+
+export interface MediaRemoveAudioFromVideoRequestDTO {
+  source: MediaSourceDTO;
+  output_format?: MediaVideoFormatDTO;
+  video_codec?: MediaVideoCodecDTO;
+  filename?: string;
+  /** For operator-scoped extensions. */
+  userId?: string;
+}
+
+export interface MediaAddAudioToVideoRequestDTO {
+  video: MediaSourceDTO;
+  audio: MediaSourceDTO;
+  output_format?: MediaVideoFormatDTO;
+  video_codec?: MediaVideoCodecDTO;
+  audio_codec?: MediaAudioCodecDTO;
+  /** Defaults to true: replace any existing audio track on the source video. */
+  replace_existing_audio?: boolean;
+  /** When true, clamp the output duration to the shorter input stream. */
+  shortest?: boolean;
+  /** Optional positive offset, in milliseconds, before the new audio starts. */
+  audio_start_ms?: number;
+  filename?: string;
+  /** For operator-scoped extensions. */
+  userId?: string;
+}
+
+export interface MediaCreateVideoFromImageAndAudioRequestDTO {
+  image: MediaSourceDTO;
+  audio: MediaSourceDTO;
+  output_format?: MediaVideoFormatDTO;
+  video_codec?: Exclude<MediaVideoCodecDTO, "copy">;
+  audio_codec?: MediaAudioCodecDTO;
+  width?: number;
+  height?: number;
+  fps?: number;
+  fit_mode?: MediaFitModeDTO;
+  background_color?: string;
+  filename?: string;
+  /** For operator-scoped extensions. */
+  userId?: string;
+}
+
 // ─── Character DTOs ─────────────────────────────────────────────────────
 
 /**
@@ -2347,6 +2507,13 @@ export interface TokenCountResultDTO {
   approximate: boolean;
 }
 
+/** A completed resumable upload read back by `spindle.uploads.get`. */
+export interface SpindleUploadDTO {
+  fileName: string;
+  size: number;
+  data: Uint8Array;
+}
+
 /** Context delivered to an on-request shared RPC endpoint handler. */
 export interface SharedRpcRequestContextDTO {
   /** Fully-qualified endpoint name (for example `weather_ext.status.current`). */
@@ -2845,6 +3012,17 @@ export type WorkerToHost =
       userId?: string;
     }
   | { type: "images_delete"; requestId: string; imageId: string; userId?: string }
+  // ─── Media (gated: "media") ─────────────────────────────────────────
+  | { type: "media_audio_convert"; requestId: string; input: MediaConvertAudioRequestDTO }
+  | { type: "media_video_convert"; requestId: string; input: MediaConvertVideoRequestDTO }
+  | { type: "media_video_transcode"; requestId: string; input: MediaTranscodeVideoRequestDTO }
+  | { type: "media_video_remove_audio"; requestId: string; input: MediaRemoveAudioFromVideoRequestDTO }
+  | { type: "media_video_add_audio"; requestId: string; input: MediaAddAudioToVideoRequestDTO }
+  | {
+      type: "media_video_from_image_audio";
+      requestId: string;
+      input: MediaCreateVideoFromImageAndAudioRequestDTO;
+    }
   // ─── Theme (gated: "app_manipulation") ──────────────────────────────────
   | { type: "theme_apply"; requestId: string; overrides: ThemeOverrideDTO; userId?: string }
   | { type: "theme_apply_palette"; requestId: string; palette: ThemePaletteConfigDTO | null; userId?: string }

package/src/index.ts

@@ -119,8 +119,22 @@ export type {
     ImageDTO,
     ImageListOptionsDTO,
     ImageSpecificityDTO,
+    ImageVideoCodecDTO,
     ImageUploadDTO,
     ImageUploadFromDataUrlOptionsDTO,
+    MediaSourceDTO,
+    MediaAudioFormatDTO,
+    MediaVideoFormatDTO,
+    MediaVideoCodecDTO,
+    MediaAudioCodecDTO,
+    MediaFitModeDTO,
+    MediaTransformResultDTO,
+    MediaConvertAudioRequestDTO,
+    MediaConvertVideoRequestDTO,
+    MediaTranscodeVideoRequestDTO,
+    MediaRemoveAudioFromVideoRequestDTO,
+    MediaAddAudioToVideoRequestDTO,
+    MediaCreateVideoFromImageAndAudioRequestDTO,
     ThemeOverrideDTO,
   ThemeInfoDTO,
   ThemePaletteConfigDTO,
@@ -160,6 +174,7 @@ export type {
   TokenModelSourceDTO,
   TokenCountOptionsDTO,
   TokenCountResultDTO,
+  SpindleUploadDTO,
   SharedRpcRequestContextDTO,
   SharedRpcEndpointPolicyDTO,
   MacroInterceptorPhase,

package/src/permissions.ts

@@ -18,6 +18,8 @@
  * - "memories"             — CRUD on the Memory Cortex (entities, relations, vaults, chat
  *                            links, consolidations) and long-term chat memory (vectorized
  *                            chat-chunk retrieval, warmup, cache).
+ * - "media"                — invoke the backend media pipeline for audio/video conversion,
+ *                            transcoding, muxing, and simple image+audio composition.
  * - "macro_interceptor"    — transform raw templates before macro parsing/dispatch
  * - "web_search"           — execute searches via the user's configured web search
  *                            provider (e.g. SearXNG) and read the safe view of their
@@ -44,6 +46,7 @@ export type SpindlePermission =
   | "regex_scripts"
   | "databanks"
   | "memories"
+  | "media"
   | "personas"
   | "push_notification"
   | "image_gen"
@@ -72,6 +75,7 @@ export const ALL_PERMISSIONS: readonly SpindlePermission[] = [
   "regex_scripts",
   "databanks",
   "memories",
+  "media",
   "personas",
   "push_notification",
   "image_gen",

package/src/spindle-api.ts

@@ -65,6 +65,13 @@ import type {
   ImageListOptionsDTO,
   ImageUploadDTO,
   ImageUploadFromDataUrlOptionsDTO,
+  MediaConvertAudioRequestDTO,
+  MediaConvertVideoRequestDTO,
+  MediaTranscodeVideoRequestDTO,
+  MediaRemoveAudioFromVideoRequestDTO,
+  MediaAddAudioToVideoRequestDTO,
+  MediaCreateVideoFromImageAndAudioRequestDTO,
+  MediaTransformResultDTO,
   ThemeOverrideDTO,
   ThemeInfoDTO,
   ThemePaletteConfigDTO,
@@ -100,6 +107,7 @@ import type {
   StreamChunkDTO,
   TokenCountOptionsDTO,
   TokenCountResultDTO,
+  SpindleUploadDTO,
   MacroInterceptorCtxDTO,
   MacroInterceptorResultDTO,
   WorldInfoInterceptorCtxDTO,
@@ -719,6 +727,17 @@ export interface SpindleAPI {
     countChat(chatId: string, options?: TokenCountOptionsDTO): Promise<TokenCountResultDTO>;
   };
 
+  /**
+   * Resumable uploads (free tier). Consume a file the user streamed to the host
+   * tus endpoint (`/api/v1/spindle-uploads`), scoped to this extension + user.
+   */
+  uploads: {
+    /** Read a completed upload's bytes. Null if missing, expired, or not owned by this extension + user. */
+    get(uploadId: string, userId?: string): Promise<SpindleUploadDTO | null>;
+    /** Delete a staged upload once consumed. Returns false if it was already gone. */
+    delete(uploadId: string, userId?: string): Promise<boolean>;
+  };
+
   /**
    * Image generation (permission: "image_gen").
    * Generate images via the user's configured image gen connection profiles.
@@ -754,8 +773,8 @@ export interface SpindleAPI {
   };
 
   /**
-   * Image CRUD (permission: "images").
-   * Manage images stored in Lumiverse's image system on behalf of the user.
+   * Image/video asset CRUD (permission: "images").
+   * Manage assets stored in Lumiverse's image system on behalf of the user.
    * For user-scoped extensions, userId is inferred from the extension owner.
    * For operator-scoped extensions, pass userId to scope to a specific user.
    */
@@ -773,6 +792,20 @@ export interface SpindleAPI {
     delete(imageId: string, userId?: string): Promise<boolean>;
   };
 
+  /**
+   * Backend media conversion and composition (permission: "media").
+   * Uses Lumiverse's host-side media pipeline for audio/video transforms.
+   * Sources may be inline bytes, staged uploads, or existing Lumiverse assets.
+   */
+  media: {
+    convertAudio(input: MediaConvertAudioRequestDTO): Promise<MediaTransformResultDTO>;
+    convertVideo(input: MediaConvertVideoRequestDTO): Promise<MediaTransformResultDTO>;
+    transcodeVideo(input: MediaTranscodeVideoRequestDTO): Promise<MediaTransformResultDTO>;
+    removeAudioFromVideo(input: MediaRemoveAudioFromVideoRequestDTO): Promise<MediaTransformResultDTO>;
+    addAudioToVideo(input: MediaAddAudioToVideoRequestDTO): Promise<MediaTransformResultDTO>;
+    createVideoFromImageAndAudio(input: MediaCreateVideoFromImageAndAudioRequestDTO): Promise<MediaTransformResultDTO>;
+  };
+
   /**
    * Local (transient), global (user-scoped), and chat (persisted) variable access
    * (free tier — no permission needed).