mulmoclaude 0.1.0

package/src/plugins/presentMulmoScript/definition.ts

@@ -0,0 +1,95 @@
+import type { ToolDefinition } from "gui-chat-protocol";
+
+export const TOOL_NAME = "presentMulmoScript";
+
+const toolDefinition: ToolDefinition = {
+  type: "function",
+  name: TOOL_NAME,
+  description: `Save and present a MulmoScript story or presentation as a visual storyboard in the canvas.
+
+Always use Google providers. Required structure:
+
+{
+  "$mulmocast": { "version": "1.1" },
+  "title": "The Life of a Star",
+  "description": "A short educational explainer about stellar evolution",
+  "lang": "en",
+  "speechParams": {
+    "speakers": {
+      "Presenter": {
+        "provider": "gemini",
+        "voiceId": "Kore",
+        "displayName": { "en": "Presenter" }
+      }
+    }
+  },
+  "imageParams": { "provider": "google", "model": "gemini-3.1-flash-image" },
+  "movieParams": { "provider": "google", "model": "veo-3.1-generate" },
+  "beats": [
+    {
+      "speaker": "Presenter",
+      "text": "Narration spoken aloud for this beat.",
+      "imagePrompt": "Detailed description — AI generates the image"
+    },
+    {
+      "speaker": "Presenter",
+      "text": "Bullet point beat.",
+      "image": { "type": "textSlide", "slide": { "title": "Slide Title", "bullets": ["Point one", "Point two"] } }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "Markdown beat.",
+      "image": { "type": "markdown", "markdown": "## Heading\\n\\nBody text here." }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "Chart beat — use for data, comparisons, trends.",
+      "image": { "type": "chart", "title": "Chart Title", "chartData": { "type": "bar", "data": { "labels": ["A", "B", "C"], "datasets": [{ "label": "Series", "data": [10, 20, 30] }] } } }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "Diagram beat — use for flows, architectures, relationships.",
+      "image": { "type": "mermaid", "title": "Diagram Title", "code": { "kind": "text", "text": "graph TD\\n  A[Start] --> B[Process] --> C[End]" } }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "Rich interactive beat — use for custom layouts, animations, or anything that benefits from HTML/CSS.",
+      "image": { "type": "html_tailwind", "html": "<div class=\\"flex items-center justify-center h-full text-4xl font-bold text-blue-600\\">Hello World</div>" }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "AI video beat.",
+      "moviePrompt": "Detailed description — AI generates the video clip"
+    }
+  ]
+}
+
+Beat visual options (choose one per beat):
+- "imagePrompt": "..."  → top-level string field — AI generates an image from the prompt
+- "moviePrompt": "..."  → top-level string field — AI generates a video clip from the prompt
+- "image": { "type": "textSlide", "slide": { "title", "subtitle"?, "bullets"? } }
+- "image": { "type": "markdown", "markdown": "..." }
+- "image": { "type": "chart", "title": "...", "chartData": { "type": "bar"|"line"|"pie"|..., "data": { "labels": [...], "datasets": [...] } } }  ← PREFER for data/numbers/comparisons. chartData is a full Chart.js config: labels/datasets go under "data", not at the top level.
+- "image": { "type": "mermaid", "title": "...", "code": { "kind": "text", "text": "..." } }  ← PREFER for flows/diagrams/relationships
+- "image": { "type": "html_tailwind", "html": "...", "script"?: "..." }  ← PREFER for rich layouts, animations, custom visuals
+
+IMPORTANT: "imagePrompt" and "moviePrompt" are plain string fields on the beat, NOT nested under "image".`,
+  parameters: {
+    type: "object",
+    properties: {
+      script: {
+        type: "object",
+        description:
+          "Complete MulmoScript JSON. Must include $mulmocast, speechParams, imageParams, movieParams, and beats array. Always populate the top-level 'description' field with a concise 1–2 sentence summary of the presentation.",
+        additionalProperties: true,
+      },
+      filename: {
+        type: "string",
+        description: "Optional filename without extension. Defaults to a slug of the script title.",
+      },
+    },
+    required: ["script"],
+  },
+};
+
+export default toolDefinition;

package/src/plugins/presentMulmoScript/helpers.ts

@@ -0,0 +1,162 @@
+// Pure helpers for presentMulmoScript View.vue. Kept separate so
+// their logic is unit-testable without mounting the Vue component.
+
+import { errorMessage } from "../../utils/errors";
+import { isRecord } from "../../utils/types";
+
+export type SSEEvent =
+  | { type: "beat_image_done"; beatIndex: number }
+  | { type: "beat_audio_done"; beatIndex: number }
+  | { type: "done"; moviePath: string }
+  | { type: "error"; message: string }
+  | { type: "unknown" };
+
+/**
+ * Parse a single SSE line of the form `data: {json}`. Returns
+ * null for non-data lines (comments, blank) or lines whose JSON
+ * payload fails to parse. Unrecognised event types still parse
+ * but resolve to `{ type: "unknown" }` so the caller can ignore
+ * them without crashing.
+ */
+export function parseSSEEventLine(line: string): SSEEvent | null {
+  if (!line.startsWith("data: ")) return null;
+  let obj: unknown;
+  try {
+    obj = JSON.parse(line.slice(6));
+  } catch {
+    return null;
+  }
+  if (!isRecord(obj)) return null;
+  const event = obj;
+  if (event.type === "beat_image_done" && typeof event.beatIndex === "number") {
+    return { type: "beat_image_done", beatIndex: event.beatIndex };
+  }
+  if (event.type === "beat_audio_done" && typeof event.beatIndex === "number") {
+    return { type: "beat_audio_done", beatIndex: event.beatIndex };
+  }
+  if (event.type === "done" && typeof event.moviePath === "string") {
+    return { type: "done", moviePath: event.moviePath };
+  }
+  if (event.type === "error" && typeof event.message === "string") {
+    return { type: "error", message: event.message };
+  }
+  return { type: "unknown" };
+}
+
+/**
+ * Decide whether a beat should be rendered automatically at
+ * script load time. Text-based beats (slides, charts, etc.) are
+ * auto-rendered only when the script has no characters —
+ * characters must be rendered first so they can be referenced by
+ * any character-using beat.
+ */
+export function shouldAutoRenderBeat(beat: { image?: { type?: string } | undefined }, hasCharacters: boolean, autoRenderTypes: readonly string[]): boolean {
+  if (hasCharacters) return false;
+  const type = beat.image?.type;
+  if (typeof type !== "string") return false;
+  return autoRenderTypes.includes(type);
+}
+
+/**
+ * Of the given character keys, return those whose image is not
+ * yet loaded and is not currently rendering. Used to fetch only
+ * what's missing after a movie-generation event arrives.
+ */
+export function getMissingCharacterKeys(keys: readonly string[], images: Record<string, unknown>, renderState: Record<string, string | undefined>): string[] {
+  return keys.filter((k) => !images[k] && renderState[k] !== "rendering");
+}
+
+/**
+ * A schema shape that exposes `safeParse` — matches Zod's API
+ * without pulling the dep into this module.
+ */
+export interface SafeParseSchema {
+  safeParse(value: unknown): { success: boolean };
+}
+
+/**
+ * Validate a candidate Beat JSON string against a schema.
+ * Returns false on any JSON parse error or schema mismatch.
+ */
+export function validateBeatJSON(json: string, schema: SafeParseSchema): boolean {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(json);
+  } catch {
+    return false;
+  }
+  return schema.safeParse(parsed).success;
+}
+
+/** Convert an unknown thrown value into a human-readable string. */
+export function extractErrorMessage(err: unknown): string {
+  return errorMessage(err);
+}
+
+/**
+ * Callback set for `applyMovieEvent` / `streamMovieEvents`. Each
+ * handler is scoped to one event shape; the dispatcher routes to
+ * the right one based on the discriminated union's `type` field.
+ * Keeping this as named handlers (rather than one big switch in
+ * the caller) lets `generateMovie` stay under the
+ * sonarjs/cognitive-complexity threshold.
+ */
+export interface MovieEventHandlers {
+  onBeatImageDone(beatIndex: number): void;
+  onBeatAudioDone(beatIndex: number): void;
+  onDone(moviePath: string): void;
+}
+
+/**
+ * Dispatch a single already-parsed SSE event from the movie
+ * generation stream to the matching handler. `"error"` events
+ * throw so the caller's try/catch can surface the message the
+ * same way a network failure would. `"unknown"` events are
+ * silently ignored — the server occasionally introduces new
+ * event types before the client catches up, and we don't want
+ * those to tear down an otherwise-healthy stream.
+ */
+export function applyMovieEvent(event: SSEEvent, handlers: MovieEventHandlers): void {
+  switch (event.type) {
+    case "beat_image_done":
+      handlers.onBeatImageDone(event.beatIndex);
+      return;
+    case "beat_audio_done":
+      handlers.onBeatAudioDone(event.beatIndex);
+      return;
+    case "done":
+      handlers.onDone(event.moviePath);
+      return;
+    case "error":
+      throw new Error(event.message);
+    case "unknown":
+      return;
+  }
+}
+
+/**
+ * Read the SSE stream body from the movie-generation endpoint
+ * and dispatch every parsed event into the given handlers.
+ * Returns when the server closes the stream; throws through any
+ * `"error"` event or unhandled read error. Kept here (rather
+ * than inline in `generateMovie`) so the reader + decoder +
+ * line-buffer state machine is a single named unit instead of a
+ * pyramid of `while` / `for` / `if` inside a Vue component.
+ */
+export async function streamMovieEvents(body: ReadableStream<Uint8Array>, handlers: MovieEventHandlers): Promise<void> {
+  const reader = body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    buffer += decoder.decode(value, { stream: true });
+    const lines = buffer.split("\n");
+    buffer = lines.pop() ?? "";
+    for (const line of lines) {
+      const event = parseSSEEventLine(line);
+      if (!event) continue;
+      applyMovieEvent(event, handlers);
+    }
+  }
+}

package/src/plugins/presentMulmoScript/index.ts

@@ -0,0 +1,40 @@
+import type { ToolPlugin } from "../../tools/types";
+import type { ToolResult } from "gui-chat-protocol";
+import type { MulmoScript } from "mulmocast";
+import toolDefinition, { TOOL_NAME } from "./definition";
+import View from "./View.vue";
+import Preview from "./Preview.vue";
+import { apiPost } from "../../utils/api";
+import { API_ROUTES } from "../../config/apiRoutes";
+
+export interface MulmoScriptData {
+  script: MulmoScript;
+  filePath: string;
+}
+
+const presentMulmoScriptPlugin: ToolPlugin<MulmoScriptData> = {
+  toolDefinition,
+
+  async execute(_context, args) {
+    const result = await apiPost<ToolResult<MulmoScriptData>>(API_ROUTES.mulmoScript.save, args);
+    if (!result.ok) {
+      return {
+        toolName: TOOL_NAME,
+        uuid: crypto.randomUUID(),
+        message: result.error,
+      };
+    }
+    return {
+      ...result.data,
+      toolName: TOOL_NAME,
+      uuid: crypto.randomUUID(),
+    };
+  },
+
+  isEnabled: () => true,
+  generatingMessage: "Generating MulmoScript storyboard…",
+  viewComponent: View,
+  previewComponent: Preview,
+};
+
+export default presentMulmoScriptPlugin;

package/src/plugins/scheduler/Preview.vue

@@ -0,0 +1,67 @@
+<template>
+  <div class="text-sm">
+    <div class="flex items-center gap-1 font-medium text-gray-700 mb-1">
+      <span>📅</span>
+      <span>{{ upcomingItems.length }} upcoming</span>
+    </div>
+    <div v-for="item in preview" :key="item.id" class="text-xs truncate text-gray-600">
+      <span v-if="item.props.date" class="text-gray-400 mr-1">{{ item.props.date }}</span>
+      {{ item.title }}
+    </div>
+    <div v-if="more > 0" class="text-xs text-gray-400">+ {{ more }} more…</div>
+  </div>
+</template>
+
+<script setup lang="ts">
+import { computed, ref, watch } from "vue";
+import type { ToolResultComplete } from "gui-chat-protocol/vue";
+import type { SchedulerData, ScheduledItem } from "./index";
+import { useFreshPluginData } from "../../composables/useFreshPluginData";
+import { API_ROUTES } from "../../config/apiRoutes";
+
+const props = defineProps<{ result: ToolResultComplete<SchedulerData> }>();
+
+const items = ref<ScheduledItem[]>(props.result.data?.items ?? []);
+
+const { refresh } = useFreshPluginData<ScheduledItem[]>({
+  endpoint: () => API_ROUTES.scheduler.base,
+  extract: (json) => {
+    const v = (json as { data?: { items?: ScheduledItem[] } }).data?.items;
+    return Array.isArray(v) ? v : null;
+  },
+  apply: (data) => {
+    items.value = data;
+  },
+});
+
+watch(
+  () => props.result.uuid,
+  () => {
+    items.value = props.result.data?.items ?? [];
+    void refresh();
+  },
+);
+
+const today = new Date().toISOString().slice(0, 10);
+
+const upcomingItems = computed(() => {
+  const withDate: ScheduledItem[] = [];
+  const noDate: ScheduledItem[] = [];
+
+  for (const item of items.value) {
+    const d = item.props.date;
+    if (typeof d === "string") {
+      if (d >= today) withDate.push(item);
+    } else {
+      noDate.push(item);
+    }
+  }
+
+  withDate.sort((a, b) => (String(a.props.date) < String(b.props.date) ? -1 : 1));
+
+  return [...withDate, ...noDate];
+});
+
+const preview = computed(() => upcomingItems.value.slice(0, 3));
+const more = computed(() => Math.max(0, upcomingItems.value.length - 3));
+</script>

package/src/plugins/scheduler/TasksTab.vue

@@ -0,0 +1,205 @@
+<template>
+  <div class="flex-1 overflow-y-auto min-h-0 p-4">
+    <!-- Mutation error banner -->
+    <div v-if="mutationError" class="mb-3 px-4 py-2 bg-red-50 text-red-700 rounded text-sm" data-testid="scheduler-task-error">
+      {{ mutationError }}
+    </div>
+
+    <!-- Loading -->
+    <div v-if="loading" class="flex items-center justify-center h-32 text-gray-400">Loading...</div>
+
+    <!-- Error -->
+    <div v-else-if="error" class="px-4 py-2 bg-red-50 text-red-700 rounded text-sm">
+      {{ error }}
+    </div>
+
+    <!-- Task list -->
+    <div v-else>
+      <div v-if="tasks.length === 0" class="flex items-center justify-center h-32 text-gray-400">No scheduled tasks</div>
+
+      <div v-else class="space-y-2">
+        <div
+          v-for="task in tasks"
+          :key="task.id"
+          :data-testid="`scheduler-task-${task.id}`"
+          class="border border-gray-200 rounded-lg p-3 hover:bg-gray-50"
+          :class="{ 'opacity-50': task.enabled === false }"
+        >
+          <div class="flex items-center justify-between">
+            <div class="flex items-center gap-2 min-w-0">
+              <!-- Origin badge -->
+              <span class="text-xs px-1.5 py-0.5 rounded font-medium shrink-0" :class="originClass(task.origin)">
+                {{ originLabel(task.origin) }}
+              </span>
+              <span class="font-medium text-gray-800 truncate">
+                {{ task.name }}
+              </span>
+            </div>
+            <div class="flex items-center gap-1 shrink-0">
+              <!-- Run now -->
+              <button
+                v-if="task.origin === 'user'"
+                class="px-2 py-1 text-xs text-blue-600 hover:bg-blue-50 rounded"
+                title="Run now"
+                aria-label="Run now"
+                data-testid="scheduler-task-run"
+                @click="runTask(task.id)"
+              >
+                <span class="material-icons text-sm">play_arrow</span>
+              </button>
+              <!-- Enable/disable toggle -->
+              <button
+                v-if="task.origin === 'user'"
+                class="px-2 py-1 text-xs rounded"
+                :class="task.enabled !== false ? 'text-green-600 hover:bg-green-50' : 'text-gray-400 hover:bg-gray-100'"
+                :title="task.enabled !== false ? 'Disable' : 'Enable'"
+                @click="toggleEnabled(task)"
+              >
+                <span class="material-icons text-sm">
+                  {{ task.enabled !== false ? "toggle_on" : "toggle_off" }}
+                </span>
+              </button>
+              <!-- Delete -->
+              <button
+                v-if="task.origin === 'user'"
+                class="px-2 py-1 text-xs text-red-500 hover:bg-red-50 rounded"
+                title="Delete"
+                aria-label="Delete"
+                data-testid="scheduler-task-delete"
+                @click="deleteTask(task.id)"
+              >
+                <span class="material-icons text-sm">delete</span>
+              </button>
+            </div>
+          </div>
+
+          <!-- Details row -->
+          <div class="mt-1 flex items-center gap-3 text-xs text-gray-500">
+            <span>{{ formatSchedule(task.schedule) }}</span>
+            <span v-if="task.state?.lastRunResult" class="flex items-center gap-1">
+              <span class="inline-block w-2 h-2 rounded-full" :class="resultDotClass(task.state.lastRunResult)"></span>
+              {{ task.state.lastRunResult }}
+            </span>
+            <span v-if="task.state?.nextScheduledAt"> Next: {{ formatShortTime(task.state.nextScheduledAt) }} </span>
+          </div>
+
+          <!-- Description -->
+          <div v-if="task.description" class="mt-1 text-xs text-gray-400 truncate">
+            {{ task.description }}
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</template>
+
+<script setup lang="ts">
+import { ref, onMounted } from "vue";
+import { apiGet, apiPost, apiPut, apiDelete } from "../../utils/api";
+import { API_ROUTES } from "../../config/apiRoutes";
+import { formatShortTime } from "../../utils/format/date";
+
+interface TaskSchedule {
+  type: string;
+  intervalMs?: number;
+  time?: string;
+}
+
+interface TaskState {
+  lastRunAt?: string | null;
+  lastRunResult?: string | null;
+  nextScheduledAt?: string | null;
+}
+
+interface SchedulerTask {
+  id: string;
+  name: string;
+  description?: string;
+  schedule: TaskSchedule;
+  origin: string;
+  enabled?: boolean;
+  state?: TaskState;
+}
+
+const tasks = ref<SchedulerTask[]>([]);
+const loading = ref(true);
+const error = ref("");
+const mutationError = ref("");
+
+async function fetchTasks(): Promise<void> {
+  loading.value = true;
+  error.value = "";
+  const result = await apiGet<{ tasks: SchedulerTask[] }>(API_ROUTES.scheduler.tasks);
+  loading.value = false;
+  if (!result.ok) {
+    error.value = result.error;
+    return;
+  }
+  tasks.value = result.data.tasks;
+}
+
+function originLabel(origin: string): string {
+  if (origin === "system") return "System";
+  if (origin === "user") return "User";
+  return "Skill";
+}
+
+function originClass(origin: string): string {
+  if (origin === "system") return "bg-gray-100 text-gray-600";
+  if (origin === "user") return "bg-blue-100 text-blue-700";
+  return "bg-purple-100 text-purple-700";
+}
+
+function resultDotClass(result: string): string {
+  if (result === "success") return "bg-green-500";
+  if (result === "error") return "bg-red-500";
+  return "bg-gray-400";
+}
+
+function formatSchedule(schedule: TaskSchedule): string {
+  if (schedule.type === "interval" && schedule.intervalMs) {
+    const mins = Math.round(schedule.intervalMs / 60000);
+    if (mins >= 60) return `Every ${Math.round(mins / 60)}h`;
+    return `Every ${mins}m`;
+  }
+  if (schedule.type === "daily" && schedule.time) {
+    return `Daily ${schedule.time} UTC`;
+  }
+  return JSON.stringify(schedule);
+}
+
+async function runTask(id: string): Promise<void> {
+  mutationError.value = "";
+  const url = API_ROUTES.scheduler.taskRun.replace(":id", id);
+  const result = await apiPost(url, {});
+  if (!result.ok) {
+    mutationError.value = `Run failed: ${result.error}`;
+    return;
+  }
+  await fetchTasks();
+}
+
+async function toggleEnabled(task: SchedulerTask): Promise<void> {
+  mutationError.value = "";
+  const url = API_ROUTES.scheduler.task.replace(":id", task.id);
+  const result = await apiPut(url, { enabled: task.enabled === false });
+  if (!result.ok) {
+    mutationError.value = `Toggle failed: ${result.error}`;
+    return;
+  }
+  await fetchTasks();
+}
+
+async function deleteTask(id: string): Promise<void> {
+  mutationError.value = "";
+  const url = API_ROUTES.scheduler.task.replace(":id", id);
+  const result = await apiDelete(url);
+  if (!result.ok) {
+    mutationError.value = `Delete failed: ${result.error}`;
+    return;
+  }
+  await fetchTasks();
+}
+
+onMounted(fetchTasks);
+</script>