@lightcone-ai/daemon 0.14.15 → 0.14.16

package/src/_vendor/video/recorder/plan-executor.js

@@ -0,0 +1,424 @@
+function normalizeText(value) {
+  if (typeof value !== 'string') return '';
+  return value.trim();
+}
+
+function normalizeInteger(value, fallback = null) {
+  const parsed = Number.parseInt(String(value ?? ''), 10);
+  if (!Number.isFinite(parsed)) return fallback;
+  return parsed;
+}
+
+function normalizeRange(value) {
+  if (!Array.isArray(value) || value.length !== 2) return null;
+  const start = Number(value[0]);
+  const end = Number(value[1]);
+  if (!Number.isFinite(start) || !Number.isFinite(end)) return null;
+  const low = Math.round(Math.min(start, end));
+  const high = Math.round(Math.max(start, end));
+  return [low, high];
+}
+
+function inferActionFromCameraMotion(phase = {}) {
+  const motion = normalizeText(phase.camera_motion ?? phase.cameraMotion).toLowerCase();
+  if (motion === 'narrated_pan') return 'linear_scroll_during';
+  if (motion === 'return_anchor') return 'scroll_back';
+  if (motion === 'cursor_focus') return 'cursor_focus';
+  if (motion === 'focus_hold') return 'scroll_to_dwell';
+  return '';
+}
+
+function normalizeSectionAsPhase(section = {}, index = 0) {
+  const phaseId = normalizeText(section.id ?? section.phase_id) || `phase_${index + 1}`;
+  const visualAction = section.visual_action && typeof section.visual_action === 'object'
+    ? section.visual_action
+    : {};
+  const focusRegion = normalizeRange(
+    section.focus_region
+    ?? section.focusRegion
+    ?? visualAction.focus_region
+    ?? visualAction.focusRegion
+  );
+  const explicitAction = normalizeText(section.action ?? visualAction.type).toLowerCase();
+  const action = explicitAction || inferActionFromCameraMotion(section) || 'scroll_to_dwell';
+
+  return {
+    ...section,
+    id: phaseId,
+    phase_id: phaseId,
+    action,
+    focus_region: focusRegion ?? null,
+    visual_action: visualAction,
+    target_y: section.target_y ?? visualAction.target_y ?? visualAction.to_y ?? null,
+    from_y: section.from_y ?? visualAction.from_y ?? null,
+    to_y: section.to_y ?? visualAction.to_y ?? null,
+    transition_ms: section.transition_ms ?? visualAction.transition_ms ?? null,
+    duration_ms: section.duration_ms ?? section.dwell_ms ?? null,
+  };
+}
+
+export function normalizePlanPhases(plan = {}) {
+  const phases = Array.isArray(plan?.phases) ? plan.phases : [];
+  if (phases.length > 0) return phases;
+
+  const sections = Array.isArray(plan?.sections) ? plan.sections : [];
+  if (sections.length > 0) {
+    return sections.map((section, index) => normalizeSectionAsPhase(section, index));
+  }
+
+  const error = new Error('plan_phases_required');
+  error.code = 'PLAN_PHASES_REQUIRED';
+  throw error;
+}
+
+function resolvePhaseAction(phase = {}) {
+  const explicit = normalizeText(phase.action ?? phase.visual_action?.type).toLowerCase();
+  if (explicit) return explicit;
+  return inferActionFromCameraMotion(phase);
+}
+
+function resolvePhaseId(phase = {}, index = 0) {
+  return normalizeText(phase.id ?? phase.phase_id) || `phase_${index + 1}`;
+}
+
+function nowMs(getNowMs) {
+  return Number(getNowMs?.()) || Date.now();
+}
+
+function resolveDurationMs(phase, fallback = 0) {
+  const parsed = normalizeInteger(phase?.duration_ms, null);
+  if (parsed !== null && parsed >= 0) return parsed;
+  const dwellMs = normalizeInteger(phase?.dwell_ms, null);
+  if (dwellMs !== null && dwellMs >= 0) return dwellMs;
+  const secs = Number(phase?.duration_s);
+  if (Number.isFinite(secs) && secs >= 0) return Math.round(secs * 1000);
+  return fallback;
+}
+
+function resolveTransitionMs(phase, fallback) {
+  const parsed = normalizeInteger(phase?.transition_ms, null);
+  if (parsed !== null && parsed >= 0) return parsed;
+  return fallback;
+}
+
+function resolveTargetY(phase, fallback = null) {
+  const raw = phase?.target_y ?? phase?.to_y ?? phase?.visual_action?.target_y ?? phase?.visual_action?.to_y;
+  const parsed = Number(raw);
+  if (Number.isFinite(parsed)) return Math.round(parsed);
+
+  const focusRegion = normalizeRange(
+    phase?.focus_region
+    ?? phase?.focusRegion
+    ?? phase?.highlight?.y_range
+    ?? phase?.semantic_focus_region
+    ?? phase?.semanticFocusRegion
+    ?? phase?.visual_action?.focus_region
+    ?? phase?.visual_action?.focusRegion
+  );
+  if (focusRegion) {
+    return Math.round((focusRegion[0] + focusRegion[1]) / 2);
+  }
+  return fallback;
+}
+
+function resolveFromY(phase, fallback = null) {
+  const raw = phase?.from_y ?? phase?.visual_action?.from_y;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed)) return fallback;
+  return Math.round(parsed);
+}
+
+async function animateScroll(page, {
+  startY = null,
+  targetY,
+  durationMs,
+  easing = 'easeInOutQuad',
+  jitterPx = 0,
+  minSteps = 10,
+} = {}) {
+  if (!Number.isFinite(Number(targetY))) {
+    const error = new Error('phase_target_y_required');
+    error.code = 'PHASE_TARGET_Y_REQUIRED';
+    throw error;
+  }
+
+  const normalizedDurationMs = Math.max(0, Number(durationMs) || 0);
+  const normalizedMinSteps = Math.max(1, Number(minSteps) || 1);
+
+  await page.evaluate(async ({
+    startY: evaluateStartY,
+    targetY: evaluateTargetY,
+    durationMs: evaluateDurationMs,
+    easing: evaluateEasing,
+    jitterPx: evaluateJitterPx,
+    minSteps: evaluateMinSteps,
+  }) => {
+    const root = document.scrollingElement || document.documentElement;
+    const wait = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+    const fromY = Number.isFinite(evaluateStartY) ? evaluateStartY : root.scrollTop;
+    const toY = evaluateTargetY;
+    const delta = toY - fromY;
+
+    const steps = Math.max(evaluateMinSteps, Math.round(Math.max(1, evaluateDurationMs) / 16));
+    const stepDurationMs = evaluateDurationMs <= 0 ? 0 : evaluateDurationMs / steps;
+
+    const applyEasing = (t) => {
+      if (evaluateEasing === 'linear') return t;
+      if (evaluateEasing === 'easeOutQuad') return 1 - ((1 - t) * (1 - t));
+      return t < 0.5
+        ? 2 * t * t
+        : 1 - (Math.pow(-2 * t + 2, 2) / 2);
+    };
+
+    root.scrollTo(0, fromY);
+    for (let index = 1; index <= steps; index += 1) {
+      const t = index / steps;
+      const eased = applyEasing(t);
+      const jitter = evaluateJitterPx > 0 ? ((Math.random() * 2 - 1) * evaluateJitterPx) : 0;
+      root.scrollTo(0, fromY + (delta * eased) + jitter);
+      if (stepDurationMs > 0) {
+        await wait(stepDurationMs);
+      }
+    }
+    root.scrollTo(0, toY);
+  }, {
+    startY,
+    targetY,
+    durationMs: normalizedDurationMs,
+    easing,
+    jitterPx: Math.max(0, Number(jitterPx) || 0),
+    minSteps: normalizedMinSteps,
+  });
+}
+
+async function executeHold(page, phase) {
+  const holdMs = resolveDurationMs(phase, 0);
+  if (holdMs > 0) {
+    await page.waitForTimeout(holdMs);
+  }
+  return { anchorY: null };
+}
+
+async function executeSmoothScroll(page, phase, { fallbackTargetY = null } = {}) {
+  const targetY = resolveTargetY(phase, fallbackTargetY);
+  const transitionMs = resolveTransitionMs(phase, 900);
+  await animateScroll(page, {
+    targetY,
+    durationMs: transitionMs,
+    easing: 'easeInOutQuad',
+    jitterPx: 2,
+    minSteps: 18,
+  });
+  return { anchorY: targetY };
+}
+
+async function executeFastScroll(page, phase, { fallbackTargetY = null } = {}) {
+  const targetY = resolveTargetY(phase, fallbackTargetY);
+  const transitionMs = resolveTransitionMs(phase, 420);
+  await animateScroll(page, {
+    targetY,
+    durationMs: transitionMs,
+    easing: 'easeOutQuad',
+    jitterPx: 3,
+    minSteps: 10,
+  });
+  return { anchorY: targetY };
+}
+
+async function executeLinearScrollDuring(page, phase, {
+  fallbackFromY = null,
+  fallbackTargetY = null,
+} = {}) {
+  const fromY = resolveFromY(phase, fallbackFromY);
+  const toY = resolveTargetY(phase, fallbackTargetY);
+  const durationMs = resolveDurationMs(phase, null);
+  if (!Number.isFinite(Number(durationMs)) || Number(durationMs) <= 0) {
+    const error = new Error('linear_scroll_duration_required');
+    error.code = 'LINEAR_SCROLL_DURATION_REQUIRED';
+    throw error;
+  }
+
+  await animateScroll(page, {
+    startY: fromY,
+    targetY: toY,
+    durationMs,
+    easing: 'linear',
+    jitterPx: 0,
+    minSteps: 12,
+  });
+
+  const dwellMs = normalizeInteger(phase?.dwell_ms, null);
+  if (Number.isFinite(dwellMs) && dwellMs > durationMs) {
+    await page.waitForTimeout(dwellMs - durationMs);
+  }
+  return { anchorY: toY };
+}
+
+async function executeScrollToDwell(page, phase, { fallbackTargetY = null } = {}) {
+  const targetY = resolveTargetY(phase, fallbackTargetY);
+  const transitionMs = resolveTransitionMs(phase, 820);
+  await animateScroll(page, {
+    targetY,
+    durationMs: transitionMs,
+    easing: 'easeInOutQuad',
+    jitterPx: 2,
+    minSteps: 16,
+  });
+  const dwellMs = normalizeInteger(phase?.dwell_ms, null);
+  if (Number.isFinite(dwellMs) && dwellMs > 0) {
+    await page.waitForTimeout(dwellMs);
+  } else {
+    const holdMs = resolveDurationMs(phase, 0);
+    if (holdMs > 0) {
+      await page.waitForTimeout(holdMs);
+    }
+  }
+  return { anchorY: targetY };
+}
+
+async function executeScrollBack(page, phase, { fallbackTargetY = 0 } = {}) {
+  const targetY = resolveTargetY(phase, fallbackTargetY);
+  const transitionMs = resolveTransitionMs(phase, 900);
+  await animateScroll(page, {
+    targetY,
+    durationMs: transitionMs,
+    easing: 'easeOutQuad',
+    jitterPx: 1,
+    minSteps: 14,
+  });
+  const dwellMs = normalizeInteger(phase?.dwell_ms, null);
+  if (Number.isFinite(dwellMs) && dwellMs > 0) {
+    await page.waitForTimeout(dwellMs);
+  }
+  return { anchorY: targetY };
+}
+
+async function executeCursorFocus(page, phase, { fallbackTargetY = null } = {}) {
+  const targetY = resolveTargetY(phase, fallbackTargetY);
+  const transitionMs = resolveTransitionMs(phase, 650);
+  await animateScroll(page, {
+    targetY,
+    durationMs: transitionMs,
+    easing: 'easeInOutQuad',
+    jitterPx: 1,
+    minSteps: 12,
+  });
+  const dwellMs = normalizeInteger(phase?.dwell_ms, null);
+  if (Number.isFinite(dwellMs) && dwellMs > 0) {
+    await page.waitForTimeout(dwellMs);
+  } else {
+    const holdMs = resolveDurationMs(phase, 360);
+    if (holdMs > 0) {
+      await page.waitForTimeout(holdMs);
+    }
+  }
+  return { anchorY: targetY };
+}
+
+async function executePhase(page, phase, {
+  lastAnchorY = null,
+  initialAnchorY = 0,
+} = {}) {
+  const action = resolvePhaseAction(phase);
+  const fallbackY = lastAnchorY ?? initialAnchorY;
+
+  if (action === 'hold') {
+    return executeHold(page, phase);
+  }
+  if (action === 'smooth_scroll') {
+    return executeSmoothScroll(page, phase, { fallbackTargetY: fallbackY });
+  }
+  if (action === 'fast_scroll') {
+    return executeFastScroll(page, phase, { fallbackTargetY: fallbackY });
+  }
+  if (action === 'linear_scroll_during') {
+    return executeLinearScrollDuring(page, phase, {
+      fallbackFromY: fallbackY,
+      fallbackTargetY: fallbackY,
+    });
+  }
+  if (action === 'scroll_to_dwell') {
+    return executeScrollToDwell(page, phase, { fallbackTargetY: fallbackY });
+  }
+  if (action === 'scroll_back') {
+    return executeScrollBack(page, phase, { fallbackTargetY: 0 });
+  }
+  if (action === 'cursor_focus') {
+    return executeCursorFocus(page, phase, { fallbackTargetY: fallbackY });
+  }
+
+  const error = new Error(`phase_action_unsupported:${action || 'empty'}`);
+  error.code = 'PHASE_ACTION_UNSUPPORTED';
+  throw error;
+}
+
+function createEvent({ tMs, action, phaseId, phaseAction, detail = {} }) {
+  return {
+    t_ms: Math.max(0, Math.round(tMs)),
+    action,
+    phase_id: phaseId,
+    phase_action: phaseAction,
+    ...detail,
+  };
+}
+
+export async function executePlanPhases(page, plan, {
+  getNowMs = () => Date.now(),
+  onEvent = null,
+} = {}) {
+  const phases = normalizePlanPhases(plan);
+  const startedAt = nowMs(getNowMs);
+  const eventsLog = [];
+  let lastAnchorY = 0;
+
+  for (let index = 0; index < phases.length; index += 1) {
+    const phase = phases[index];
+    const phaseId = resolvePhaseId(phase, index);
+    const phaseAction = resolvePhaseAction(phase);
+
+    const startEvent = createEvent({
+      tMs: nowMs(getNowMs) - startedAt,
+      action: 'phase_start',
+      phaseId,
+      phaseAction,
+    });
+    eventsLog.push(startEvent);
+    if (typeof onEvent === 'function') onEvent(startEvent);
+
+    try {
+      const executionResult = await executePhase(page, phase, {
+        lastAnchorY,
+        initialAnchorY: 0,
+      });
+      const anchorY = Number(executionResult?.anchorY);
+      if (Number.isFinite(anchorY)) {
+        lastAnchorY = Math.round(anchorY);
+      }
+    } catch (error) {
+      const errorEvent = createEvent({
+        tMs: nowMs(getNowMs) - startedAt,
+        action: 'phase_error',
+        phaseId,
+        phaseAction,
+        detail: {
+          error: error?.message ?? 'unknown_error',
+        },
+      });
+      eventsLog.push(errorEvent);
+      if (typeof onEvent === 'function') onEvent(errorEvent);
+      throw error;
+    }
+
+    const endEvent = createEvent({
+      tMs: nowMs(getNowMs) - startedAt,
+      action: 'phase_end',
+      phaseId,
+      phaseAction,
+    });
+    eventsLog.push(endEvent);
+    if (typeof onEvent === 'function') onEvent(endEvent);
+  }
+
+  return eventsLog;
+}

package/src/chat-bridge.js

@@ -12,11 +12,13 @@ import {
   probeDurationMs,
   transcodeForPlatform,
 } from './_vendor/video/composer/index.js';
+import { recordUrlNarration } from './_vendor/video/recorder/index.js';
 import {
   VIDEO_EXT,
   VIDEO_MIME,
   writeLocalFileToWorkspace,
 } from './workspace-file-upload.js';
+import { runRecordUrlNarrationTool } from './record-url-narration-tool.js';
 import { runSubmitToLibraryTool } from './submit-to-library-tool.js';
 import { isLeaseInvalidated, clearInvalidatedLease } from './governance-state.js';
 import { classifyLeaseWindow } from './lease-window.js';
@@ -197,6 +199,7 @@ const DEFAULT_TOOL_CLASSIFICATION = {
   supersede_goal_field: 'mandatory',
   request_credential_auth: 'mandatory',
   generate_voiceover: 'mandatory',
+  record_url_narration: 'mandatory',
   compose_video: 'mandatory',
   submit_to_library: 'mandatory',
   register_data_source: 'mandatory',
@@ -1407,6 +1410,30 @@ server.tool('generate_voiceover',
   }
 );
 
+// ── record_url_narration ────────────────────────────────────────────────────────
+server.tool('record_url_narration',
+  'Record a silent video of a URL by orchestrating Xvfb + Chromium + ffmpeg, driven by a video plan. Outputs a silent mp4 plus an events.json timestamp log that compose_video can use to align audio segments.\n\nUse this as the canonical recording step for URL-narration videos. Falls back: if the page needs interactions outside the visual_action vocabulary (clicks, waits, OCR loops), use Monitor (Bash) with custom Playwright instead.\n\nRuntime requirements: this tool only works on a Linux daemon machine with Xvfb + ffmpeg (x11grab) + Chromium installed. macOS / Windows daemons will fail at startup.',
+  {
+    url: z.string().describe('Page URL to record'),
+    plan: z.record(z.any()).describe('Video plan from plan_video / detail_sections — containing phases array with visual_action per phase'),
+    output_path: z.string().optional().describe('Workspace-relative output mp4 path. Default tmp/wx3_video/recorded-{ts}.mp4'),
+    events_path: z.string().optional().describe('Workspace-relative events.json path. Default ${output_path}.events.json'),
+    viewport: z.object({
+      width: z.number().optional(),
+      height: z.number().optional(),
+    }).optional().describe('Default 1080x1920 (mobile portrait). Override only if the plan requires a different shape.'),
+    fps: z.number().optional().describe('Default 30. Do not change unless needed.'),
+    settle_ms: z.number().optional().describe('Default 4000. Settle wait after navigation before recording starts.'),
+  },
+  async (args) => runRecordUrlNarrationTool({
+    args,
+    currentWorkspaceId,
+    workspaceDir: WORKSPACE_DIR,
+    runMandatoryLocalToolFn: runMandatoryLocalTool,
+    recordUrlNarrationFn: recordUrlNarration,
+  })
+);
+
 // ── compose_video ───────────────────────────────────────────────────────────────
 server.tool('compose_video',
   'Compose a final short video by muxing audio onto a base video, optionally concatenating an outro, and transcoding to platform spec.',

package/src/record-url-narration-tool.js

@@ -0,0 +1,165 @@
+import { mkdirSync } from 'fs';
+import path from 'path';
+
+function toolText(text) {
+  return { content: [{ type: 'text', text }] };
+}
+
+function toolError(text) {
+  return { isError: true, content: [{ type: 'text', text }] };
+}
+
+function normalizeText(value) {
+  if (typeof value !== 'string') return '';
+  return value.trim();
+}
+
+function isPlainObject(value) {
+  return !!value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function normalizeNumberOrNull(value) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return null;
+  return Math.floor(parsed);
+}
+
+function deriveDurationMs(recorderOutput) {
+  const directDurationMs = normalizeNumberOrNull(recorderOutput?.durationMs ?? recorderOutput?.duration_ms);
+  if (directDurationMs != null) return directDurationMs;
+
+  const eventsLog = Array.isArray(recorderOutput?.events_log) ? recorderOutput.events_log : null;
+  if (!eventsLog || eventsLog.length === 0) return null;
+
+  const lastTms = eventsLog.reduce((max, event) => {
+    const next = normalizeNumberOrNull(event?.t_ms);
+    if (next == null) return max;
+    return Math.max(max, next);
+  }, 0);
+  return lastTms > 0 ? lastTms : null;
+}
+
+function derivePhaseCount({ plan, recorderOutput }) {
+  const explicit = normalizeNumberOrNull(recorderOutput?.phases);
+  if (explicit != null) return explicit;
+
+  if (Array.isArray(plan?.phases)) return plan.phases.length;
+  if (Array.isArray(plan?.sections)) return plan.sections.length;
+  return null;
+}
+
+export function validateRecordUrlNarrationArgs(args = {}) {
+  const normalizedUrl = normalizeText(args.url);
+  if (!normalizedUrl) {
+    const error = new Error('url is required for record_url_narration.');
+    error.code = 'RECORD_URL_REQUIRED';
+    throw error;
+  }
+
+  if (!isPlainObject(args.plan)) {
+    const error = new Error('plan is required for record_url_narration.');
+    error.code = 'RECORD_PLAN_REQUIRED';
+    throw error;
+  }
+
+  return {
+    ...(args ?? {}),
+    url: normalizedUrl,
+  };
+}
+
+export function resolveRecordUrlNarrationPaths({
+  workspaceDir,
+  outputPath,
+  eventsPath,
+  nowMs = () => Date.now(),
+} = {}) {
+  const normalizedOutputPath = normalizeText(outputPath) || `tmp/wx3_video/recorded-${nowMs()}.mp4`;
+  const resolvedOutputPath = path.resolve(workspaceDir, normalizedOutputPath);
+
+  const normalizedEventsPath = normalizeText(eventsPath);
+  const resolvedEventsPath = normalizedEventsPath
+    ? path.resolve(workspaceDir, normalizedEventsPath)
+    : `${resolvedOutputPath}.events.json`;
+
+  return {
+    resolvedOutputPath,
+    resolvedEventsPath,
+  };
+}
+
+export async function runRecordUrlNarrationTool({
+  args = {},
+  currentWorkspaceId = '',
+  workspaceDir = process.cwd(),
+  runMandatoryLocalToolFn,
+  recordUrlNarrationFn,
+  nowMs = () => Date.now(),
+} = {}) {
+  if (!currentWorkspaceId) {
+    return toolError('No workspace context.');
+  }
+  if (typeof runMandatoryLocalToolFn !== 'function') {
+    return toolError('Error: record_url_narration runMandatoryLocalToolFn is required.');
+  }
+  if (typeof recordUrlNarrationFn !== 'function') {
+    return toolError('Error: record_url_narration executor is required.');
+  }
+
+  let validatedInput;
+  try {
+    validatedInput = validateRecordUrlNarrationArgs(args);
+  } catch (error) {
+    return toolError(`Error: ${error.message}`);
+  }
+
+  try {
+    const result = await runMandatoryLocalToolFn({
+      toolName: 'record_url_narration',
+      toolInput: validatedInput,
+      executor: async (checkedInput = {}) => {
+        const mergedInput = {
+          ...validatedInput,
+          ...checkedInput,
+        };
+        const finalInput = validateRecordUrlNarrationArgs(mergedInput);
+        const { resolvedOutputPath, resolvedEventsPath } = resolveRecordUrlNarrationPaths({
+          workspaceDir,
+          outputPath: finalInput.output_path,
+          eventsPath: finalInput.events_path,
+          nowMs,
+        });
+
+        mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
+        mkdirSync(path.dirname(resolvedEventsPath), { recursive: true });
+
+        const recorderOutput = await recordUrlNarrationFn({
+          url: finalInput.url,
+          plan: finalInput.plan,
+          output_path: resolvedOutputPath,
+          events_path: resolvedEventsPath,
+          viewport: finalInput.viewport,
+          fps: finalInput.fps,
+          settle_ms: finalInput.settle_ms,
+        });
+
+        return {
+          videoPath: resolvedOutputPath,
+          eventsPath: resolvedEventsPath,
+          durationMs: deriveDurationMs(recorderOutput),
+          phases: derivePhaseCount({ plan: finalInput.plan, recorderOutput }),
+        };
+      },
+    });
+
+    return toolText(
+      `Recorded URL narration.\n`
+      + `video_path=${result.videoPath}\n`
+      + `events_path=${result.eventsPath}\n`
+      + `duration_ms=${result.durationMs ?? 'unknown'}\n`
+      + `phases=${result.phases ?? 'n/a'}`
+    );
+  } catch (error) {
+    return toolError(`Error: ${error.message}`);
+  }
+}