@telepat/rilo 0.1.0

package/src/policy/contentGuardrails.js

@@ -0,0 +1,30 @@
+const NAME_PATTERNS = [
+  /\bMr\.\s+[A-Z][a-z]+\b/g,
+  /\bMrs\.\s+[A-Z][a-z]+\b/g,
+  /\bMs\.\s+[A-Z][a-z]+\b/g,
+  /\bJudge\s+[A-Z][a-z]+\b/g,
+  /\b[A-Z][a-z]+\s+[A-Z][a-z]+\b/g
+];
+
+const SENSITIVE_PATTERNS = [
+  /\b\d{3}-\d{2}-\d{4}\b/g,
+  /\b\d{10,}\b/g,
+  /\b\d{1,5}\s+[A-Za-z]+\s+(Street|St|Avenue|Ave|Road|Rd|Boulevard|Blvd)\b/gi
+];
+
+export function sanitizeStoryInput(story) {
+  let sanitized = story;
+  for (const pattern of NAME_PATTERNS) {
+    sanitized = sanitized.replace(pattern, 'a person');
+  }
+  for (const pattern of SENSITIVE_PATTERNS) {
+    sanitized = sanitized.replace(pattern, '[redacted]');
+  }
+  return sanitized;
+}
+
+export function assertSafeStory(story) {
+  if (!story || story.trim().length < 50) {
+    throw new Error('Story input must be at least 50 characters');
+  }
+}

package/src/providers/predictions.js

@@ -0,0 +1,188 @@
+import { env } from '../config/env.js';
+import { getReplicateClient } from './replicateClient.js';
+import { logError, logInfo } from '../observability/logger.js';
+import { appendApiTrace } from '../observability/apiTrace.js';
+
+function pickPredictionFields(prediction) {
+  if (!prediction || typeof prediction !== 'object') {
+    return null;
+  }
+
+  return {
+    id: prediction.id || null,
+    status: prediction.status || null,
+    createdAt: prediction.created_at || null,
+    startedAt: prediction.started_at || null,
+    completedAt: prediction.completed_at || null,
+    error: prediction.error || null,
+    metrics: prediction.metrics || null
+  };
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function pickErrorFields(error) {
+  if (!error || typeof error !== 'object') {
+    return {
+      name: 'Error',
+      message: String(error || ''),
+      nonRetryable: false
+    };
+  }
+
+  return {
+    name: error.name || 'Error',
+    message: error.message || '',
+    code: error.code || null,
+    status: error.status || error.statusCode || null,
+    nonRetryable: Boolean(error.nonRetryable)
+  };
+}
+
+async function withRetries(fn, label, { logErrorFn = logError, sleepFn = sleep } = {}) {
+  let lastError;
+  const maxAttempts = env.maxRetries + 1;
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      const shouldRetry = !error?.nonRetryable && attempt < maxAttempts;
+      const retryDelayMs = shouldRetry ? env.retryDelayMs * attempt : 0;
+      logErrorFn('prediction_attempt_failed', {
+        label,
+        attempt,
+        maxAttempts,
+        willRetry: shouldRetry,
+        retryDelayMs,
+        error: pickErrorFields(error)
+      });
+      if (error?.nonRetryable) {
+        throw error;
+      }
+      if (shouldRetry) {
+        await sleepFn(retryDelayMs);
+      }
+    }
+  }
+  throw lastError;
+}
+
+export async function runModel({ model, input, trace = null, deps = {} }) {
+  const getReplicateClientFn = deps.getReplicateClient || getReplicateClient;
+  const appendApiTraceFn = deps.appendApiTrace || appendApiTrace;
+  const logInfoFn = deps.logInfo || logInfo;
+  const sleepFn = deps.sleep || sleep;
+  const nowFn = deps.now || (() => Date.now());
+  const logErrorFn = deps.logError || logError;
+
+  if (env.useWebhooks) {
+    const error = new Error('USE_WEBHOOKS=true is currently disabled until webhook verification and queue reconciliation are implemented');
+    error.nonRetryable = true;
+    throw error;
+  }
+
+  const replicate = getReplicateClientFn();
+
+  return withRetries(async () => {
+    const startedAt = new Date().toISOString();
+    logInfoFn('prediction_start', { model });
+    await appendApiTraceFn(trace?.projectDir, {
+      type: 'request_start',
+      ts: startedAt,
+      model,
+      input,
+      trace
+    });
+
+    const prediction = await replicate.predictions.create({
+      model,
+      input,
+    });
+
+    await appendApiTraceFn(trace?.projectDir, {
+      type: 'request_created',
+      ts: new Date().toISOString(),
+      model,
+      predictionId: prediction.id,
+      status: prediction.status,
+      prediction: pickPredictionFields(prediction),
+      trace
+    });
+
+    let current = prediction;
+    const pollStartedAt = nowFn();
+    while (current.status !== 'succeeded' && current.status !== 'failed' && current.status !== 'canceled') {
+      if (nowFn() - pollStartedAt > env.predictionMaxWaitMs) {
+        await appendApiTraceFn(trace?.projectDir, {
+          type: 'request_timeout',
+          ts: new Date().toISOString(),
+          model,
+          predictionId: current.id,
+          status: current.status,
+          prediction: pickPredictionFields(current),
+          trace
+        });
+        const timeoutError = new Error(`Prediction timed out after ${env.predictionMaxWaitMs}ms`);
+        timeoutError.nonRetryable = true;
+        throw timeoutError;
+      }
+
+      await sleepFn(env.predictionPollIntervalMs);
+      current = await replicate.predictions.get(current.id);
+    }
+
+    if (current.status !== 'succeeded') {
+      await appendApiTraceFn(trace?.projectDir, {
+        type: 'request_failed',
+        ts: new Date().toISOString(),
+        model,
+        input,
+        predictionId: current.id,
+        status: current.status,
+        prediction: pickPredictionFields(current),
+        trace
+      });
+      const statusError = new Error(`Prediction failed with status ${current.status}`);
+      statusError.nonRetryable = true;
+      throw statusError;
+    }
+
+    logInfoFn('prediction_succeeded', { model, predictionId: current.id });
+    await appendApiTraceFn(trace?.projectDir, {
+      type: 'request_succeeded',
+      ts: new Date().toISOString(),
+      model,
+      input,
+      predictionId: current.id,
+      status: current.status,
+      prediction: pickPredictionFields(current),
+      output: current.output,
+      trace
+    });
+    return current;
+  }, model, { logErrorFn, sleepFn });
+}
+
+export function extractOutputText(output) {
+  if (Array.isArray(output)) {
+    return output.join('').trim();
+  }
+  if (typeof output === 'string') {
+    return output.trim();
+  }
+  return JSON.stringify(output || '');
+}
+
+export function extractOutputUri(output) {
+  if (typeof output === 'string') return output;
+  if (Array.isArray(output) && output.length > 0) {
+    return typeof output[0] === 'string' ? output[0] : '';
+  }
+  if (output && typeof output === 'object' && output.url) {
+    return output.url;
+  }
+  return '';
+}

package/src/providers/replicateClient.js

@@ -0,0 +1,12 @@
+import Replicate from 'replicate';
+import { env, assertRequiredEnv } from '../config/env.js';
+
+let client;
+
+export function getReplicateClient() {
+  if (!client) {
+    assertRequiredEnv();
+    client = new Replicate({ auth: env.replicateApiToken });
+  }
+  return client;
+}

package/src/steps/alignSubtitles.js

@@ -0,0 +1,156 @@
+import { spawn } from 'node:child_process';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { env } from '../config/env.js';
+import { ensureDir } from '../media/files.js';
+import { parseSrtCues, writeSeedSrtFromScript, writeAssFromSrt } from '../media/subtitles.js';
+
+function runCommand(command, args) {
+  return new Promise((resolve, reject) => {
+    const proc = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+    let stderr = '';
+
+    proc.stderr.on('data', (chunk) => {
+      stderr += chunk.toString();
+    });
+
+    proc.on('error', reject);
+    proc.on('exit', (code) => {
+      if (code === 0) {
+        resolve();
+      } else {
+        reject(new Error((stderr || `${command} exited with code ${code}`).trim()));
+      }
+    });
+  });
+}
+
+function normalizeCueText(text) {
+  return String(text || '').replace(/\s+/g, ' ').trim().toLowerCase();
+}
+
+function formatSrtTimestamp(ms) {
+  const total = Math.max(0, Math.round(ms));
+  const hours = Math.floor(total / 3600000);
+  const minutes = Math.floor((total % 3600000) / 60000);
+  const seconds = Math.floor((total % 60000) / 1000);
+  const millis = total % 1000;
+  return `${String(hours).padStart(2, '0')}:${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')},${String(millis).padStart(3, '0')}`;
+}
+
+function serializeSrtCues(cues) {
+  return cues.map((cue, index) => {
+    return `${index + 1}\n${formatSrtTimestamp(cue.startMs)} --> ${formatSrtTimestamp(cue.endMs)}\n${cue.text}`;
+  }).join('\n\n');
+}
+
+function restoreDroppedLeadingCues(seedCues, alignedCues) {
+  if (!Array.isArray(seedCues) || !Array.isArray(alignedCues) || !alignedCues.length || alignedCues.length >= seedCues.length) {
+    return alignedCues;
+  }
+
+  const firstAligned = alignedCues[0];
+  const firstAlignedText = normalizeCueText(firstAligned.text);
+  if (!firstAlignedText) {
+    return alignedCues;
+  }
+
+  const firstSeedMatchIndex = seedCues.findIndex((cue) => normalizeCueText(cue.text) === firstAlignedText);
+  if (firstSeedMatchIndex <= 0) {
+    return alignedCues;
+  }
+
+  const shiftMs = firstAligned.startMs - seedCues[firstSeedMatchIndex].startMs;
+  const recoveredLeading = seedCues.slice(0, firstSeedMatchIndex).map((cue) => {
+    const shiftedStartMs = cue.startMs + shiftMs;
+    const shiftedEndMs = cue.endMs + shiftMs;
+    const startMs = Math.max(0, shiftedStartMs);
+    const endMs = Math.max(startMs + 120, shiftedEndMs);
+    return {
+      startMs,
+      endMs,
+      text: cue.text
+    };
+  });
+
+  return [...recoveredLeading, ...alignedCues];
+}
+
+async function reconcileAlignedSrt(seedSrtPath, alignedSrtPath, deps = {}) {
+  const readFileFn = deps.readFile || fs.readFile;
+  const writeFileFn = deps.writeFile || fs.writeFile;
+
+  const [seedRaw, alignedRaw] = await Promise.all([
+    readFileFn(seedSrtPath, 'utf8'),
+    readFileFn(alignedSrtPath, 'utf8')
+  ]);
+
+  const seedCues = parseSrtCues(seedRaw);
+  const alignedCues = parseSrtCues(alignedRaw);
+
+  if (!seedCues.length || !alignedCues.length) {
+    return;
+  }
+
+  const reconciled = restoreDroppedLeadingCues(seedCues, alignedCues);
+  if (reconciled.length === alignedCues.length) {
+    return;
+  }
+
+  await writeFileFn(alignedSrtPath, `${serializeSrtCues(reconciled)}\n`, 'utf8');
+}
+
+export async function alignSubtitlesToVideo({
+  projectDir,
+  videoPath,
+  script,
+  totalDurationSec,
+  subtitleOptions,
+  deps = {}
+}) {
+  const ensureDirFn = deps.ensureDir || ensureDir;
+  const runCommandFn = deps.runCommand || runCommand;
+  const reconcileAlignedSrtFn = deps.reconcileAlignedSrt || reconcileAlignedSrt;
+  const writeSeedSrtFn = deps.writeSeedSrtFromScript || writeSeedSrtFromScript;
+  const writeAssFromSrtFn = deps.writeAssFromSrt || writeAssFromSrt;
+
+  if (!script || !String(script).trim()) {
+    throw new Error('Cannot align subtitles: script is empty');
+  }
+
+  const subtitlesDir = path.join(projectDir, 'assets', 'subtitles');
+  await ensureDirFn(subtitlesDir);
+
+  const seedSrtPath = path.join(subtitlesDir, 'seed.srt');
+  const alignedSrtPath = path.join(subtitlesDir, 'aligned.srt');
+  const alignedAssPath = path.join(subtitlesDir, 'aligned.ass');
+
+  await writeSeedSrtFn({
+    script,
+    totalDurationSec,
+    outputPath: seedSrtPath,
+    maxWordsPerLine: subtitleOptions?.maxWordsPerLine
+  });
+
+  await runCommandFn(env.ffsubsyncBin, [
+    videoPath,
+    '-i',
+    seedSrtPath,
+    '-o',
+    alignedSrtPath
+  ]);
+
+  await reconcileAlignedSrtFn(seedSrtPath, alignedSrtPath, deps);
+
+  await writeAssFromSrtFn({
+    sourceSrtPath: alignedSrtPath,
+    outputAssPath: alignedAssPath,
+    subtitleOptions
+  });
+
+  return {
+    subtitleSeedPath: seedSrtPath,
+    subtitleAlignedSrtPath: alignedSrtPath,
+    subtitleAssPath: alignedAssPath
+  };
+}

package/src/steps/burnInSubtitles.js

@@ -0,0 +1,22 @@
+import path from 'node:path';
+import { burnInAssSubtitles } from '../media/ffmpeg.js';
+
+export async function burnInSubtitles({
+  projectDir,
+  videoPath,
+  subtitleAssPath,
+  deps = {}
+}) {
+  const burnInAssSubtitlesFn = deps.burnInAssSubtitles || burnInAssSubtitles;
+
+  if (!subtitleAssPath) {
+    throw new Error('Cannot burn subtitles: subtitle ASS path is missing');
+  }
+
+  const outputPath = path.join(projectDir, 'final_captioned.mp4');
+  await burnInAssSubtitlesFn(videoPath, subtitleAssPath, outputPath);
+
+  return {
+    finalCaptionedVideoPath: outputPath
+  };
+}

package/src/steps/composeFinalVideo.js

@@ -0,0 +1,57 @@
+import path from 'node:path';
+import { concatSegments, muxVoiceover } from '../media/ffmpeg.js';
+import { downloadToFile, ensureDir } from '../media/files.js';
+
+export async function composeFinalVideo({
+  projectDir,
+  segmentUrls,
+  segmentPaths = [],
+  voiceoverPath = '',
+  voiceoverUrl,
+  keyframePaths = [],
+  finalDurationMode = 'match_audio',
+  deps = {}
+}) {
+  const ensureDirFn = deps.ensureDir || ensureDir;
+  const downloadToFileFn = deps.downloadToFile || downloadToFile;
+  const concatSegmentsFn = deps.concatSegments || concatSegments;
+  const muxVoiceoverFn = deps.muxVoiceover || muxVoiceover;
+
+  const assetsDir = path.join(projectDir, 'assets');
+  const segmentsDir = path.join(assetsDir, 'segments');
+  const audioDir = path.join(assetsDir, 'audio');
+  await ensureDirFn(segmentsDir);
+  await ensureDirFn(audioDir);
+
+  const localSegmentPaths = [];
+  if (segmentPaths.length === segmentUrls.length && segmentPaths.length > 0) {
+    localSegmentPaths.push(...segmentPaths);
+  } else {
+    for (let i = 0; i < segmentUrls.length; i += 1) {
+      const segmentPath = path.join(segmentsDir, `segment_${String(i + 1).padStart(2, '0')}.mp4`);
+      await downloadToFileFn(segmentUrls[i], segmentPath);
+      localSegmentPaths.push(segmentPath);
+    }
+  }
+
+  let voicePath = voiceoverPath;
+  if (!voicePath) {
+    voicePath = path.join(audioDir, 'voiceover.mp3');
+    await downloadToFileFn(voiceoverUrl, voicePath);
+  }
+
+  const concatPath = path.join(assetsDir, 'video_concat.mp4');
+  const finalPath = path.join(projectDir, 'final.mp4');
+
+  await concatSegmentsFn(localSegmentPaths, concatPath);
+  await muxVoiceoverFn(concatPath, voicePath, finalPath, {
+    trimToAudio: finalDurationMode !== 'match_visual'
+  });
+
+  return {
+    finalVideoPath: finalPath,
+    keyframePaths,
+    segmentPaths: localSegmentPaths,
+    voiceoverPath: voicePath
+  };
+}

package/src/steps/generateKeyframes.js

@@ -0,0 +1,70 @@
+import { ASPECT_RATIO_PRESETS, MODEL_CATEGORIES, resolveModelForCategory } from '../config/models.js';
+import { runModel, extractOutputUri } from '../providers/predictions.js';
+import path from 'node:path';
+import { downloadToFile, ensureDir } from '../media/files.js';
+import { resolveTextToImageAdapter } from './textToImageAdapters.js';
+
+export async function generateKeyframe(
+  promptText,
+  tone,
+  aspectRatio = '9:16',
+  index = 0,
+  trace = null,
+  sizeOverride = null,
+  options = {}
+) {
+  const deps = options.deps || {};
+  const runModelFn = deps.runModel || runModel;
+  const extractOutputUriFn = deps.extractOutputUri || extractOutputUri;
+
+  const preset = ASPECT_RATIO_PRESETS[aspectRatio] || ASPECT_RATIO_PRESETS['9:16'];
+  const width = sizeOverride?.width || preset.keyframeWidth || ASPECT_RATIO_PRESETS['9:16'].keyframeWidth;
+  const height = sizeOverride?.height || preset.keyframeHeight || ASPECT_RATIO_PRESETS['9:16'].keyframeHeight;
+  const modelId = options.modelId || resolveModelForCategory(MODEL_CATEGORIES.textToImage);
+  const modelOptions = options.modelOptions;
+  const adapter = resolveTextToImageAdapter(modelId);
+
+  const prediction = await runModelFn({
+    model: modelId,
+    input: adapter.buildInput({ promptText, tone, index, aspectRatio, width, height, modelOptions }),
+    trace: trace ? { ...trace, step: 'keyframe', index } : null
+  });
+
+  const imageUrl = extractOutputUriFn(prediction.output);
+  if (!imageUrl) {
+    throw new Error(`Missing keyframe output for shot ${index + 1}`);
+  }
+
+  return imageUrl;
+}
+
+export async function generateKeyframes(shots, tone, aspectRatio = '9:16', trace = null, options = {}) {
+  const urls = [];
+  for (let i = 0; i < shots.length; i += 1) {
+    const imageUrl = await generateKeyframe(shots[i], tone, aspectRatio, i, trace, null, options);
+    urls.push(imageUrl);
+  }
+  return urls;
+}
+
+export async function persistKeyframe(projectDir, keyframeUrl, index, options = {}) {
+  const deps = options.deps || {};
+  const ensureDirFn = deps.ensureDir || ensureDir;
+  const downloadToFileFn = deps.downloadToFile || downloadToFile;
+
+  const keyframesDir = path.join(projectDir, 'assets', 'keyframes');
+  await ensureDirFn(keyframesDir);
+  const keyframePath = path.join(keyframesDir, `keyframe_${String(index + 1).padStart(2, '0')}.png`);
+  await downloadToFileFn(keyframeUrl, keyframePath);
+  return keyframePath;
+}
+
+export async function persistKeyframes(projectDir, keyframeUrls, options = {}) {
+  const keyframePaths = [];
+  for (let i = 0; i < keyframeUrls.length; i += 1) {
+    const keyframePath = await persistKeyframe(projectDir, keyframeUrls[i], i, options);
+    keyframePaths.push(keyframePath);
+  }
+
+  return keyframePaths;
+}

package/src/steps/generateVideoSegments.js

@@ -0,0 +1,95 @@
+import {
+  ASPECT_RATIO_PRESETS,
+  MODEL_CATEGORIES,
+  resolveModelForCategory
+} from '../config/models.js';
+import { runModel, extractOutputUri } from '../providers/predictions.js';
+import path from 'node:path';
+import { downloadToFile, ensureDir } from '../media/files.js';
+import { resolveImageToVideoAdapter } from './imageToVideoAdapters.js';
+
+function asModelOptions(candidate) {
+  if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) {
+    return {};
+  }
+  return candidate;
+}
+
+export async function generateVideoSegmentAtIndex(
+  segmentIndex,
+  keyframeUrls,
+  timeline,
+  shots = [],
+  aspectRatio = '9:16',
+  trace = null,
+  options = {}
+) {
+  const deps = options.deps || {};
+  const runModelFn = deps.runModel || runModel;
+  const extractOutputUriFn = deps.extractOutputUri || extractOutputUri;
+
+  const preset = ASPECT_RATIO_PRESETS[aspectRatio] || ASPECT_RATIO_PRESETS['9:16'];
+  const totalKeyframes = keyframeUrls.length;
+  const durationSec = timeline[segmentIndex]?.durationSec || 5;
+  const modelId = options.modelId || resolveModelForCategory(MODEL_CATEGORIES.imageTextToVideo);
+  const modelOptions = asModelOptions(options.modelOptions);
+  const adapter = resolveImageToVideoAdapter(modelId);
+
+  if (segmentIndex < 0 || segmentIndex >= totalKeyframes - 1) {
+    throw new Error(`segment index ${segmentIndex} out of range for ${Math.max(0, totalKeyframes - 1)} segments`);
+  }
+
+  const prompt = shots[segmentIndex] || `Cinematic continuity shot ${segmentIndex + 1}`;
+  const prediction = await runModelFn({
+    model: modelId,
+    input: adapter.buildInput({
+      prompt,
+      startImage: keyframeUrls[segmentIndex],
+      endImage: keyframeUrls[segmentIndex + 1],
+      durationSec,
+      videoResolution: preset.videoResolution,
+      aspectRatio,
+      modelOptions
+    }),
+    trace: trace ? { ...trace, step: 'segment', index: segmentIndex } : null
+  });
+
+  const url = extractOutputUriFn(prediction.output);
+  if (!url) {
+    throw new Error(`Missing segment output for segment ${segmentIndex + 1}`);
+  }
+  return url;
+}
+
+export async function generateVideoSegments(keyframeUrls, timeline, shots = [], aspectRatio = '9:16', trace = null, options = {}) {
+  const segmentUrls = [];
+
+  for (let i = 0; i < Math.max(0, keyframeUrls.length - 1); i += 1) {
+    const url = await generateVideoSegmentAtIndex(i, keyframeUrls, timeline, shots, aspectRatio, trace, options);
+    segmentUrls.push(url);
+  }
+
+  return segmentUrls;
+}
+
+export async function persistSegment(projectDir, segmentUrl, index, options = {}) {
+  const deps = options.deps || {};
+  const ensureDirFn = deps.ensureDir || ensureDir;
+  const downloadToFileFn = deps.downloadToFile || downloadToFile;
+
+  const segmentsDir = path.join(projectDir, 'assets', 'segments');
+  await ensureDirFn(segmentsDir);
+  const segmentPath = path.join(segmentsDir, `segment_${String(index + 1).padStart(2, '0')}.mp4`);
+  await downloadToFileFn(segmentUrl, segmentPath);
+  return segmentPath;
+}
+
+export async function persistSegments(projectDir, segmentUrls, options = {}) {
+  const segmentPaths = [];
+  for (let i = 0; i < segmentUrls.length; i += 1) {
+    const segmentPath = await persistSegment(projectDir, segmentUrls[i], i, options);
+    segmentPaths.push(segmentPath);
+  }
+
+  return segmentPaths;
+}