@alchemint/fuse-cli 0.0.0-stage → 0.1.0

package/src/batch.mjs

@@ -0,0 +1,523 @@
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import {
+  CliError,
+  need,
+  id,
+  readJson,
+  writeJson,
+  exists,
+  digest,
+  sameSubject,
+  subject,
+  duration,
+  pause,
+  withWriter,
+  errorData,
+} from './core.mjs';
+import {
+  acceptVideo,
+  observeVideo,
+  validateIntent,
+  terminal,
+  cancelJob,
+} from './video.mjs';
+import { download, fileDigest } from './download.mjs';
+
+const receiptPath = (directory, index) =>
+  path.join(directory, `item-${String(index + 1).padStart(4, '0')}.json`);
+const planHash = ({ plan_sha256, ...plan }) => digest(plan);
+const validPlan = (plan) => {
+  if (plan.schema_version !== 1 || plan.plan_sha256 !== planHash(plan))
+    throw new CliError(
+      2,
+      'PLAN_CHANGED',
+      'The frozen plan hash/version does not match. Do not edit a started plan.',
+    );
+};
+async function prepare(inputFile, out, context) {
+  const manifest = await readJson(inputFile);
+  if (
+    manifest.schema_version !== 1 ||
+    !Array.isArray(manifest.items) ||
+    !manifest.items.length
+  )
+    throw new CliError(
+      2,
+      'INVALID_MANIFEST',
+      'Expected schema_version:1 and a nonempty items array.',
+    );
+  const owner = await subject(context.client);
+  const options = await context.client.get('/generation/options');
+  const targets = new Map();
+  const keys = new Set();
+  const idem = new Set();
+  const items = [];
+  for (const input of manifest.items) {
+    if (!input.key || keys.has(input.key))
+      throw new CliError(
+        2,
+        'DUPLICATE_ITEM',
+        'Each manifest item needs a unique key.',
+      );
+    keys.add(input.key);
+    const request = await readJson(
+      path.resolve(
+        path.dirname(inputFile),
+        need(input.request_file, 'item.request_file'),
+      ),
+    );
+    const key = input.idempotency_key || randomUUID();
+    validateIntent(request, key);
+    if (idem.has(key.toLowerCase()))
+      throw new CliError(
+        2,
+        'DUPLICATE_IDEMPOTENCY_KEY',
+        'Each generation intent needs its own fixed key.',
+      );
+    idem.add(key.toLowerCase());
+    const profile = options.profiles.find(
+      (item) =>
+        item.pricing_key === request.generation_pricing_key &&
+        item.stage === 'i2v',
+    );
+    if (!profile)
+      throw new CliError(
+        2,
+        'PROFILE_NOT_FOUND',
+        `No video pricing profile for ${input.key}.`,
+      );
+    if (!targets.has(input.creative_id))
+      targets.set(
+        input.creative_id,
+        await context.client.get(
+          `/storyboards/${id(input.creative_id, 'item.creative_id')}`,
+        ),
+      );
+    const creative = targets.get(input.creative_id);
+    if (!creative.shots.some((shot) => shot.id === request.shot_id))
+      throw new CliError(
+        2,
+        'TARGET_MISMATCH',
+        `Shot does not belong to the specified Creative: ${input.key}`,
+      );
+    const item = {
+      key: input.key,
+      creative_id: input.creative_id,
+      idempotency_key: key,
+      request,
+      request_sha256: digest(request),
+      model: profile.model,
+      ...(input.source ? { source: input.source } : {}),
+    };
+    // Import an existing exact intent; never create a second canary key.
+    if (input.receipt_file) {
+      const receipt = await readJson(
+        path.resolve(path.dirname(inputFile), input.receipt_file),
+      );
+      sameSubject(receipt, owner);
+      if (
+        receipt.idempotency_key !== key ||
+        receipt.creative_id !== item.creative_id ||
+        digest(receipt.request) !== item.request_sha256 ||
+        !receipt.accepted
+      )
+        throw new CliError(
+          2,
+          'IMPORT_MISMATCH',
+          `Existing receipt does not match this exact intent: ${input.key}`,
+        );
+      await observeVideo(context.client, receipt);
+      item.accepted = receipt.accepted;
+    } else if (input.accepted) {
+      item.accepted = {
+        job_id: need(input.accepted.job_id, 'accepted.job_id'),
+        result_id: need(input.accepted.result_id, 'accepted.result_id'),
+        shot_id: request.shot_id,
+        replayed: false,
+      };
+      await observeVideo(context.client, item);
+    }
+    items.push(item);
+  }
+  const plan = {
+    schema_version: 1,
+    kind: 'video_batch',
+    name: manifest.name,
+    ...owner,
+    created_at: new Date().toISOString(),
+    items,
+  };
+  plan.plan_sha256 = planHash(plan);
+  await writeJson(out, plan, true);
+  return {
+    plan: path.resolve(out),
+    plan_sha256: plan.plan_sha256,
+    count: items.length,
+    imported: items.filter((item) => item.accepted).length,
+    models: [...new Set(items.map((item) => item.model))],
+    items: items.map((item) => ({
+      key: item.key,
+      creative_id: item.creative_id,
+      shot_id: item.request.shot_id,
+      model: item.model,
+      pricing_key: item.request.generation_pricing_key,
+      requested_duration: item.request.params?.duration,
+      request_sha256: item.request_sha256,
+      references: {
+        images: item.request.reference_images?.length || 0,
+        videos: item.request.reference_videos?.length || 0,
+        audios: item.request.reference_audios?.length || 0,
+      },
+    })),
+  };
+}
+async function loadState(directory, context) {
+  const plan = await readJson(path.join(directory, 'plan.json'));
+  validPlan(plan);
+  sameSubject(plan, await subject(context.client));
+  return plan;
+}
+function newReceipt(plan, item) {
+  return {
+    schema_version: 1,
+    kind: 'video',
+    api_url: plan.api_url,
+    user_id: plan.user_id,
+    workspace_id: plan.workspace_id,
+    ...item,
+    state: item.accepted ? 'accepted' : 'prepared',
+    created_at: plan.created_at,
+  };
+}
+function matchReceipt(plan, item, receipt) {
+  sameSubject(plan, receipt);
+  if (
+    item.idempotency_key !== receipt.idempotency_key ||
+    item.creative_id !== receipt.creative_id ||
+    item.request_sha256 !== digest(receipt.request)
+  )
+    throw new CliError(
+      2,
+      'INTENT_CHANGED',
+      `State differs from its frozen plan: ${item.key}`,
+    );
+}
+async function status(directory, plan, context) {
+  const rows = [];
+  for (let index = 0; index < plan.items.length; index++) {
+    const item = plan.items[index];
+    const file = receiptPath(directory, index);
+    const receipt = (await exists(file))
+      ? await readJson(file)
+      : newReceipt(plan, item);
+    matchReceipt(plan, item, receipt);
+    const observation = receipt.accepted
+      ? await observeVideo(context.client, receipt)
+      : null;
+    rows.push({
+      key: item.key,
+      state: receipt.state,
+      ...(receipt.accepted || {}),
+      observation,
+      receipt: file,
+    });
+  }
+  return {
+    state: path.resolve(directory),
+    count: rows.length,
+    accepted: rows.filter((row) => row.job_id).length,
+    completed: rows.filter((row) => row.observation?.status === 'succeeded')
+      .length,
+    failed: rows.filter(
+      (row) => row.observation?.status === 'failed' || row.state === 'rejected',
+    ).length,
+    cancelled: rows.filter(
+      (row) =>
+        row.state === 'cancelled' || row.observation?.status === 'cancelled',
+    ).length,
+    unknown: rows.filter((row) => ['unknown', 'submitting'].includes(row.state))
+      .length,
+    items: rows,
+  };
+}
+async function run(directory, plan, options, context) {
+  const deadline = Date.now() + duration(options.timeout || '2h');
+  const limit = options.limit ? Number(options.limit) : plan.items.length;
+  if (!Number.isInteger(limit) || limit < 1)
+    throw new CliError(
+      2,
+      'INVALID_LIMIT',
+      '--limit must be a positive number of new submissions.',
+    );
+  let submitted = 0;
+  for (let index = 0; index < plan.items.length; index++) {
+    if (context.signal.aborted)
+      throw new CliError(
+        6,
+        'INTERRUPTED',
+        'Stopped local submissions; accepted jobs continue.',
+      );
+    const item = plan.items[index];
+    const file = receiptPath(directory, index);
+    const receipt = (await exists(file))
+      ? await readJson(file)
+      : newReceipt(plan, item);
+    matchReceipt(plan, item, receipt);
+    if (!(await exists(file))) await writeJson(file, receipt, true);
+    if (receipt.accepted) {
+      receipt.observation = await observeVideo(context.client, receipt);
+      await writeJson(file, receipt);
+      continue;
+    }
+    if (receipt.state === 'rejected' || receipt.state === 'cancelled') continue;
+    if (submitted >= limit) break;
+    if (receipt.state === 'unknown' || receipt.state === 'submitting')
+      await observeVideo(context.client, receipt);
+    for (;;) {
+      if (Date.now() >= deadline)
+        throw new CliError(
+          6,
+          'BATCH_TIMEOUT',
+          'Batch submission deadline reached; resume the same state.',
+          { receipt: file },
+        );
+      const current = await context.client.get('/generation/options');
+      const profile = current.profiles.find(
+        (entry) => entry.pricing_key === receipt.request.generation_pricing_key,
+      );
+      if (!profile || profile.model !== item.model)
+        throw new CliError(
+          4,
+          'PROFILE_CHANGED',
+          `The pricing profile identity changed for ${item.key}; keep this plan unchanged.`,
+        );
+      try {
+        await acceptVideo(context.client, receipt, file);
+        submitted++;
+        context.progress(`${item.key}: accepted ${receipt.accepted.job_id}`);
+        break;
+      } catch (error) {
+        if (receipt.state !== 'waiting') throw error;
+        context.progress(`${item.key}: waiting for server admission`);
+        await pause(
+          Math.min(
+            (error.retry_after || 5) * 1000,
+            Math.max(0, deadline - Date.now()),
+          ),
+          context.signal,
+        );
+      }
+    }
+  }
+  const result = await status(directory, plan, context);
+  if (result.items.some((item) => item.state === 'rejected'))
+    throw new CliError(
+      4,
+      'BATCH_REJECTED',
+      'Some fixed intents were rejected; they were not replaced.',
+      { data: result },
+    );
+  if (result.items.some((item) => !item.job_id && item.state !== 'cancelled'))
+    throw new CliError(
+      6,
+      'BATCH_PARTIAL',
+      'Submission limit reached; resume this state to continue.',
+      { data: result },
+    );
+  return result;
+}
+async function collect(directory, plan, options, context) {
+  const rows = [];
+  let pending = 0,
+    failed = 0;
+  for (let index = 0; index < plan.items.length; index++) {
+    const file = receiptPath(directory, index),
+      item = plan.items[index];
+    const receipt = (await exists(file))
+      ? await readJson(file)
+      : newReceipt(plan, item);
+    matchReceipt(plan, item, receipt);
+    if (!receipt.accepted) {
+      if (receipt.state === 'rejected') failed++;
+      else if (receipt.state !== 'cancelled') pending++;
+      rows.push({ key: item.key, state: receipt.state });
+      continue;
+    }
+    const observation = await observeVideo(context.client, receipt);
+    if (!terminal(observation.status)) {
+      pending++;
+      rows.push({ key: item.key, status: observation.status });
+      continue;
+    }
+    if (observation.status !== 'succeeded') {
+      if (observation.status === 'failed') failed++;
+      rows.push({ key: item.key, status: observation.status });
+      continue;
+    }
+    const output = path.resolve(
+      need(options.out, '--out'),
+      `${String(index + 1).padStart(4, '0')}-${item.key.replace(/[^\p{L}\p{N}._-]/gu, '_')}.mp4`,
+    );
+    let media;
+    if (receipt.download?.file === output && (await exists(output))) {
+      const current = await fileDigest(output);
+      if (
+        current.bytes !== receipt.download.bytes ||
+        current.sha256 !== receipt.download.sha256
+      )
+        throw new CliError(
+          2,
+          'OUTPUT_CHANGED',
+          `Existing downloaded file changed: ${output}`,
+        );
+      media = receipt.download;
+    } else {
+      media = await download(
+        'take',
+        {
+          creative: item.creative_id,
+          shot: item.request.shot_id,
+          result: receipt.accepted.result_id,
+          out: output,
+        },
+        context,
+      );
+      receipt.download = media;
+      await writeJson(file, receipt);
+    }
+    rows.push({ key: item.key, ...media });
+  }
+  const result = {
+    downloaded: rows.filter((row) => row.downloaded).length,
+    pending,
+    failed,
+    items: rows,
+  };
+  if (pending || failed)
+    throw new CliError(
+      pending ? 6 : 7,
+      pending ? 'BATCH_PENDING' : 'BATCH_FAILED',
+      'Available completed Takes were downloaded; remaining rows retain their actual state.',
+      { data: result },
+    );
+  return result;
+}
+// 批次取消：已受理、未終局的 Job 交給伺服器取消（FUSE 排隊中的，以及已在機器上的開源 take）；
+// 尚未送出的項目在本地撤回，resume 不再送出。閉源已送到 provider（伺服器拒絕）、已結束或
+// 受理結果不明的照實列出，不重送、不猜測。
+async function cancelBatch(directory, plan, context) {
+  const rows = [];
+  for (let index = 0; index < plan.items.length; index++) {
+    const item = plan.items[index];
+    const file = receiptPath(directory, index);
+    const saved = await exists(file);
+    const receipt = saved ? await readJson(file) : newReceipt(plan, item);
+    matchReceipt(plan, item, receipt);
+    if (!receipt.accepted) {
+      if (['prepared', 'waiting', 'auth_required'].includes(receipt.state)) {
+        receipt.state = 'cancelled';
+        await writeJson(file, receipt, !saved);
+        rows.push({ key: item.key, outcome: 'withdrawn', receipt: file });
+      } else
+        rows.push({
+          key: item.key,
+          outcome: receipt.state === 'cancelled' ? 'withdrawn' : receipt.state,
+          receipt: file,
+        });
+      continue;
+    }
+    const observation = await observeVideo(context.client, receipt);
+    if (terminal(observation.status)) {
+      rows.push({
+        key: item.key,
+        job_id: receipt.accepted.job_id,
+        outcome: observation.status,
+      });
+      continue;
+    }
+    try {
+      const result = await cancelJob(context.client, receipt.accepted.job_id);
+      rows.push({
+        key: item.key,
+        job_id: receipt.accepted.job_id,
+        outcome: result.status,
+      });
+    } catch (error) {
+      if (!(error instanceof CliError)) throw error;
+      rows.push({
+        key: item.key,
+        job_id: receipt.accepted.job_id,
+        outcome:
+          error.code === 'GENERATION_JOB_CANCEL_FORBIDDEN'
+            ? 'already_submitted'
+            : 'error',
+        status: observation.status,
+        ...(error.code === 'GENERATION_JOB_CANCEL_FORBIDDEN'
+          ? {}
+          : { error: errorData(error) }),
+      });
+    }
+  }
+  const count = (outcome) =>
+    rows.filter((row) => row.outcome === outcome).length;
+  const result = {
+    state: path.resolve(directory),
+    count: rows.length,
+    cancelled: count('cancelled'),
+    withdrawn: count('withdrawn'),
+    already_submitted: count('already_submitted'),
+    unknown: rows.filter((row) =>
+      ['unknown', 'submitting', 'error'].includes(row.outcome),
+    ).length,
+    items: rows,
+  };
+  if (result.unknown)
+    throw new CliError(
+      5,
+      'BATCH_CANCEL_UNKNOWN',
+      'Some items could not be confirmed; resolve them (batch resume for unknown acceptance) and run batch cancel again.',
+      { data: result },
+    );
+  return result;
+}
+export async function batch(action, positional, options, context) {
+  if (action === 'plan')
+    return prepare(
+      path.resolve(need(positional, 'manifest file')),
+      need(options.out, '--out'),
+      context,
+    );
+  const directory = path.resolve(
+    need(action === 'run' ? options.state : positional, 'state directory'),
+  );
+  if (action === 'status')
+    return status(directory, await loadState(directory, context), context);
+  return withWriter(directory, async () => {
+    const savedPlan = path.join(directory, 'plan.json');
+    if (action === 'run') {
+      const input = await readJson(need(positional, 'plan file'));
+      validPlan(input);
+      sameSubject(input, await subject(context.client));
+      if (await exists(savedPlan)) {
+        if ((await readJson(savedPlan)).plan_sha256 !== input.plan_sha256)
+          throw new CliError(
+            2,
+            'STATE_PLAN_MISMATCH',
+            'This state belongs to a different plan.',
+          );
+      } else await writeJson(savedPlan, input, true);
+    }
+    const plan = await loadState(directory, context);
+    if (action === 'download')
+      return collect(directory, plan, options, context);
+    if (action === 'cancel') return cancelBatch(directory, plan, context);
+    if (!['run', 'resume'].includes(action))
+      throw new CliError(
+        2,
+        'UNKNOWN_COMMAND',
+        'Use batch plan, run, status, resume, download, or cancel.',
+      );
+    return run(directory, plan, options, context);
+  });
+}