@crievo/omni 0.0.0-stage → 0.1.0

package/tools/job-queue.mjs

@@ -0,0 +1,180 @@
+/* ============================================================================
+   Machine queue turn for the Node tools: final render, sound and
+   Premiere. The protocol lives in tools/job_queue.py; here it runs as a
+   guardian, a Python process that waits for the turn on behalf of this process,
+   holds the flock while it lives and releases it when it exits, even on kill -9
+   (the stdin pipe closes and the guardian sees the end).
+
+     import { turn } from "./job-queue.mjs";
+     await turn("render", dir);      // returns on our turn; the turn ends when the process exits
+
+   During the turn, stdout and stderr also go to .queue/<id>.log, which the
+   studio shows live. Without contention, terminal output is the same as before.
+   ============================================================================ */
+
+import { spawn, spawnSync } from "node:child_process";
+import {
+  readFileSync,
+  writeFileSync,
+  renameSync,
+  openSync,
+  writeSync,
+  existsSync,
+  readdirSync,
+  statSync,
+} from "node:fs";
+import { join, dirname } from "node:path";
+import { CODE_ROOT, DATA_ROOT as ROOT } from "./lib/paths.mjs";
+
+const CFG = JSON.parse(readFileSync(join(ROOT, "omnivideo.config.json"), "utf8"));
+const QUEUE_DIR = join(ROOT, ".queue");
+
+/* a zombie counts as dead (see studio/api/services/queue.ts) */
+function alive(pid) {
+  try {
+    process.kill(pid, 0);
+  } catch (e) {
+    if (e.code !== "EPERM") return false;
+  }
+  const ps = spawnSync("ps", ["-o", "stat=", "-p", String(pid)], { encoding: "utf8" });
+  return !String(ps.stdout).trim().startsWith("Z");
+}
+
+/* a child of a task that already holds the turn doesn't wait for its parent's turn */
+function insideTurn() {
+  const id = process.env.OMNIVIDEO_TURN;
+  if (!id) return false;
+  try {
+    const ticket = JSON.parse(readFileSync(join(QUEUE_DIR, `${id}.json`), "utf8"));
+    return ticket.state === "running" && alive(ticket.pid);
+  } catch {
+    return false;
+  }
+}
+
+/* tasks holding the turn now, with the same liveness rule as job_queue.py:
+   process alive and ticket heartbeat less than 15 s ago */
+export function running() {
+  if (!existsSync(QUEUE_DIR)) return [];
+  const now = Date.now();
+  return readdirSync(QUEUE_DIR)
+    .filter(n => n.endsWith(".json"))
+    .flatMap(n => {
+      try {
+        const ticket = JSON.parse(readFileSync(join(QUEUE_DIR, n), "utf8"));
+        const live =
+          ticket.state === "running" && alive(ticket.pid) && now - statSync(join(QUEUE_DIR, n)).mtimeMs < 15_000;
+        return live ? [ticket] : [];
+      } catch {
+        return [];
+      }
+    });
+}
+
+function mirror(log) {
+  const fd = openSync(log, "a");
+  for (const out of [process.stdout, process.stderr]) {
+    const write = out.write.bind(out);
+    out.write = (chunk, ...rest) => {
+      try {
+        writeSync(fd, chunk);
+      } catch {
+        /* the log is a copy; the terminal is what counts */
+      }
+      return write(chunk, ...rest);
+    };
+  }
+}
+
+/* on 'exit' only synchronous code runs; death by signal doesn't pass here, and
+   the guardian closes the ticket as interrupted */
+function close(file, exitCode) {
+  try {
+    const ticket = JSON.parse(readFileSync(file, "utf8"));
+    Object.assign(ticket, {
+      state: exitCode === 0 ? "ok" : "error",
+      exit_code: exitCode,
+      ended_at: new Date().toISOString(),
+    });
+    const tmp = join(dirname(file), `.${ticket.id}.${process.pid}.tmp`);
+    writeFileSync(tmp, JSON.stringify(ticket, null, 1) + "\n");
+    renameSync(tmp, file);
+  } catch {
+    /* the guardian closes it */
+  }
+}
+
+/* what comes after the voice doesn't run while the user swaps voices in the UI
+   (voice-pause.json in the video dir; "Seguir com estas vozes" removes it).
+   The same rule is in job_queue.py and studio/api/services/voice-pause.ts */
+const AFTER_VOICE = new Set([
+  "render",
+  "sound",
+  "captions",
+  "assembly",
+  "premiere-assemble",
+  "premiere-check",
+  "premiere-export",
+]);
+function checkVoicePause(tool, dir) {
+  if (!AFTER_VOICE.has(tool) || !dir || !existsSync(join(dir, "voice-pause.json"))) return;
+  process.stderr.write(
+    `pausado: o usuário está trocando vozes deste vídeo na tela do studio (voice-pause.json). ` +
+      `${tool} só roda quando ele clicar em "Seguir com estas vozes" na etapa Voz. Pare aqui e não refaça vozes.\n`,
+  );
+  process.exit(3);
+}
+
+export async function turn(tool, dir) {
+  checkVoicePause(tool, dir);
+  if (process.env.OMNIVIDEO_QUEUE === "0" || insideTurn()) return null;
+  /* the project's Python (.venv); without it, the config's and the system python3 */
+  const python =
+    [join(CODE_ROOT, ".venv", "bin", "python"), CFG.voicestudio?.python ?? ""].find(p => p && existsSync(p)) ??
+    "python3";
+  const guardian = spawn(
+    python,
+    [
+      join(CODE_ROOT, "tools", "job_queue.py"),
+      "guardian",
+      tool,
+      dir ?? "-",
+      String(process.pid),
+      "--",
+      ...process.argv.slice(1),
+    ],
+    { stdio: ["pipe", "pipe", "inherit"] },
+  );
+  const ticket = await new Promise((resolveTicket, reject) => {
+    let buf = "";
+    guardian.stdout.on("data", d => {
+      buf += d;
+      const i = buf.indexOf("\n");
+      if (i >= 0) resolveTicket(JSON.parse(buf.slice(0, i)));
+    });
+    guardian.on("error", reject);
+    guardian.on("exit", c =>
+      reject(
+        new Error(
+          `a fila da máquina falhou: o guardião saiu com ${c} antes da vez ` + "(OMNIVIDEO_QUEUE=0 roda sem fila)",
+        ),
+      ),
+    );
+  });
+  /* the guardian lives until this process exits, without holding the event
+     loop. If it dies earlier (kill), the lock was released: warn, because
+     another heavy task may start alongside this one */
+  guardian.stdout.destroy();
+  guardian.stdin.unref();
+  guardian.unref();
+  guardian.removeAllListeners("exit");
+  guardian.on("exit", () =>
+    process.stderr.write(
+      "fila: ⚠ o guardião saiu antes desta tarefa terminar; a vez foi solta e outra tarefa pesada pode rodar junto\n",
+    ),
+  );
+  process.env.OMNIVIDEO_TURN = ticket.id;
+  mirror(ticket.log);
+  process.on("exit", code => close(ticket.file, code));
+  return ticket;
+}

package/tools/job_queue.py

@@ -0,0 +1,430 @@
+#!/usr/bin/env python3
+"""Machine queue: one heavy task at a time, with the studio running or not.
+
+    job_queue.py            who runs, who waits and the last ones that finished
+    job_queue.py --json     the same as data
+    job_queue.py guardian <tool> <dir|-> <pid> -- <command…>
+                            holds the turn of a Node tool (tools/job-queue.mjs)
+
+In the Python tools:
+
+    from job_queue import turn
+    with turn("voice", video_dir):
+        ...                 # runs only on this task's turn
+
+Heavy: voice, captions, final render, sound, assembly and Premiere
+(assemble, check, export). Each writes a ticket, .queue/<id>.json, and waits to
+be the oldest live one waiting; then it takes the flock on .queue/machine.lock.
+The kernel releases the flock when the process dies, even on kill -9: there is
+no forgotten lock. The task output also goes to .queue/<id>.log.
+
+The studio only reads this dir. Running, stopped or restarting in the middle of
+a task, the queue is the same.
+
+OMNIVIDEO_QUEUE=0 disables the queue (emergency). OMNIVIDEO_TURN, which the task
+holding the turn passes to its children, keeps a child from waiting for its own
+parent's turn.
+"""
+
+import fcntl
+import json
+import os
+import select
+import shlex
+import signal
+import subprocess
+import sys
+import threading
+import time
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+
+from omni_paths import DATA_ROOT as ROOT
+
+QUEUE_DIR = ROOT / ".queue"
+LOCK = QUEUE_DIR / "machine.lock"
+
+POLL_S = 0.4  # between one look at the queue and the next
+HEARTBEAT_S = 2.0  # a live ticket renews its mtime at this pace
+STALE_S = 15.0  # no heartbeat for longer than this: process stopped or pid reused
+KEPT = 100  # finished tickets left in the dir, with their log
+ACTIVE = ("waiting", "running")
+
+
+def now():
+    return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def alive(pid):
+    if not pid:
+        return False
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        pass
+    # a zombie counts as dead (see studio/api/services/queue.ts)
+    ps = subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True)
+    return not ps.stdout.strip().startswith("Z")
+
+
+def ticket_path(id_):
+    return QUEUE_DIR / f"{id_}.json"
+
+
+def write(ticket):
+    """Atomic swap: a reader never gets a half-written ticket."""
+    dest = ticket_path(ticket["id"])
+    tmp = dest.with_name(f".{ticket['id']}.{os.getpid()}.tmp")
+    tmp.write_text(json.dumps(ticket, ensure_ascii=False, indent=1) + "\n")
+    tmp.replace(dest)
+
+
+def read_all():
+    """Tickets with their effective state: an active one whose process died, or with no heartbeat, counts as interrupted."""
+    if not QUEUE_DIR.is_dir():
+        return []
+    instant = time.time()
+    tickets = []
+    for p in QUEUE_DIR.glob("*.json"):
+        try:
+            ticket = json.loads(p.read_text())
+            mtime = p.stat().st_mtime
+        except (OSError, ValueError):
+            continue
+        effective = ticket.get("state")
+        if effective in ACTIVE and (not alive(ticket.get("pid")) or instant - mtime > STALE_S):
+            effective = "interrupted"
+        ticket["effective"] = effective
+        tickets.append(ticket)
+    return tickets
+
+
+def sort_key(ticket):
+    return (ticket.get("order", 0), ticket.get("pid", 0))
+
+
+def close_dead(id_):
+    """Re-read after learning the owner died: what it wrote before exiting stands."""
+    try:
+        ticket = json.loads(ticket_path(id_).read_text())
+    except (OSError, ValueError):
+        return
+    if ticket.get("state") in ACTIVE:
+        ticket.update(state="interrupted", ended_at=now())
+        write(ticket)
+
+
+def prune():
+    """Closes tickets of dead processes and deletes the oldest finished ones."""
+    for ticket in read_all():
+        if ticket["state"] in ACTIVE and not alive(ticket.get("pid")):
+            close_dead(ticket["id"])
+    finished = [t for t in read_all() if t["state"] not in ACTIVE]
+    finished.sort(key=lambda t: t.get("ended_at") or t.get("arrived_at") or "", reverse=True)
+    for ticket in finished[KEPT:]:
+        for p in (ticket_path(ticket["id"]), QUEUE_DIR / f"{ticket['id']}.log"):
+            try:
+                p.unlink()
+            except FileNotFoundError:
+                pass
+
+
+def origin():
+    if os.environ.get("OMNIVIDEO_ORIGIN"):
+        return os.environ["OMNIVIDEO_ORIGIN"]
+    return "chat" if os.environ.get("CLAUDECODE") else "terminal"
+
+
+def new_ticket(tool, dir, command, pid, guardian=None):
+    QUEUE_DIR.mkdir(exist_ok=True)
+    ns = time.time_ns()
+    id_ = f"{time.strftime('%Y%m%d-%H%M%S', time.localtime(ns // 1_000_000_000))}-{ns // 1_000_000 % 1000:03d}-{pid}"
+    video = Path(dir).resolve() if dir else None
+    ticket = {
+        "id": id_,
+        "tool": tool,
+        "video": video.name if video else None,
+        "dir": str(video) if video else None,
+        "command": shlex.join([Path(command[0]).name, *command[1:]]) if command else "",
+        "pid": pid,
+        "guardian": guardian,
+        "origin": origin(),
+        "session": os.environ.get("CLAUDE_CODE_SESSION_ID"),
+        "state": "waiting",
+        "order": ns,
+        "arrived_at": now(),
+        "started_at": None,
+        "ended_at": None,
+        "exit_code": None,
+        "log": f".queue/{id_}.log",
+    }
+    write(ticket)
+    return ticket
+
+
+class Heartbeat(threading.Thread):
+    """Renews the ticket's mtime: this is how a reused pid does not pass for alive."""
+
+    def __init__(self, id_):
+        super().__init__(daemon=True)
+        self.path = ticket_path(id_)
+        self.done = threading.Event()
+        self.start()
+
+    def run(self):
+        while not self.done.wait(HEARTBEAT_S):
+            try:
+                os.utime(self.path)
+            except OSError:
+                pass
+
+    def stop(self):
+        self.done.set()
+
+
+def duration(seconds):
+    s = int(seconds)
+    if s < 60:
+        return f"{s}s"
+    if s < 3600:
+        return f"{s // 60}min{s % 60:02d}s"
+    return f"{s // 3600}h{s % 3600 // 60:02d}min"
+
+
+def since(iso):
+    if not iso:
+        return 0
+    return time.time() - datetime.fromisoformat(iso.replace("Z", "+00:00")).timestamp()
+
+
+def summary(ticket):
+    who = ticket["origin"] + (f" {ticket['session'][:8]}" if ticket.get("session") else "")
+    return f"{ticket['tool']} · {ticket.get('video') or '-'} ({who}, pid {ticket['pid']})"
+
+
+def wait(ticket, fd, pause):
+    """Returns the seconds waited when this ticket is the oldest live one waiting
+    and the flock is its own; None if `pause` gives up midway."""
+    t0 = time.time()
+    said = None
+    lock_held_since = None
+    while True:
+        tickets = read_all()
+        ahead = [
+            t
+            for t in tickets
+            if t["effective"] == "waiting" and t["id"] != ticket["id"] and sort_key(t) < sort_key(ticket)
+        ]
+        if not ahead:
+            try:
+                fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                return time.time() - t0
+            except BlockingIOError:
+                pass
+        running = [t for t in tickets if t["effective"] == "running"]
+        # lock busy with nobody running is usually the handover between two tasks
+        # (Node closes the ticket an instant before the guardian releases the lock)
+        lock_held_since = (lock_held_since or time.time()) if not running and not ahead else None
+        snapshot = (len(ahead), tuple(t["id"] for t in running))
+        # nobody running with someone ahead is also a handover: no notice
+        if snapshot != said and (running or (not ahead and time.time() - lock_held_since > 3)):
+            said = snapshot
+            parts = [f"rodando {summary(t)} há {duration(since(t.get('started_at')))}" for t in running]
+            if not running:
+                parts.append("o lock está com um processo fora da fila")
+            if ahead:
+                parts.append(f"{len(ahead)} na frente")
+            print(f"fila: esperando a vez de {ticket['tool']} · {'; '.join(parts)}", file=sys.stderr, flush=True)
+        if not pause(POLL_S):
+            return None
+
+
+def begin(ticket, waited):
+    ticket.update(state="running", started_at=now())
+    write(ticket)
+    # with the flock in hand, another "running" ticket can only belong to a dead process
+    for t in read_all():
+        if t["id"] != ticket["id"] and t["state"] == "running" and not alive(t.get("pid")):
+            close_dead(t["id"])
+    if waited >= 1:
+        print(f"fila: vez de {ticket['tool']} depois de {duration(waited)} de espera", file=sys.stderr, flush=True)
+
+
+class Tee:
+    """Writes to the terminal as before and copies to the ticket's log, which the studio shows live."""
+
+    def __init__(self, original, log):
+        self._original = original
+        self._log = log
+
+    def write(self, text):
+        n = self._original.write(text)
+        try:
+            self._log.write(text)
+            self._log.flush()
+        except (OSError, ValueError):
+            pass
+        return n
+
+    def __getattr__(self, name):
+        return getattr(self._original, name)
+
+
+def tee(path):
+    log = open(path, "a", encoding="utf-8")
+    originals = sys.stdout, sys.stderr
+    sys.stdout, sys.stderr = Tee(sys.stdout, log), Tee(sys.stderr, log)
+
+    def restore():
+        sys.stdout, sys.stderr = originals
+        log.close()
+
+    return restore
+
+
+def inside_turn():
+    """A child of a task that already holds the turn does not wait for its parent's turn."""
+    id_ = os.environ.get("OMNIVIDEO_TURN")
+    if not id_:
+        return False
+    try:
+        ticket = json.loads(ticket_path(id_).read_text())
+    except (OSError, ValueError):
+        return False
+    return ticket.get("state") == "running" and alive(ticket.get("pid"))
+
+
+# what comes after the voice does not run while the user swaps voices in the studio
+# (voice-pause.json in the video dir; "Seguir com estas vozes" removes it).
+# The same rule is in job-queue.mjs and studio/api/services/voice-pause.ts
+AFTER_VOICE = {"render", "sound", "captions", "assembly", "premiere-assemble", "premiere-check", "premiere-export"}
+
+
+def check_voice_pause(tool, dir):
+    if tool in AFTER_VOICE and dir and (Path(dir) / "voice-pause.json").exists():
+        sys.stderr.write(
+            f"pausado: o usuário está trocando vozes deste vídeo na tela do studio (voice-pause.json). "
+            f'{tool} só roda quando ele clicar em "Seguir com estas vozes" na etapa Voz. '
+            "Pare aqui e não refaça vozes.\n"
+        )
+        sys.exit(3)
+
+
+@contextmanager
+def turn(tool, dir=None):
+    check_voice_pause(tool, dir)
+    if os.environ.get("OMNIVIDEO_QUEUE") == "0" or inside_turn():
+        yield None
+        return
+    prune()
+    ticket = new_ticket(tool, dir, sys.argv, os.getpid())
+    heartbeat = Heartbeat(ticket["id"])
+    fd = os.open(LOCK, os.O_RDWR | os.O_CREAT, 0o644)
+    restore = None
+    state, exit_code = "ok", 0
+    try:
+        begin(ticket, wait(ticket, fd, lambda s: time.sleep(s) or True))
+        os.environ["OMNIVIDEO_TURN"] = ticket["id"]
+        restore = tee(ROOT / ticket["log"])
+        yield ticket
+    except KeyboardInterrupt:
+        state, exit_code = "interrupted", 130
+        raise
+    except SystemExit as e:
+        exit_code = e.code if isinstance(e.code, int) else (0 if e.code is None else 1)
+        state = "ok" if exit_code == 0 else "error"
+        raise
+    except BaseException:
+        state, exit_code = "error", 1
+        raise
+    finally:
+        if restore:
+            restore()
+        if ticket["started_at"]:
+            ticket.update(state=state, exit_code=exit_code, ended_at=now())
+        else:
+            ticket.update(state="interrupted", ended_at=now())
+        write(ticket)
+        heartbeat.stop()
+        os.close(fd)
+        if os.environ.get("OMNIVIDEO_TURN") == ticket["id"]:
+            del os.environ["OMNIVIDEO_TURN"]
+
+
+def guardian(tool, dir, pid, command):
+    """Holds the turn of a Node process (tools/job-queue.mjs).
+
+    Waits like any task; on its turn, writes one JSON line to stdout and keeps
+    reading stdin, which is a pipe from the Node process. The pipe closes when
+    it exits, in any way, and then the ticket is closed and the flock released.
+    """
+    # Ctrl-C kills Node; the guardian stays to close the ticket when the pipe closes
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    prune()
+    ticket = new_ticket(tool, dir, command, pid, guardian=os.getpid())
+    heartbeat = Heartbeat(ticket["id"])
+    fd = os.open(LOCK, os.O_RDWR | os.O_CREAT, 0o644)
+
+    def pause(seconds):
+        readable, _, _ = select.select([0], [], [], seconds)
+        return not (readable and not os.read(0, 4096))
+
+    waited = wait(ticket, fd, pause)
+    if waited is None:
+        ticket.update(state="interrupted", ended_at=now())
+        write(ticket)
+    else:
+        begin(ticket, waited)
+        print(
+            json.dumps({"id": ticket["id"], "file": str(ticket_path(ticket["id"])), "log": str(ROOT / ticket["log"])}),
+            flush=True,
+        )
+        while os.read(0, 4096):
+            pass
+        close_dead(ticket["id"])
+    heartbeat.stop()
+    os.close(fd)
+
+
+def show(as_json):
+    tickets = read_all()
+    running = sorted((t for t in tickets if t["effective"] == "running"), key=sort_key)
+    waiting = sorted((t for t in tickets if t["effective"] == "waiting"), key=sort_key)
+    recent = sorted(
+        (t for t in tickets if t["effective"] not in ACTIVE), key=lambda t: t.get("ended_at") or "", reverse=True
+    )[:10]
+    if as_json:
+        print(json.dumps({"running": running, "waiting": waiting, "recent": recent}, ensure_ascii=False, indent=1))
+        return
+    if not running and not waiting:
+        print("fila livre")
+    for t in running:
+        print(f"rodando  {summary(t)} há {duration(since(t.get('started_at')))}")
+    for n, t in enumerate(waiting, 1):
+        print(f"espera   {n}. {summary(t)} há {duration(since(t.get('arrived_at')))}")
+    for t in recent:
+        mark = {"ok": "✓", "error": "✗"}.get(t["effective"], "·")
+        took = (
+            duration(since(t["started_at"]) - since(t["ended_at"]))
+            if t.get("started_at") and t.get("ended_at")
+            else "sem rodar"
+        )
+        print(f"{mark} {t['effective']:<12} {summary(t)} · {took}")
+
+
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    if args[:1] == ["guardian"]:
+        if len(args) < 5 or args[4] != "--":
+            print(__doc__)
+            sys.exit(2)
+        guardian(args[1], None if args[2] == "-" else args[2], int(args[3]), args[5:])
+    elif args in ([], ["--json"]):
+        show(as_json=bool(args))
+    else:
+        print(__doc__)
+        sys.exit(2)