@zzclub/pipeline 0.10.2 → 0.12.0

package/src/monitor/server.ts

@@ -0,0 +1,170 @@
+import { closeSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { timingSafeEqual, randomBytes } from "node:crypto";
+import { MonitorStore, type MonitorFilter } from "./store";
+import { monitorDir, processIdentity, processStatus } from "./runtime";
+import { monitorHealth, readMonitorDescriptor } from "./client";
+import type { MonitorDescriptor, MonitorUpdate } from "./types";
+
+function json(data: unknown, status = 200): Response {
+  return Response.json(data, { status, headers: { "Cache-Control": "no-store" } });
+}
+
+/** 原子锁只属于监控进程，与任何工作流锁互不关联。 */
+async function claimServer(root: string, instanceId: string): Promise<(() => void) | null> {
+  const lockPath = join(root, "server.lock");
+  for (let attempt = 0; attempt < 50; attempt++) {
+    try {
+      const fd = openSync(lockPath, "wx", 0o600);
+      try { writeFileSync(fd, JSON.stringify({ pid: process.pid, identity: processIdentity(process.pid), instance_id: instanceId })); }
+      finally { closeSync(fd); }
+      return () => {
+        try {
+          if (JSON.parse(readFileSync(lockPath, "utf8")).instance_id === instanceId) unlinkSync(lockPath);
+        } catch { /* 已被本实例清理。 */ }
+      };
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      const descriptor = readMonitorDescriptor(root);
+      if (descriptor && await monitorHealth(descriptor)) return null;
+      try {
+        const lock = JSON.parse(readFileSync(lockPath, "utf8")) as { pid: number; identity: string | null };
+        if (Number.isSafeInteger(lock.pid) && lock.pid > 0 && processStatus(lock.pid, lock.identity) === "interrupted") {
+          unlinkSync(lockPath);
+          continue;
+        }
+      } catch { /* 写锁过程中短暂空文件不能被抢占。 */ }
+      await new Promise((done) => setTimeout(done, 100));
+    }
+  }
+  throw new Error("监控启动锁仍被占用；请检查已有 monitor serve 进程。");
+}
+
+/** 一个本机 HTTP 服务承载快照、分页日志和多任务 SSE。 */
+export async function serveMonitor(): Promise<{ descriptor: MonitorDescriptor; stop: () => void } | null> {
+  const root = monitorDir();
+  mkdirSync(root, { recursive: true, mode: 0o700 });
+  const instanceId = crypto.randomUUID();
+  const release = await claimServer(root, instanceId);
+  if (!release) return null;
+  const token = randomBytes(32).toString("hex");
+  const store = new MonitorStore(instanceId, root);
+  let lastAccess = Date.now();
+  let ready = false;
+  let stopped = false;
+  const connections = new Set<() => void>();
+  let stop = () => {};
+  let server: ReturnType<typeof Bun.serve>;
+  try {
+    server = Bun.serve({
+      hostname: "127.0.0.1", port: 0,
+      async fetch(request, runtime) {
+        const authorization = request.headers.get("authorization") || "";
+        const expected = `Bearer ${token}`;
+        if (Buffer.byteLength(authorization) !== Buffer.byteLength(expected) || !timingSafeEqual(Buffer.from(authorization), Buffer.from(expected))) return json({ error: "unauthorized" }, 401);
+        const url = new URL(request.url);
+        // 首版只接收同机原生客户端，不向任意网页开放本地凭据与日志。
+        if (request.headers.has("origin") && request.headers.get("origin") !== url.origin) return json({ error: "origin not allowed" }, 403);
+        lastAccess = Date.now();
+        if (url.pathname === "/v1/health" && request.method === "GET") return json({ service: "zzhub-pipeline-monitor", version: 1, instance_id: instanceId, ready, issues: [...store.issues] });
+        if (url.pathname === "/v1/stop" && request.method === "POST") { setTimeout(() => stop(), 25); return json({ stopped: true }); }
+        if (request.method !== "GET") return json({ error: "method not allowed" }, 405);
+        if (!ready) return json({ error: "monitor starting" }, 503);
+        const filter: MonitorFilter = {
+          workspace: url.searchParams.get("workspace") || undefined,
+          task_id: url.searchParams.get("task_id") || undefined,
+          execution_id: url.searchParams.get("execution_id") || undefined,
+        };
+        try {
+          if (url.pathname === "/v1/snapshot") { await store.refresh(); return json(store.snapshot(filter)); }
+          const match = url.pathname.match(/^\/v1\/executions\/([0-9a-f-]+)(\/logs)?$/);
+          if (match) {
+            const execution = store.executions.get(match[1]!);
+            if (!execution) return json({ error: "execution not found" }, 404);
+            if (!match[2]) return json(execution);
+            const after = Number(url.searchParams.get("after_seq") || "0");
+            const limit = Number(url.searchParams.get("limit") || "200");
+            if (!Number.isSafeInteger(after) || after < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > 500) return json({ error: "invalid log cursor or limit" }, 400);
+            return json(await store.logs(execution.id, after, limit));
+          }
+          if (url.pathname !== "/v1/events") return json({ error: "not found" }, 404);
+          runtime.timeout(request, 0);
+          const cursor = request.headers.get("last-event-id") || url.searchParams.get("after") || "";
+          const events = cursor ? store.eventsAfter(cursor) : null;
+          let cleanup = () => {};
+          const encoder = new TextEncoder();
+          const stream = new ReadableStream<Uint8Array>({
+            start(controller) {
+              let closed = false;
+              let heartbeat: ReturnType<typeof setInterval> | undefined;
+              const close = () => {
+                if (closed) return;
+                closed = true;
+                clearInterval(heartbeat);
+                store.listeners.delete(send);
+                connections.delete(close);
+                request.signal.removeEventListener("abort", close);
+                try { controller.close(); } catch { /* 客户端可能已取消。 */ }
+              };
+              const enqueue = (text: string) => {
+                if (closed) return;
+                if ((controller.desiredSize ?? 0) < 0) { close(); return; }
+                try { controller.enqueue(encoder.encode(text)); } catch { close(); }
+              };
+              const send = (event: MonitorUpdate) => {
+                if (store.matchesUpdate(event, filter)) enqueue(`id: ${event.cursor}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
+              };
+              cleanup = close;
+              if (events === null) {
+                enqueue(`event: resync\ndata: ${JSON.stringify({ reason: "snapshot_required", instance_id: instanceId })}\n\n`);
+                close();
+                return;
+              }
+              // 注册与重放之间不 await，避免丢失快照之后的事件。
+              connections.add(close);
+              store.listeners.add(send);
+              enqueue(": connected\n\n");
+              for (const event of events) send(event);
+              if (closed) return;
+              heartbeat = setInterval(() => enqueue(": keepalive\n\n"), 15_000);
+              request.signal.addEventListener("abort", close, { once: true });
+              if (request.signal.aborted) close();
+            },
+            cancel() { cleanup(); },
+          }, new ByteLengthQueuingStrategy({ highWaterMark: 256 * 1024 }));
+          return new Response(stream, { headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-cache, no-transform", "X-Accel-Buffering": "no" } });
+        } catch { return json({ error: "monitor data unavailable" }, 500); }
+      },
+    });
+  } catch (error) { release(); throw error; }
+
+  const descriptor: MonitorDescriptor = { version: 1, instance_id: instanceId, pid: process.pid, url: `http://127.0.0.1:${server.port}`, token };
+  const timer = setInterval(() => {
+    void store.refresh();
+    if (!connections.size && Date.now() - lastAccess > 5 * 60_000) stop();
+  }, 500);
+  const onSignal = () => stop();
+  stop = () => {
+    if (stopped) return;
+    stopped = true;
+    clearInterval(timer);
+    connections.forEach((close) => close());
+    server.stop(true);
+    process.removeListener("SIGINT", onSignal);
+    process.removeListener("SIGTERM", onSignal);
+    process.removeListener("exit", stop);
+    try { if (readMonitorDescriptor(root)?.instance_id === instanceId) unlinkSync(join(root, "server.json")); } catch { /* 描述文件已清理。 */ }
+    release();
+  };
+  try {
+    const temporary = join(root, `server.${instanceId}.tmp`);
+    writeFileSync(temporary, JSON.stringify(descriptor), { mode: 0o600 });
+    renameSync(temporary, join(root, "server.json"));
+    process.once("SIGINT", onSignal);
+    process.once("SIGTERM", onSignal);
+    process.once("exit", stop);
+    await store.refresh();
+    ready = true;
+    return { descriptor, stop };
+  } catch (error) { stop(); throw error; }
+}

package/src/monitor/store.ts

@@ -0,0 +1,235 @@
+import { open, readdir, stat, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { StringDecoder } from "node:string_decoder";
+import { getTaskByStatePath } from "../task-manager";
+import { canonicalPath, monitorDir, processStatus, redact, RETENTION_MS, STORAGE_LIMIT } from "./runtime";
+import type { MonitorEvent, MonitorExecution, MonitorProgress, MonitorSnapshot, MonitorTask, MonitorUpdate } from "./types";
+import type { CommandOutcome } from "../command-outcome";
+
+const ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f-]{27}$/;
+const OUTCOMES = new Set(["success", "skipped", "waiting", "partial_failure", "failed"]);
+export interface MonitorFilter { workspace?: string; task_id?: string; execution_id?: string }
+interface FileCursor { offset: number; tail: string; seq: number; decoder?: StringDecoder }
+
+/** 内存投影可重建；事件文件是执行历史，工作流文件是业务状态真相源。 */
+export class MonitorStore {
+  readonly executions = new Map<string, MonitorExecution>();
+  readonly tasks = new Map<string, MonitorTask>();
+  readonly issues: string[] = [];
+  readonly listeners = new Set<(event: MonitorUpdate) => void>();
+  private files = new Map<string, FileCursor>();
+  private dirtyTasks = new Set<string>();
+  private sequence = 0;
+  private replay: MonitorUpdate[] = [];
+  private replayBytes = 0;
+  private pending: Promise<void> | null = null;
+  private lastProbe = 0;
+  private lastPrune = 0;
+
+  constructor(readonly instanceId: string, readonly root = monitorDir()) {}
+
+  get cursor(): string { return `${this.instanceId}:${this.sequence}`; }
+
+  private issue(error: unknown): void {
+    this.issues.push(redact(error instanceof Error ? error.message : String(error)));
+    if (this.issues.length > 20) this.issues.shift();
+  }
+
+  private broadcast(type: MonitorUpdate["type"], data: Record<string, unknown>, executionId?: string, taskId?: string): void {
+    this.sequence++;
+    const update: MonitorUpdate = { version: 1, cursor: this.cursor, type, data: structuredClone(data), execution_id: executionId, task_id: taskId };
+    this.replay.push(update);
+    this.replayBytes += Buffer.byteLength(JSON.stringify(update));
+    while (this.replay.length > 1000 || this.replayBytes > 2 * 1024 * 1024) {
+      this.replayBytes -= Buffer.byteLength(JSON.stringify(this.replay.shift()));
+    }
+    for (const listener of this.listeners) {
+      try { listener(update); } catch { this.listeners.delete(listener); }
+    }
+  }
+
+  matches(execution: MonitorExecution, filter: MonitorFilter): boolean {
+    return (!filter.execution_id || filter.execution_id === execution.id)
+      && (!filter.task_id || execution.task_ids.includes(filter.task_id))
+      && (!filter.workspace || execution.task_ids.some((id) => this.tasks.get(id)?.workspace === canonicalPath(filter.workspace!)));
+  }
+
+  matchesUpdate(update: MonitorUpdate, filter: MonitorFilter): boolean {
+    if (update.execution_id) {
+      const execution = this.executions.get(update.execution_id);
+      return !!execution && this.matches(execution, filter);
+    }
+    const task = update.task_id ? this.tasks.get(update.task_id) : null;
+    return !!task && (!filter.task_id || task.id === filter.task_id)
+      && (!filter.workspace || task.workspace === canonicalPath(filter.workspace))
+      && (!filter.execution_id || this.executions.get(filter.execution_id)?.task_ids.includes(task.id) === true);
+  }
+
+  snapshot(filter: MonitorFilter = {}): MonitorSnapshot {
+    const executions = [...this.executions.values()].filter((item) => this.matches(item, filter))
+      .sort((a, b) => b.started_at.localeCompare(a.started_at));
+    const tasks = [...this.tasks.values()].filter((task) => (!filter.workspace || task.workspace === canonicalPath(filter.workspace))
+      && (!filter.task_id || task.id === filter.task_id)
+      && (!filter.execution_id || this.executions.get(filter.execution_id)?.task_ids.includes(task.id) === true))
+      .map((task) => ({ ...task, active_execution_ids: executions.filter((execution) => !execution.is_query
+        && (execution.status === "running" || execution.status === "unknown") && execution.task_ids.includes(task.id)).map((execution) => execution.id) }));
+    return structuredClone({ version: 1, instance_id: this.instanceId, cursor: this.cursor, tasks,
+      executions: executions.filter((execution, index) => index < 100 || execution.status === "running" || execution.status === "unknown") });
+  }
+
+  /** 返回 null 表示游标已失效，客户端必须重新读取快照。 */
+  eventsAfter(cursor: string): MonitorUpdate[] | null {
+    if (cursor === this.cursor) return [];
+    const prefix = `${this.instanceId}:`;
+    if (!cursor.startsWith(prefix) || !/^\d+$/.test(cursor.slice(prefix.length))) return null;
+    const sequence = Number(cursor.slice(prefix.length));
+    const oldest = this.sequence - this.replay.length;
+    return sequence < oldest || sequence > this.sequence ? null : this.replay.slice(sequence - oldest);
+  }
+
+  refresh(): Promise<void> {
+    if (!this.pending) this.pending = this.scan().catch((error) => this.issue(error)).finally(() => { this.pending = null; });
+    return this.pending;
+  }
+
+  private apply(event: MonitorEvent): void {
+    const data = event.data;
+    if (event.type === "started") {
+      if (typeof data.command !== "string" || !Number.isSafeInteger(data.pid) || Number(data.pid) <= 0) return;
+      this.executions.set(event.execution_id, {
+        id: event.execution_id, command: data.command, pid: Number(data.pid),
+        process_identity: typeof data.process_identity === "string" ? data.process_identity : null,
+        is_query: data.is_query === true, started_at: event.at, ended_at: null, last_event_at: event.at,
+        status: "running", outcome: null, exit_code: null, task_ids: [],
+      });
+    }
+    const execution = this.executions.get(event.execution_id);
+    if (!execution) return;
+    execution.last_event_at = event.at;
+    if (event.type === "task") {
+      if (![data.id, data.workspace, data.run_id, data.state_path].every((value) => typeof value === "string" && value.length > 0)) return;
+      const task = data as unknown as MonitorTask;
+      this.tasks.set(task.id, { ...this.tasks.get(task.id), ...task });
+      if (!execution.task_ids.includes(task.id)) execution.task_ids.push(task.id);
+      this.dirtyTasks.add(task.id);
+    } else if (event.type === "progress") {
+      execution.progress = data as unknown as MonitorProgress;
+    } else if (event.type === "finished") {
+      const outcome = data.outcome as CommandOutcome | undefined;
+      if (!outcome || !OUTCOMES.has(outcome.status)) return;
+      execution.outcome = outcome;
+      execution.exit_code = typeof data.exit_code === "number" ? data.exit_code : null;
+      execution.status = "exited";
+      execution.ended_at = event.at;
+      execution.task_ids.forEach((id) => this.dirtyTasks.add(id));
+    } else if (event.type === "log") {
+      if (data.truncated) execution.logs_truncated = true;
+      this.broadcast("log", { ...data, seq: event.seq, at: event.at }, execution.id);
+      if (!data.truncated) return;
+    }
+    this.broadcast("execution.updated", execution as unknown as Record<string, unknown>, execution.id);
+  }
+
+  private async readEvents(id: string, cursor: FileCursor, consume: (event: MonitorEvent) => boolean | void): Promise<void> {
+    const handle = await open(join(this.root, "events", `${id}.jsonl`), "r");
+    try {
+      const size = (await handle.stat()).size;
+      if (size < cursor.offset) throw new Error(`监控事件文件被截短：${id}`);
+      // 分块扫描，不把所有任务的日志装入内存。
+      const buffer = Buffer.alloc(64 * 1024);
+      cursor.decoder ??= new StringDecoder("utf8");
+      while (cursor.offset < size) {
+        const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, size - cursor.offset), cursor.offset);
+        if (!bytesRead) break;
+        cursor.offset += bytesRead;
+        const text = cursor.tail + cursor.decoder.write(buffer.subarray(0, bytesRead));
+        const lines = text.split("\n");
+        cursor.tail = lines.pop() || "";
+        for (const line of lines) {
+          try {
+            const event = JSON.parse(line) as MonitorEvent;
+            if (event.version !== 1 || event.execution_id !== id || !Number.isSafeInteger(event.seq)
+              || event.seq <= cursor.seq || !event.data || typeof event.data !== "object" || typeof event.at !== "string") continue;
+            cursor.seq = event.seq;
+            if (consume(event) === false) return;
+          } catch { this.issue(`忽略损坏的监控事件：${id}`); }
+        }
+      }
+    } finally { await handle.close(); }
+  }
+
+  private async scan(): Promise<void> {
+    const names = await readdir(join(this.root, "events")).catch(() => [] as string[]);
+    for (const name of names) {
+      const id = name.replace(/\.jsonl$/, "");
+      if (!name.endsWith(".jsonl") || !ID_PATTERN.test(id)) continue;
+      // 已完整读到终态的日志不可再追加，历史越多也不会反复打开所有文件。
+      if (this.executions.get(id)?.status === "exited") continue;
+      const cursor = this.files.get(id) || { offset: 0, tail: "", seq: 0 };
+      this.files.set(id, cursor);
+      await this.readEvents(id, cursor, (event) => this.apply(event)).catch((error) => this.issue(error));
+    }
+    if (Date.now() - this.lastProbe > 2000) {
+      this.lastProbe = Date.now();
+      for (const execution of this.executions.values()) {
+        if (execution.status !== "running" && execution.status !== "unknown") continue;
+        const status = processStatus(execution.pid, execution.process_identity);
+        if (status === execution.status) continue;
+        execution.status = status;
+        if (status === "interrupted") {
+          execution.ended_at = new Date().toISOString();
+          execution.task_ids.forEach((id) => this.dirtyTasks.add(id));
+        }
+        this.broadcast("execution.updated", execution as unknown as Record<string, unknown>, execution.id);
+      }
+    }
+    for (const id of this.dirtyTasks) {
+      const task = this.tasks.get(id)!;
+      try {
+        const report = await getTaskByStatePath(task.state_path);
+        if (report.summary.run_id !== task.run_id) throw new Error(`监控状态文件身份不匹配：${id}`);
+        const updated: MonitorTask = { ...task, state_path: report.summary.state_path,
+          title: redact(report.summary.metadata.title || "") || null, mode: report.summary.mode, phase: report.summary.phase.current,
+          next_action: { action: report.next_action.action, reason: redact(report.next_action.reason), executor: report.next_action.executor },
+          publish_results: report.summary.publish.results.map((result) => ({ route: result.route, account: result.account,
+            status: result.status, detail: result.detail ? redact(result.detail) : null })),
+        };
+        this.tasks.set(id, updated);
+        this.broadcast("task.updated", updated as unknown as Record<string, unknown>, undefined, id);
+      } catch (error) { this.issue(error); }
+    }
+    this.dirtyTasks.clear();
+    if (Date.now() - this.lastPrune > 60_000) { this.lastPrune = Date.now(); await this.prune(); }
+  }
+
+  async logs(id: string, afterSeq: number, limit: number): Promise<{ logs: MonitorEvent[]; next_seq: number; truncated: boolean }> {
+    if (!ID_PATTERN.test(id) || !this.executions.has(id)) throw new Error("execution not found");
+    const logs: MonitorEvent[] = [];
+    const cursor = { offset: 0, tail: "", seq: 0 };
+    await this.readEvents(id, cursor, (event) => {
+      if (event.type === "log" && event.seq > afterSeq) logs.push(event);
+      return logs.length < limit;
+    });
+    return { logs, next_seq: logs.at(-1)?.seq ?? Math.max(afterSeq, cursor.seq), truncated: this.executions.get(id)?.logs_truncated === true };
+  }
+
+  private async prune(): Promise<void> {
+    const files = await Promise.all([...this.executions.values()].map(async (execution) => {
+      const path = join(this.root, "events", `${execution.id}.jsonl`);
+      const info = await stat(path).catch(() => null);
+      return { execution, path, size: info?.size ?? 0 };
+    }));
+    let total = files.reduce((sum, file) => sum + file.size, 0);
+    for (const file of files.sort((a, b) => a.execution.started_at.localeCompare(b.execution.started_at))) {
+      if (file.execution.status !== "exited" && file.execution.status !== "interrupted") continue;
+      if (Date.now() - Date.parse(file.execution.ended_at || file.execution.started_at) < RETENTION_MS && total <= STORAGE_LIMIT) continue;
+      try { await unlink(file.path); } catch (error) { this.issue(error); continue; }
+      total -= file.size;
+      this.executions.delete(file.execution.id);
+      this.files.delete(file.execution.id);
+    }
+    // ponytail: 历史元数据随文件重建；本机几十个并行任务先用内存投影。
+    const referenced = new Set([...this.executions.values()].flatMap((execution) => execution.task_ids));
+    for (const id of this.tasks.keys()) if (!referenced.has(id)) this.tasks.delete(id);
+  }
+}

package/src/monitor/types.ts

@@ -0,0 +1,77 @@
+import type { CommandOutcome } from "../command-outcome";
+
+/** 只报告能测量的进度；未知总量不生成百分比。 */
+export interface MonitorProgress {
+  stage: string;
+  message?: string;
+  current?: number;
+  total?: number;
+  unit?: "pages" | "files" | "bytes" | "targets";
+  route?: string;
+  account?: string;
+}
+
+export interface MonitorEvent {
+  version: 1;
+  execution_id: string;
+  seq: number;
+  at: string;
+  type: "started" | "task" | "progress" | "log" | "finished";
+  data: Record<string, unknown>;
+}
+
+export interface MonitorExecution {
+  id: string;
+  command: string;
+  pid: number;
+  process_identity: string | null;
+  is_query: boolean;
+  started_at: string;
+  ended_at: string | null;
+  last_event_at: string;
+  status: "running" | "exited" | "interrupted" | "unknown";
+  outcome: CommandOutcome | null;
+  exit_code: number | null;
+  task_ids: string[];
+  progress?: MonitorProgress;
+  logs_truncated?: boolean;
+}
+
+export interface MonitorTask {
+  id: string;
+  workspace: string;
+  run_id: string;
+  state_path: string;
+  title: string | null;
+  mode?: string;
+  phase?: string;
+  next_action?: { action: string; reason: string; executor: string };
+  publish_results?: Array<{ route: string; account: string; status: string; detail: string | null }>;
+  active_execution_ids?: string[];
+}
+
+export interface MonitorUpdate {
+  version: 1;
+  cursor: string;
+  type: "execution.updated" | "task.updated" | "log";
+  execution_id?: string;
+  task_id?: string;
+  data: Record<string, unknown>;
+}
+
+export interface MonitorSnapshot {
+  version: 1;
+  instance_id: string;
+  cursor: string;
+  tasks: MonitorTask[];
+  executions: MonitorExecution[];
+}
+
+/** 描述文件只能由当前用户读取，令牌不得发送到渲染进程。 */
+export interface MonitorDescriptor {
+  version: 1;
+  instance_id: string;
+  pid: number;
+  url: string;
+  token: string;
+}