@akairos/pipeline 0.22.0 → 0.23.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@akairos/pipeline",
-  "version": "0.22.0",
+  "version": "0.23.0",
   "type": "module",
   "license": "MIT",
   "bin": {

package/src/logger.test.ts

@@ -7,10 +7,13 @@ import { getDailyLogPath, getLogDir, runWithCommandLog } from "./logger";
 describe("pipeline daily logger", () => {
   const previousLogDir = process.env.ZZHUB_PIPELINE_LOG_DIR;
   const previousMonitor = process.env.ZZHUB_PIPELINE_MONITOR;
+  const previousBudget = process.env.ZZHUB_PIPELINE_WORKER_BUDGET;
   beforeEach(() => { process.env.ZZHUB_PIPELINE_MONITOR = "0"; });
   let tempDir: string | null = null;
 
   afterEach(() => {
+    if (previousBudget === undefined) delete process.env.ZZHUB_PIPELINE_WORKER_BUDGET;
+    else process.env.ZZHUB_PIPELINE_WORKER_BUDGET = previousBudget;
     if (previousMonitor === undefined) delete process.env.ZZHUB_PIPELINE_MONITOR;
     else process.env.ZZHUB_PIPELINE_MONITOR = previousMonitor;
     if (previousLogDir === undefined) {
@@ -80,4 +83,16 @@ describe("pipeline daily logger", () => {
     expect(body).toContain("Error: boom-upload");
     expect(body).toContain("boom-upload");
   });
+
+  test("trusted Worker truncates diagnostic lines without changing the original exception", async () => {
+    tempDir = mkdtempSync(join(tmpdir(), "zzhub-pipeline-log-"));
+    process.env.ZZHUB_PIPELINE_LOG_DIR = tempDir;
+    process.env.ZZHUB_PIPELINE_WORKER_BUDGET = "1";
+    const message = `original-error-${"x".repeat(20_000)}`;
+    await expect(runWithCommandLog("publish", [], async () => { throw new Error(message); })).rejects.toThrow(message);
+    const body = readFileSync(getDailyLogPath(), "utf8");
+    expect(body).toContain("[truncated]");
+    expect(body).toContain("=== FAIL command=publish");
+    expect(Buffer.byteLength(body)).toBeLessThan(6000);
+  });
 });

package/src/logger.ts

@@ -14,6 +14,7 @@ import { homedir, platform } from "node:os";
 import { join } from "node:path";
 import { outcomeExitCode, type CommandOutcome } from "./command-outcome";
 import { observeCommand, reportLog } from "./monitor/recorder";
+import { LOG_LIMIT, redact } from "./monitor/runtime";
 
 const APP_DIR = "zzhub-pipeline";
 const LOG_DIR_NAME = "logs";
@@ -158,7 +159,20 @@ export async function runWithCommandLog(
     const startedAt = new Date();
     const startedMs = Date.now();
     const logPath = getDailyLogPath(startedAt);
-    const write = (message: string) => {
+    let logBytes = 0;
+    let truncated = false;
+    const write = (message: string, terminal = false) => {
+      if (process.env.ZZHUB_PIPELINE_WORKER_BUDGET === "1") {
+        if (truncated && !terminal) return;
+        const oversized = message.length > 4096;
+        message = redact(message);
+        if (oversized) message = `${message} [truncated]`;
+        if (logBytes + Buffer.byteLength(message) > LOG_LIMIT && !terminal) {
+          truncated = true;
+          message = "[truncated] Worker diagnostic log limit reached; command outcome remains available.";
+        }
+        logBytes += Buffer.byteLength(message) + 30;
+      }
       appendLogLine(`[${new Date().toISOString()}] ${message}`, startedAt);
     };
 
@@ -177,12 +191,14 @@ export async function runWithCommandLog(
         outcome = { status: "failed", errors: [{ code: "PROCESS_EXIT", message: `进程退出：${process.exitCode}` }] };
       }
       const label = outcomeExitCode(outcome) ? "FAIL" : outcome.status === "success" ? "OK" : outcome.status.toUpperCase();
-      write(`=== ${label} command=${command} duration_ms=${Date.now() - startedMs} ===`);
-      for (const error of outcome.errors ?? []) write(`${error.code}: ${error.message}${error.cause ? `\n${error.cause}` : ""}`);
+      write(`=== ${label} command=${command} duration_ms=${Date.now() - startedMs} ===`, true);
+      const errors = process.env.ZZHUB_PIPELINE_WORKER_BUDGET === "1" ? outcome.errors?.slice(0, 3) : outcome.errors;
+      for (const error of errors ?? []) write(`${error.code}: ${error.message}${error.cause ? `\n${error.cause}` : ""}`, true);
+      if (errors && outcome.errors!.length > errors.length) write("[truncated] Additional errors remain in the command result.", true);
       return outcome;
     } catch (err) {
-      write(`=== FAIL command=${command} duration_ms=${Date.now() - startedMs} ===`);
-      write(formatError(err));
+      write(`=== FAIL command=${command} duration_ms=${Date.now() - startedMs} ===`, true);
+      write(formatError(err), true);
       throw err;
     } finally {
       restoreConsole();

package/src/monitor/recorder.ts

@@ -19,6 +19,8 @@ interface Recording {
   progressAccount?: string;
   ended?: boolean;
   tasks: Set<string>;
+  eventBytes: number;
+  eventTruncated: boolean;
 }
 
 /** 调用上下文自动穿过内置异步流程，不要求业务函数传监控对象。 */
@@ -29,8 +31,25 @@ function emit(type: MonitorEvent["type"], data: Record<string, unknown>): void {
   const current = recording.getStore();
   if (!current || current.disabled || current.ended) return;
   try {
-    const event: MonitorEvent = { version: 1, execution_id: current.id, seq: ++current.seq, at: new Date().toISOString(), type, data };
-    appendFileSync(current.file, `${JSON.stringify(event)}\n`, { encoding: "utf8", mode: 0o600 });
+    let event: MonitorEvent = { version: 1, execution_id: current.id, seq: ++current.seq, at: new Date().toISOString(), type, data };
+    if (process.env.ZZHUB_PIPELINE_WORKER_BUDGET === "1") {
+      // 仅截断观测内容，终态与调用方原异常、退出码和业务 receipt 不受影响。
+      event = JSON.parse(JSON.stringify(event, (_key, value) => typeof value === "string" && value.length > 4096 ? `${redact(value.slice(0, 4096))} [truncated]` : value)) as MonitorEvent;
+      if (Buffer.byteLength(JSON.stringify(event)) > 64 * 1024) {
+        const outcome = event.data.outcome as CommandOutcome | undefined;
+        event.data = type === "finished" ? { exit_code: event.data.exit_code, outcome: { status: outcome?.status, errors: outcome?.errors?.slice(0, 3).map(error => ({ code: error.code, message: error.message })) }, truncated: true }
+          : { level: "warn", message: "Worker 观测事件超过单条上限，内容已截断。", truncated: true };
+        if (type !== "finished") event.type = "log";
+      }
+      if (current.eventTruncated && type !== "finished") return;
+      if (current.eventBytes + Buffer.byteLength(JSON.stringify(event)) > LOG_LIMIT && type !== "finished") {
+        current.eventTruncated = true;
+        event.type = "log"; event.data = { level: "warn", message: "Worker 观测事件达到上限，后续事件截断；业务错误和终态保留。", truncated: true };
+      }
+    }
+    const line = `${JSON.stringify(event)}\n`;
+    appendFileSync(current.file, line, { encoding: "utf8", mode: 0o600 });
+    current.eventBytes += Buffer.byteLength(line);
   } catch {
     // 只停用本次记录，不重试、不递归写日志，也不影响业务结果。
     current.disabled = true;
@@ -115,7 +134,7 @@ export async function observeCommand(command: string, handler: () => Promise<Com
     const dir = join(monitorDir(), "events");
     mkdirSync(dir, { recursive: true, mode: 0o700 });
     const id = crypto.randomUUID();
-    context = { id, file: join(dir, `${id}.jsonl`), seq: 0, disabled: false, logBytes: 0, truncated: false, lastProgressAt: 0, progressKey: "", tasks: new Set() };
+    context = { id, file: join(dir, `${id}.jsonl`), seq: 0, disabled: false, logBytes: 0, truncated: false, lastProgressAt: 0, progressKey: "", tasks: new Set(), eventBytes: 0, eventTruncated: false };
   } catch { return handler(); }
 
   return recording.run(context, async () => {

package/src/monitor.test.ts

@@ -28,7 +28,7 @@ beforeEach(async () => {
 afterEach(async () => {
   stop?.();
   stop = undefined;
-  for (const key of ["ZZHUB_PIPELINE_MONITOR_DIR", "ZZHUB_PIPELINE_LOG_DIR", "ZZHUB_PIPELINE_CONFIG", "ZZHUB_PIPELINE_MONITOR", "NO_COLOR"]) {
+  for (const key of ["ZZHUB_PIPELINE_MONITOR_DIR", "ZZHUB_PIPELINE_LOG_DIR", "ZZHUB_PIPELINE_CONFIG", "ZZHUB_PIPELINE_MONITOR", "ZZHUB_PIPELINE_WORKER_BUDGET", "NO_COLOR"]) {
     if (previousEnv[key] === undefined) delete process.env[key];
     else process.env[key] = previousEnv[key];
   }
@@ -178,6 +178,22 @@ test("incomplete final records resume on the next read and state aliases keep on
   expect(store.snapshot().tasks[0]?.state_path).toContain("posts/test/workflow-state.json");
 });
 
+test("trusted Worker caps individual and cumulative events while returning the original failure", async () => {
+  process.env.ZZHUB_PIPELINE_WORKER_BUDGET = "1";
+  const message = "字".repeat(5000);
+  const outcome = await runWithCommandLog("render", [], async () => {
+    for (let index = 0; index < 1100; index++) reportProgress({ stage: `stage-${index}`, message });
+    return { status: "failed", errors: [{ code: "RENDER_FAILED", message: "原始最终错误" }] };
+  });
+  expect(outcome.errors?.[0]?.message).toBe("原始最终错误");
+  const directory = join(process.env.ZZHUB_PIPELINE_MONITOR_DIR!, "events");
+  const raw = await readFile(join(directory, (await readdir(directory))[0]!), "utf8");
+  const events = raw.trim().split("\n").map(line => JSON.parse(line));
+  expect(raw).toContain("truncated");
+  expect(Buffer.byteLength(raw)).toBeLessThan(LOG_LIMIT + 64 * 1024);
+  expect(events.at(-1)).toMatchObject({ type: "finished", data: { exit_code: 1, outcome: { errors: [{ message: "原始最终错误" }] } } });
+});
+
 test("HTTP/SSE support auth, shared subscriptions, cursor replay, resync and server restart", async () => {
   const server = await serveMonitor();
   expect(server).not.toBeNull();

package/src/wechat-preview/index.ts

@@ -548,6 +548,9 @@ export async function exportMarkdownToWechatHtml(
     }
 
     const html = result.html;
+    if (process.env.ZZHUB_PIPELINE_WORKER_BUDGET === "1" && Buffer.byteLength(html, "utf8") > 50 * 1024 * 1024) {
+      throw makeError("render_error", "Worker HTML export exceeds 50 MiB");
+    }
     const semanticHtml =
       "semanticHtml" in result && typeof result.semanticHtml === "string"
         ? result.semanticHtml