backtrace-console 0.0.3 → 0.0.5

package/lib/backtrace/options.js

@@ -0,0 +1,278 @@
+const path = require("node:path");
+const {
+  DEFAULT_QUERY_URL,
+  DEFAULT_BACKTRACE_PROJECT,
+  DEFAULT_BACKTRACE_FORMAT,
+  DEFAULT_WORKDIR,
+  DEFAULT_PROXY,
+  DEFAULT_SELECT,
+  DEFAULT_LIMIT,
+  DEFAULT_DOWNLOAD_CONCURRENCY,
+  DEFAULT_RETRIES,
+  DEFAULT_STORAGE_DIR,
+} = require("./constants");
+const { nowRange, formatBatchName } = require("./utils");
+
+function normalizeBacktraceQueryUrl(value) {
+  const rawValue = String(value || "").trim();
+  const fallbackUrl = new URL(DEFAULT_QUERY_URL);
+  const url = rawValue ? new URL(rawValue) : new URL(DEFAULT_QUERY_URL);
+  if (!url.pathname || url.pathname === "/") {
+    url.pathname = "/api/query";
+  }
+  if (!url.searchParams.get("project")) {
+    url.searchParams.set("project", process.env.BACKTRACE_PROJECT || DEFAULT_BACKTRACE_PROJECT);
+  }
+  if (!url.searchParams.get("format")) {
+    url.searchParams.set("format", process.env.BACKTRACE_FORMAT || DEFAULT_BACKTRACE_FORMAT);
+  }
+  if (!url.searchParams.get("token") && fallbackUrl.searchParams.get("token")) {
+    url.searchParams.set("token", fallbackUrl.searchParams.get("token"));
+  }
+  return url.toString();
+}
+
+function normalizeOptionalInteger(value, fallback) {
+  if (value === undefined || value === null || value === "") {
+    return fallback;
+  }
+  return Number(value);
+}
+
+// 顶层工具只暴露固定命令集合，便于尽早做参数校验。
+const SUPPORTED_COMMANDS = ["list", "fingerprint", "collect-all", "summarize-fingerprint-errors"];
+
+// 输出旧入口 codex_demo.js 和库调用共用的帮助信息。
+function printUsage() {
+  console.log(
+    [
+      "Usage:",
+      "  backtrace <command> [options]",
+      "",
+      "Commands:",
+      "  list                          list local fingerprint directories and meta information",
+      "  fingerprint                   query all grouped fingerprints and print them",
+      "  collect-all                   download all objects only, do not generate reports",
+      "  summarize-fingerprint-errors  summarize and deduplicate error lines for one local fingerprint directory",
+      "  fix-plan                      generate a repair plan from the smallest AboveLand.LOG under one fingerprint",
+      "",
+      "Environment:",
+      "  BACKTRACE_USERNAME     required Backtrace console login username",
+      "  BACKTRACE_PASSWORD     required Backtrace console login password",
+      `  BACKTRACE_PROJECT      default: ${DEFAULT_BACKTRACE_PROJECT}`,
+      `  BACKTRACE_FORMAT       default: ${DEFAULT_BACKTRACE_FORMAT}`,
+      "",
+      "Options:",
+      `  --query-url <url>      default base url: ${DEFAULT_QUERY_URL}`,
+      `  --workdir <dir>        default: ${DEFAULT_WORKDIR}`,
+      "  --fingerprint <value[,value2,...]>  optional query filter; collect supports multiple values",
+      "  --from <unixTs>        default: 1",
+      "  --to <unixTs>          default: now",
+      `  --limit <n>            default: ${DEFAULT_LIMIT}`,
+      "  --offset <n>           default: 0",
+      `  --select <a,b,c>       default: ${DEFAULT_SELECT.join(",")}`,
+      `  --proxy <url>          default: ${DEFAULT_PROXY}`,
+      `  --retries <n>          default: ${DEFAULT_RETRIES} (0 means infinite retry)`,
+      `  --download-concurrency <n> default: ${DEFAULT_DOWNLOAD_CONCURRENCY}`,
+      "  --storage-dir <dir>    default: ./fingerprints",
+      "  --report-dir <dir>     deprecated alias of --storage-dir",
+      "  --logs-dir <dir>       deprecated alias of --storage-dir",
+      "  --help                 show this message",
+    ].join("\n"),
+  );
+}
+
+// 对标准化选项做一次集中校验，
+// 后续流程就可以默认命令前置条件和数值范围都已成立。
+function validateOptions(options) {
+  if (!SUPPORTED_COMMANDS.includes(options.command)) {
+    throw new Error(`Command must be one of: ${SUPPORTED_COMMANDS.join(", ")}`);
+  }
+  if (options.command === "summarize-fingerprint-errors" && !options.fingerprint) {
+    throw new Error("--fingerprint is required for summarize-fingerprint-errors");
+  }
+  if (!["list", "summarize-fingerprint-errors"].includes(options.command) && (!options.username || !options.password)) {
+    throw new Error("BACKTRACE_USERNAME and BACKTRACE_PASSWORD are required");
+  }
+  if (!Number.isInteger(options.limit) || options.limit <= 0) {
+    throw new Error("--limit must be a positive integer");
+  }
+  if (!Number.isInteger(options.offset) || options.offset < 0) {
+    throw new Error("--offset must be a non-negative integer");
+  }
+  if (!Number.isInteger(options.retries) || options.retries < 0) {
+    throw new Error("--retries must be a non-negative integer; use 0 for infinite retry");
+  }
+  if (!Number.isInteger(options.downloadConcurrency) || options.downloadConcurrency <= 0) {
+    throw new Error("--download-concurrency must be a positive integer");
+  }
+  if (options.select.length === 0) {
+    throw new Error("--select must contain at least one field");
+  }
+  return options;
+}
+
+// 把旧版 Node 入口传入的 argv 解析为已校验的运行时选项。
+function parseArgs(argv) {
+  const range = nowRange();
+  const batchName = formatBatchName(new Date());
+  // 这里先填满默认值，后续只覆盖用户显式传入的字段。
+  const options = {
+    command: "",
+    queryUrl: normalizeBacktraceQueryUrl(DEFAULT_QUERY_URL),
+    workdir: DEFAULT_WORKDIR,
+    objectId: "",
+    fingerprint: "",
+    from: range.from,
+    to: range.to,
+    limit: DEFAULT_LIMIT,
+    offset: 0,
+    select: [...DEFAULT_SELECT],
+    proxy: DEFAULT_PROXY,
+    retries: DEFAULT_RETRIES,
+    downloadConcurrency: DEFAULT_DOWNLOAD_CONCURRENCY,
+    batchName,
+    storageDir: path.resolve(DEFAULT_STORAGE_DIR),
+    reportDir: path.resolve(DEFAULT_STORAGE_DIR),
+    logsDir: path.resolve(DEFAULT_STORAGE_DIR),
+    username: process.env.BACKTRACE_USERNAME || "",
+    password: process.env.BACKTRACE_PASSWORD || "",
+  };
+
+  let index = 0;
+  if (argv[0] && !argv[0].startsWith("-")) {
+    // 第一个非 flag 参数视为主命令。
+    options.command = argv[0];
+    index = 1;
+  }
+
+  // 按顺序解析参数，因为这里采用的是简单的成对参数约定。
+  for (; index < argv.length; index += 1) {
+    const arg = argv[index];
+
+    if (arg === "--query-url") {
+      options.queryUrl = argv[index + 1] || "";
+      index += 1;
+      continue;
+    }
+    if (arg === "--workdir") {
+      options.workdir = argv[index + 1] || "";
+      index += 1;
+      continue;
+    }
+    if (arg === "--object-id") {
+      options.objectId = argv[index + 1] || "";
+      index += 1;
+      continue;
+    }
+    if (arg === "--fingerprint") {
+      options.fingerprint = argv[index + 1] || "";
+      index += 1;
+      continue;
+    }
+    if (arg === "--from") {
+      options.from = argv[index + 1] || options.from;
+      index += 1;
+      continue;
+    }
+    if (arg === "--to") {
+      options.to = argv[index + 1] || options.to;
+      index += 1;
+      continue;
+    }
+    if (arg === "--limit") {
+      options.limit = Number(argv[index + 1] || String(DEFAULT_LIMIT));
+      index += 1;
+      continue;
+    }
+    if (arg === "--offset") {
+      options.offset = Number(argv[index + 1] || "0");
+      index += 1;
+      continue;
+    }
+    if (arg === "--select") {
+      options.select = (argv[index + 1] || DEFAULT_SELECT.join(","))
+        .split(",")
+        .map((item) => item.trim())
+        .filter(Boolean);
+      index += 1;
+      continue;
+    }
+    if (arg === "--proxy") {
+      options.proxy = argv[index + 1] || "";
+      index += 1;
+      continue;
+    }
+    if (arg === "--retries") {
+      options.retries = Number(argv[index + 1] || String(DEFAULT_RETRIES));
+      index += 1;
+      continue;
+    }
+    if (arg === "--download-concurrency") {
+      options.downloadConcurrency = Number(argv[index + 1] || String(DEFAULT_DOWNLOAD_CONCURRENCY));
+      index += 1;
+      continue;
+    }
+    if (arg === "--storage-dir" || arg === "--report-dir" || arg === "--logs-dir") {
+      options.storageDir = path.resolve(argv[index + 1] || DEFAULT_STORAGE_DIR);
+      options.reportDir = options.storageDir;
+      options.logsDir = options.storageDir;
+      index += 1;
+      continue;
+    }
+    if (arg === "--help" || arg === "-h") {
+      printUsage();
+      process.exit(0);
+    }
+  }
+
+  options.queryUrl = normalizeBacktraceQueryUrl(options.queryUrl || DEFAULT_QUERY_URL);
+  options.proxy = String(options.proxy || DEFAULT_PROXY || "").trim();
+
+  return validateOptions(options);
+}
+
+// 给直接调用库的场景构造选项，跳过原始 argv 解析。
+function createOptions(overrides = {}) {
+  const range = nowRange();
+  const batchName = overrides.batchName || formatBatchName(new Date());
+  // 支持调用方直接传数组，也支持传逗号分隔字符串。
+  const selectValue = Array.isArray(overrides.select)
+    ? overrides.select
+    : String(overrides.select || DEFAULT_SELECT.join(","))
+      .split(",")
+      .map((item) => item.trim())
+      .filter(Boolean);
+
+  const options = {
+    command: overrides.command || "",
+    queryUrl: normalizeBacktraceQueryUrl(overrides.queryUrl || DEFAULT_QUERY_URL),
+    workdir: overrides.workdir || DEFAULT_WORKDIR,
+    objectId: overrides.objectId ? String(overrides.objectId) : "",
+    fingerprint: overrides.fingerprint ? String(overrides.fingerprint) : "",
+    from: String(overrides.from || range.from),
+    to: String(overrides.to || range.to),
+    limit: normalizeOptionalInteger(overrides.limit, DEFAULT_LIMIT),
+    offset: normalizeOptionalInteger(overrides.offset, 0),
+    select: selectValue.length > 0 ? selectValue : [...DEFAULT_SELECT],
+    proxy: String(overrides.proxy || DEFAULT_PROXY || "").trim(),
+    retries: normalizeOptionalInteger(overrides.retries, DEFAULT_RETRIES),
+    downloadConcurrency: normalizeOptionalInteger(overrides.downloadConcurrency, DEFAULT_DOWNLOAD_CONCURRENCY),
+    batchName,
+    storageDir: path.resolve(overrides.storageDir || overrides.reportDir || overrides.logsDir || DEFAULT_STORAGE_DIR),
+    reportDir: path.resolve(overrides.storageDir || overrides.reportDir || overrides.logsDir || DEFAULT_STORAGE_DIR),
+    logsDir: path.resolve(overrides.storageDir || overrides.reportDir || overrides.logsDir || DEFAULT_STORAGE_DIR),
+    username: overrides.username || process.env.BACKTRACE_USERNAME || "",
+    password: overrides.password || process.env.BACKTRACE_PASSWORD || "",
+  };
+
+  return validateOptions(options);
+}
+
+module.exports = {
+  SUPPORTED_COMMANDS,
+  printUsage,
+  parseArgs,
+  createOptions,
+  validateOptions,
+};

package/lib/backtrace/query-download.js

@@ -0,0 +1,117 @@
+const fs = require("node:fs/promises");
+const fsNative = require("node:fs");
+const path = require("node:path");
+const { logStep } = require("./utils");
+
+function formatBytes(bytes) {
+  const numeric = Number(bytes || 0);
+  if (!Number.isFinite(numeric) || numeric <= 0) return "0 B";
+  const units = ["B", "KB", "MB", "GB"];
+  let value = numeric;
+  let unitIndex = 0;
+  while (value >= 1024 && unitIndex < units.length - 1) {
+    value /= 1024;
+    unitIndex += 1;
+  }
+  return `${value.toFixed(value >= 10 || unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
+}
+
+function formatAttachmentSpeed(bytes, elapsedMs) {
+  const seconds = Math.max(0.001, Number(elapsedMs || 0) / 1000);
+  return `${formatBytes(Number(bytes || 0) / seconds)}/s`;
+}
+
+function formatAttachmentProgress(downloadedBytes, totalBytes) {
+  if (!totalBytes) return formatBytes(downloadedBytes);
+  const percent = Math.min(100, Math.max(0, (downloadedBytes / totalBytes) * 100));
+  return `${formatBytes(downloadedBytes)} / ${formatBytes(totalBytes)} (${percent.toFixed(1)}%)`;
+}
+
+function formatAttachmentStatus(downloadedBytes, totalBytes, elapsedMs) {
+  const progress = formatAttachmentProgress(downloadedBytes, totalBytes);
+  const transferred = `${formatBytes(downloadedBytes)} transferred`;
+  if (!elapsedMs) return `${progress}, ${transferred}`;
+  return [progress, transferred, formatAttachmentSpeed(downloadedBytes, elapsedMs)].join(", ");
+}
+
+async function downloadAttachment(session, queryUrl, objectHexId, targetDir, attachment, buildSiblingUrl) {
+  const savedPath = path.join(targetDir, attachment.name);
+  const existing = await fs.stat(savedPath).catch(() => null);
+  if (existing && existing.isFile()) {
+    logStep("download", "attachment already exists, skipping", savedPath);
+    return { ...attachment, savedPath, skipped: true, sizeBytes: existing.size || 0 };
+  }
+  const source = new URL(queryUrl);
+  const url = buildSiblingUrl(queryUrl, "/api/get");
+  const project = source.searchParams.get("project");
+  if (project) url.searchParams.set("project", project);
+  url.searchParams.set("object", objectHexId);
+  url.searchParams.set("attachment_id", attachment.id);
+  url.searchParams.set("attachment_inline", "false");
+  logStep("download", "downloading attachment", { objectId: objectHexId, attachment: attachment.name });
+  const response = await session.requestStream(url.toString(), {}, `download ${objectHexId}/${attachment.name}`);
+  const totalBytes = Number(response.headers.get("content-length") || attachment.size || 0);
+  const fileStream = fsNative.createWriteStream(savedPath);
+  const startedAt = Date.now();
+  let downloadedBytes = 0;
+  let progressTimer = null;
+  const printProgress = () => {
+    const elapsedMs = Math.max(1, Date.now() - startedAt);
+    logStep("download", `file ${formatAttachmentStatus(downloadedBytes, totalBytes, elapsedMs)}`, {
+      objectId: objectHexId,
+      attachment: attachment.name,
+    });
+  };
+  try {
+    printProgress();
+    progressTimer = setInterval(printProgress, 1000);
+    for await (const chunk of response.body) {
+      downloadedBytes += chunk.length;
+      if (!fileStream.write(chunk)) {
+        await new Promise((resolve) => fileStream.once("drain", resolve));
+      }
+    }
+    await new Promise((resolve, reject) => {
+      fileStream.end((error) => (error ? reject(error) : resolve()));
+    });
+  } catch (error) {
+    if (progressTimer) clearInterval(progressTimer);
+    fileStream.destroy();
+    await fs.unlink(savedPath).catch(() => {});
+    throw error;
+  }
+  if (progressTimer) clearInterval(progressTimer);
+  printProgress();
+  logStep("download", "attachment saved", savedPath);
+  return { ...attachment, savedPath, sizeBytes: downloadedBytes };
+}
+
+async function downloadAttachmentWithRetry(session, queryUrl, objectHexId, targetDir, attachment, buildSiblingUrl, maxAttempts = 5) {
+  let lastError = null;
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      return await downloadAttachment(session, queryUrl, objectHexId, targetDir, attachment, buildSiblingUrl);
+    } catch (error) {
+      lastError = error;
+      logStep("download", "attachment download failed", {
+        objectId: objectHexId,
+        attachment: attachment.name,
+        attempt,
+        maxAttempts,
+        message: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+  logStep("download", "attachment skipped after retries", {
+    objectId: objectHexId,
+    attachment: attachment.name,
+    maxAttempts,
+    message: lastError instanceof Error ? lastError.message : String(lastError || ""),
+  });
+  return { ...attachment, savedPath: "", skipped: true, failed: true };
+}
+
+module.exports = {
+  downloadAttachment: downloadAttachment,
+  downloadAttachmentWithRetry: downloadAttachmentWithRetry,
+};

package/lib/backtrace/query-session.js

@@ -0,0 +1,229 @@
+const { logStep, retry } = require("./utils");
+
+// 进程级 token 缓存，key = loginUrl，有效期 1 小时
+const TOKEN_TTL_MS = 60 * 60 * 1000;
+const tokenCache = new Map(); // loginUrl -> { token, expiresAt }
+
+function getCachedToken(loginUrl) {
+  const entry = tokenCache.get(loginUrl);
+  if (!entry) return null;
+  if (Date.now() >= entry.expiresAt) {
+    tokenCache.delete(loginUrl);
+    return null;
+  }
+  return entry.token;
+}
+
+function setCachedToken(loginUrl, token) {
+  tokenCache.set(loginUrl, { token, expiresAt: Date.now() + TOKEN_TTL_MS });
+}
+
+function clearCachedToken(loginUrl) {
+  tokenCache.delete(loginUrl);
+}
+
+let undiciModule = null;
+
+function getUndici() {
+  if (!undiciModule) {
+    undiciModule = require("undici");
+  }
+  return undiciModule;
+}
+
+function createProxyDispatcher(proxyUrl) {
+  const normalizedProxy = String(proxyUrl || "").trim();
+  if (!normalizedProxy) {
+    return null;
+  }
+  try {
+    const { ProxyAgent } = getUndici();
+    return new ProxyAgent(normalizedProxy);
+  } catch (error) {
+    throw new Error(`Proxy support requires dependency "undici": ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
+function withDispatcher(options, dispatcher) {
+  if (!dispatcher) {
+    return options || {};
+  }
+  return { ...(options || {}), dispatcher };
+}
+
+function buildSiblingUrl(sourceUrl, pathname) {
+  const url = new URL(sourceUrl);
+  url.pathname = pathname;
+  url.search = "";
+  return url;
+}
+
+function buildLoginUrl(queryUrl) {
+  return buildSiblingUrl(queryUrl, "/api/login").toString();
+}
+
+function withSessionToken(urlString, sessionToken) {
+  const url = new URL(urlString);
+  url.searchParams.set("token", sessionToken);
+  return url;
+}
+
+class AuthenticationError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "AuthenticationError";
+  }
+}
+
+class BacktraceSession {
+  constructor(options) {
+    this.username = options.username;
+    this.password = options.password;
+    this.retries = options.retries;
+    this.proxy = String(options.proxy || "").trim();
+    this.dispatcher = createProxyDispatcher(this.proxy);
+    this.token = "";
+    this.loginUrl = buildLoginUrl(options.queryUrl);
+  }
+
+  async login() {
+    const cached = getCachedToken(this.loginUrl);
+    if (cached) {
+      this.token = cached;
+      logStep("auth", "using cached token");
+      return this.token;
+    }
+    const form = new URLSearchParams();
+    form.set("username", this.username);
+    form.set("password", this.password);
+    logStep("auth", "logging in", { loginUrl: this.loginUrl, username: this.username });
+    const payload = await retry("backtrace login", this.retries, async () => {
+      const response = await fetch(this.loginUrl, {
+        method: "POST",
+        headers: { "content-type": "application/x-www-form-urlencoded" },
+        body: form.toString(),
+        ...(this.dispatcher ? { dispatcher: this.dispatcher } : {}),
+      });
+      const text = await response.text();
+      let data = null;
+      try {
+        data = text ? JSON.parse(text) : null;
+      } catch (error) {
+        data = null;
+      }
+      if (!response.ok) {
+        throw new Error(`Login failed: ${response.status} ${response.statusText}`);
+      }
+      if (!data || !data.token) {
+        throw new Error("Login response did not contain a token");
+      }
+      return data;
+    });
+    this.token = payload.token;
+    setCachedToken(this.loginUrl, this.token);
+    logStep("auth", "login succeeded");
+    return this.token;
+  }
+
+  async getToken() {
+    if (!this.token) {
+      await this.login();
+    }
+    return this.token;
+  }
+
+  async refreshToken() {
+    logStep("auth", "refreshing session token");
+    clearCachedToken(this.loginUrl);
+    this.token = "";
+    return this.login();
+  }
+
+  buildHeaders(extraHeaders, token) {
+    return { ...(extraHeaders || {}), Cookie: `token=${token}` };
+  }
+
+  parseResponseBody(text) {
+    try {
+      return text ? JSON.parse(text) : null;
+    } catch (error) {
+      return text;
+    }
+  }
+
+  isAuthFailure(response, parsedBody) {
+    const normalizedBody = typeof parsedBody === "string" ? parsedBody.toLowerCase() : JSON.stringify(parsedBody || {}).toLowerCase();
+    return response.status === 401
+      || response.status === 403
+      || normalizedBody.includes("invalid token")
+      || normalizedBody.includes("authentication")
+      || normalizedBody.includes("not authorized");
+  }
+
+  async request(urlString, options, label, mode) {
+    return retry(label, this.retries, async () => {
+      let token = await this.getToken();
+      let authAttempt = 0;
+      while (authAttempt < 2) {
+        const url = withSessionToken(urlString, token);
+        const response = await fetch(url.toString(), withDispatcher({ ...options, headers: this.buildHeaders(options?.headers, token) }, this.dispatcher));
+        if (mode === "buffer" && response.ok) {
+          return Buffer.from(await response.arrayBuffer());
+        }
+        const text = await response.text();
+        const parsed = this.parseResponseBody(text);
+        if (response.ok) {
+          return parsed;
+        }
+        if (this.isAuthFailure(response, parsed) && authAttempt === 0) {
+          await this.refreshToken();
+          token = await this.getToken();
+          authAttempt += 1;
+          continue;
+        }
+        throw new Error(`Request failed: ${response.status} ${response.statusText}`);
+      }
+      throw new AuthenticationError("Authentication failed after token refresh");
+    });
+  }
+
+  async requestJson(urlString, options, label) {
+    return this.request(urlString, options, label, "json");
+  }
+
+  async requestBuffer(urlString, options, label) {
+    return this.request(urlString, options, label, "buffer");
+  }
+
+  async requestStream(urlString, options, label) {
+    return retry(label, this.retries, async () => {
+      let token = await this.getToken();
+      let authAttempt = 0;
+      while (authAttempt < 2) {
+        const url = withSessionToken(urlString, token);
+        const response = await fetch(url.toString(), withDispatcher({ ...options, headers: this.buildHeaders(options?.headers, token) }, this.dispatcher));
+        if (response.ok) {
+          return response;
+        }
+        const text = await response.text();
+        const parsed = this.parseResponseBody(text);
+        if (this.isAuthFailure(response, parsed) && authAttempt === 0) {
+          await this.refreshToken();
+          token = await this.getToken();
+          authAttempt += 1;
+          continue;
+        }
+        throw new Error(`Request failed: ${response.status} ${response.statusText}`);
+      }
+      throw new AuthenticationError("Authentication failed after token refresh");
+    });
+  }
+}
+
+module.exports = {
+  AuthenticationError,
+  BacktraceSession,
+  buildSiblingUrl,
+  buildLoginUrl,
+  withSessionToken,
+};