backtrace-console 0.0.1 → 0.0.2

package/app.js

@@ -10,13 +10,14 @@ var backtraceRouter = require('./routes/backtrace');
 var app = express();
 
 app.use(logger('dev'));
-app.use(express.json());
-app.use(express.urlencoded({ extended: false }));
+app.use(express.json({ limit: '50mb' }));
+app.use(express.urlencoded({ extended: false, limit: '50mb' }));
 app.use(cookieParser());
 app.use(express.static(path.join(__dirname, 'public')));
 
 app.use('/', indexRouter);
 app.use('/users', usersRouter);
+
 app.use('/api/backtrace', backtraceRouter);
 
 module.exports = app;

package/bin/backtrace-server.js

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+
+'use strict';
+
+require('dotenv').config();
+
+var path = require('node:path');
+var fs = require('node:fs');
+var { spawn } = require('node:child_process');
+
+var ROOT = path.resolve(__dirname, '..');
+var PID_FILE = path.join(ROOT, '.backtrace-server.pid');
+var LOG_FILE = path.join(ROOT, '.backtrace-server.log');
+var WWW = path.join(__dirname, 'www');
+
+var cmd = process.argv[2];
+
+if (cmd === 'run') {
+  // 前台运行，直接 require www（同进程）
+  require('./www');
+  return;
+}
+
+if (cmd === 'start') {
+  // 检查是否已在运行
+  if (fs.existsSync(PID_FILE)) {
+    var existingPid = parseInt(fs.readFileSync(PID_FILE, 'utf8').trim(), 10);
+    if (existingPid && isRunning(existingPid)) {
+      console.log('[backtrace] server already running, pid=' + existingPid);
+      process.exit(0);
+    }
+    fs.unlinkSync(PID_FILE);
+  }
+
+  var logFd = fs.openSync(LOG_FILE, 'a');
+  var child = spawn(process.execPath, [WWW], {
+    detached: true,
+    stdio: ['ignore', logFd, logFd],
+    env: process.env,
+    cwd: ROOT,
+  });
+  child.unref();
+  fs.writeFileSync(PID_FILE, String(child.pid), 'utf8');
+  console.log('[backtrace] server started, pid=' + child.pid + ', log=' + LOG_FILE);
+  process.exit(0);
+}
+
+if (cmd === 'stop') {
+  if (!fs.existsSync(PID_FILE)) {
+    console.log('[backtrace] server is not running (no pid file)');
+    process.exit(0);
+  }
+  var pid = parseInt(fs.readFileSync(PID_FILE, 'utf8').trim(), 10);
+  if (!pid || !isRunning(pid)) {
+    console.log('[backtrace] server is not running (pid=' + pid + ')');
+    fs.unlinkSync(PID_FILE);
+    process.exit(0);
+  }
+  try {
+    process.kill(pid, 'SIGTERM');
+    fs.unlinkSync(PID_FILE);
+    console.log('[backtrace] server stopped, pid=' + pid);
+  } catch (err) {
+    console.error('[backtrace] failed to stop server:', err.message);
+    process.exit(1);
+  }
+  process.exit(0);
+}
+
+if (cmd === 'status') {
+  if (!fs.existsSync(PID_FILE)) {
+    console.log('[backtrace] server is not running');
+    process.exit(0);
+  }
+  var statusPid = parseInt(fs.readFileSync(PID_FILE, 'utf8').trim(), 10);
+  if (statusPid && isRunning(statusPid)) {
+    console.log('[backtrace] server is running, pid=' + statusPid);
+  } else {
+    console.log('[backtrace] server is not running (stale pid=' + statusPid + ')');
+    fs.unlinkSync(PID_FILE);
+  }
+  process.exit(0);
+}
+
+console.log([
+  'Usage:',
+  '  backtrace-server run     前台启动服务器',
+  '  backtrace-server start   后台守护进程启动',
+  '  backtrace-server stop    停止后台进程',
+  '  backtrace-server status  查看运行状态',
+].join('\n'));
+process.exit(1);
+
+function isRunning(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (_) {
+    return false;
+  }
+}

package/bin/www

@@ -3,6 +3,7 @@
 /**
  * Module dependencies.
  */
+require("dotenv").config();
 
 var app = require('../app');
 var debug = require('debug')('backtrace-console:server');
@@ -29,6 +30,8 @@ server.listen(port);
 server.on('error', onError);
 server.on('listening', onListening);
 
+require('../lib/scheduler').start();
+
 /**
  * Normalize a port into a number, string, or false.
  */

package/lib/backtrace/constants.js

@@ -8,6 +8,8 @@ const DEFAULT_LIMIT = Number(process.env.BACKTRACE_LIMIT || 20);
 const DEFAULT_DOWNLOAD_CONCURRENCY = Number(process.env.BACKTRACE_DOWNLOAD_CONCURRENCY || 4);
 const DEFAULT_RETRIES = Number(process.env.BACKTRACE_RETRIES || 0);
 const DEFAULT_STORAGE_DIR = process.env.BACKTRACE_STORAGE_DIR || "fingerprints";
+const FEISHU_WEBHOOK_URL = process.env.FEISHU_WEBHOOK_URL || "";
+const SCHEDULER_CRON = process.env.SCHEDULER_CRON || "*/5 * * * *";
 
 module.exports = {
   DEFAULT_QUERY_URL,
@@ -20,4 +22,6 @@ module.exports = {
   DEFAULT_DOWNLOAD_CONCURRENCY,
   DEFAULT_RETRIES,
   DEFAULT_STORAGE_DIR,
+  FEISHU_WEBHOOK_URL,
+  SCHEDULER_CRON,
 };

package/lib/backtrace/query-download.js

@@ -0,0 +1,117 @@
+const fs = require("node:fs/promises");
+const fsNative = require("node:fs");
+const path = require("node:path");
+const { logStep } = require("./utils");
+
+function formatBytes(bytes) {
+  const numeric = Number(bytes || 0);
+  if (!Number.isFinite(numeric) || numeric <= 0) return "0 B";
+  const units = ["B", "KB", "MB", "GB"];
+  let value = numeric;
+  let unitIndex = 0;
+  while (value >= 1024 && unitIndex < units.length - 1) {
+    value /= 1024;
+    unitIndex += 1;
+  }
+  return `${value.toFixed(value >= 10 || unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
+}
+
+function formatAttachmentSpeed(bytes, elapsedMs) {
+  const seconds = Math.max(0.001, Number(elapsedMs || 0) / 1000);
+  return `${formatBytes(Number(bytes || 0) / seconds)}/s`;
+}
+
+function formatAttachmentProgress(downloadedBytes, totalBytes) {
+  if (!totalBytes) return formatBytes(downloadedBytes);
+  const percent = Math.min(100, Math.max(0, (downloadedBytes / totalBytes) * 100));
+  return `${formatBytes(downloadedBytes)} / ${formatBytes(totalBytes)} (${percent.toFixed(1)}%)`;
+}
+
+function formatAttachmentStatus(downloadedBytes, totalBytes, elapsedMs) {
+  const progress = formatAttachmentProgress(downloadedBytes, totalBytes);
+  const transferred = `${formatBytes(downloadedBytes)} transferred`;
+  if (!elapsedMs) return `${progress}, ${transferred}`;
+  return [progress, transferred, formatAttachmentSpeed(downloadedBytes, elapsedMs)].join(", ");
+}
+
+async function downloadAttachment(session, queryUrl, objectHexId, targetDir, attachment, buildSiblingUrl) {
+  const savedPath = path.join(targetDir, attachment.name);
+  const existing = await fs.stat(savedPath).catch(() => null);
+  if (existing && existing.isFile()) {
+    logStep("download", "attachment already exists, skipping", savedPath);
+    return { ...attachment, savedPath, skipped: true, sizeBytes: existing.size || 0 };
+  }
+  const source = new URL(queryUrl);
+  const url = buildSiblingUrl(queryUrl, "/api/get");
+  const project = source.searchParams.get("project");
+  if (project) url.searchParams.set("project", project);
+  url.searchParams.set("object", objectHexId);
+  url.searchParams.set("attachment_id", attachment.id);
+  url.searchParams.set("attachment_inline", "false");
+  logStep("download", "downloading attachment", { objectId: objectHexId, attachment: attachment.name });
+  const response = await session.requestStream(url.toString(), {}, `download ${objectHexId}/${attachment.name}`);
+  const totalBytes = Number(response.headers.get("content-length") || attachment.size || 0);
+  const fileStream = fsNative.createWriteStream(savedPath);
+  const startedAt = Date.now();
+  let downloadedBytes = 0;
+  let progressTimer = null;
+  const printProgress = () => {
+    const elapsedMs = Math.max(1, Date.now() - startedAt);
+    logStep("download", `file ${formatAttachmentStatus(downloadedBytes, totalBytes, elapsedMs)}`, {
+      objectId: objectHexId,
+      attachment: attachment.name,
+    });
+  };
+  try {
+    printProgress();
+    progressTimer = setInterval(printProgress, 1000);
+    for await (const chunk of response.body) {
+      downloadedBytes += chunk.length;
+      if (!fileStream.write(chunk)) {
+        await new Promise((resolve) => fileStream.once("drain", resolve));
+      }
+    }
+    await new Promise((resolve, reject) => {
+      fileStream.end((error) => (error ? reject(error) : resolve()));
+    });
+  } catch (error) {
+    if (progressTimer) clearInterval(progressTimer);
+    fileStream.destroy();
+    await fs.unlink(savedPath).catch(() => {});
+    throw error;
+  }
+  if (progressTimer) clearInterval(progressTimer);
+  printProgress();
+  logStep("download", "attachment saved", savedPath);
+  return { ...attachment, savedPath, sizeBytes: downloadedBytes };
+}
+
+async function downloadAttachmentWithRetry(session, queryUrl, objectHexId, targetDir, attachment, buildSiblingUrl, maxAttempts = 5) {
+  let lastError = null;
+  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+    try {
+      return await downloadAttachment(session, queryUrl, objectHexId, targetDir, attachment, buildSiblingUrl);
+    } catch (error) {
+      lastError = error;
+      logStep("download", "attachment download failed", {
+        objectId: objectHexId,
+        attachment: attachment.name,
+        attempt,
+        maxAttempts,
+        message: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+  logStep("download", "attachment skipped after retries", {
+    objectId: objectHexId,
+    attachment: attachment.name,
+    maxAttempts,
+    message: lastError instanceof Error ? lastError.message : String(lastError || ""),
+  });
+  return { ...attachment, savedPath: "", skipped: true, failed: true };
+}
+
+module.exports = {
+  downloadAttachment: downloadAttachment,
+  downloadAttachmentWithRetry: downloadAttachmentWithRetry,
+};

package/lib/backtrace/query-session.js

@@ -0,0 +1,229 @@
+const { logStep, retry } = require("./utils");
+
+// 进程级 token 缓存，key = loginUrl，有效期 1 小时
+const TOKEN_TTL_MS = 60 * 60 * 1000;
+const tokenCache = new Map(); // loginUrl -> { token, expiresAt }
+
+function getCachedToken(loginUrl) {
+  const entry = tokenCache.get(loginUrl);
+  if (!entry) return null;
+  if (Date.now() >= entry.expiresAt) {
+    tokenCache.delete(loginUrl);
+    return null;
+  }
+  return entry.token;
+}
+
+function setCachedToken(loginUrl, token) {
+  tokenCache.set(loginUrl, { token, expiresAt: Date.now() + TOKEN_TTL_MS });
+}
+
+function clearCachedToken(loginUrl) {
+  tokenCache.delete(loginUrl);
+}
+
+let undiciModule = null;
+
+function getUndici() {
+  if (!undiciModule) {
+    undiciModule = require("undici");
+  }
+  return undiciModule;
+}
+
+function createProxyDispatcher(proxyUrl) {
+  const normalizedProxy = String(proxyUrl || "").trim();
+  if (!normalizedProxy) {
+    return null;
+  }
+  try {
+    const { ProxyAgent } = getUndici();
+    return new ProxyAgent(normalizedProxy);
+  } catch (error) {
+    throw new Error(`Proxy support requires dependency "undici": ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
+function withDispatcher(options, dispatcher) {
+  if (!dispatcher) {
+    return options || {};
+  }
+  return { ...(options || {}), dispatcher };
+}
+
+function buildSiblingUrl(sourceUrl, pathname) {
+  const url = new URL(sourceUrl);
+  url.pathname = pathname;
+  url.search = "";
+  return url;
+}
+
+function buildLoginUrl(queryUrl) {
+  return buildSiblingUrl(queryUrl, "/api/login").toString();
+}
+
+function withSessionToken(urlString, sessionToken) {
+  const url = new URL(urlString);
+  url.searchParams.set("token", sessionToken);
+  return url;
+}
+
+class AuthenticationError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "AuthenticationError";
+  }
+}
+
+class BacktraceSession {
+  constructor(options) {
+    this.username = options.username;
+    this.password = options.password;
+    this.retries = options.retries;
+    this.proxy = String(options.proxy || "").trim();
+    this.dispatcher = createProxyDispatcher(this.proxy);
+    this.token = "";
+    this.loginUrl = buildLoginUrl(options.queryUrl);
+  }
+
+  async login() {
+    const cached = getCachedToken(this.loginUrl);
+    if (cached) {
+      this.token = cached;
+      logStep("auth", "using cached token");
+      return this.token;
+    }
+    const form = new URLSearchParams();
+    form.set("username", this.username);
+    form.set("password", this.password);
+    logStep("auth", "logging in", { loginUrl: this.loginUrl, username: this.username });
+    const payload = await retry("backtrace login", this.retries, async () => {
+      const response = await fetch(this.loginUrl, {
+        method: "POST",
+        headers: { "content-type": "application/x-www-form-urlencoded" },
+        body: form.toString(),
+        ...(this.dispatcher ? { dispatcher: this.dispatcher } : {}),
+      });
+      const text = await response.text();
+      let data = null;
+      try {
+        data = text ? JSON.parse(text) : null;
+      } catch (error) {
+        data = null;
+      }
+      if (!response.ok) {
+        throw new Error(`Login failed: ${response.status} ${response.statusText}`);
+      }
+      if (!data || !data.token) {
+        throw new Error("Login response did not contain a token");
+      }
+      return data;
+    });
+    this.token = payload.token;
+    setCachedToken(this.loginUrl, this.token);
+    logStep("auth", "login succeeded");
+    return this.token;
+  }
+
+  async getToken() {
+    if (!this.token) {
+      await this.login();
+    }
+    return this.token;
+  }
+
+  async refreshToken() {
+    logStep("auth", "refreshing session token");
+    clearCachedToken(this.loginUrl);
+    this.token = "";
+    return this.login();
+  }
+
+  buildHeaders(extraHeaders, token) {
+    return { ...(extraHeaders || {}), Cookie: `token=${token}` };
+  }
+
+  parseResponseBody(text) {
+    try {
+      return text ? JSON.parse(text) : null;
+    } catch (error) {
+      return text;
+    }
+  }
+
+  isAuthFailure(response, parsedBody) {
+    const normalizedBody = typeof parsedBody === "string" ? parsedBody.toLowerCase() : JSON.stringify(parsedBody || {}).toLowerCase();
+    return response.status === 401
+      || response.status === 403
+      || normalizedBody.includes("invalid token")
+      || normalizedBody.includes("authentication")
+      || normalizedBody.includes("not authorized");
+  }
+
+  async request(urlString, options, label, mode) {
+    return retry(label, this.retries, async () => {
+      let token = await this.getToken();
+      let authAttempt = 0;
+      while (authAttempt < 2) {
+        const url = withSessionToken(urlString, token);
+        const response = await fetch(url.toString(), withDispatcher({ ...options, headers: this.buildHeaders(options?.headers, token) }, this.dispatcher));
+        if (mode === "buffer" && response.ok) {
+          return Buffer.from(await response.arrayBuffer());
+        }
+        const text = await response.text();
+        const parsed = this.parseResponseBody(text);
+        if (response.ok) {
+          return parsed;
+        }
+        if (this.isAuthFailure(response, parsed) && authAttempt === 0) {
+          await this.refreshToken();
+          token = await this.getToken();
+          authAttempt += 1;
+          continue;
+        }
+        throw new Error(`Request failed: ${response.status} ${response.statusText}`);
+      }
+      throw new AuthenticationError("Authentication failed after token refresh");
+    });
+  }
+
+  async requestJson(urlString, options, label) {
+    return this.request(urlString, options, label, "json");
+  }
+
+  async requestBuffer(urlString, options, label) {
+    return this.request(urlString, options, label, "buffer");
+  }
+
+  async requestStream(urlString, options, label) {
+    return retry(label, this.retries, async () => {
+      let token = await this.getToken();
+      let authAttempt = 0;
+      while (authAttempt < 2) {
+        const url = withSessionToken(urlString, token);
+        const response = await fetch(url.toString(), withDispatcher({ ...options, headers: this.buildHeaders(options?.headers, token) }, this.dispatcher));
+        if (response.ok) {
+          return response;
+        }
+        const text = await response.text();
+        const parsed = this.parseResponseBody(text);
+        if (this.isAuthFailure(response, parsed) && authAttempt === 0) {
+          await this.refreshToken();
+          token = await this.getToken();
+          authAttempt += 1;
+          continue;
+        }
+        throw new Error(`Request failed: ${response.status} ${response.statusText}`);
+      }
+      throw new AuthenticationError("Authentication failed after token refresh");
+    });
+  }
+}
+
+module.exports = {
+  AuthenticationError,
+  BacktraceSession,
+  buildSiblingUrl,
+  buildLoginUrl,
+  withSessionToken,
+};