@spzhongwin/skill-logger-plugin 1.0.11 → 1.0.13

package/src/reporter.ts

@@ -1,298 +1,298 @@
-/**
- * 本地落盘 + 日志轮转 (Log Rotation) + 每 3 分钟批量上报。
- *
- * 设计原则：
- * - events.jsonl 是活跃的写入日志，通过 appendFile 追加。
- * - 上报时，将其 rename 轮转为带有时间戳的文件，隔离写和读，根绝并发读写丢失数据的竞态条件。
- * - 后台处理所有的轮转文件并上报，上报成功后通过 fs.unlink 清理文件。
- * - 失败则保留待下次重试（因为底层 DB 依赖 INSERT IGNORE 处理重复 event_id，所以即使重试时存在部分重复上报也是安全的）。
- * - 所有异常吞掉，绝不阻塞 agent / gateway。
- */
-import fs from "node:fs/promises";
-import path from "node:path";
-import type { PluginConfig, SkillEvent, UserInfoSnapshot } from "./types.ts";
-import type { PluginPaths } from "./paths.ts";
-import { GitIdentityProvider, type IdentityProvider } from "./identity.ts";
-import { defaultFetch } from "./http.ts";
-
-/** 注入 fetch 便于测试；默认用全局 fetch。 */
-type FetchLike = (url: string, init: RequestInit) => Promise<{ ok: boolean; status: number; json?: () => Promise<unknown> }>;
-
-const FLUSH_INTERVAL_MS = 3 * 60 * 1000;
-const BATCH_SIZE = 500;
-
-export type ReporterOptions = {
-  paths: Pick<PluginPaths, "eventsLogPath">;
-  /** 读取最新插件配置（上报地址/鉴权可能在 gateway_start 后才注入）。 */
-  getConfig: () => PluginConfig;
-  identityProvider?: IdentityProvider;
-  fetchImpl?: FetchLike;
-  now?: () => number;
-};
-
-export class Reporter {
-  private readonly paths: Pick<PluginPaths, "eventsLogPath">;
-  private readonly getConfig: () => PluginConfig;
-  private readonly identityProvider: IdentityProvider;
-  private readonly fetchImpl: FetchLike;
-  private timer: ReturnType<typeof setInterval> | undefined;
-  /** 防止多次 flush 重入。 */
-  private flushing = false;
-
-  constructor(opts: ReporterOptions) {
-    this.paths = opts.paths;
-    this.getConfig = opts.getConfig;
-    this.identityProvider = opts.identityProvider ?? new GitIdentityProvider();
-    this.fetchImpl = opts.fetchImpl ?? defaultFetch();
-  }
-
-  private get isDebug(): boolean {
-    return this.getConfig().debugLogging !== false;
-  }
-
-  private async writeFallbackLog(level: "INFO" | "WARN" | "ERROR", ...args: any[]): Promise<void> {
-    try {
-      const msg = args.map(a => (a instanceof Error) ? (a.stack || a.toString()) : (typeof a === "object" ? JSON.stringify(a) : String(a))).join(" ");
-      const ts = new Date().toISOString();
-      const logLine = `[${ts}] [${level}] [skill-logger-plugin] ${msg}\n`;
-      
-      const logDir = path.dirname(this.paths.eventsLogPath);
-      await fs.mkdir(logDir, { recursive: true });
-      await fs.appendFile(path.join(logDir, "skill-logger.err.log"), logLine);
-      
-      if (level === "INFO" && this.isDebug) {
-        console.log("[skill-logger-plugin/reporter]", ...args);
-      } else if (level === "WARN" || level === "ERROR") {
-        console.warn("[skill-logger-plugin]", ...args);
-      }
-    } catch {
-      // ignore
-    }
-  }
-
-  private debug(...args: any[]): void {
-    if (this.isDebug) {
-      void this.writeFallbackLog("INFO", ...args);
-    }
-  }
-
-  /** 追加一行事件到 events.jsonl（目录不存在自动建）。失败不抛。 */
-  async appendEvent(event: SkillEvent): Promise<void> {
-    try {
-      // 防御：截断会导致 MySQL Data Truncation 宕机的超长字段
-      if (typeof event.error_message === "string" && event.error_message.length > 15000) {
-        event.error_message = event.error_message.substring(0, 15000) + "...(truncated)";
-      }
-      if (typeof event.command === "string" && event.command.length > 15000) {
-        event.command = event.command.substring(0, 15000) + "...(truncated)";
-      }
-
-      let line = "";
-      try {
-        line = JSON.stringify(event);
-        // 防御：防止整个 JSON 过大（如带有 base64 图片的 args）导致 Express 413 Payload Too Large
-        if (line.length > 100000) {
-          const safeEvent = { ...event, args: { _warning: "args omitted due to excessive size" } };
-          line = JSON.stringify(safeEvent);
-        }
-      } catch {
-        return; // JSON 序列化失败直接丢弃
-      }
-
-      await fs.mkdir(path.dirname(this.paths.eventsLogPath), { recursive: true });
-      await fs.appendFile(this.paths.eventsLogPath, line + "\n");
-    } catch (err) {
-      void this.writeFallbackLog("ERROR", "写事件日志失败", this.paths.eventsLogPath, err);
-    }
-  }
-
-  /** 启动 3 分钟定时上报。 */
-  startTimer(): void {
-    if (this.timer) return;
-    this.timer = setInterval(() => void this.flush(), FLUSH_INTERVAL_MS);
-    // 不阻止进程退出
-    if (typeof this.timer.unref === "function") this.timer.unref();
-  }
-
-  /** 停止定时器，并尽力做最后一次 flush。 */
-  async stopTimer(): Promise<void> {
-    if (this.timer) {
-      clearInterval(this.timer);
-      this.timer = undefined;
-    }
-    await this.flush();
-  }
-
-  private linesOf(content: string): string[] {
-    return content.split("\n").filter((l) => l.trim().length > 0);
-  }
-
-  private async resolveUserInfo(appKey: string, config: PluginConfig): Promise<UserInfoSnapshot> {
-    if (!config.reportBaseUrl) return {};
-
-    try {
-      const url = config.reportBaseUrl.replace(/\/$/, "") + "/skill_user/resolve";
-      const headers: Record<string, string> = { "Content-Type": "application/json" };
-      if (config.authToken) headers.Authorization = config.authToken;
-
-      const res = await this.fetchImpl(url, {
-        method: "POST",
-        headers,
-        body: JSON.stringify({ appKey }),
-      });
-
-      if (!res.ok) {
-        return { error_message: `resolve user info failed: HTTP ${res.status}` };
-      }
-
-      const data = typeof (res as any).json === "function" ? await (res as any).json() : {};
-      return (data.user_info || data.userInfo || data.data || {}) as UserInfoSnapshot;
-    } catch (err) {
-      return { error_message: `resolve user info failed: ${err instanceof Error ? err.message : String(err)}` };
-    }
-  }
-
-  private async attachUserInfo(events: SkillEvent[], config: PluginConfig): Promise<SkillEvent[]> {
-    const appKeys = [...new Set(events.map((event) => event.app_key || "").filter(Boolean))];
-    if (appKeys.length === 0) return events;
-
-    const byAppKey = new Map<string, UserInfoSnapshot>();
-    await Promise.all(appKeys.map(async (appKey) => {
-      byAppKey.set(appKey, await this.resolveUserInfo(appKey, config));
-    }));
-
-    return events.map((event) => {
-      if (!event.app_key) return event;
-      const userInfo = byAppKey.get(event.app_key);
-      if (!userInfo || Object.keys(userInfo).length === 0) return event;
-      return { ...event, user_info: userInfo };
-    });
-  }
-
-  /**
-   * 读本地队列并分批 POST；每批成功后清理对应本地文件。
-   * 采用日志轮转（Rename）规避读写竞态条件。
-   * 未配置 reportBaseUrl → 直接返回（只落本地，不上报）。
-   */
-  async flush(): Promise<void> {
-    if (this.flushing) return;
-    this.flushing = true;
-    try {
-      const config = this.getConfig();
-      if (!config.reportBaseUrl) {
-        void this.writeFallbackLog("WARN", "flush skipped: reportBaseUrl 没有配置！请检查 openclaw.json 插件配置是否正确加载。");
-        return;
-      }
-
-      const logDir = path.dirname(this.paths.eventsLogPath);
-
-      // 1. 重命名当前的活跃日志文件为时间戳格式（原子操作，解决读写冲突）
-      try {
-        await fs.access(this.paths.eventsLogPath);
-        const timestamp = Date.now();
-        const rotatedPath = path.join(logDir, `events.${timestamp}.jsonl`);
-        await fs.rename(this.paths.eventsLogPath, rotatedPath);
-        this.debug(`Rotated active log to ${path.basename(rotatedPath)}`);
-      } catch {
-        // 文件不存在，跳过重命名
-      }
-
-      // 2. 扫描所有轮转的日志文件
-      let files: string[] = [];
-      try {
-        const dirEntries = await fs.readdir(logDir);
-        files = dirEntries
-          .filter(f => f.startsWith("events.") && f.endsWith(".jsonl") && f !== "events.jsonl")
-          .map(f => path.join(logDir, f));
-      } catch {
-        return; // 目录不存在直接返回
-      }
-
-      if (files.length === 0) {
-        this.debug("No rotated log files found. flush finished.");
-        return;
-      }
-
-      this.debug(`Found ${files.length} rotated log files to process.`);
-      const identity = await this.identityProvider.getIdentity();
-      const url = config.reportBaseUrl.replace(/\/$/, "") + "/skill_report/batch";
-
-      // 3. 逐个处理文件上报
-      for (const filePath of files) {
-        try {
-          const content = await fs.readFile(filePath, "utf-8");
-          const lines = this.linesOf(content);
-          if (lines.length === 0) {
-            await fs.unlink(filePath); // 空文件直接删除
-            continue;
-          }
-
-          let cursor = 0;
-          let allSuccess = true;
-
-          while (cursor < lines.length) {
-            const slice = lines.slice(cursor, cursor + BATCH_SIZE);
-            const events: SkillEvent[] = [];
-            for (const line of slice) {
-              try {
-                events.push(JSON.parse(line));
-              } catch {
-                // 跳过坏行
-              }
-            }
-            
-            if (events.length === 0) {
-              cursor += slice.length;
-              continue;
-            }
-
-            const eventsWithUserInfo = await this.attachUserInfo(events, config);
-            const body = JSON.stringify({
-              identity,
-              ide: "openclaw",
-              marketplace: "openclaw",
-              events: eventsWithUserInfo,
-            });
-            const headers: Record<string, string> = { "Content-Type": "application/json" };
-            if (config.authToken) headers.Authorization = config.authToken;
-
-            const res = await this.fetchImpl(url, { method: "POST", headers, body });
-            if (!res.ok) {
-              const errBody = await res.text().catch(() => "无法读取响应体");
-              await this.writeFallbackLog("ERROR", `批量上报失败 (文件 ${path.basename(filePath)}), HTTP`, res.status, "服务端返回信息:", errBody);
-              // 防御死循环：如果是由于报文过大(413)或格式错误(400)等业务级拒绝，放弃该批次，不要死锁整个本地队列
-              if (res.status === 400 || res.status === 413 || res.status === 422) {
-                await this.writeFallbackLog("WARN", `报文被服务器永久拒绝，丢弃该批次以释放队列`);
-                cursor += slice.length;
-                continue;
-              }
-              allSuccess = false;
-              break; // 其他网络或 500 错误：本文件终止处理，留待下轮重试
-            }
-            
-            this.debug(`Successfully reported batch of ${events.length} events from ${path.basename(filePath)}`);
-            cursor += slice.length;
-          }
-
-          // 如果该文件所有的 batch 都上报成功了，将其物理删除
-          if (allSuccess) {
-            await fs.unlink(filePath);
-            this.debug(`Deleted fully processed file: ${path.basename(filePath)}`);
-          } else {
-            // 如果某一批次失败，文件保留。下轮 flush 会把前面成功批次的事件再报一次。
-            // 但因为 DB 的 skill_logger_events 表 event_id 是唯一键且使用了 INSERT IGNORE，所以数据去重是绝对安全的。
-            this.debug(`File ${path.basename(filePath)} partially failed. Keeping it for next flush.`);
-          }
-
-        } catch (err) {
-          await this.writeFallbackLog("ERROR", `处理文件 ${path.basename(filePath)} 异常:`, err);
-        }
-      }
-
-    } catch (err) {
-      await this.writeFallbackLog("ERROR", "flush 整体异常", err);
-    } finally {
-      this.flushing = false;
-    }
-  }
-}
+/**
+ * 本地落盘 + 日志轮转 (Log Rotation) + 每 3 分钟批量上报。
+ *
+ * 设计原则：
+ * - events.jsonl 是活跃的写入日志，通过 appendFile 追加。
+ * - 上报时，将其 rename 轮转为带有时间戳的文件，隔离写和读，根绝并发读写丢失数据的竞态条件。
+ * - 后台处理所有的轮转文件并上报，上报成功后通过 fs.unlink 清理文件。
+ * - 失败则保留待下次重试（因为底层 DB 依赖 INSERT IGNORE 处理重复 event_id，所以即使重试时存在部分重复上报也是安全的）。
+ * - 所有异常吞掉，绝不阻塞 agent / gateway。
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { PluginConfig, SkillEvent, UserInfoSnapshot } from "./types.ts";
+import type { PluginPaths } from "./paths.ts";
+import { GitIdentityProvider, type IdentityProvider } from "./identity.ts";
+import { defaultFetch } from "./http.ts";
+
+/** 注入 fetch 便于测试；默认用全局 fetch。 */
+type FetchLike = (url: string, init: RequestInit) => Promise<{ ok: boolean; status: number; json?: () => Promise<unknown> }>;
+
+const FLUSH_INTERVAL_MS = 3 * 60 * 1000;
+const BATCH_SIZE = 500;
+
+export type ReporterOptions = {
+  paths: Pick<PluginPaths, "eventsLogPath">;
+  /** 读取最新插件配置（上报地址/鉴权可能在 gateway_start 后才注入）。 */
+  getConfig: () => PluginConfig;
+  identityProvider?: IdentityProvider;
+  fetchImpl?: FetchLike;
+  now?: () => number;
+};
+
+export class Reporter {
+  private readonly paths: Pick<PluginPaths, "eventsLogPath">;
+  private readonly getConfig: () => PluginConfig;
+  private readonly identityProvider: IdentityProvider;
+  private readonly fetchImpl: FetchLike;
+  private timer: ReturnType<typeof setInterval> | undefined;
+  /** 防止多次 flush 重入。 */
+  private flushing = false;
+
+  constructor(opts: ReporterOptions) {
+    this.paths = opts.paths;
+    this.getConfig = opts.getConfig;
+    this.identityProvider = opts.identityProvider ?? new GitIdentityProvider();
+    this.fetchImpl = opts.fetchImpl ?? defaultFetch();
+  }
+
+  private get isDebug(): boolean {
+    return this.getConfig().debugLogging !== false;
+  }
+
+  private async writeFallbackLog(level: "INFO" | "WARN" | "ERROR", ...args: any[]): Promise<void> {
+    try {
+      const msg = args.map(a => (a instanceof Error) ? (a.stack || a.toString()) : (typeof a === "object" ? JSON.stringify(a) : String(a))).join(" ");
+      const ts = new Date().toISOString();
+      const logLine = `[${ts}] [${level}] [skill-logger-plugin] ${msg}\n`;
+      
+      const logDir = path.dirname(this.paths.eventsLogPath);
+      await fs.mkdir(logDir, { recursive: true });
+      await fs.appendFile(path.join(logDir, "skill-logger.err.log"), logLine);
+      
+      if (level === "INFO" && this.isDebug) {
+        console.log("[skill-logger-plugin/reporter]", ...args);
+      } else if (level === "WARN" || level === "ERROR") {
+        console.warn("[skill-logger-plugin]", ...args);
+      }
+    } catch {
+      // ignore
+    }
+  }
+
+  private debug(...args: any[]): void {
+    if (this.isDebug) {
+      void this.writeFallbackLog("INFO", ...args);
+    }
+  }
+
+  /** 追加一行事件到 events.jsonl（目录不存在自动建）。失败不抛。 */
+  async appendEvent(event: SkillEvent): Promise<void> {
+    try {
+      // 防御：截断会导致 MySQL Data Truncation 宕机的超长字段
+      if (typeof event.error_message === "string" && event.error_message.length > 15000) {
+        event.error_message = event.error_message.substring(0, 15000) + "...(truncated)";
+      }
+      if (typeof event.command === "string" && event.command.length > 15000) {
+        event.command = event.command.substring(0, 15000) + "...(truncated)";
+      }
+
+      let line = "";
+      try {
+        line = JSON.stringify(event);
+        // 防御：防止整个 JSON 过大（如带有 base64 图片的 args）导致 Express 413 Payload Too Large
+        if (line.length > 100000) {
+          const safeEvent = { ...event, args: { _warning: "args omitted due to excessive size" } };
+          line = JSON.stringify(safeEvent);
+        }
+      } catch {
+        return; // JSON 序列化失败直接丢弃
+      }
+
+      await fs.mkdir(path.dirname(this.paths.eventsLogPath), { recursive: true });
+      await fs.appendFile(this.paths.eventsLogPath, line + "\n");
+    } catch (err) {
+      void this.writeFallbackLog("ERROR", "写事件日志失败", this.paths.eventsLogPath, err);
+    }
+  }
+
+  /** 启动 3 分钟定时上报。 */
+  startTimer(): void {
+    if (this.timer) return;
+    this.timer = setInterval(() => void this.flush(), FLUSH_INTERVAL_MS);
+    // 不阻止进程退出
+    if (typeof this.timer.unref === "function") this.timer.unref();
+  }
+
+  /** 停止定时器，并尽力做最后一次 flush。 */
+  async stopTimer(): Promise<void> {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+    await this.flush();
+  }
+
+  private linesOf(content: string): string[] {
+    return content.split("\n").filter((l) => l.trim().length > 0);
+  }
+
+  private async resolveUserInfo(appKey: string, config: PluginConfig): Promise<UserInfoSnapshot> {
+    if (!config.reportBaseUrl) return {};
+
+    try {
+      const url = config.reportBaseUrl.replace(/\/$/, "") + "/skill_user/resolve";
+      const headers: Record<string, string> = { "Content-Type": "application/json" };
+      if (config.authToken) headers.Authorization = config.authToken;
+
+      const res = await this.fetchImpl(url, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({ appKey }),
+      });
+
+      if (!res.ok) {
+        return { error_message: `resolve user info failed: HTTP ${res.status}` };
+      }
+
+      const data = typeof (res as any).json === "function" ? await (res as any).json() : {};
+      return (data.user_info || data.userInfo || data.data || {}) as UserInfoSnapshot;
+    } catch (err) {
+      return { error_message: `resolve user info failed: ${err instanceof Error ? err.message : String(err)}` };
+    }
+  }
+
+  private async attachUserInfo(events: SkillEvent[], config: PluginConfig): Promise<SkillEvent[]> {
+    const appKeys = [...new Set(events.map((event) => event.app_key || "").filter(Boolean))];
+    if (appKeys.length === 0) return events;
+
+    const byAppKey = new Map<string, UserInfoSnapshot>();
+    await Promise.all(appKeys.map(async (appKey) => {
+      byAppKey.set(appKey, await this.resolveUserInfo(appKey, config));
+    }));
+
+    return events.map((event) => {
+      if (!event.app_key) return event;
+      const userInfo = byAppKey.get(event.app_key);
+      if (!userInfo || Object.keys(userInfo).length === 0) return event;
+      return { ...event, user_info: userInfo };
+    });
+  }
+
+  /**
+   * 读本地队列并分批 POST；每批成功后清理对应本地文件。
+   * 采用日志轮转（Rename）规避读写竞态条件。
+   * 未配置 reportBaseUrl → 直接返回（只落本地，不上报）。
+   */
+  async flush(): Promise<void> {
+    if (this.flushing) return;
+    this.flushing = true;
+    try {
+      const config = this.getConfig();
+      if (!config.reportBaseUrl) {
+        void this.writeFallbackLog("WARN", "flush skipped: reportBaseUrl 没有配置！请检查 openclaw.json 插件配置是否正确加载。");
+        return;
+      }
+
+      const logDir = path.dirname(this.paths.eventsLogPath);
+
+      // 1. 重命名当前的活跃日志文件为时间戳格式（原子操作，解决读写冲突）
+      try {
+        await fs.access(this.paths.eventsLogPath);
+        const timestamp = Date.now();
+        const rotatedPath = path.join(logDir, `events.${timestamp}.jsonl`);
+        await fs.rename(this.paths.eventsLogPath, rotatedPath);
+        this.debug(`Rotated active log to ${path.basename(rotatedPath)}`);
+      } catch {
+        // 文件不存在，跳过重命名
+      }
+
+      // 2. 扫描所有轮转的日志文件
+      let files: string[] = [];
+      try {
+        const dirEntries = await fs.readdir(logDir);
+        files = dirEntries
+          .filter(f => f.startsWith("events.") && f.endsWith(".jsonl") && f !== "events.jsonl")
+          .map(f => path.join(logDir, f));
+      } catch {
+        return; // 目录不存在直接返回
+      }
+
+      if (files.length === 0) {
+        this.debug("No rotated log files found. flush finished.");
+        return;
+      }
+
+      this.debug(`Found ${files.length} rotated log files to process.`);
+      const identity = await this.identityProvider.getIdentity();
+      const url = config.reportBaseUrl.replace(/\/$/, "") + "/skill_report/batch";
+
+      // 3. 逐个处理文件上报
+      for (const filePath of files) {
+        try {
+          const content = await fs.readFile(filePath, "utf-8");
+          const lines = this.linesOf(content);
+          if (lines.length === 0) {
+            await fs.unlink(filePath); // 空文件直接删除
+            continue;
+          }
+
+          let cursor = 0;
+          let allSuccess = true;
+
+          while (cursor < lines.length) {
+            const slice = lines.slice(cursor, cursor + BATCH_SIZE);
+            const events: SkillEvent[] = [];
+            for (const line of slice) {
+              try {
+                events.push(JSON.parse(line));
+              } catch {
+                // 跳过坏行
+              }
+            }
+            
+            if (events.length === 0) {
+              cursor += slice.length;
+              continue;
+            }
+
+            const eventsWithUserInfo = await this.attachUserInfo(events, config);
+            const body = JSON.stringify({
+              identity,
+              ide: "openclaw",
+              marketplace: "openclaw",
+              events: eventsWithUserInfo,
+            });
+            const headers: Record<string, string> = { "Content-Type": "application/json" };
+            if (config.authToken) headers.Authorization = config.authToken;
+
+            const res = await this.fetchImpl(url, { method: "POST", headers, body });
+            if (!res.ok) {
+              const errBody = await res.text().catch(() => "无法读取响应体");
+              await this.writeFallbackLog("ERROR", `批量上报失败 (文件 ${path.basename(filePath)}), HTTP`, res.status, "服务端返回信息:", errBody);
+              // 防御死循环：如果是由于报文过大(413)或格式错误(400)等业务级拒绝，放弃该批次，不要死锁整个本地队列
+              if (res.status === 400 || res.status === 413 || res.status === 422) {
+                await this.writeFallbackLog("WARN", `报文被服务器永久拒绝，丢弃该批次以释放队列`);
+                cursor += slice.length;
+                continue;
+              }
+              allSuccess = false;
+              break; // 其他网络或 500 错误：本文件终止处理，留待下轮重试
+            }
+            
+            this.debug(`Successfully reported batch of ${events.length} events from ${path.basename(filePath)}`);
+            cursor += slice.length;
+          }
+
+          // 如果该文件所有的 batch 都上报成功了，将其物理删除
+          if (allSuccess) {
+            await fs.unlink(filePath);
+            this.debug(`Deleted fully processed file: ${path.basename(filePath)}`);
+          } else {
+            // 如果某一批次失败，文件保留。下轮 flush 会把前面成功批次的事件再报一次。
+            // 但因为 DB 的 skill_logger_events 表 event_id 是唯一键且使用了 INSERT IGNORE，所以数据去重是绝对安全的。
+            this.debug(`File ${path.basename(filePath)} partially failed. Keeping it for next flush.`);
+          }
+
+        } catch (err) {
+          await this.writeFallbackLog("ERROR", `处理文件 ${path.basename(filePath)} 异常:`, err);
+        }
+      }
+
+    } catch (err) {
+      await this.writeFallbackLog("ERROR", "flush 整体异常", err);
+    } finally {
+      this.flushing = false;
+    }
+  }
+}