openclaw-db9-audit 0.1.0

package/src/event-log-sync.ts

@@ -0,0 +1,166 @@
+import type { AgentEventPayload, PluginLogger } from "openclaw/plugin-sdk/core";
+import { Db9AuditRunTracker } from "./run-tracker.js";
+import { redactUnknown } from "./redact.js";
+import type { Db9AuditDiagnosticEvent, Db9AuditRedactConfig } from "./types.js";
+import { joinPosixPath, normalizeLogRoot, resolveAgentIdFromSessionKey, toDatePartition } from "./utils.js";
+
+export type EventLogAppender = {
+  appendText: (path: string, text: string) => Promise<void>;
+};
+
+const MAX_BUFFERED_LINES = 10_000;
+
+function normalizeStreamName(stream: string): string {
+  const trimmed = stream.trim().toLowerCase();
+  if (!trimmed) {
+    return "misc";
+  }
+  if (trimmed === "lifecycle" || trimmed === "tool" || trimmed === "assistant" || trimmed === "error") {
+    return trimmed;
+  }
+  return "misc";
+}
+
+export function resolveEventLogPath(params: {
+  logRoot: string;
+  stream: string;
+  agentId: string;
+  timestampMs: number;
+}): string {
+  return joinPosixPath(
+    normalizeLogRoot(params.logRoot),
+    normalizeStreamName(params.stream),
+    toDatePartition(params.timestampMs),
+    `${params.agentId}.jsonl`,
+  );
+}
+
+export class Db9AuditEventLogSync {
+  private readonly getAppender: () => Promise<EventLogAppender | null>;
+  private readonly logger?: PluginLogger | undefined;
+  private readonly logRoot: string;
+  private readonly batchSize: number;
+  private readonly redactConfig: Db9AuditRedactConfig;
+  private readonly runTracker: Db9AuditRunTracker;
+  private readonly pending = new Map<string, string[]>();
+  private flushPromise: Promise<{ flushedFiles: number; flushedLines: number }> | null = null;
+
+  constructor(params: {
+    getAppender: () => Promise<EventLogAppender | null>;
+    logger?: PluginLogger | undefined;
+    logRoot: string;
+    batchSize: number;
+    redactConfig: Db9AuditRedactConfig;
+    runTracker: Db9AuditRunTracker;
+  }) {
+    this.getAppender = params.getAppender;
+    this.logger = params.logger;
+    this.logRoot = params.logRoot;
+    this.batchSize = params.batchSize;
+    this.redactConfig = params.redactConfig;
+    this.runTracker = params.runTracker;
+  }
+
+  enqueue(event: AgentEventPayload): void {
+    const sessionKey = event.sessionKey?.trim() || undefined;
+    const agentId = this.runTracker.resolveAgentIdForRun(event.runId, sessionKey);
+    const path = resolveEventLogPath({
+      logRoot: this.logRoot,
+      stream: event.stream,
+      agentId,
+      timestampMs: event.ts,
+    });
+    const line = JSON.stringify({
+      runId: event.runId,
+      seq: event.seq,
+      stream: event.stream,
+      ts: event.ts,
+      agentId,
+      ...(sessionKey ? { sessionKey } : {}),
+      data: redactUnknown(event.data, this.redactConfig),
+    });
+    this.pushLine(path, line);
+  }
+
+  enqueueDiagnostic(event: Db9AuditDiagnosticEvent): void {
+    const sessionKey = event.sessionKey?.trim() || undefined;
+    const agentId =
+      event.agentId?.trim() ||
+      resolveAgentIdFromSessionKey(sessionKey) ||
+      "db9-audit";
+    const timestampMs = Date.now();
+    const path = resolveEventLogPath({
+      logRoot: this.logRoot,
+      stream: event.stream ?? "error",
+      agentId,
+      timestampMs,
+    });
+    const line = JSON.stringify({
+      runId: event.runId,
+      ts: timestampMs,
+      stream: event.stream ?? "error",
+      agentId,
+      ...(sessionKey ? { sessionKey } : {}),
+      data: redactUnknown(event.data, this.redactConfig),
+    });
+    this.pushLine(path, line);
+  }
+
+  private pushLine(path: string, line: string): void {
+    const bucket = this.pending.get(path) ?? [];
+    bucket.push(`${line}\n`);
+    if (bucket.length > MAX_BUFFERED_LINES) {
+      bucket.splice(0, bucket.length - MAX_BUFFERED_LINES);
+      this.logger?.warn(`db9-audit: dropping old buffered FS log lines for ${path}`);
+    }
+    this.pending.set(path, bucket);
+  }
+
+  async flushPending(): Promise<{ flushedFiles: number; flushedLines: number }> {
+    if (this.flushPromise) {
+      return await this.flushPromise;
+    }
+    this.flushPromise = this.doFlushPending();
+    try {
+      return await this.flushPromise;
+    } finally {
+      this.flushPromise = null;
+    }
+  }
+
+  private async doFlushPending(): Promise<{ flushedFiles: number; flushedLines: number }> {
+    const appender = await this.getAppender();
+    if (!appender || this.pending.size === 0) {
+      return { flushedFiles: 0, flushedLines: 0 };
+    }
+
+    let flushedFiles = 0;
+    let flushedLines = 0;
+    const buckets = [...this.pending.entries()];
+    for (const [path, lines] of buckets) {
+      if (lines.length === 0) {
+        this.pending.delete(path);
+        continue;
+      }
+      const toFlush = lines.splice(0, this.batchSize);
+      if (toFlush.length === 0) {
+        continue;
+      }
+      try {
+        await appender.appendText(path, toFlush.join(""));
+        flushedFiles += 1;
+        flushedLines += toFlush.length;
+      } catch (error) {
+        lines.unshift(...toFlush);
+        this.logger?.warn(`db9-audit: FS append failed for ${path}: ${String(error)}`);
+      }
+      if (lines.length === 0) {
+        this.pending.delete(path);
+      } else {
+        this.pending.set(path, lines);
+      }
+    }
+
+    return { flushedFiles, flushedLines };
+  }
+}

package/src/fs-client.ts

@@ -0,0 +1,317 @@
+import type { PluginLogger } from "openclaw/plugin-sdk/core";
+import WebSocket from "ws";
+import { Buffer } from "node:buffer";
+import { dirnamePosix, extractErrorMessage } from "./utils.js";
+
+type FsResponseEnvelope = {
+  id?: string;
+  ok?: boolean;
+  data?: unknown;
+  error?: {
+    code?: string;
+    message?: string;
+  };
+};
+
+export class Db9FsClientError extends Error {
+  readonly code?: string | undefined;
+  readonly retriable: boolean;
+
+  constructor(params: {
+    message: string;
+    code?: string | undefined;
+    retriable: boolean;
+    cause?: unknown;
+  }) {
+    super(params.message, params.cause !== undefined ? { cause: params.cause } : undefined);
+    this.name = "Db9FsClientError";
+    this.code = params.code;
+    this.retriable = params.retriable;
+  }
+}
+
+function isOpen(socket: WebSocket | null): socket is WebSocket {
+  return socket !== null && socket.readyState === WebSocket.OPEN;
+}
+
+export class Db9FsClient {
+  readonly wsUrl: string;
+  readonly username: string;
+  readonly password: string;
+  private readonly logger?: PluginLogger | undefined;
+  private readonly requestTimeoutMs: number;
+  private socket: WebSocket | null = null;
+  private connectPromise: Promise<void> | null = null;
+  private requestChain: Promise<void> = Promise.resolve();
+  private requestId = 0;
+  private authenticated = false;
+  private readonly knownDirs = new Set<string>();
+
+  constructor(params: {
+    wsUrl: string;
+    username: string;
+    password: string;
+    logger?: PluginLogger | undefined;
+    requestTimeoutMs?: number | undefined;
+  }) {
+    this.wsUrl = params.wsUrl;
+    this.username = params.username;
+    this.password = params.password;
+    this.logger = params.logger;
+    this.requestTimeoutMs = params.requestTimeoutMs ?? 10_000;
+  }
+
+  async connect(): Promise<void> {
+    if (this.authenticated && isOpen(this.socket)) {
+      return;
+    }
+    if (this.connectPromise) {
+      return await this.connectPromise;
+    }
+    this.connectPromise = this.openAndAuthenticate();
+    try {
+      await this.connectPromise;
+    } finally {
+      this.connectPromise = null;
+    }
+  }
+
+  private async openAndAuthenticate(): Promise<void> {
+    await this.close();
+    const socket = new WebSocket(this.wsUrl);
+    this.socket = socket;
+    this.authenticated = false;
+
+    await new Promise<void>((resolve, reject) => {
+      const onOpen = () => {
+        cleanup();
+        resolve();
+      };
+      const onError = (error: Error) => {
+        cleanup();
+        reject(
+          new Db9FsClientError({
+            message: `FS WebSocket connection failed: ${error.message}`,
+            retriable: true,
+            cause: error,
+          }),
+        );
+      };
+      const onClose = () => {
+        cleanup();
+        reject(
+          new Db9FsClientError({
+            message: "FS WebSocket closed before opening completed",
+            retriable: true,
+          }),
+        );
+      };
+      const cleanup = () => {
+        socket.off("open", onOpen);
+        socket.off("error", onError);
+        socket.off("close", onClose);
+      };
+
+      socket.once("open", onOpen);
+      socket.once("error", onError);
+      socket.once("close", onClose);
+    });
+
+    socket.on("close", () => {
+      this.authenticated = false;
+      this.socket = null;
+      this.logger?.warn(`db9-audit: FS WebSocket closed: ${this.wsUrl}`);
+    });
+    socket.on("error", (error) => {
+      this.logger?.warn(`db9-audit: FS WebSocket error: ${extractErrorMessage(error) ?? String(error)}`);
+    });
+
+    await this.sendOnSocket(socket, "auth", {
+      username: this.username,
+      password: this.password,
+    });
+    this.authenticated = true;
+    this.logger?.debug?.(`db9-audit: FS authenticated as ${this.username}`);
+  }
+
+  async stat(pathValue: string): Promise<Record<string, unknown> | null> {
+    try {
+      const data = await this.sendRequest("stat", { path: pathValue.trim() });
+      return data && typeof data === "object" ? (data as Record<string, unknown>) : {};
+    } catch (error) {
+      if (error instanceof Db9FsClientError && error.code === "ENOENT") {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  async mkdirp(pathValue: string): Promise<void> {
+    const normalized = pathValue.trim();
+    if (!normalized || normalized === "/" || this.knownDirs.has(normalized)) {
+      return;
+    }
+    await this.sendRequest("mkdir", {
+      path: normalized,
+      recursive: true,
+    });
+    this.knownDirs.add(normalized);
+  }
+
+  async appendText(pathValue: string, text: string): Promise<void> {
+    const normalized = pathValue.trim();
+    if (!normalized) {
+      throw new Error("FS append path must not be empty");
+    }
+    await this.mkdirp(dirnamePosix(normalized));
+    try {
+      await this.sendRequest("append", {
+        path: normalized,
+        content: Buffer.from(text, "utf8").toString("base64"),
+        encoding: "base64",
+      });
+    } catch (error) {
+      if (error instanceof Db9FsClientError && error.code === "ENOENT") {
+        await this.mkdirp(dirnamePosix(normalized));
+        await this.sendRequest("append", {
+          path: normalized,
+          content: Buffer.from(text, "utf8").toString("base64"),
+          encoding: "base64",
+        });
+        return;
+      }
+      throw error;
+    }
+  }
+
+  private async sendRequest(op: string, payload: Record<string, unknown>): Promise<unknown> {
+    return await this.enqueue(async () => {
+      await this.connect();
+      const socket = this.socket;
+      if (!isOpen(socket)) {
+        throw new Db9FsClientError({
+          message: "FS WebSocket is not connected",
+          retriable: true,
+        });
+      }
+
+      return await this.sendOnSocket(socket, op, payload);
+    });
+  }
+
+  private async sendOnSocket(socket: WebSocket, op: string, payload: Record<string, unknown>): Promise<unknown> {
+    const id = String(++this.requestId);
+    const frame = JSON.stringify({ id, op, ...payload });
+    return await this.awaitResponse(socket, id, frame);
+  }
+
+  private async awaitResponse(socket: WebSocket, id: string, frame: string): Promise<unknown> {
+    return await new Promise<unknown>((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        cleanup();
+        reject(
+          new Db9FsClientError({
+            message: `FS request timed out: ${id}`,
+            retriable: true,
+          }),
+        );
+      }, this.requestTimeoutMs);
+
+      const onMessage = (message: WebSocket.RawData) => {
+        cleanup();
+        try {
+          const parsed = JSON.parse(message.toString("utf8")) as FsResponseEnvelope;
+          if (parsed.ok !== true) {
+            reject(
+              new Db9FsClientError({
+                code: parsed.error?.code,
+                message: parsed.error?.message ?? `FS request failed: ${id}`,
+                retriable: parsed.error?.code === "ENOENT" || parsed.error?.code === "ENETUNREACH",
+              }),
+            );
+            return;
+          }
+          resolve(parsed.data);
+        } catch (error) {
+          reject(
+            new Db9FsClientError({
+              message: `FS returned invalid response: ${extractErrorMessage(error) ?? String(error)}`,
+              retriable: true,
+              cause: error,
+            }),
+          );
+        }
+      };
+
+      const onClose = () => {
+        cleanup();
+        reject(
+          new Db9FsClientError({
+            message: "FS WebSocket closed while waiting for response",
+            retriable: true,
+          }),
+        );
+      };
+
+      const onError = (error: Error) => {
+        cleanup();
+        reject(
+          new Db9FsClientError({
+            message: `FS WebSocket send failed: ${error.message}`,
+            retriable: true,
+            cause: error,
+          }),
+        );
+      };
+
+      const cleanup = () => {
+        clearTimeout(timeout);
+        socket.off("message", onMessage);
+        socket.off("close", onClose);
+        socket.off("error", onError);
+      };
+
+      socket.once("message", onMessage);
+      socket.once("close", onClose);
+      socket.once("error", onError);
+      socket.send(frame, (error) => {
+        if (!error) {
+          return;
+        }
+        cleanup();
+        reject(
+          new Db9FsClientError({
+            message: `FS WebSocket send failed: ${error.message}`,
+            retriable: true,
+            cause: error,
+          }),
+        );
+      });
+    });
+  }
+
+  private async enqueue<T>(fn: () => Promise<T>): Promise<T> {
+    const next = this.requestChain.catch(() => undefined).then(fn);
+    this.requestChain = next.then(() => undefined, () => undefined);
+    return await next;
+  }
+
+  async close(): Promise<void> {
+    const socket = this.socket;
+    this.socket = null;
+    this.authenticated = false;
+    this.knownDirs.clear();
+    if (!socket) {
+      return;
+    }
+    await new Promise<void>((resolve) => {
+      if (socket.readyState === WebSocket.CLOSED) {
+        resolve();
+        return;
+      }
+      socket.once("close", () => resolve());
+      socket.close();
+      setTimeout(resolve, 1_000).unref();
+    });
+  }
+}