openclaw-db9-audit 0.1.0

package/src/service.ts

@@ -0,0 +1,356 @@
+import type {
+  OpenClawPluginApi,
+  OpenClawPluginService,
+  OpenClawPluginServiceContext,
+  PluginHookAgentContext,
+  PluginHookAgentEndEvent,
+} from "openclaw/plugin-sdk/core";
+import { bootstrapFreshState, recoverDb9AuditState } from "./bootstrap.js";
+import { Db9ControlPlaneClient } from "./control-plane.js";
+import { Db9AuditEventLogSync } from "./event-log-sync.js";
+import { Db9FsClient } from "./fs-client.js";
+import { Db9AuditPostgres } from "./postgres.js";
+import { Db9AuditRunTracker } from "./run-tracker.js";
+import { Db9AuditSessionLookup, discoverTranscriptFiles } from "./session-store.js";
+import { Db9AuditStateStore } from "./state-store.js";
+import { Db9AuditTranscriptSync } from "./transcript-sync.js";
+import type { Db9AuditPluginConfig, Db9AuditState } from "./types.js";
+import { extractErrorMessage } from "./utils.js";
+
+export class Db9AuditServiceRuntime implements OpenClawPluginService {
+  readonly id = "db9-audit";
+  private readonly api: OpenClawPluginApi;
+  private readonly config: Db9AuditPluginConfig;
+  private readonly controlPlane: Db9ControlPlaneClient;
+  private serviceContext: OpenClawPluginServiceContext | null = null;
+  private stateStore: Db9AuditStateStore | null = null;
+  private state: Db9AuditState | null = null;
+  private pg: Db9AuditPostgres | null = null;
+  private fsClient: Db9FsClient | null = null;
+  private readonly runTracker = new Db9AuditRunTracker();
+  private readonly sessionLookup = new Db9AuditSessionLookup();
+  private transcriptSync: Db9AuditTranscriptSync | null = null;
+  private eventLogSync: Db9AuditEventLogSync | null = null;
+  private reconnectingStatePromise: Promise<Db9AuditState | null> | null = null;
+  private pgConnectPromise: Promise<Db9AuditPostgres | null> | null = null;
+  private fsConnectPromise: Promise<Db9FsClient | null> | null = null;
+  private flushTimer: NodeJS.Timeout | null = null;
+  private stopAgentEvents: (() => void) | null = null;
+  private stopTranscriptEvents: (() => void) | null = null;
+  private runSummaryQueue: Promise<void> = Promise.resolve();
+  private started = false;
+  private stopped = false;
+
+  constructor(params: {
+    api: OpenClawPluginApi;
+    config: Db9AuditPluginConfig;
+  }) {
+    this.api = params.api;
+    this.config = params.config;
+    this.controlPlane = new Db9ControlPlaneClient(params.config.apiBase);
+  }
+
+  async start(ctx: OpenClawPluginServiceContext): Promise<void> {
+    this.serviceContext = ctx;
+    this.stateStore = new Db9AuditStateStore(ctx.stateDir);
+    this.stopped = false;
+
+    if (!this.config.enabled) {
+      ctx.logger.info?.("db9-audit: disabled");
+      return;
+    }
+
+    this.state = await this.loadOrBootstrapState();
+    this.eventLogSync = new Db9AuditEventLogSync({
+      getAppender: async () => {
+        const client = await this.ensureFs();
+        return client ? { appendText: (path, text) => client.appendText(path, text) } : null;
+      },
+      logger: ctx.logger,
+      logRoot: this.config.logRoot,
+      batchSize: this.config.batchSize,
+      redactConfig: this.config.redact,
+      runTracker: this.runTracker,
+    });
+    this.transcriptSync = new Db9AuditTranscriptSync({
+      sessionLookup: this.sessionLookup,
+      redactConfig: this.config.redact,
+      batchSize: this.config.batchSize,
+      logger: ctx.logger,
+      getStore: async () => {
+        const pg = await this.ensurePg();
+        return pg
+          ? {
+              getOffset: (sessionFile) => pg.getOffset(sessionFile),
+              syncTranscriptBatch: (batch) => pg.syncTranscriptBatch(batch),
+            }
+          : null;
+      },
+      onDiagnostic: async (event) => {
+        this.eventLogSync?.enqueueDiagnostic(event);
+      },
+    });
+
+    await Promise.allSettled([this.ensurePg(), this.ensureFs()]);
+
+    this.stopAgentEvents = this.api.runtime.events.onAgentEvent((event) => {
+      this.runTracker.recordAgentEvent(event);
+      this.eventLogSync?.enqueue(event);
+    });
+    this.stopTranscriptEvents = this.api.runtime.events.onSessionTranscriptUpdate(({ sessionFile }) => {
+      this.transcriptSync?.enqueue(sessionFile);
+    });
+
+    this.flushTimer = setInterval(() => {
+      void this.flushAll();
+    }, this.config.flushIntervalMs);
+    this.flushTimer.unref?.();
+
+    this.started = true;
+    this.api.logger.info?.("db9-audit: service started");
+
+    if (this.config.backfillOnStart) {
+      void this.backfillExistingTranscripts();
+    }
+  }
+
+  async stop(): Promise<void> {
+    this.stopAgentEvents?.();
+    this.stopAgentEvents = null;
+    this.stopTranscriptEvents?.();
+    this.stopTranscriptEvents = null;
+    if (this.flushTimer) {
+      clearInterval(this.flushTimer);
+      this.flushTimer = null;
+    }
+    await this.flushAll();
+    this.stopped = true;
+    await this.runSummaryQueue.catch(() => undefined);
+    await this.fsClient?.close().catch(() => undefined);
+    await this.pg?.close().catch(() => undefined);
+    this.fsClient = null;
+    this.pg = null;
+    this.started = false;
+    this.api.logger.info?.("db9-audit: stopped");
+  }
+
+  handleAgentEnd(event: PluginHookAgentEndEvent, ctx: PluginHookAgentContext): void {
+    if (!this.config.enabled || !this.started) {
+      return;
+    }
+    const summary = this.runTracker.buildRunSummaryFromHook(event, ctx);
+    if (!summary) {
+      return;
+    }
+    this.runSummaryQueue = this.runSummaryQueue
+      .then(async () => {
+        const pg = await this.ensurePg();
+        if (!pg) {
+          this.eventLogSync?.enqueueDiagnostic({
+            stream: "error",
+            sessionKey: summary.sessionKey,
+            agentId: summary.agentId,
+            runId: summary.runId,
+            data: {
+              type: "run_summary_skipped",
+              reason: "pg_unavailable",
+            },
+          });
+          return;
+        }
+        await pg.upsertRunSummary(summary);
+      })
+      .catch((error) => {
+        this.api.logger.warn(`db9-audit: run summary upsert failed: ${extractErrorMessage(error) ?? String(error)}`);
+      });
+  }
+
+  private async loadOrBootstrapState(): Promise<Db9AuditState> {
+    if (!this.stateStore || !this.serviceContext) {
+      throw new Error("Service context is not initialized");
+    }
+    const existing = await this.stateStore.read();
+    if (existing) {
+      return existing;
+    }
+    const state = await bootstrapFreshState({
+      client: this.controlPlane,
+      config: this.config,
+      logger: this.serviceContext.logger,
+    });
+    await this.stateStore.write(state);
+    return state;
+  }
+
+  private async refreshStateViaControlPlane(): Promise<Db9AuditState | null> {
+    if (this.reconnectingStatePromise) {
+      return await this.reconnectingStatePromise;
+    }
+    if (!this.stateStore || !this.state || !this.serviceContext) {
+      return null;
+    }
+
+    this.reconnectingStatePromise = recoverDb9AuditState({
+      client: this.controlPlane,
+      config: this.config,
+      state: this.state,
+      logger: this.serviceContext.logger,
+    })
+      .then(async (nextState) => {
+        this.state = nextState;
+        await this.stateStore?.write(nextState);
+        await this.pg?.close().catch(() => undefined);
+        await this.fsClient?.close().catch(() => undefined);
+        this.pg = null;
+        this.fsClient = null;
+        return nextState;
+      })
+      .catch((error) => {
+        this.serviceContext?.logger.warn(`db9-audit: state recovery failed: ${extractErrorMessage(error) ?? String(error)}`);
+        return null;
+      })
+      .finally(() => {
+        this.reconnectingStatePromise = null;
+      });
+
+    return await this.reconnectingStatePromise;
+  }
+
+  private async ensurePg(): Promise<Db9AuditPostgres | null> {
+    if (this.pg) {
+      return this.pg;
+    }
+    if (this.pgConnectPromise) {
+      return await this.pgConnectPromise;
+    }
+    this.pgConnectPromise = this.connectPg();
+    try {
+      return await this.pgConnectPromise;
+    } finally {
+      this.pgConnectPromise = null;
+    }
+  }
+
+  private async connectPg(): Promise<Db9AuditPostgres | null> {
+    if (!this.state || !this.serviceContext) {
+      return null;
+    }
+    const attempt = async (state: Db9AuditState): Promise<Db9AuditPostgres> => {
+      const pg = new Db9AuditPostgres({
+        connectionString: state.database.connectionString,
+        schema: this.config.schema,
+        logger: this.serviceContext?.logger,
+      });
+      try {
+        await pg.ensureSchema();
+        return pg;
+      } catch (error) {
+        await pg.close().catch(() => undefined);
+        throw error;
+      }
+    };
+
+    try {
+      this.pg = await attempt(this.state);
+      return this.pg;
+    } catch (initialError) {
+      this.serviceContext.logger.warn(`db9-audit: PG connect failed, attempting recovery: ${extractErrorMessage(initialError) ?? String(initialError)}`);
+      const refreshed = await this.refreshStateViaControlPlane();
+      if (!refreshed) {
+        return null;
+      }
+      try {
+        this.pg = await attempt(refreshed);
+        return this.pg;
+      } catch (recoveredError) {
+        this.serviceContext.logger.warn(`db9-audit: PG still unavailable after recovery: ${extractErrorMessage(recoveredError) ?? String(recoveredError)}`);
+        return null;
+      }
+    }
+  }
+
+  private async ensureFs(): Promise<Db9FsClient | null> {
+    if (this.fsClient) {
+      return this.fsClient;
+    }
+    if (this.fsConnectPromise) {
+      return await this.fsConnectPromise;
+    }
+    this.fsConnectPromise = this.connectFs();
+    try {
+      return await this.fsConnectPromise;
+    } finally {
+      this.fsConnectPromise = null;
+    }
+  }
+
+  private async connectFs(): Promise<Db9FsClient | null> {
+    if (!this.state || !this.serviceContext) {
+      return null;
+    }
+    const attempt = async (state: Db9AuditState): Promise<Db9FsClient> => {
+      const client = new Db9FsClient({
+        wsUrl: state.fs.wsUrl,
+        username: state.fs.username,
+        password: state.fs.password,
+        logger: this.serviceContext?.logger,
+      });
+      try {
+        await client.connect();
+        return client;
+      } catch (error) {
+        await client.close().catch(() => undefined);
+        throw error;
+      }
+    };
+
+    try {
+      this.fsClient = await attempt(this.state);
+      return this.fsClient;
+    } catch (initialError) {
+      this.serviceContext.logger.warn(`db9-audit: FS connect failed, attempting recovery: ${extractErrorMessage(initialError) ?? String(initialError)}`);
+      const refreshed = await this.refreshStateViaControlPlane();
+      if (!refreshed) {
+        return null;
+      }
+      try {
+        this.fsClient = await attempt(refreshed);
+        return this.fsClient;
+      } catch (recoveredError) {
+        this.serviceContext.logger.warn(`db9-audit: FS still unavailable after recovery: ${extractErrorMessage(recoveredError) ?? String(recoveredError)}`);
+        return null;
+      }
+    }
+  }
+
+  private async backfillExistingTranscripts(): Promise<void> {
+    if (!this.serviceContext || !this.transcriptSync) {
+      return;
+    }
+    const files = await discoverTranscriptFiles({
+      stateDir: this.serviceContext.stateDir,
+    });
+    for (const filePath of files) {
+      this.transcriptSync.enqueue(filePath);
+    }
+    await this.flushAll();
+  }
+
+  private async flushAll(): Promise<void> {
+    if (this.stopped) {
+      return;
+    }
+    await Promise.allSettled([
+      this.transcriptSync?.flushPending() ?? Promise.resolve({ flushedSessions: 0, insertedMessages: 0 }),
+      this.eventLogSync?.flushPending() ?? Promise.resolve({ flushedFiles: 0, flushedLines: 0 }),
+      this.runSummaryQueue,
+    ]);
+  }
+}
+
+export function createDb9AuditService(params: {
+  api: OpenClawPluginApi;
+  config: Db9AuditPluginConfig;
+}): Db9AuditServiceRuntime {
+  return new Db9AuditServiceRuntime(params);
+}

package/src/session-store.ts

@@ -0,0 +1,150 @@
+import fs from "node:fs";
+import path from "node:path";
+import type { SessionLookupResult, SessionStoreEntry } from "./types.js";
+import {
+  normalizeSessionPath,
+  resolveAgentIdFromSessionPath,
+  resolveSessionIdFromSessionPath,
+  sanitizeAgentId,
+} from "./utils.js";
+
+type SessionStoreCacheEntry = {
+  mtimeMs: number;
+  entries: Record<string, SessionStoreEntry>;
+};
+
+function normalizeStoredSessionFile(sessionsDir: string, value: string): string {
+  const trimmed = value.trim();
+  if (!trimmed) {
+    return "";
+  }
+  return path.isAbsolute(trimmed)
+    ? normalizeSessionPath(trimmed)
+    : normalizeSessionPath(path.join(sessionsDir, trimmed));
+}
+
+export class Db9AuditSessionLookup {
+  private readonly storeCache = new Map<string, SessionStoreCacheEntry>();
+
+  async lookup(sessionFile: string): Promise<SessionLookupResult> {
+    const normalizedSessionFile = normalizeSessionPath(sessionFile);
+    const sessionsDir = path.dirname(normalizedSessionFile);
+    const sessionStorePath = path.join(sessionsDir, "sessions.json");
+    const cached = await this.readSessionStore(sessionStorePath);
+
+    for (const [sessionKey, entry] of Object.entries(cached.entries)) {
+      if (!entry.sessionFile) {
+        continue;
+      }
+      const candidatePath = normalizeStoredSessionFile(sessionsDir, entry.sessionFile);
+      if (candidatePath !== normalizedSessionFile) {
+        continue;
+      }
+      return {
+        agentId:
+          resolveAgentIdFromSessionPath(normalizedSessionFile) ??
+          sanitizeAgentId(sessionKey.split(":")[1]),
+        ...(entry.sessionId ? { sessionId: entry.sessionId } : {}),
+        sessionKey,
+        sessionStorePath,
+      };
+    }
+
+    const fallbackAgentId = resolveAgentIdFromSessionPath(normalizedSessionFile) ?? "unknown";
+    const fallbackSessionId = resolveSessionIdFromSessionPath(normalizedSessionFile);
+    return {
+      agentId: fallbackAgentId,
+      ...(fallbackSessionId ? { sessionId: fallbackSessionId } : {}),
+      ...(fs.existsSync(sessionStorePath) ? { sessionStorePath } : {}),
+    };
+  }
+
+  private async readSessionStore(storePath: string): Promise<SessionStoreCacheEntry> {
+    const stat = await fs.promises.stat(storePath).catch(() => null);
+    if (!stat) {
+      return { mtimeMs: 0, entries: {} };
+    }
+    const cached = this.storeCache.get(storePath);
+    if (cached && cached.mtimeMs === stat.mtimeMs) {
+      return cached;
+    }
+    const raw = await fs.promises.readFile(storePath, "utf8");
+    const parsed = JSON.parse(raw) as unknown;
+    const entries: Record<string, SessionStoreEntry> = {};
+    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+      for (const [key, value] of Object.entries(parsed as Record<string, unknown>)) {
+        if (!value || typeof value !== "object" || Array.isArray(value)) {
+          continue;
+        }
+        const record = value as Record<string, unknown>;
+        entries[key] = {
+          ...(typeof record.sessionId === "string" && record.sessionId.trim()
+            ? { sessionId: record.sessionId.trim() }
+            : {}),
+          ...(typeof record.sessionFile === "string" && record.sessionFile.trim()
+            ? { sessionFile: record.sessionFile.trim() }
+            : {}),
+          ...(typeof record.updatedAt === "number" && Number.isFinite(record.updatedAt)
+            ? { updatedAt: record.updatedAt }
+            : {}),
+        };
+      }
+    }
+    const next = { mtimeMs: stat.mtimeMs, entries };
+    this.storeCache.set(storePath, next);
+    return next;
+  }
+}
+
+async function walkTranscriptDirs(root: string, output: string[]): Promise<void> {
+  const entries = await fs.promises.readdir(root, { withFileTypes: true }).catch(() => []);
+  for (const entry of entries) {
+    const fullPath = path.join(root, entry.name);
+    if (entry.isDirectory()) {
+      await walkTranscriptDirs(fullPath, output);
+      continue;
+    }
+    if (entry.isFile() && entry.name.endsWith(".jsonl")) {
+      output.push(normalizeSessionPath(fullPath));
+    }
+  }
+}
+
+export async function discoverTranscriptFiles(params: {
+  stateDir: string;
+  agentId?: string | undefined;
+  since?: Date | undefined;
+}): Promise<string[]> {
+  const roots: string[] = [];
+  const agentsRoot = path.join(params.stateDir, "agents");
+  if (params.agentId) {
+    roots.push(path.join(agentsRoot, sanitizeAgentId(params.agentId), "sessions"));
+  } else {
+    const agentDirs = await fs.promises.readdir(agentsRoot, { withFileTypes: true }).catch(() => []);
+    for (const entry of agentDirs) {
+      if (!entry.isDirectory()) {
+        continue;
+      }
+      roots.push(path.join(agentsRoot, entry.name, "sessions"));
+    }
+  }
+
+  const files: string[] = [];
+  for (const root of roots) {
+    await walkTranscriptDirs(root, files);
+  }
+
+  if (!params.since) {
+    return files.toSorted();
+  }
+
+  const sinceMs = params.since.getTime();
+  const filtered: string[] = [];
+  for (const filePath of files) {
+    const stat = await fs.promises.stat(filePath).catch(() => null);
+    if (stat && stat.mtimeMs >= sinceMs) {
+      filtered.push(filePath);
+    }
+  }
+  return filtered.toSorted();
+}

package/src/state-store.ts

@@ -0,0 +1,165 @@
+import fs from "node:fs";
+import path from "node:path";
+import type { Db9AuditState } from "./types.js";
+import { isNonEmptyString, isRecord, maskSecret } from "./utils.js";
+
+const STATE_VERSION = 1;
+const PLUGIN_DIR = "db9-audit";
+const STATE_FILE = "state.json";
+
+function assertString(record: Record<string, unknown>, key: string): string {
+  const value = record[key];
+  if (!isNonEmptyString(value)) {
+    throw new Error(`Invalid state field: ${key}`);
+  }
+  return value.trim();
+}
+
+function assertOptionalString(record: Record<string, unknown>, key: string): string | undefined {
+  const value = record[key];
+  if (value === undefined || value === null || value === "") {
+    return undefined;
+  }
+  if (!isNonEmptyString(value)) {
+    throw new Error(`Invalid state field: ${key}`);
+  }
+  return value.trim();
+}
+
+function assertNumber(record: Record<string, unknown>, key: string): number {
+  const value = record[key];
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    throw new Error(`Invalid state field: ${key}`);
+  }
+  return value;
+}
+
+export function parseDb9AuditState(value: unknown): Db9AuditState {
+  if (!isRecord(value)) {
+    throw new Error("State payload must be an object");
+  }
+  if (value.version !== STATE_VERSION) {
+    throw new Error(`Unsupported state version: ${String(value.version)}`);
+  }
+
+  const customer = value.customer;
+  const database = value.database;
+  const fsState = value.fs;
+  const plugin = value.plugin;
+
+  if (!isRecord(customer) || !isRecord(database) || !isRecord(fsState) || !isRecord(plugin)) {
+    throw new Error("State payload is missing required sections");
+  }
+
+  return {
+    version: STATE_VERSION,
+    apiBase: assertString(value, "apiBase"),
+    customer: {
+      id: assertString(customer, "id"),
+      token: assertString(customer, "token"),
+      ...(assertOptionalString(customer, "tokenExpiresAt")
+        ? { tokenExpiresAt: assertOptionalString(customer, "tokenExpiresAt") }
+        : {}),
+      isAnonymous: customer.isAnonymous === true,
+      ...(assertOptionalString(customer, "anonymousId")
+        ? { anonymousId: assertOptionalString(customer, "anonymousId") }
+        : {}),
+      ...(assertOptionalString(customer, "anonymousSecret")
+        ? { anonymousSecret: assertOptionalString(customer, "anonymousSecret") }
+        : {}),
+    },
+    database: {
+      id: assertString(database, "id"),
+      name: assertString(database, "name"),
+      state: assertString(database, "state"),
+      adminUser: assertString(database, "adminUser"),
+      adminPassword: assertString(database, "adminPassword"),
+      connectionString: assertString(database, "connectionString"),
+      host: assertString(database, "host"),
+      port: assertNumber(database, "port"),
+      database: assertString(database, "database"),
+      ...(assertOptionalString(database, "region")
+        ? { region: assertOptionalString(database, "region") }
+        : {}),
+    },
+    fs: {
+      wsUrl: assertString(fsState, "wsUrl"),
+      username: assertString(fsState, "username"),
+      password: assertString(fsState, "password"),
+    },
+    plugin: {
+      schema: assertString(plugin, "schema"),
+      logRoot: assertString(plugin, "logRoot"),
+      initializedAt: assertString(plugin, "initializedAt"),
+      lastBootstrapAt: assertString(plugin, "lastBootstrapAt"),
+    },
+  };
+}
+
+export function summarizeDb9AuditState(state: Db9AuditState): Record<string, unknown> {
+  return {
+    version: state.version,
+    apiBase: state.apiBase,
+    customer: {
+      id: state.customer.id,
+      token: maskSecret(state.customer.token),
+      tokenExpiresAt: state.customer.tokenExpiresAt,
+      isAnonymous: state.customer.isAnonymous,
+      anonymousId: state.customer.anonymousId,
+      anonymousSecret: state.customer.anonymousSecret ? maskSecret(state.customer.anonymousSecret) : undefined,
+    },
+    database: {
+      id: state.database.id,
+      name: state.database.name,
+      state: state.database.state,
+      adminUser: state.database.adminUser,
+      adminPassword: maskSecret(state.database.adminPassword),
+      connectionString: `${state.database.host}:${state.database.port}/${state.database.database}`,
+    },
+    fs: {
+      wsUrl: state.fs.wsUrl,
+      username: state.fs.username,
+      password: maskSecret(state.fs.password),
+    },
+    plugin: state.plugin,
+  };
+}
+
+export class Db9AuditStateStore {
+  readonly stateFilePath: string;
+
+  constructor(stateDir: string) {
+    this.stateFilePath = path.join(stateDir, "plugins", PLUGIN_DIR, STATE_FILE);
+  }
+
+  async read(): Promise<Db9AuditState | null> {
+    let raw: string;
+    try {
+      raw = await fs.promises.readFile(this.stateFilePath, "utf8");
+    } catch (error) {
+      const nodeError = error as NodeJS.ErrnoException;
+      if (nodeError.code === "ENOENT") {
+        return null;
+      }
+      throw error;
+    }
+    return parseDb9AuditState(JSON.parse(raw));
+  }
+
+  async write(state: Db9AuditState): Promise<void> {
+    const directory = path.dirname(this.stateFilePath);
+    await fs.promises.mkdir(directory, { recursive: true, mode: 0o700 });
+    await fs.promises.chmod(directory, 0o700).catch(() => undefined);
+    const tempFile = `${this.stateFilePath}.tmp`;
+    const payload = `${JSON.stringify(state, null, 2)}\n`;
+    const handle = await fs.promises.open(tempFile, "w", 0o600);
+    try {
+      await handle.writeFile(payload, "utf8");
+      await handle.sync();
+    } finally {
+      await handle.close();
+    }
+    await fs.promises.rename(tempFile, this.stateFilePath);
+    await fs.promises.chmod(this.stateFilePath, 0o600).catch(() => undefined);
+  }
+}