openclaw-db9-audit 0.1.0

package/src/postgres.ts

@@ -0,0 +1,430 @@
+import type { PluginLogger } from "openclaw/plugin-sdk/core";
+import { Pool } from "pg";
+import type {
+  AuditRunRecord,
+  TranscriptFileOffset,
+  TranscriptSyncBatch,
+  TranscriptSyncResult,
+} from "./types.js";
+
+function buildSchemaSql(schema: string): string[] {
+  return [
+    `create schema if not exists ${schema}`,
+    `
+      create table if not exists ${schema}.schema_version (
+        version integer primary key,
+        applied_at timestamptz not null default now()
+      )
+    `,
+    `
+      create table if not exists ${schema}.audit_sessions (
+        session_key text primary key,
+        agent_id text not null,
+        session_id text,
+        created_at timestamptz not null default now(),
+        updated_at timestamptz not null default now(),
+        last_run_id text,
+        message_count integer not null default 0,
+        status text not null default 'active',
+        meta jsonb not null default '{}'::jsonb
+      )
+    `,
+    `
+      create table if not exists ${schema}.audit_runs (
+        run_id text primary key,
+        session_key text,
+        agent_id text,
+        started_at timestamptz,
+        ended_at timestamptz,
+        success boolean,
+        duration_ms integer,
+        error text,
+        trigger text,
+        message_provider text,
+        stats jsonb not null default '{}'::jsonb,
+        created_at timestamptz not null default now(),
+        updated_at timestamptz not null default now()
+      )
+    `,
+    `
+      create index if not exists audit_runs_session_ended_idx
+        on ${schema}.audit_runs(session_key, ended_at desc)
+    `,
+    `
+      create index if not exists audit_runs_agent_ended_idx
+        on ${schema}.audit_runs(agent_id, ended_at desc)
+    `,
+    `
+      create table if not exists ${schema}.audit_messages (
+        id bigserial primary key,
+        session_key text not null,
+        agent_id text,
+        session_file text not null,
+        line_no integer not null,
+        run_id text,
+        role text,
+        source text not null,
+        tool_name text,
+        tool_call_id text,
+        content_text text,
+        content_json jsonb,
+        raw_json jsonb not null,
+        created_at timestamptz,
+        inserted_at timestamptz not null default now(),
+        unique(session_file, line_no)
+      )
+    `,
+    `
+      create index if not exists audit_messages_session_idx
+        on ${schema}.audit_messages(session_key, line_no)
+    `,
+    `
+      create index if not exists audit_messages_run_idx
+        on ${schema}.audit_messages(run_id, line_no)
+    `,
+    `
+      create table if not exists ${schema}.audit_offsets (
+        session_file text primary key,
+        agent_id text,
+        session_key text,
+        last_offset bigint not null default 0,
+        last_line_no integer not null default 0,
+        updated_at timestamptz not null default now()
+      )
+    `,
+    `insert into ${schema}.schema_version(version) values (1) on conflict (version) do nothing`,
+  ];
+}
+
+export class Db9AuditPostgres {
+  readonly schema: string;
+  private readonly pool: Pool;
+  private readonly logger?: PluginLogger | undefined;
+  private ensureSchemaPromise: Promise<void> | null = null;
+  private schemaReady = false;
+
+  constructor(params: {
+    connectionString: string;
+    schema: string;
+    logger?: PluginLogger | undefined;
+  }) {
+    this.schema = params.schema;
+    this.logger = params.logger;
+    this.pool = new Pool({
+      connectionString: params.connectionString,
+      max: 2,
+      idleTimeoutMillis: 30_000,
+      connectionTimeoutMillis: 10_000,
+      allowExitOnIdle: true,
+    });
+  }
+
+  async ping(): Promise<void> {
+    await this.pool.query("select 1");
+  }
+
+  async ensureSchema(): Promise<void> {
+    if (this.schemaReady) {
+      return;
+    }
+    if (!this.ensureSchemaPromise) {
+      this.ensureSchemaPromise = this.runEnsureSchema().catch((error) => {
+        this.ensureSchemaPromise = null;
+        throw error;
+      });
+    }
+    return await this.ensureSchemaPromise;
+  }
+
+  private async runEnsureSchema(): Promise<void> {
+    const client = await this.pool.connect();
+    try {
+      for (const statement of buildSchemaSql(this.schema)) {
+        await client.query(statement);
+      }
+      this.schemaReady = true;
+    } finally {
+      client.release();
+    }
+  }
+
+  async hasSchemaVersion(): Promise<boolean> {
+    const tableResult = await this.pool.query(
+      `
+        select 1
+        from information_schema.tables
+        where table_schema = $1
+          and table_name = 'schema_version'
+        limit 1
+      `,
+      [this.schema],
+    );
+    if ((tableResult.rowCount ?? 0) === 0) {
+      return false;
+    }
+    const result = await this.pool.query(
+      `select 1 from ${this.schema}.schema_version where version = 1 limit 1`,
+    );
+    return (result.rowCount ?? 0) > 0;
+  }
+
+  async getOffset(sessionFile: string): Promise<TranscriptFileOffset | null> {
+    await this.ensureSchema();
+    const result = await this.pool.query(
+      `
+        select session_file, agent_id, session_key, last_offset, last_line_no
+        from ${this.schema}.audit_offsets
+        where session_file = $1
+      `,
+      [sessionFile],
+    );
+    const row = result.rows[0];
+    if (!row) {
+      return null;
+    }
+    return {
+      sessionFile: String(row.session_file),
+      ...(row.agent_id ? { agentId: String(row.agent_id) } : {}),
+      ...(row.session_key ? { sessionKey: String(row.session_key) } : {}),
+      lastOffset: Number(row.last_offset),
+      lastLineNo: Number(row.last_line_no),
+    };
+  }
+
+  async syncTranscriptBatch(batch: TranscriptSyncBatch): Promise<TranscriptSyncResult> {
+    await this.ensureSchema();
+    const client = await this.pool.connect();
+    try {
+      await client.query("begin");
+
+      if (batch.resetExistingFile) {
+        const deletedCounts = await client.query(
+          `
+            select session_key, count(*)::int as count
+            from ${this.schema}.audit_messages
+            where session_file = $1
+            group by session_key
+          `,
+          [batch.sessionFile],
+        );
+
+        await client.query(`delete from ${this.schema}.audit_messages where session_file = $1`, [
+          batch.sessionFile,
+        ]);
+
+        for (const row of deletedCounts.rows) {
+          const sessionKey = row.session_key ? String(row.session_key) : "";
+          const count = Number(row.count ?? 0);
+          if (!sessionKey || count <= 0) {
+            continue;
+          }
+          await client.query(
+            `
+              update ${this.schema}.audit_sessions
+              set message_count = greatest(0, message_count - $2), updated_at = now()
+              where session_key = $1
+            `,
+            [sessionKey, count],
+          );
+        }
+      }
+
+      let insertedMessages = 0;
+      if (batch.messages.length > 0) {
+        const values: unknown[] = [];
+        const rowsSql = batch.messages
+          .map((message, index) => {
+            const base = index * 13;
+            values.push(
+              message.sessionKey,
+              message.agentId,
+              message.sessionFile,
+              message.lineNo,
+              message.runId ?? null,
+              message.role ?? null,
+              message.source,
+              message.toolName ?? null,
+              message.toolCallId ?? null,
+              message.contentText ?? null,
+              message.contentJson === undefined ? null : JSON.stringify(message.contentJson),
+              JSON.stringify(message.rawJson),
+              message.createdAt ?? null,
+            );
+            return `(
+              $${base + 1},
+              $${base + 2},
+              $${base + 3},
+              $${base + 4},
+              $${base + 5},
+              $${base + 6},
+              $${base + 7},
+              $${base + 8},
+              $${base + 9},
+              $${base + 10},
+              $${base + 11}::jsonb,
+              $${base + 12}::jsonb,
+              $${base + 13}
+            )`;
+          })
+          .join(",\n");
+
+        const result = await client.query(
+          `
+            insert into ${this.schema}.audit_messages (
+              session_key,
+              agent_id,
+              session_file,
+              line_no,
+              run_id,
+              role,
+              source,
+              tool_name,
+              tool_call_id,
+              content_text,
+              content_json,
+              raw_json,
+              created_at
+            )
+            values ${rowsSql}
+            on conflict (session_file, line_no) do nothing
+          `,
+          values,
+        );
+        insertedMessages = result.rowCount ?? 0;
+      }
+
+      await client.query(
+        `
+          insert into ${this.schema}.audit_sessions (
+            session_key,
+            agent_id,
+            session_id,
+            updated_at,
+            message_count,
+            meta
+          )
+          values ($1, $2, $3, now(), $4, $5::jsonb)
+          on conflict (session_key) do update
+          set
+            agent_id = excluded.agent_id,
+            session_id = coalesce(excluded.session_id, ${this.schema}.audit_sessions.session_id),
+            updated_at = now(),
+            message_count = ${this.schema}.audit_sessions.message_count + excluded.message_count,
+            meta = ${this.schema}.audit_sessions.meta || excluded.meta
+        `,
+        [
+          batch.sessionMeta.sessionKey,
+          batch.sessionMeta.agentId,
+          batch.sessionMeta.sessionId ?? batch.sessionHeader?.id ?? null,
+          insertedMessages,
+          JSON.stringify(batch.sessionMetaJson ?? {}),
+        ],
+      );
+
+      await client.query(
+        `
+          insert into ${this.schema}.audit_offsets (
+            session_file,
+            agent_id,
+            session_key,
+            last_offset,
+            last_line_no,
+            updated_at
+          )
+          values ($1, $2, $3, $4, $5, now())
+          on conflict (session_file) do update
+          set
+            agent_id = excluded.agent_id,
+            session_key = excluded.session_key,
+            last_offset = excluded.last_offset,
+            last_line_no = excluded.last_line_no,
+            updated_at = now()
+        `,
+        [
+          batch.sessionFile,
+          batch.sessionMeta.agentId,
+          batch.sessionMeta.sessionKey,
+          batch.lastOffset,
+          batch.lastLineNo,
+        ],
+      );
+
+      await client.query("commit");
+      return {
+        insertedMessages,
+        lastOffset: batch.lastOffset,
+        lastLineNo: batch.lastLineNo,
+      };
+    } catch (error) {
+      await client.query("rollback").catch(() => undefined);
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  async upsertRunSummary(run: AuditRunRecord): Promise<void> {
+    await this.ensureSchema();
+    await this.pool.query(
+      `
+        insert into ${this.schema}.audit_runs (
+          run_id,
+          session_key,
+          agent_id,
+          started_at,
+          ended_at,
+          success,
+          duration_ms,
+          error,
+          trigger,
+          message_provider,
+          stats,
+          updated_at
+        )
+        values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb, now())
+        on conflict (run_id) do update
+        set
+          session_key = excluded.session_key,
+          agent_id = excluded.agent_id,
+          started_at = coalesce(excluded.started_at, ${this.schema}.audit_runs.started_at),
+          ended_at = coalesce(excluded.ended_at, ${this.schema}.audit_runs.ended_at),
+          success = excluded.success,
+          duration_ms = excluded.duration_ms,
+          error = excluded.error,
+          trigger = excluded.trigger,
+          message_provider = excluded.message_provider,
+          stats = excluded.stats,
+          updated_at = now()
+      `,
+      [
+        run.runId,
+        run.sessionKey ?? null,
+        run.agentId ?? null,
+        run.startedAt ?? null,
+        run.endedAt ?? null,
+        run.success ?? null,
+        run.durationMs ?? null,
+        run.error ?? null,
+        run.trigger ?? null,
+        run.messageProvider ?? null,
+        JSON.stringify(run.stats),
+      ],
+    );
+
+    if (run.sessionKey && run.runId) {
+      await this.pool.query(
+        `
+          update ${this.schema}.audit_sessions
+          set last_run_id = $2, updated_at = now()
+          where session_key = $1
+        `,
+        [run.sessionKey, run.runId],
+      );
+    }
+  }
+
+  async close(): Promise<void> {
+    await this.pool.end();
+    this.logger?.debug?.("db9-audit: postgres pool closed");
+  }
+}

package/src/redact.ts

@@ -0,0 +1,37 @@
+import { isRecord, truncateUtf8Bytes } from "./utils.js";
+import type { Db9AuditRedactConfig } from "./types.js";
+
+const SENSITIVE_KEY_PARTS = ["token", "secret", "authorization", "cookie", "password"];
+
+function isSensitiveKey(key: string): boolean {
+  const lowered = key.trim().toLowerCase();
+  return SENSITIVE_KEY_PARTS.some((part) => lowered.includes(part));
+}
+
+function redactString(value: string, config: Db9AuditRedactConfig): string {
+  return truncateUtf8Bytes(value, config.maxFieldBytes);
+}
+
+export function redactUnknown(value: unknown, config: Db9AuditRedactConfig): unknown {
+  if (!config.enabled) {
+    return value;
+  }
+  if (typeof value === "string") {
+    return redactString(value, config);
+  }
+  if (Array.isArray(value)) {
+    return value.map((entry) => redactUnknown(entry, config));
+  }
+  if (!isRecord(value)) {
+    return value;
+  }
+  const redacted: Record<string, unknown> = {};
+  for (const [key, entry] of Object.entries(value)) {
+    if (isSensitiveKey(key)) {
+      redacted[key] = "[REDACTED]";
+      continue;
+    }
+    redacted[key] = redactUnknown(entry, config);
+  }
+  return redacted;
+}

package/src/run-tracker.ts

@@ -0,0 +1,158 @@
+import type {
+  AgentEventPayload,
+  PluginHookAgentContext,
+  PluginHookAgentEndEvent,
+} from "openclaw/plugin-sdk/core";
+import type { AuditRunRecord } from "./types.js";
+import { coerceInteger, resolveAgentIdFromSessionKey, toIsoString } from "./utils.js";
+
+type RunState = {
+  runId: string;
+  sessionKey?: string | undefined;
+  agentId: string;
+  startedAt?: number | undefined;
+  endedAt?: number | undefined;
+  lastSeenAt: number;
+  lastSeq: number;
+  terminalStream?: string | undefined;
+  terminalPhase?: string | undefined;
+  terminalError?: string | undefined;
+  finalizedAt?: number | undefined;
+};
+
+function extractPhase(event: AgentEventPayload): string | undefined {
+  const phase = event.data.phase;
+  return typeof phase === "string" && phase.trim() ? phase : undefined;
+}
+
+function extractEventError(event: AgentEventPayload): string | undefined {
+  const error = event.data.error;
+  return typeof error === "string" && error.trim() ? error.trim() : undefined;
+}
+
+export class Db9AuditRunTracker {
+  private readonly runs = new Map<string, RunState>();
+
+  recordAgentEvent(event: AgentEventPayload): void {
+    this.evictOldRuns();
+    const existing = this.runs.get(event.runId);
+    const sessionKey =
+      typeof event.sessionKey === "string" && event.sessionKey.trim() ? event.sessionKey : existing?.sessionKey;
+    const phase = extractPhase(event);
+    const startedAtValue = coerceInteger(event.data.startedAt);
+    const endedAtValue = coerceInteger(event.data.endedAt);
+    const next: RunState = existing ?? {
+      runId: event.runId,
+      agentId: resolveAgentIdFromSessionKey(sessionKey),
+      lastSeenAt: event.ts,
+      lastSeq: event.seq,
+    };
+
+    next.lastSeenAt = event.ts;
+    next.lastSeq = event.seq;
+    if (sessionKey) {
+      next.sessionKey = sessionKey;
+      next.agentId = resolveAgentIdFromSessionKey(sessionKey);
+    }
+    if (phase === "start" && next.startedAt === undefined) {
+      next.startedAt = startedAtValue ?? event.ts;
+    }
+    if (phase === "end" || phase === "error") {
+      next.endedAt = endedAtValue ?? event.ts;
+      next.terminalPhase = phase;
+      next.terminalStream = event.stream;
+      next.terminalError = extractEventError(event);
+    }
+    this.runs.set(event.runId, next);
+  }
+
+  resolveAgentIdForRun(runId: string, sessionKey?: string | undefined): string {
+    const tracked = this.runs.get(runId);
+    if (sessionKey?.trim()) {
+      return resolveAgentIdFromSessionKey(sessionKey);
+    }
+    return tracked?.agentId ?? "main";
+  }
+
+  buildRunSummaryFromHook(
+    event: PluginHookAgentEndEvent,
+    ctx: PluginHookAgentContext,
+  ): AuditRunRecord | null {
+    this.evictOldRuns();
+    const match = this.matchRun(ctx);
+    if (!match) {
+      return null;
+    }
+    const endedAt = match.endedAt ?? Date.now();
+    match.finalizedAt = Date.now();
+
+    return {
+      runId: match.runId,
+      ...(match.sessionKey ? { sessionKey: match.sessionKey } : {}),
+      agentId: match.agentId,
+      ...(match.startedAt ? { startedAt: toIsoString(match.startedAt) } : {}),
+      endedAt: toIsoString(endedAt),
+      success: event.success,
+      ...(typeof event.durationMs === "number" ? { durationMs: event.durationMs } : {}),
+      ...(event.error ? { error: event.error } : match.terminalError ? { error: match.terminalError } : {}),
+      ...(ctx.trigger ? { trigger: ctx.trigger } : {}),
+      ...(ctx.messageProvider ? { messageProvider: ctx.messageProvider } : {}),
+      stats: {
+        matchedBy: this.describeMatch(match, ctx),
+        messageCount: Array.isArray(event.messages) ? event.messages.length : 0,
+        lastSeq: match.lastSeq,
+        terminalStream: match.terminalStream,
+        terminalPhase: match.terminalPhase,
+      },
+    };
+  }
+
+  private describeMatch(match: RunState, ctx: PluginHookAgentContext): string {
+    if (ctx.sessionKey?.trim() && match.sessionKey === ctx.sessionKey) {
+      return "sessionKey";
+    }
+    if (ctx.agentId?.trim() && match.agentId === ctx.agentId.trim().toLowerCase()) {
+      return "agentId";
+    }
+    return "fallback";
+  }
+
+  private matchRun(ctx: PluginHookAgentContext): RunState | null {
+    const candidates = [...this.runs.values()].filter((run) => run.finalizedAt === undefined);
+    if (candidates.length === 0) {
+      return null;
+    }
+
+    const sessionKey = ctx.sessionKey?.trim();
+    if (sessionKey) {
+      const matched = candidates
+        .filter((run) => run.sessionKey === sessionKey)
+        .toSorted((a, b) => b.lastSeenAt - a.lastSeenAt);
+      if (matched[0]) {
+        return matched[0];
+      }
+    }
+
+    const agentId = ctx.agentId?.trim().toLowerCase();
+    if (agentId) {
+      const matched = candidates
+        .filter((run) => run.agentId === agentId)
+        .toSorted((a, b) => b.lastSeenAt - a.lastSeenAt);
+      if (matched[0]) {
+        return matched[0];
+      }
+    }
+
+    return candidates.toSorted((a, b) => b.lastSeenAt - a.lastSeenAt)[0] ?? null;
+  }
+
+  private evictOldRuns(): void {
+    const cutoff = Date.now() - 6 * 60 * 60 * 1_000;
+    for (const [runId, run] of this.runs.entries()) {
+      const lastRelevantAt = run.finalizedAt ?? run.lastSeenAt;
+      if (lastRelevantAt < cutoff) {
+        this.runs.delete(runId);
+      }
+    }
+  }
+}