openclaw-db9-audit 0.1.0

package/src/transcript-sync.ts

@@ -0,0 +1,514 @@
+import fs from "node:fs";
+import type { PluginLogger } from "openclaw/plugin-sdk/core";
+import { redactUnknown } from "./redact.js";
+import { Db9AuditSessionLookup } from "./session-store.js";
+import type {
+  Db9AuditDiagnosticEvent,
+  Db9AuditRedactConfig,
+  TranscriptFileOffset,
+  TranscriptMessageRecord,
+  TranscriptSessionHeader,
+  TranscriptSyncBatch,
+  TranscriptSyncResult,
+  TranscriptSyncSessionMeta,
+} from "./types.js";
+import {
+  coerceIsoTimestamp,
+  extractErrorMessage,
+  isRecord,
+  normalizeSessionPath,
+  safeJsonParse,
+} from "./utils.js";
+
+export type TranscriptSyncStore = {
+  getOffset: (sessionFile: string) => Promise<TranscriptFileOffset | null>;
+  syncTranscriptBatch: (batch: TranscriptSyncBatch) => Promise<TranscriptSyncResult>;
+};
+
+function resolveSessionKey(params: { sessionKey?: string | undefined; sessionFile: string }): {
+  sessionKey: string;
+  derived: boolean;
+} {
+  const sessionKey = params.sessionKey?.trim();
+  if (sessionKey) {
+    return { sessionKey, derived: false };
+  }
+  return { sessionKey: `file:${normalizeSessionPath(params.sessionFile)}`, derived: true };
+}
+
+function normalizeSource(role: string | undefined, record: Record<string, unknown>): string {
+  if (role) {
+    return role;
+  }
+  const type = typeof record.type === "string" ? record.type.trim() : "";
+  return type ? "meta" : "unknown";
+}
+
+function extractTextFromContent(value: unknown): string | undefined {
+  if (typeof value === "string") {
+    return value.trim() || undefined;
+  }
+  if (Array.isArray(value)) {
+    const texts = value
+      .map((entry) => {
+        if (typeof entry === "string") {
+          return entry.trim();
+        }
+        if (!isRecord(entry)) {
+          return "";
+        }
+        if (typeof entry.text === "string") {
+          return entry.text.trim();
+        }
+        if (typeof entry.content === "string") {
+          return entry.content.trim();
+        }
+        if (typeof entry.output_text === "string") {
+          return entry.output_text.trim();
+        }
+        if (typeof entry.input_text === "string") {
+          return entry.input_text.trim();
+        }
+        return "";
+      })
+      .filter(Boolean);
+    return texts.length > 0 ? texts.join("\n") : undefined;
+  }
+  if (isRecord(value)) {
+    if (typeof value.text === "string" && value.text.trim()) {
+      return value.text.trim();
+    }
+    if (typeof value.content === "string" && value.content.trim()) {
+      return value.content.trim();
+    }
+  }
+  return undefined;
+}
+
+function extractToolName(record: Record<string, unknown>, message: Record<string, unknown> | undefined): string | undefined {
+  const candidates = [
+    message?.toolName,
+    message?.tool_name,
+    message?.name,
+    record.toolName,
+    record.tool_name,
+    record.name,
+  ];
+  for (const value of candidates) {
+    if (typeof value === "string" && value.trim()) {
+      return value.trim();
+    }
+  }
+  const content = message?.content;
+  if (Array.isArray(content)) {
+    for (const entry of content) {
+      if (!isRecord(entry)) {
+        continue;
+      }
+      const candidate = entry.toolName ?? entry.tool_name ?? entry.name;
+      if (typeof candidate === "string" && candidate.trim()) {
+        return candidate.trim();
+      }
+    }
+  }
+  return undefined;
+}
+
+function extractToolCallId(record: Record<string, unknown>, message: Record<string, unknown> | undefined): string | undefined {
+  const candidates = [
+    message?.toolCallId,
+    message?.tool_call_id,
+    message?.callId,
+    record.toolCallId,
+    record.tool_call_id,
+    record.callId,
+  ];
+  for (const value of candidates) {
+    if (typeof value === "string" && value.trim()) {
+      return value.trim();
+    }
+  }
+  return undefined;
+}
+
+function extractRunId(record: Record<string, unknown>, message: Record<string, unknown> | undefined): string | undefined {
+  const candidates = [record.runId, record.run_id, message?.runId, message?.run_id];
+  for (const value of candidates) {
+    if (typeof value === "string" && value.trim()) {
+      return value.trim();
+    }
+  }
+  return undefined;
+}
+
+function extractCreatedAt(record: Record<string, unknown>, message: Record<string, unknown> | undefined): string | undefined {
+  return (
+    coerceIsoTimestamp(record.timestamp) ??
+    coerceIsoTimestamp(message?.timestamp) ??
+    coerceIsoTimestamp(record.createdAt) ??
+    coerceIsoTimestamp(record.created_at)
+  );
+}
+
+export type ParsedTranscriptLine =
+  | { kind: "header"; header: TranscriptSessionHeader }
+  | { kind: "message"; message: TranscriptMessageRecord }
+  | { kind: "skip" };
+
+export function parseTranscriptRecord(params: {
+  record: Record<string, unknown>;
+  lineNo: number;
+  sessionFile: string;
+  sessionMeta: TranscriptSyncSessionMeta;
+  redactConfig: Db9AuditRedactConfig;
+}): ParsedTranscriptLine {
+  if (params.record.type === "session") {
+    return {
+      kind: "header",
+      header: {
+        type: "session",
+        ...(typeof params.record.version === "string" || typeof params.record.version === "number"
+          ? { version: params.record.version }
+          : {}),
+        ...(typeof params.record.id === "string" && params.record.id.trim()
+          ? { id: params.record.id.trim() }
+          : {}),
+        ...(typeof params.record.timestamp === "string" && params.record.timestamp.trim()
+          ? { timestamp: params.record.timestamp.trim() }
+          : {}),
+        ...(typeof params.record.cwd === "string" && params.record.cwd.trim()
+          ? { cwd: params.record.cwd.trim() }
+          : {}),
+      },
+    };
+  }
+
+  const redactedRecord = redactUnknown(params.record, params.redactConfig);
+  if (!isRecord(redactedRecord)) {
+    return { kind: "skip" };
+  }
+
+  const rawMessage = isRecord(params.record.message) ? params.record.message : undefined;
+  const redactedMessage = isRecord(redactedRecord.message) ? redactedRecord.message : undefined;
+  const role = typeof rawMessage?.role === "string" && rawMessage.role.trim() ? rawMessage.role.trim() : undefined;
+  const contentValue = redactedMessage?.content;
+  const contentText = extractTextFromContent(contentValue);
+  const source = normalizeSource(role, params.record);
+  const runId = extractRunId(params.record, rawMessage);
+  const toolName = extractToolName(params.record, rawMessage);
+  const toolCallId = extractToolCallId(params.record, rawMessage);
+  const createdAt = extractCreatedAt(params.record, rawMessage);
+
+  return {
+    kind: "message",
+    message: {
+      sessionKey: params.sessionMeta.sessionKey,
+      agentId: params.sessionMeta.agentId,
+      sessionFile: params.sessionFile,
+      lineNo: params.lineNo,
+      ...(runId ? { runId } : {}),
+      ...(role ? { role } : {}),
+      source,
+      ...(toolName ? { toolName } : {}),
+      ...(toolCallId ? { toolCallId } : {}),
+      ...(contentText ? { contentText } : {}),
+      ...(contentValue !== undefined && typeof contentValue !== "string"
+        ? { contentJson: contentValue }
+        : {}),
+      rawJson: redactedRecord,
+      ...(createdAt ? { createdAt } : {}),
+    },
+  };
+}
+
+type JsonlLine = {
+  line: string;
+  bytes: number;
+};
+
+async function* readJsonlLines(params: {
+  sessionFile: string;
+  offset: number;
+  chunkSizeBytes?: number | undefined;
+}): AsyncGenerator<JsonlLine> {
+  const handle = await fs.promises.open(params.sessionFile, "r");
+  try {
+    let position = params.offset;
+    const chunkSize = Math.max(1024, params.chunkSizeBytes ?? 64 * 1024);
+    const buffer = Buffer.alloc(chunkSize);
+
+    // Buffers for the current line when it spans multiple reads.
+    // We only materialize a full line buffer once a trailing \n is observed.
+    let carryParts: Buffer[] = [];
+    let carryLength = 0;
+
+    while (true) {
+      const { bytesRead } = await handle.read(buffer, 0, buffer.length, position);
+      if (bytesRead <= 0) {
+        return;
+      }
+      position += bytesRead;
+      const chunk = buffer.subarray(0, bytesRead);
+
+      let start = 0;
+      for (let index = 0; index < chunk.length; index += 1) {
+        if (chunk[index] !== 0x0a) {
+          continue;
+        }
+
+        const linePart = chunk.subarray(start, index);
+        const bytes = carryLength + linePart.length + 1;
+
+        let lineBuffer: Buffer;
+        if (carryLength > 0) {
+          carryParts.push(linePart);
+          carryLength += linePart.length;
+          lineBuffer = Buffer.concat(carryParts, carryLength);
+          carryParts = [];
+          carryLength = 0;
+        } else {
+          lineBuffer = linePart;
+        }
+
+        let end = lineBuffer.length;
+        if (end > 0 && lineBuffer[end - 1] === 0x0d) {
+          end -= 1;
+        }
+        const line = end > 0 ? lineBuffer.subarray(0, end).toString("utf8") : "";
+        yield { line, bytes };
+
+        start = index + 1;
+      }
+
+      if (start < chunk.length) {
+        // Copy to avoid holding onto the shared read buffer across iterations.
+        const tail = Buffer.from(chunk.subarray(start));
+        carryParts.push(tail);
+        carryLength += tail.length;
+      }
+    }
+  } finally {
+    await handle.close();
+  }
+}
+
+export class Db9AuditTranscriptSync {
+  private readonly sessionLookup: Db9AuditSessionLookup;
+  private readonly redactConfig: Db9AuditRedactConfig;
+  private readonly batchSize: number;
+  private readonly logger?: PluginLogger | undefined;
+  private readonly getStore: () => Promise<TranscriptSyncStore | null>;
+  private readonly onDiagnostic?: ((event: Db9AuditDiagnosticEvent) => Promise<void> | void) | undefined;
+  private readonly pending = new Set<string>();
+  private flushPromise: Promise<{ flushedSessions: number; insertedMessages: number }> | null = null;
+
+  constructor(params: {
+    sessionLookup: Db9AuditSessionLookup;
+    redactConfig: Db9AuditRedactConfig;
+    batchSize: number;
+    logger?: PluginLogger | undefined;
+    getStore: () => Promise<TranscriptSyncStore | null>;
+    onDiagnostic?: ((event: Db9AuditDiagnosticEvent) => Promise<void> | void) | undefined;
+  }) {
+    this.sessionLookup = params.sessionLookup;
+    this.redactConfig = params.redactConfig;
+    this.batchSize = params.batchSize;
+    this.logger = params.logger;
+    this.getStore = params.getStore;
+    this.onDiagnostic = params.onDiagnostic;
+  }
+
+  enqueue(sessionFile: string): void {
+    this.pending.add(normalizeSessionPath(sessionFile));
+  }
+
+  async flushPending(): Promise<{ flushedSessions: number; insertedMessages: number }> {
+    if (this.flushPromise) {
+      return await this.flushPromise;
+    }
+    this.flushPromise = this.doFlushPending();
+    try {
+      return await this.flushPromise;
+    } finally {
+      this.flushPromise = null;
+    }
+  }
+
+  private async doFlushPending(): Promise<{ flushedSessions: number; insertedMessages: number }> {
+    const store = await this.getStore();
+    if (!store || this.pending.size === 0) {
+      return { flushedSessions: 0, insertedMessages: 0 };
+    }
+
+    const sessionFiles = [...this.pending];
+    this.pending.clear();
+    let flushedSessions = 0;
+    let insertedMessages = 0;
+
+    for (const sessionFile of sessionFiles) {
+      try {
+        const result = await this.syncSessionFile(sessionFile, { store });
+        flushedSessions += 1;
+        insertedMessages += result.insertedMessages;
+      } catch (error) {
+        this.pending.add(sessionFile);
+        this.logger?.warn(`db9-audit: transcript sync failed for ${sessionFile}: ${extractErrorMessage(error) ?? String(error)}`);
+      }
+    }
+
+    return { flushedSessions, insertedMessages };
+  }
+
+  async syncSessionFile(
+    sessionFile: string,
+    options: {
+      resetOffsets?: boolean | undefined;
+      store?: TranscriptSyncStore | undefined;
+    } = {},
+  ): Promise<TranscriptSyncResult> {
+    const normalizedSessionFile = normalizeSessionPath(sessionFile);
+    const store = options.store ?? (await this.getStore());
+    if (!store) {
+      throw new Error("Postgres store is not available");
+    }
+
+    const lookup = await this.sessionLookup.lookup(normalizedSessionFile);
+    const sessionKeyInfo = resolveSessionKey({
+      sessionKey: lookup.sessionKey,
+      sessionFile: normalizedSessionFile,
+    });
+    const sessionMeta: TranscriptSyncSessionMeta = {
+      sessionKey: sessionKeyInfo.sessionKey,
+      agentId: lookup.agentId,
+      ...(lookup.sessionId ? { sessionId: lookup.sessionId } : {}),
+    };
+
+    let offset = options.resetOffsets ? null : await store.getOffset(normalizedSessionFile);
+    const stat = await fs.promises.stat(normalizedSessionFile);
+    let resetExistingFile = Boolean(options.resetOffsets);
+    if (!resetExistingFile && offset && stat.size < offset.lastOffset) {
+      resetExistingFile = true;
+      await this.onDiagnostic?.({
+        stream: "error",
+        sessionKey: sessionMeta.sessionKey,
+        agentId: sessionMeta.agentId,
+        data: {
+          type: "transcript_truncated",
+          sessionFile: normalizedSessionFile,
+          previousOffset: offset.lastOffset,
+          currentSize: stat.size,
+        },
+      });
+    }
+
+    if (resetExistingFile) {
+      offset = null;
+    }
+
+    const startOffset = offset?.lastOffset ?? 0;
+
+    let currentOffset = resetExistingFile ? 0 : startOffset;
+    let lineNo = resetExistingFile ? 0 : (offset?.lastLineNo ?? 0);
+    let insertedMessages = 0;
+    let sessionHeader: TranscriptSessionHeader | undefined;
+    let resetBeforeInsert = resetExistingFile;
+    let pendingMessages: TranscriptMessageRecord[] = [];
+    let lastFlushedOffset = currentOffset;
+    let lastFlushedLineNo = lineNo;
+
+    const flushBatch = async (): Promise<void> => {
+      const batch: TranscriptSyncBatch = {
+        sessionFile: normalizedSessionFile,
+        sessionMeta: {
+          ...sessionMeta,
+          ...(sessionHeader?.id && !sessionMeta.sessionId ? { sessionId: sessionHeader.id } : {}),
+        },
+        messages: pendingMessages,
+        lastOffset: currentOffset,
+        lastLineNo: lineNo,
+        resetExistingFile: resetBeforeInsert,
+        ...(sessionHeader ? { sessionHeader } : {}),
+        sessionMetaJson: {
+          sessionFile: normalizedSessionFile,
+          ...(lookup.sessionStorePath ? { sessionStorePath: lookup.sessionStorePath } : {}),
+          sessionKeyDerived: sessionKeyInfo.derived,
+          batchSizeHint: this.batchSize,
+        },
+      };
+
+      const result = await store.syncTranscriptBatch(batch);
+      insertedMessages += result.insertedMessages;
+      resetBeforeInsert = false;
+      pendingMessages = [];
+      lastFlushedOffset = currentOffset;
+      lastFlushedLineNo = lineNo;
+    };
+
+    for await (const entry of readJsonlLines({
+      sessionFile: normalizedSessionFile,
+      offset: currentOffset,
+    })) {
+      currentOffset += entry.bytes;
+      lineNo += 1;
+
+      const trimmed = entry.line.trim();
+      if (!trimmed) {
+        continue;
+      }
+
+      let parsed: unknown;
+      try {
+        parsed = safeJsonParse(trimmed);
+      } catch (error) {
+        await this.onDiagnostic?.({
+          stream: "error",
+          sessionKey: sessionMeta.sessionKey,
+          agentId: sessionMeta.agentId,
+          data: {
+            type: "transcript_parse_error",
+            sessionFile: normalizedSessionFile,
+            lineNo,
+            error: extractErrorMessage(error) ?? String(error),
+          },
+        });
+        continue;
+      }
+      if (!isRecord(parsed)) {
+        continue;
+      }
+
+      const parsedLine = parseTranscriptRecord({
+        record: parsed,
+        lineNo,
+        sessionFile: normalizedSessionFile,
+        sessionMeta,
+        redactConfig: this.redactConfig,
+      });
+      if (parsedLine.kind === "header") {
+        sessionHeader = parsedLine.header;
+        continue;
+      }
+      if (parsedLine.kind === "message") {
+        pendingMessages.push(parsedLine.message);
+        if (pendingMessages.length >= this.batchSize) {
+          await flushBatch();
+        }
+      }
+    }
+
+    if (
+      resetBeforeInsert ||
+      pendingMessages.length > 0 ||
+      currentOffset !== lastFlushedOffset ||
+      lineNo !== lastFlushedLineNo
+    ) {
+      await flushBatch();
+    }
+
+    return {
+      insertedMessages,
+      lastOffset: currentOffset,
+      lastLineNo: lineNo,
+    };
+  }
+}