@karmaniverous/jeeves-server 3.0.0-0

package/src/services/eventLog.test.ts

@@ -0,0 +1,144 @@
+/**
+ * Event log service tests
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { describe, expect, it } from 'vitest';
+
+import type { EventLogEntry } from '../config/types.js';
+import { nowIso } from '../util/formatters.js';
+
+// Test helper functions (isolated from config system)
+function parseJsonl(filePath: string): EventLogEntry[] {
+  if (!fs.existsSync(filePath)) return [];
+
+  const content = fs.readFileSync(filePath, 'utf8');
+  return content
+    .split('\n')
+    .filter((line) => line.trim())
+    .map((line) => JSON.parse(line) as EventLogEntry);
+}
+
+function writeJsonl(filePath: string, entries: EventLogEntry[]): void {
+  const content = entries.map((entry) => JSON.stringify(entry)).join('\n');
+  fs.writeFileSync(filePath, content + '\n', 'utf8');
+}
+
+function purgeOldEntries(
+  entries: EventLogEntry[],
+  retentionMs: number,
+): EventLogEntry[] {
+  const cutoff = new Date(Date.now() - retentionMs);
+  return entries.filter((entry) => new Date(entry.ts) >= cutoff);
+}
+
+describe('eventLog', () => {
+  it('should parse JSONL entries correctly', () => {
+    const entry1: EventLogEntry = {
+      ts: nowIso(),
+      event: 'test-event',
+      matched: true,
+      exitCode: 0,
+      durationMs: 123,
+    };
+    const entry2: EventLogEntry = {
+      ts: nowIso(),
+      event: null,
+      matched: false,
+      bodyPreview: '{"type":"unknown"}',
+    };
+
+    const tempFile = path.join(process.cwd(), 'test-temp-log.jsonl');
+    writeJsonl(tempFile, [entry1, entry2]);
+
+    const parsed = parseJsonl(tempFile);
+
+    expect(parsed).toHaveLength(2);
+    expect(parsed[0].event).toBe('test-event');
+    expect(parsed[0].matched).toBe(true);
+    expect(parsed[1].event).toBeNull();
+    expect(parsed[1].matched).toBe(false);
+
+    // Cleanup
+    fs.unlinkSync(tempFile);
+  });
+
+  it('should purge entries older than retention period', () => {
+    const now = Date.now();
+    const retentionMs = 86400000; // 1 day
+
+    const oldEntry: EventLogEntry = {
+      ts: new Date(now - 2 * retentionMs).toISOString(),
+      event: 'old-event',
+      matched: true,
+      exitCode: 0,
+      durationMs: 100,
+    };
+
+    const recentEntry: EventLogEntry = {
+      ts: new Date(now - 0.5 * retentionMs).toISOString(),
+      event: 'recent-event',
+      matched: true,
+      exitCode: 0,
+      durationMs: 200,
+    };
+
+    const currentEntry: EventLogEntry = {
+      ts: new Date(now).toISOString(),
+      event: 'current-event',
+      matched: true,
+      exitCode: 0,
+      durationMs: 300,
+    };
+
+    const purged = purgeOldEntries(
+      [oldEntry, recentEntry, currentEntry],
+      retentionMs,
+    );
+
+    expect(purged).toHaveLength(2);
+    expect(purged[0].event).toBe('recent-event');
+    expect(purged[1].event).toBe('current-event');
+  });
+
+  it('should write JSONL with proper newline formatting', () => {
+    const entries: EventLogEntry[] = [
+      {
+        ts: nowIso(),
+        event: 'event1',
+        matched: true,
+        exitCode: 0,
+        durationMs: 100,
+      },
+      {
+        ts: nowIso(),
+        event: 'event2',
+        matched: true,
+        exitCode: 1,
+        durationMs: 200,
+      },
+    ];
+
+    const tempFile = path.join(process.cwd(), 'test-temp-write.jsonl');
+    writeJsonl(tempFile, entries);
+
+    const content = fs.readFileSync(tempFile, 'utf8');
+    const lines = content.split('\n').filter((line) => line.trim());
+
+    expect(lines).toHaveLength(2);
+    expect((JSON.parse(lines[0]) as EventLogEntry).event).toBe('event1');
+    expect((JSON.parse(lines[1]) as EventLogEntry).event).toBe('event2');
+
+    // Cleanup
+    fs.unlinkSync(tempFile);
+  });
+
+  it('should handle empty log file', () => {
+    const tempFile = path.join(process.cwd(), 'test-temp-empty.jsonl');
+    const parsed = parseJsonl(tempFile);
+
+    expect(parsed).toHaveLength(0);
+  });
+});

package/src/services/eventLog.ts

@@ -0,0 +1,68 @@
+/**
+ * Event logging service - logs all events (matched and unmatched)
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { getConfig } from '../config/index.js';
+import type { EventLogEntry } from '../config/types.js';
+import { nowIso } from '../util/formatters.js';
+
+/**
+ * Ensure directory exists
+ */
+function ensureDir(dirPath: string): void {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+/**
+ * Parse JSONL file into entries
+ */
+function parseJsonl(filePath: string): EventLogEntry[] {
+  if (!fs.existsSync(filePath)) return [];
+
+  const content = fs.readFileSync(filePath, 'utf8');
+  return content
+    .split('\n')
+    .filter((line) => line.trim())
+    .map((line) => JSON.parse(line) as EventLogEntry);
+}
+
+/**
+ * Write entries to JSONL file
+ */
+function writeJsonl(filePath: string, entries: EventLogEntry[]): void {
+  const content = entries.map((entry) => JSON.stringify(entry)).join('\n');
+  fs.writeFileSync(filePath, content + '\n', 'utf8');
+}
+
+/**
+ * Purge entries older than configured retention period
+ */
+function purgeOldEntries(entries: EventLogEntry[]): EventLogEntry[] {
+  const { eventLogPurgeMs } = getConfig();
+  const cutoff = new Date(Date.now() - eventLogPurgeMs);
+
+  return entries.filter((entry) => new Date(entry.ts) >= cutoff);
+}
+
+/**
+ * Append an event to the log and purge old entries
+ */
+export function logEvent(entry: Omit<EventLogEntry, 'ts'>): void {
+  const { eventLogPath } = getConfig();
+  ensureDir(path.dirname(eventLogPath));
+
+  // Read existing entries
+  const entries = parseJsonl(eventLogPath);
+
+  // Add new entry with timestamp
+  entries.push({ ts: nowIso(), ...entry });
+
+  // Purge old entries
+  const purgedEntries = purgeOldEntries(entries);
+
+  // Write back
+  writeJsonl(eventLogPath, purgedEntries);
+}

package/src/services/eventQueue.test.ts

@@ -0,0 +1,127 @@
+/**
+ * Event queue service tests
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { describe, expect, it } from 'vitest';
+
+import type { QueueEntry } from '../config/types.js';
+import { nowIso } from '../util/formatters.js';
+
+describe('eventQueue', () => {
+  it('should format queue entries correctly', () => {
+    const entry: QueueEntry = {
+      ts: nowIso(),
+      event: 'test-event',
+      cmd: 'node test.js',
+      body: { foo: 'bar', baz: 123 },
+      timeoutMs: 30000,
+    };
+
+    const json = JSON.stringify(entry);
+    const parsed = JSON.parse(json) as QueueEntry;
+
+    expect(parsed.event).toBe('test-event');
+    expect(parsed.cmd).toBe('node test.js');
+    expect(parsed.body).toEqual({ foo: 'bar', baz: 123 });
+    expect(parsed.timeoutMs).toBe(30000);
+    expect(parsed.ts).toMatch(/^\d{4}-\d{2}-\d{2}T/);
+  });
+
+  it('should write entries as JSONL', () => {
+    const entry1: QueueEntry = {
+      ts: nowIso(),
+      event: 'event1',
+      cmd: 'cmd1',
+      body: { a: 1 },
+      timeoutMs: 10000,
+    };
+
+    const entry2: QueueEntry = {
+      ts: nowIso(),
+      event: 'event2',
+      cmd: 'cmd2',
+      body: { b: 2 },
+      timeoutMs: 20000,
+    };
+
+    const tempFile = path.join(process.cwd(), 'test-queue.jsonl');
+    const lines = [entry1, entry2].map((e) => JSON.stringify(e)).join('\n');
+    fs.writeFileSync(tempFile, lines + '\n', 'utf8');
+
+    const content = fs.readFileSync(tempFile, 'utf8');
+    const parsed = content
+      .split('\n')
+      .filter((line) => line.trim())
+      .map((line) => JSON.parse(line) as QueueEntry);
+
+    expect(parsed).toHaveLength(2);
+    expect(parsed[0].event).toBe('event1');
+    expect(parsed[1].event).toBe('event2');
+
+    // Cleanup
+    fs.unlinkSync(tempFile);
+  });
+
+  it('should handle cursor-based reading', () => {
+    const entry1 = JSON.stringify({
+      ts: nowIso(),
+      event: 'e1',
+      cmd: 'c1',
+      body: {},
+      timeoutMs: 1000,
+    });
+    const entry2 = JSON.stringify({
+      ts: nowIso(),
+      event: 'e2',
+      cmd: 'c2',
+      body: {},
+      timeoutMs: 2000,
+    });
+
+    const tempFile = path.join(process.cwd(), 'test-cursor-queue.jsonl');
+    fs.writeFileSync(tempFile, entry1 + '\n' + entry2 + '\n', 'utf8');
+
+    const fullContent = fs.readFileSync(tempFile, 'utf8');
+    const cursorPosition = Buffer.byteLength(entry1 + '\n', 'utf8');
+
+    // Read from cursor
+    const remaining = fullContent.slice(cursorPosition);
+    const lines = remaining
+      .split('\n')
+      .filter((line) => line.trim())
+      .map((line) => JSON.parse(line) as QueueEntry);
+
+    expect(lines).toHaveLength(1);
+    expect(lines[0].event).toBe('e2');
+
+    // Cleanup
+    fs.unlinkSync(tempFile);
+  });
+
+  it('should preserve body structure through JSON serialization', () => {
+    const complexBody = {
+      string: 'test',
+      number: 123,
+      boolean: true,
+      null: null,
+      array: [1, 2, 3],
+      nested: { a: { b: { c: 'deep' } } },
+    };
+
+    const entry: QueueEntry = {
+      ts: nowIso(),
+      event: 'complex',
+      cmd: 'node test.js',
+      body: complexBody,
+      timeoutMs: 5000,
+    };
+
+    const json = JSON.stringify(entry);
+    const parsed = JSON.parse(json) as QueueEntry;
+
+    expect(parsed.body).toEqual(complexBody);
+  });
+});

package/src/services/eventQueue.ts

@@ -0,0 +1,196 @@
+/**
+ * Durable event queue for webhook event processing
+ */
+
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { getConfig } from '../config/index.js';
+import type { QueueEntry } from '../config/types.js';
+import { nowIso } from '../util/formatters.js';
+import { logEvent } from './eventLog.js';
+
+/**
+ * Ensure directory exists
+ */
+function ensureDir(dirPath: string): void {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+/**
+ * Append a JSON object as a line to the general event log (backward compatibility)
+ * This is separate from the Event Gateway event log
+ */
+export function appendEvent(event: Record<string, unknown>): void {
+  const { eventsLog } = getConfig();
+  ensureDir(path.dirname(eventsLog));
+  const line = JSON.stringify({ at: nowIso(), ...event }) + '\n';
+  fs.appendFileSync(eventsLog, line, 'utf8');
+}
+
+/**
+ * Append a queue entry to the durable queue
+ */
+export function enqueue(
+  event: string,
+  cmd: string,
+  body: Record<string, unknown>,
+  timeoutMs: number,
+): void {
+  const { eventQueuePath } = getConfig();
+  ensureDir(path.dirname(eventQueuePath));
+
+  const entry: QueueEntry = {
+    ts: nowIso(),
+    event,
+    cmd,
+    body,
+    timeoutMs,
+  };
+
+  const line = JSON.stringify(entry) + '\n';
+  fs.appendFileSync(eventQueuePath, line, 'utf8');
+}
+
+/**
+ * Read cursor position (byte offset of last processed entry)
+ */
+function readCursor(): number {
+  const { eventQueueCursorPath } = getConfig();
+  if (!fs.existsSync(eventQueueCursorPath)) return 0;
+
+  const content = fs.readFileSync(eventQueueCursorPath, 'utf8').trim();
+  return parseInt(content, 10) || 0;
+}
+
+/**
+ * Write cursor position
+ */
+function writeCursor(position: number): void {
+  const { eventQueueCursorPath } = getConfig();
+  ensureDir(path.dirname(eventQueueCursorPath));
+  fs.writeFileSync(eventQueueCursorPath, position.toString(), 'utf8');
+}
+
+/**
+ * Parse JSONL entries from file starting at cursor position
+ */
+function readEntriesFromCursor(): {
+  entries: QueueEntry[];
+  newPosition: number;
+} {
+  const { eventQueuePath } = getConfig();
+
+  if (!fs.existsSync(eventQueuePath)) {
+    return { entries: [], newPosition: 0 };
+  }
+
+  const cursor = readCursor();
+  const buf = fs.readFileSync(eventQueuePath);
+
+  // Read from cursor position (byte-based to match writeCursor)
+  const remaining = buf.subarray(cursor).toString('utf8');
+  const lines = remaining
+    .split('\n')
+    .filter((line) => line.trim())
+    .map((line) => JSON.parse(line) as QueueEntry);
+
+  const newPosition = buf.length;
+
+  return { entries: lines, newPosition };
+}
+
+/**
+ * Execute a queue entry (spawn command with body piped to stdin)
+ */
+async function executeEntry(entry: QueueEntry): Promise<{
+  exitCode: number;
+  durationMs: number;
+}> {
+  const startTime = Date.now();
+
+  return new Promise((resolve) => {
+    // Parse command and args
+    const parts = entry.cmd.split(/\s+/);
+    const command = parts[0];
+    const args = parts.slice(1);
+
+    const proc = spawn(command, args, {
+      stdio: ['pipe', 'inherit', 'inherit'],
+      shell: true,
+    });
+
+    // Pipe body as JSON to stdin
+    const bodyJson = JSON.stringify(entry.body);
+    proc.stdin.write(bodyJson);
+    proc.stdin.end();
+
+    // Setup timeout
+    const timeout = setTimeout(() => {
+      proc.kill('SIGTERM');
+    }, entry.timeoutMs);
+
+    proc.on('exit', (code) => {
+      clearTimeout(timeout);
+      const exitCode = code ?? -1;
+      const durationMs = Date.now() - startTime;
+      resolve({ exitCode, durationMs });
+    });
+
+    proc.on('error', () => {
+      clearTimeout(timeout);
+      const durationMs = Date.now() - startTime;
+      resolve({ exitCode: -1, durationMs });
+    });
+  });
+}
+
+/**
+ * Process one batch of queue entries
+ */
+async function processBatch(): Promise<void> {
+  const { entries, newPosition } = readEntriesFromCursor();
+
+  for (const entry of entries) {
+    try {
+      const { exitCode, durationMs } = await executeEntry(entry);
+
+      // Log to event log
+      logEvent({
+        event: entry.event,
+        matched: true,
+        exitCode,
+        durationMs,
+      });
+    } catch {
+      // Log error but continue (errors are ignored per spec)
+      logEvent({
+        event: entry.event,
+        matched: true,
+        exitCode: -1,
+        durationMs: 0,
+      });
+    }
+  }
+
+  // Update cursor
+  if (entries.length > 0) {
+    writeCursor(newPosition);
+  }
+}
+
+/**
+ * Start the queue drain loop
+ */
+export function startQueueProcessor(): void {
+  setInterval(
+    () => {
+      void processBatch();
+    },
+    5000, // Check every 5 seconds
+  );
+
+  // Process immediately on start
+  void processBatch();
+}