@karmaniverous/jeeves-server 3.0.0-0

package/dist/src/services/diagramCache.js

@@ -0,0 +1,104 @@
+/**
+ * Content-addressed diagram render cache.
+ *
+ * Cache key = sha256(type + '\\0' + source). Cached artifacts are SVG strings
+ * stored as files in a single directory. No timestamp comparison needed:
+ * if the source changes, the hash changes → automatic cache miss.
+ */
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+let cacheDir = null;
+/**
+ * Initialize the cache directory. Call once at startup.
+ * Defaults to `.diagram-cache` in the current working directory.
+ */
+export function initDiagramCache(dir) {
+    cacheDir = dir ?? path.resolve('.diagram-cache');
+    fs.mkdirSync(cacheDir, { recursive: true });
+}
+/**
+ * Get the cache directory path (for diagnostics / health endpoint).
+ */
+export function getDiagramCacheDir() {
+    return cacheDir;
+}
+/**
+ * Compute the cache key for a diagram.
+ */
+function cacheKey(type, source) {
+    return crypto.createHash('sha256').update(`${type}\0${source}`).digest('hex');
+}
+/**
+ * Look up a cached diagram. Returns the content as a string or null on miss.
+ * @param format - Output format extension (e.g. 'svg', 'png', 'pdf'). Defaults to 'svg'.
+ */
+export function getCachedDiagram(type, source, format = 'svg') {
+    if (!cacheDir)
+        return null;
+    const file = path.join(cacheDir, `${cacheKey(type, source)}.${format}`);
+    try {
+        return fs.readFileSync(file, 'utf8');
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Look up a cached diagram as a Buffer (for binary formats like PNG/PDF).
+ * @param format - Output format extension (e.g. 'png', 'pdf').
+ */
+export function getCachedDiagramBuffer(type, source, format) {
+    if (!cacheDir)
+        return null;
+    const file = path.join(cacheDir, `${cacheKey(type, source)}.${format}`);
+    try {
+        return fs.readFileSync(file);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Store a rendered diagram in the cache (string content).
+ * @param format - Output format extension. Defaults to 'svg'.
+ */
+export function cacheDiagram(type, source, content, format = 'svg') {
+    if (!cacheDir)
+        return;
+    const file = path.join(cacheDir, `${cacheKey(type, source)}.${format}`);
+    try {
+        fs.writeFileSync(file, content, 'utf8');
+    }
+    catch (err) {
+        console.error('[diagramCache] write failed:', err.message);
+    }
+}
+/**
+ * Store a rendered diagram in the cache (binary content).
+ * @param format - Output format extension (e.g. 'png', 'pdf').
+ */
+export function cacheDiagramBuffer(type, source, buffer, format) {
+    if (!cacheDir)
+        return;
+    const file = path.join(cacheDir, `${cacheKey(type, source)}.${format}`);
+    try {
+        fs.writeFileSync(file, buffer);
+    }
+    catch (err) {
+        console.error('[diagramCache] write failed:', err.message);
+    }
+}
+/**
+ * Get a cached diagram or render it and cache the result.
+ * Eliminates the repeated getCachedDiagram → render → cacheDiagram pattern.
+ */
+export async function getOrRenderDiagram(type, source, renderFn, format = 'svg') {
+    const cached = getCachedDiagram(type, source, format);
+    if (cached)
+        return cached;
+    const result = await renderFn();
+    if (result)
+        cacheDiagram(type, source, result, format);
+    return result;
+}

package/dist/src/services/embeddedDiagrams.js

@@ -0,0 +1,136 @@
+/**
+ * Embedded diagram support for Mermaid and PlantUML code blocks in markdown.
+ *
+ * Strategy: parseMarkdown() calls registerDiagram() which stores the source
+ * and returns a placeholder div. The client lazily fetches rendered SVGs
+ * via GET /api/diagram/:type/:hash.svg, which renders on-demand and caches.
+ *
+ * For PDF/DOCX export (server-side rendering), renderEmbeddedDiagrams()
+ * replaces placeholders with inline SVGs synchronously.
+ */
+import crypto from 'node:crypto';
+import { getOrRenderDiagram } from './diagramCache.js';
+import { renderMermaidFromSource } from './mermaid.js';
+import { renderPlantUmlFromSource } from './plantuml.js';
+/** Placeholder format used by the markdown renderer */
+const PLACEHOLDER_RE = /<!--DIAGRAM:(mermaid|plantuml):([a-f0-9]{64})-->/g;
+/**
+ * In-flight diagram sources, keyed by content hash.
+ * Entries are cleaned up after a TTL to prevent unbounded growth.
+ */
+const diagramSources = new Map();
+/** TTL for source map entries (10 minutes) */
+const SOURCE_TTL_MS = 10 * 60 * 1000;
+/** Periodic cleanup interval */
+let cleanupInterval = null;
+function startCleanup() {
+    if (cleanupInterval)
+        return;
+    cleanupInterval = setInterval(() => {
+        const now = Date.now();
+        for (const [hash, entry] of diagramSources) {
+            if (now - entry.createdAt > SOURCE_TTL_MS) {
+                diagramSources.delete(hash);
+            }
+        }
+    }, 60_000);
+    // Don't keep process alive just for cleanup
+    cleanupInterval.unref();
+}
+/**
+ * Compute content hash matching the cache key format.
+ */
+export function diagramHash(type, source) {
+    return crypto.createHash('sha256').update(`${type}\0${source}`).digest('hex');
+}
+/** Module-level context directory for the current markdown parse. */
+let currentContextDir;
+/**
+ * Set the context directory for diagram registration.
+ * Call before parseMarkdown() so registered diagrams know their !include context.
+ */
+export function setDiagramContext(contextDir) {
+    currentContextDir = contextDir;
+}
+/**
+ * Register a diagram source and return a placeholder HTML div.
+ * Called synchronously from the marked renderer.
+ */
+export function registerDiagram(type, source) {
+    const hash = diagramHash(type, source);
+    diagramSources.set(hash, {
+        source,
+        contextDir: currentContextDir,
+        createdAt: Date.now(),
+    });
+    startCleanup();
+    // Emit a client-side placeholder that the LazyDiagram component will hydrate
+    return `<div class="embedded-diagram-lazy" data-diagram-type="${type}" data-diagram-hash="${hash}"><!--DIAGRAM:${type}:${hash}--></div>\n`;
+}
+/**
+ * Look up a registered diagram source by hash.
+ * Used by the /api/diagram endpoint for on-demand rendering.
+ */
+export function getDiagramSource(hash) {
+    const entry = diagramSources.get(hash);
+    if (!entry)
+        return null;
+    return { type: '', source: entry.source, contextDir: entry.contextDir };
+}
+/**
+ * Render a diagram to SVG (with cache). Used by the /api/diagram endpoint.
+ */
+export async function renderDiagramToSvg(type, source, 
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+_contextDir) {
+    try {
+        return await getOrRenderDiagram(type, source, () => {
+            if (type === 'mermaid')
+                return renderMermaidFromSource(source);
+            if (type === 'plantuml')
+                return renderPlantUmlFromSource(source);
+            return null;
+        });
+    }
+    catch (err) {
+        console.error(`[embeddedDiagrams] ${type} render failed:`, err.message);
+        return null;
+    }
+}
+/**
+ * Replace all diagram placeholders in rendered HTML with inline SVGs.
+ * Used for PDF/DOCX export where client-side lazy loading isn't available.
+ */
+export async function renderEmbeddedDiagrams(html, contextDir) {
+    const matches = [...html.matchAll(PLACEHOLDER_RE)];
+    if (matches.length === 0)
+        return html;
+    let result = html;
+    for (const match of matches) {
+        const [placeholder, type, hash] = match;
+        const entry = diagramSources.get(hash);
+        if (!entry)
+            continue;
+        const source = entry.source;
+        if (!source)
+            continue;
+        const svg = await renderDiagramToSvg(type, source, contextDir ?? entry.contextDir);
+        if (svg) {
+            const wrapped = `<div class="embedded-diagram-rendered" data-type="${type}">${svg}</div>`;
+            result = result.replace(placeholder, wrapped);
+        }
+        else {
+            const escaped = escapeHtml(source);
+            const errorBlock = `<div class="embedded-diagram-error" data-type="${type}"><div class="diagram-error-label">${type} render failed</div><pre class="hljs"><code>${escaped}</code></pre></div>`;
+            result = result.replace(placeholder, errorBlock);
+        }
+    }
+    return result;
+}
+function escapeHtml(str) {
+    return str
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;');
+}

package/dist/src/services/eventLog.js

@@ -0,0 +1,55 @@
+/**
+ * Event logging service - logs all events (matched and unmatched)
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { getConfig } from '../config/index.js';
+import { nowIso } from '../util/formatters.js';
+/**
+ * Ensure directory exists
+ */
+function ensureDir(dirPath) {
+    fs.mkdirSync(dirPath, { recursive: true });
+}
+/**
+ * Parse JSONL file into entries
+ */
+function parseJsonl(filePath) {
+    if (!fs.existsSync(filePath))
+        return [];
+    const content = fs.readFileSync(filePath, 'utf8');
+    return content
+        .split('\n')
+        .filter((line) => line.trim())
+        .map((line) => JSON.parse(line));
+}
+/**
+ * Write entries to JSONL file
+ */
+function writeJsonl(filePath, entries) {
+    const content = entries.map((entry) => JSON.stringify(entry)).join('\n');
+    fs.writeFileSync(filePath, content + '\n', 'utf8');
+}
+/**
+ * Purge entries older than configured retention period
+ */
+function purgeOldEntries(entries) {
+    const { eventLogPurgeMs } = getConfig();
+    const cutoff = new Date(Date.now() - eventLogPurgeMs);
+    return entries.filter((entry) => new Date(entry.ts) >= cutoff);
+}
+/**
+ * Append an event to the log and purge old entries
+ */
+export function logEvent(entry) {
+    const { eventLogPath } = getConfig();
+    ensureDir(path.dirname(eventLogPath));
+    // Read existing entries
+    const entries = parseJsonl(eventLogPath);
+    // Add new entry with timestamp
+    entries.push({ ts: nowIso(), ...entry });
+    // Purge old entries
+    const purgedEntries = purgeOldEntries(entries);
+    // Write back
+    writeJsonl(eventLogPath, purgedEntries);
+}

package/dist/src/services/eventLog.test.js

@@ -0,0 +1,113 @@
+/**
+ * Event log service tests
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { nowIso } from '../util/formatters.js';
+// Test helper functions (isolated from config system)
+function parseJsonl(filePath) {
+    if (!fs.existsSync(filePath))
+        return [];
+    const content = fs.readFileSync(filePath, 'utf8');
+    return content
+        .split('\n')
+        .filter((line) => line.trim())
+        .map((line) => JSON.parse(line));
+}
+function writeJsonl(filePath, entries) {
+    const content = entries.map((entry) => JSON.stringify(entry)).join('\n');
+    fs.writeFileSync(filePath, content + '\n', 'utf8');
+}
+function purgeOldEntries(entries, retentionMs) {
+    const cutoff = new Date(Date.now() - retentionMs);
+    return entries.filter((entry) => new Date(entry.ts) >= cutoff);
+}
+describe('eventLog', () => {
+    it('should parse JSONL entries correctly', () => {
+        const entry1 = {
+            ts: nowIso(),
+            event: 'test-event',
+            matched: true,
+            exitCode: 0,
+            durationMs: 123,
+        };
+        const entry2 = {
+            ts: nowIso(),
+            event: null,
+            matched: false,
+            bodyPreview: '{"type":"unknown"}',
+        };
+        const tempFile = path.join(process.cwd(), 'test-temp-log.jsonl');
+        writeJsonl(tempFile, [entry1, entry2]);
+        const parsed = parseJsonl(tempFile);
+        expect(parsed).toHaveLength(2);
+        expect(parsed[0].event).toBe('test-event');
+        expect(parsed[0].matched).toBe(true);
+        expect(parsed[1].event).toBeNull();
+        expect(parsed[1].matched).toBe(false);
+        // Cleanup
+        fs.unlinkSync(tempFile);
+    });
+    it('should purge entries older than retention period', () => {
+        const now = Date.now();
+        const retentionMs = 86400000; // 1 day
+        const oldEntry = {
+            ts: new Date(now - 2 * retentionMs).toISOString(),
+            event: 'old-event',
+            matched: true,
+            exitCode: 0,
+            durationMs: 100,
+        };
+        const recentEntry = {
+            ts: new Date(now - 0.5 * retentionMs).toISOString(),
+            event: 'recent-event',
+            matched: true,
+            exitCode: 0,
+            durationMs: 200,
+        };
+        const currentEntry = {
+            ts: new Date(now).toISOString(),
+            event: 'current-event',
+            matched: true,
+            exitCode: 0,
+            durationMs: 300,
+        };
+        const purged = purgeOldEntries([oldEntry, recentEntry, currentEntry], retentionMs);
+        expect(purged).toHaveLength(2);
+        expect(purged[0].event).toBe('recent-event');
+        expect(purged[1].event).toBe('current-event');
+    });
+    it('should write JSONL with proper newline formatting', () => {
+        const entries = [
+            {
+                ts: nowIso(),
+                event: 'event1',
+                matched: true,
+                exitCode: 0,
+                durationMs: 100,
+            },
+            {
+                ts: nowIso(),
+                event: 'event2',
+                matched: true,
+                exitCode: 1,
+                durationMs: 200,
+            },
+        ];
+        const tempFile = path.join(process.cwd(), 'test-temp-write.jsonl');
+        writeJsonl(tempFile, entries);
+        const content = fs.readFileSync(tempFile, 'utf8');
+        const lines = content.split('\n').filter((line) => line.trim());
+        expect(lines).toHaveLength(2);
+        expect(JSON.parse(lines[0]).event).toBe('event1');
+        expect(JSON.parse(lines[1]).event).toBe('event2');
+        // Cleanup
+        fs.unlinkSync(tempFile);
+    });
+    it('should handle empty log file', () => {
+        const tempFile = path.join(process.cwd(), 'test-temp-empty.jsonl');
+        const parsed = parseJsonl(tempFile);
+        expect(parsed).toHaveLength(0);
+    });
+});

package/dist/src/services/eventQueue.js

@@ -0,0 +1,154 @@
+/**
+ * Durable event queue for webhook event processing
+ */
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { getConfig } from '../config/index.js';
+import { nowIso } from '../util/formatters.js';
+import { logEvent } from './eventLog.js';
+/**
+ * Ensure directory exists
+ */
+function ensureDir(dirPath) {
+    fs.mkdirSync(dirPath, { recursive: true });
+}
+/**
+ * Append a JSON object as a line to the general event log (backward compatibility)
+ * This is separate from the Event Gateway event log
+ */
+export function appendEvent(event) {
+    const { eventsLog } = getConfig();
+    ensureDir(path.dirname(eventsLog));
+    const line = JSON.stringify({ at: nowIso(), ...event }) + '\n';
+    fs.appendFileSync(eventsLog, line, 'utf8');
+}
+/**
+ * Append a queue entry to the durable queue
+ */
+export function enqueue(event, cmd, body, timeoutMs) {
+    const { eventQueuePath } = getConfig();
+    ensureDir(path.dirname(eventQueuePath));
+    const entry = {
+        ts: nowIso(),
+        event,
+        cmd,
+        body,
+        timeoutMs,
+    };
+    const line = JSON.stringify(entry) + '\n';
+    fs.appendFileSync(eventQueuePath, line, 'utf8');
+}
+/**
+ * Read cursor position (byte offset of last processed entry)
+ */
+function readCursor() {
+    const { eventQueueCursorPath } = getConfig();
+    if (!fs.existsSync(eventQueueCursorPath))
+        return 0;
+    const content = fs.readFileSync(eventQueueCursorPath, 'utf8').trim();
+    return parseInt(content, 10) || 0;
+}
+/**
+ * Write cursor position
+ */
+function writeCursor(position) {
+    const { eventQueueCursorPath } = getConfig();
+    ensureDir(path.dirname(eventQueueCursorPath));
+    fs.writeFileSync(eventQueueCursorPath, position.toString(), 'utf8');
+}
+/**
+ * Parse JSONL entries from file starting at cursor position
+ */
+function readEntriesFromCursor() {
+    const { eventQueuePath } = getConfig();
+    if (!fs.existsSync(eventQueuePath)) {
+        return { entries: [], newPosition: 0 };
+    }
+    const cursor = readCursor();
+    const buf = fs.readFileSync(eventQueuePath);
+    // Read from cursor position (byte-based to match writeCursor)
+    const remaining = buf.subarray(cursor).toString('utf8');
+    const lines = remaining
+        .split('\n')
+        .filter((line) => line.trim())
+        .map((line) => JSON.parse(line));
+    const newPosition = buf.length;
+    return { entries: lines, newPosition };
+}
+/**
+ * Execute a queue entry (spawn command with body piped to stdin)
+ */
+async function executeEntry(entry) {
+    const startTime = Date.now();
+    return new Promise((resolve) => {
+        // Parse command and args
+        const parts = entry.cmd.split(/\s+/);
+        const command = parts[0];
+        const args = parts.slice(1);
+        const proc = spawn(command, args, {
+            stdio: ['pipe', 'inherit', 'inherit'],
+            shell: true,
+        });
+        // Pipe body as JSON to stdin
+        const bodyJson = JSON.stringify(entry.body);
+        proc.stdin.write(bodyJson);
+        proc.stdin.end();
+        // Setup timeout
+        const timeout = setTimeout(() => {
+            proc.kill('SIGTERM');
+        }, entry.timeoutMs);
+        proc.on('exit', (code) => {
+            clearTimeout(timeout);
+            const exitCode = code ?? -1;
+            const durationMs = Date.now() - startTime;
+            resolve({ exitCode, durationMs });
+        });
+        proc.on('error', () => {
+            clearTimeout(timeout);
+            const durationMs = Date.now() - startTime;
+            resolve({ exitCode: -1, durationMs });
+        });
+    });
+}
+/**
+ * Process one batch of queue entries
+ */
+async function processBatch() {
+    const { entries, newPosition } = readEntriesFromCursor();
+    for (const entry of entries) {
+        try {
+            const { exitCode, durationMs } = await executeEntry(entry);
+            // Log to event log
+            logEvent({
+                event: entry.event,
+                matched: true,
+                exitCode,
+                durationMs,
+            });
+        }
+        catch {
+            // Log error but continue (errors are ignored per spec)
+            logEvent({
+                event: entry.event,
+                matched: true,
+                exitCode: -1,
+                durationMs: 0,
+            });
+        }
+    }
+    // Update cursor
+    if (entries.length > 0) {
+        writeCursor(newPosition);
+    }
+}
+/**
+ * Start the queue drain loop
+ */
+export function startQueueProcessor() {
+    setInterval(() => {
+        void processBatch();
+    }, 5000);
+    // Process immediately on start
+    void processBatch();
+}

package/dist/src/services/eventQueue.test.js

@@ -0,0 +1,104 @@
+/**
+ * Event queue service tests
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { nowIso } from '../util/formatters.js';
+describe('eventQueue', () => {
+    it('should format queue entries correctly', () => {
+        const entry = {
+            ts: nowIso(),
+            event: 'test-event',
+            cmd: 'node test.js',
+            body: { foo: 'bar', baz: 123 },
+            timeoutMs: 30000,
+        };
+        const json = JSON.stringify(entry);
+        const parsed = JSON.parse(json);
+        expect(parsed.event).toBe('test-event');
+        expect(parsed.cmd).toBe('node test.js');
+        expect(parsed.body).toEqual({ foo: 'bar', baz: 123 });
+        expect(parsed.timeoutMs).toBe(30000);
+        expect(parsed.ts).toMatch(/^\d{4}-\d{2}-\d{2}T/);
+    });
+    it('should write entries as JSONL', () => {
+        const entry1 = {
+            ts: nowIso(),
+            event: 'event1',
+            cmd: 'cmd1',
+            body: { a: 1 },
+            timeoutMs: 10000,
+        };
+        const entry2 = {
+            ts: nowIso(),
+            event: 'event2',
+            cmd: 'cmd2',
+            body: { b: 2 },
+            timeoutMs: 20000,
+        };
+        const tempFile = path.join(process.cwd(), 'test-queue.jsonl');
+        const lines = [entry1, entry2].map((e) => JSON.stringify(e)).join('\n');
+        fs.writeFileSync(tempFile, lines + '\n', 'utf8');
+        const content = fs.readFileSync(tempFile, 'utf8');
+        const parsed = content
+            .split('\n')
+            .filter((line) => line.trim())
+            .map((line) => JSON.parse(line));
+        expect(parsed).toHaveLength(2);
+        expect(parsed[0].event).toBe('event1');
+        expect(parsed[1].event).toBe('event2');
+        // Cleanup
+        fs.unlinkSync(tempFile);
+    });
+    it('should handle cursor-based reading', () => {
+        const entry1 = JSON.stringify({
+            ts: nowIso(),
+            event: 'e1',
+            cmd: 'c1',
+            body: {},
+            timeoutMs: 1000,
+        });
+        const entry2 = JSON.stringify({
+            ts: nowIso(),
+            event: 'e2',
+            cmd: 'c2',
+            body: {},
+            timeoutMs: 2000,
+        });
+        const tempFile = path.join(process.cwd(), 'test-cursor-queue.jsonl');
+        fs.writeFileSync(tempFile, entry1 + '\n' + entry2 + '\n', 'utf8');
+        const fullContent = fs.readFileSync(tempFile, 'utf8');
+        const cursorPosition = Buffer.byteLength(entry1 + '\n', 'utf8');
+        // Read from cursor
+        const remaining = fullContent.slice(cursorPosition);
+        const lines = remaining
+            .split('\n')
+            .filter((line) => line.trim())
+            .map((line) => JSON.parse(line));
+        expect(lines).toHaveLength(1);
+        expect(lines[0].event).toBe('e2');
+        // Cleanup
+        fs.unlinkSync(tempFile);
+    });
+    it('should preserve body structure through JSON serialization', () => {
+        const complexBody = {
+            string: 'test',
+            number: 123,
+            boolean: true,
+            null: null,
+            array: [1, 2, 3],
+            nested: { a: { b: { c: 'deep' } } },
+        };
+        const entry = {
+            ts: nowIso(),
+            event: 'complex',
+            cmd: 'node test.js',
+            body: complexBody,
+            timeoutMs: 5000,
+        };
+        const json = JSON.stringify(entry);
+        const parsed = JSON.parse(json);
+        expect(parsed.body).toEqual(complexBody);
+    });
+});