@jagilber-org/index-server 1.19.1

package/dist/utils/BufferRing.js

@@ -0,0 +1,551 @@
+"use strict";
+/**
+ * Configurable Buffer Ring - Generic circular buffer implementation
+ *
+ * Provides a high-performance, memory-efficient circular buffer with configurable
+ * size limits, overflow strategies, and optional persistence support.
+ *
+ * Features:
+ * - Configurable capacity with automatic overflow handling
+ * - Multiple overflow strategies (drop-oldest, drop-newest, resize)
+ * - Optional persistence to disk with atomic writes
+ * - Type-safe generic implementation
+ * - Memory usage tracking and cleanup
+ * - Event notifications for buffer state changes
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BufferRingFactory = exports.BufferRing = exports.OverflowStrategy = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+const events_1 = require("events");
+const logger_1 = require("../services/logger");
+const runtimeConfig_1 = require("../config/runtimeConfig");
+/**
+ * Buffer overflow strategies
+ */
+var OverflowStrategy;
+(function (OverflowStrategy) {
+    /** Drop the oldest entries when buffer is full (default) */
+    OverflowStrategy["DROP_OLDEST"] = "drop-oldest";
+    /** Drop the newest entry when buffer is full */
+    OverflowStrategy["DROP_NEWEST"] = "drop-newest";
+    /** Resize the buffer to accommodate new entries */
+    OverflowStrategy["RESIZE"] = "resize";
+    /** Throw an error when buffer is full */
+    OverflowStrategy["ERROR"] = "error";
+})(OverflowStrategy || (exports.OverflowStrategy = OverflowStrategy = {}));
+/**
+ * Generic configurable buffer ring implementation
+ */
+class BufferRing extends events_1.EventEmitter {
+    buffer = [];
+    writeIndex = 0;
+    isFull = false;
+    config;
+    stats;
+    /** Sequence counter for append mode records */
+    appendSeq = 0;
+    /** Track whether append mode file has been lazily loaded */
+    appendLoaded = false;
+    constructor(config = {}) {
+        super();
+        this.config = {
+            capacity: 1000,
+            overflowStrategy: OverflowStrategy.DROP_OLDEST,
+            autoPersist: false,
+            maxPersistEntries: 0,
+            enableIntegrityCheck: true,
+            appendMode: true,
+            ...config
+        };
+        // Environment override via config: allow disable or enable explicitly
+        const bufferRingCfg = (0, runtimeConfig_1.getRuntimeConfig)().bufferRing;
+        this.config.appendMode = bufferRingCfg.append;
+        this.stats = {
+            count: 0,
+            capacity: this.config.capacity,
+            totalAdded: 0,
+            totalDropped: 0,
+            resizeCount: 0,
+            memoryUsage: 0,
+            lastModified: new Date(),
+            utilization: 0
+        };
+        // Load from persistence if configured
+        if (this.config.persistPath) {
+            if (this.config.appendMode) {
+                // Lazy load for append mode occurs on first access; optionally preload if config set
+                if (bufferRingCfg.preload) {
+                    try {
+                        this.loadFromAppendSync();
+                    }
+                    catch (error) {
+                        const err = error instanceof Error ? error : new Error('Unknown append preload error');
+                        this.emit('error', err, 'load');
+                        (0, logger_1.logError)(`[BufferRing] Failed (append) to load from ${this.config.persistPath}`, err);
+                    }
+                }
+            }
+            else {
+                this.loadFromDisk().catch(error => {
+                    this.emit('error', error, 'load');
+                    (0, logger_1.logError)(`[BufferRing] Failed to load from ${this.config.persistPath}`, error);
+                });
+            }
+        }
+    }
+    /**
+     * Add an entry to the buffer
+     */
+    add(entry) {
+        try {
+            const serializedEntry = this.config.serializer ? this.config.serializer(entry) : entry;
+            let dropped;
+            if (this.isFull) {
+                switch (this.config.overflowStrategy) {
+                    case OverflowStrategy.DROP_OLDEST:
+                        dropped = this.buffer[this.writeIndex];
+                        this.buffer[this.writeIndex] = serializedEntry;
+                        this.writeIndex = (this.writeIndex + 1) % this.config.capacity;
+                        this.stats.totalDropped++;
+                        if (dropped !== undefined) {
+                            this.emit('entry-dropped', dropped, 'overflow');
+                        }
+                        break;
+                    case OverflowStrategy.DROP_NEWEST:
+                        this.emit('entry-dropped', serializedEntry, 'overflow');
+                        this.stats.totalDropped++;
+                        return false;
+                    case OverflowStrategy.RESIZE:
+                        this.resize(this.config.capacity * 2);
+                        this.buffer[this.writeIndex] = serializedEntry;
+                        this.writeIndex++;
+                        this.stats.count++;
+                        break;
+                    case OverflowStrategy.ERROR:
+                        throw new Error(`Buffer overflow: capacity ${this.config.capacity} exceeded`);
+                }
+            }
+            else {
+                this.buffer[this.writeIndex] = serializedEntry;
+                this.writeIndex++;
+                this.stats.count++;
+                if (this.writeIndex >= this.config.capacity) {
+                    this.isFull = true;
+                    this.writeIndex = 0;
+                    this.emit('buffer-full', this.config.capacity);
+                }
+            }
+            this.stats.totalAdded++;
+            this.stats.lastModified = new Date();
+            this.updateStats();
+            this.emit('entry-added', serializedEntry, this.writeIndex - 1);
+            // Auto-persist if enabled
+            if (this.config.autoPersist && this.config.persistPath) {
+                if (this.config.appendMode) {
+                    try {
+                        this.appendToDisk(serializedEntry);
+                    }
+                    catch (err) {
+                        this.emit('error', err, 'auto-append');
+                    }
+                }
+                else {
+                    this.saveToDisk().catch(error => {
+                        this.emit('error', error, 'auto-persist');
+                    });
+                }
+            }
+            return true;
+        }
+        catch (error) {
+            this.emit('error', error, 'add');
+            return false;
+        }
+    }
+    /**
+     * Get all entries in chronological order (oldest first)
+     */
+    getAll() {
+        // Lazy load append file on first read if in append mode
+        if (this.config.appendMode && this.config.persistPath && !this.appendLoaded) {
+            try {
+                this.loadFromAppendSync();
+            }
+            catch (err) {
+                this.emit('error', err, 'load-append');
+            }
+        }
+        if (!this.isFull) {
+            return this.buffer.slice(0, this.writeIndex).map(this.deserializeEntry.bind(this));
+        }
+        // Buffer is full, need to reorder from oldest to newest
+        const oldest = this.buffer.slice(this.writeIndex).map(this.deserializeEntry.bind(this));
+        const newest = this.buffer.slice(0, this.writeIndex).map(this.deserializeEntry.bind(this));
+        return [...oldest, ...newest];
+    }
+    /**
+     * Get the last N entries (newest first)
+     */
+    getLast(n) {
+        const all = this.getAll();
+        return all.slice(-n).reverse();
+    }
+    /**
+     * Get the first N entries (oldest first)
+     */
+    getFirst(n) {
+        const all = this.getAll();
+        return all.slice(0, n);
+    }
+    /**
+     * Get entries within a range
+     */
+    getRange(start, end) {
+        const all = this.getAll();
+        return all.slice(start, end);
+    }
+    /**
+     * Filter entries based on predicate
+     */
+    filter(predicate) {
+        return this.getAll().filter(predicate);
+    }
+    /**
+     * Find first entry matching predicate
+     */
+    find(predicate) {
+        return this.getAll().find(predicate);
+    }
+    /**
+     * Clear all entries
+     */
+    clear() {
+        const droppedCount = this.stats.count;
+        this.buffer.forEach(entry => {
+            if (entry !== undefined) {
+                this.emit('entry-dropped', entry, 'clear');
+            }
+        });
+        this.buffer = [];
+        this.writeIndex = 0;
+        this.isFull = false;
+        this.stats.count = 0;
+        this.stats.totalDropped += droppedCount;
+        this.stats.lastModified = new Date();
+        this.updateStats();
+        (0, logger_1.logInfo)(`[BufferRing] Cleared ${droppedCount} entries`);
+    }
+    /**
+     * Resize the buffer capacity
+     */
+    resize(newCapacity) {
+        if (newCapacity <= 0) {
+            throw new Error('Buffer capacity must be positive');
+        }
+        const oldCapacity = this.config.capacity;
+        const currentEntries = this.getAll();
+        this.config.capacity = newCapacity;
+        this.buffer = [];
+        this.writeIndex = 0;
+        this.isFull = false;
+        this.stats.count = 0;
+        this.stats.capacity = newCapacity;
+        this.stats.resizeCount++;
+        // Re-add entries up to new capacity
+        const entriesToKeep = currentEntries.slice(-newCapacity);
+        entriesToKeep.forEach(entry => this.add(entry));
+        this.emit('buffer-resized', oldCapacity, newCapacity);
+        (0, logger_1.logInfo)(`[BufferRing] Resized from ${oldCapacity} to ${newCapacity} entries`);
+    }
+    /**
+     * Get buffer statistics
+     */
+    getStats() {
+        this.updateStats();
+        return { ...this.stats };
+    }
+    /**
+     * Get buffer configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+    /**
+     * Update configuration
+     */
+    updateConfig(newConfig) {
+        const oldCapacity = this.config.capacity;
+        this.config = { ...this.config, ...newConfig };
+        // Handle capacity changes
+        if (this.config.capacity !== oldCapacity) {
+            this.resize(this.config.capacity);
+        }
+    }
+    /**
+     * Save buffer to disk
+     */
+    async saveToDisk() {
+        if (!this.config.persistPath) {
+            throw new Error('No persist path configured');
+        }
+        try {
+            const entries = this.getAll();
+            const entriesToSave = this.config.maxPersistEntries > 0
+                ? entries.slice(-this.config.maxPersistEntries)
+                : entries;
+            const data = {
+                config: this.config,
+                entries: entriesToSave,
+                stats: this.getStats(),
+                version: '1.0.0',
+                createdAt: new Date().toISOString(),
+                lastSaved: new Date().toISOString()
+            };
+            if (this.config.enableIntegrityCheck) {
+                data.checksum = this.computeChecksum(data);
+            }
+            // Ensure directory exists
+            const dir = path.dirname(this.config.persistPath);
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
+            // Atomic write
+            const tempPath = this.config.persistPath + '.tmp';
+            const serialized = JSON.stringify(data, null, 2);
+            fs.writeFileSync(tempPath, serialized);
+            fs.renameSync(tempPath, this.config.persistPath);
+            this.emit('persisted', this.config.persistPath, entriesToSave.length);
+            if (!this.config.suppressPersistLog) {
+                (0, logger_1.logInfo)(`[BufferRing] Persisted ${entriesToSave.length} entries to ${this.config.persistPath}`);
+            }
+        }
+        catch (error) {
+            this.emit('error', error, 'save');
+            throw error;
+        }
+    }
+    /**
+     * Load buffer from disk
+     */
+    async loadFromDisk() {
+        if (!this.config.persistPath || !fs.existsSync(this.config.persistPath)) {
+            return;
+        }
+        try {
+            const raw = fs.readFileSync(this.config.persistPath, 'utf8');
+            const data = JSON.parse(raw);
+            // Verify integrity if enabled
+            if (this.config.enableIntegrityCheck && data.checksum) {
+                const expectedChecksum = this.computeChecksum(data);
+                if (data.checksum !== expectedChecksum) {
+                    throw new Error('Integrity check failed: checksum mismatch');
+                }
+            }
+            // Clear current buffer
+            this.clear();
+            // Restore entries
+            data.entries.forEach(entry => this.add(entry));
+            // Update stats (preserve some current values)
+            this.stats = {
+                ...data.stats,
+                lastModified: new Date(),
+                memoryUsage: this.estimateMemoryUsage()
+            };
+            this.emit('loaded', this.config.persistPath, data.entries.length);
+            if (!this.config.suppressPersistLog) {
+                (0, logger_1.logInfo)(`[BufferRing] Loaded ${data.entries.length} entries from ${this.config.persistPath}`);
+            }
+        }
+        catch (error) {
+            this.emit('error', error, 'load');
+            throw error;
+        }
+    }
+    /**
+     * Append a single entry (JSONL) instead of rewriting full snapshot.
+     * Record shape: { t: ISO timestamp, i: sequence, v: serializedEntry }
+     */
+    appendToDisk(entry) {
+        if (!this.config.persistPath)
+            return;
+        const dir = path.dirname(this.config.persistPath);
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        const rec = { t: new Date().toISOString(), i: this.appendSeq++, v: entry };
+        fs.appendFileSync(this.config.persistPath, JSON.stringify(rec) + '\n');
+        if (!this.config.suppressPersistLog && this.appendSeq % 250 === 0) {
+            // Throttle log noise (every 250 appends)
+            (0, logger_1.logInfo)(`[BufferRing] Appended ${this.appendSeq} records to ${this.config.persistPath}`);
+        }
+    }
+    /**
+     * Synchronous load for append mode (reads JSONL file)
+     */
+    loadFromAppendSync() {
+        if (!this.config.persistPath || !fs.existsSync(this.config.persistPath)) {
+            this.appendLoaded = true;
+            return;
+        }
+        try {
+            const raw = fs.readFileSync(this.config.persistPath, 'utf8');
+            const lines = raw.split(/\r?\n/).filter(l => l.trim().length);
+            const max = this.config.maxPersistEntries > 0 ? this.config.maxPersistEntries : this.config.capacity;
+            const slice = lines.slice(-max);
+            // Clear existing buffer first (without emitting per-drop noise)
+            this.buffer = [];
+            this.writeIndex = 0;
+            this.isFull = false;
+            this.stats.count = 0;
+            slice.forEach(line => {
+                try {
+                    const rec = JSON.parse(line);
+                    this.add(rec.v); // reuse add for ordering & stats
+                    if (typeof rec.i === 'number' && rec.i >= this.appendSeq)
+                        this.appendSeq = rec.i + 1;
+                }
+                catch { /* skip malformed line */ }
+            });
+            this.appendLoaded = true;
+            if (!this.config.suppressPersistLog) {
+                (0, logger_1.logInfo)(`[BufferRing] Loaded (append mode) ${slice.length}/${lines.length} entries from ${this.config.persistPath}`);
+            }
+        }
+        catch (err) {
+            this.appendLoaded = true; // avoid retry loop
+            throw err;
+        }
+    }
+    /**
+     * Get memory usage estimate
+     */
+    getMemoryUsage() {
+        return this.estimateMemoryUsage();
+    }
+    /**
+     * Compact buffer by removing undefined entries
+     */
+    compact() {
+        const validEntries = this.getAll();
+        this.clear();
+        validEntries.forEach(entry => this.add(entry));
+        (0, logger_1.logInfo)(`[BufferRing] Compacted buffer, retained ${validEntries.length} entries`);
+    }
+    /**
+     * Private helper methods
+     */
+    deserializeEntry(entry) {
+        return this.config.deserializer ? this.config.deserializer(entry) : entry;
+    }
+    updateStats() {
+        this.stats.utilization = (this.stats.count / this.config.capacity) * 100;
+        this.stats.memoryUsage = this.estimateMemoryUsage();
+    }
+    estimateMemoryUsage() {
+        // Rough estimate: 8 bytes per pointer + JSON stringified size
+        const baseSize = this.buffer.length * 8;
+        const dataSize = JSON.stringify(this.buffer.filter(x => x !== undefined)).length * 2; // UTF-16
+        return baseSize + dataSize;
+    }
+    computeChecksum(data) {
+        const dataWithoutChecksum = { ...data };
+        delete dataWithoutChecksum.checksum;
+        const serialized = JSON.stringify(dataWithoutChecksum, Object.keys(dataWithoutChecksum).sort());
+        return crypto.createHash('sha256').update(serialized).digest('hex');
+    }
+}
+exports.BufferRing = BufferRing;
+/**
+ * Factory function for creating commonly used buffer rings
+ */
+class BufferRingFactory {
+    /**
+     * Create a log buffer ring
+     */
+    static createLogBuffer(capacity = 1000, persistPath) {
+        return new BufferRing({
+            capacity,
+            overflowStrategy: OverflowStrategy.DROP_OLDEST,
+            persistPath,
+            autoPersist: !!persistPath,
+            maxPersistEntries: capacity * 0.8, // Persist 80% of capacity
+            enableIntegrityCheck: true
+        });
+    }
+    /**
+     * Create a metrics buffer ring
+     */
+    static createMetricsBuffer(capacity = 500, persistPath) {
+        return new BufferRing({
+            capacity,
+            overflowStrategy: OverflowStrategy.DROP_OLDEST,
+            persistPath,
+            autoPersist: false, // Manual persistence for performance
+            maxPersistEntries: 0, // Persist all
+            enableIntegrityCheck: true,
+            serializer: (entry) => ({
+                ...entry,
+                timestamp: entry.timestamp || new Date().toISOString()
+            })
+        });
+    }
+    /**
+     * Create an event buffer ring
+     */
+    static createEventBuffer(capacity = 200) {
+        return new BufferRing({
+            capacity,
+            overflowStrategy: OverflowStrategy.DROP_OLDEST,
+            autoPersist: false,
+            enableIntegrityCheck: false // Events are transient
+        });
+    }
+    /**
+     * Create a resizable buffer ring
+     */
+    static createResizableBuffer(initialCapacity = 100, _maxCapacity = 10000) {
+        return new BufferRing({
+            capacity: initialCapacity,
+            overflowStrategy: OverflowStrategy.RESIZE,
+            autoPersist: false,
+            enableIntegrityCheck: false
+        });
+    }
+}
+exports.BufferRingFactory = BufferRingFactory;
+exports.default = BufferRing;

package/dist/utils/BufferRingExamples.d.ts

@@ -0,0 +1,55 @@
+/**
+ * BufferRing Usage Examples
+ *
+ * This file demonstrates various use cases for the configurable BufferRing utility,
+ * particularly for log management, metrics collection, and event handling in the
+ * Index Server.
+ */
+import { BufferRing, BufferRingFactory, OverflowStrategy } from './BufferRing';
+export declare function createLogViewerBuffer(logDir: string): BufferRing<string>;
+export declare function createMetricsCollector(dataDir: string): BufferRing<Record<string, unknown>>;
+export declare function createEventTracker(): BufferRing<Record<string, unknown>>;
+export declare function createToolMetricsBuffer(): BufferRing<{
+    toolName: string;
+    duration: number;
+    timestamp: string;
+    success: boolean;
+}>;
+export interface BufferConfig {
+    type: 'log' | 'metrics' | 'events' | 'custom';
+    capacity: number;
+    persistPath?: string;
+    overflowStrategy?: OverflowStrategy;
+    autoPersist?: boolean;
+}
+export declare function createConfiguredBuffer(config: BufferConfig): BufferRing<unknown>;
+export declare class LogViewerService {
+    private logBuffer;
+    constructor(logDir: string);
+    addLogEntry(logLine: string): void;
+    getRecentLogs(count?: number): string[];
+    getLogRange(start: number, end: number): string[];
+    searchLogs(pattern: string): string[];
+    getLogStats(): {
+        totalLines: number;
+        bufferUtilization: number;
+        memoryUsage: number;
+        totalProcessed: number;
+        droppedLines: number;
+    };
+    clearLogs(): void;
+    persistLogs(): Promise<void>;
+}
+export declare class LogStreamer {
+    private logBuffer;
+    private subscribers;
+    constructor(bufferSize?: number);
+    addLog(logEntry: string): void;
+    subscribe(callback: (log: string) => void): () => void;
+    getBufferSnapshot(): string[];
+    getStats(): {
+        bufferStats: import("./BufferRing").BufferRingStats;
+        subscriberCount: number;
+    };
+}
+export { BufferRing, BufferRingFactory, OverflowStrategy };