@llmaudit/logship 1.0.0 → 1.0.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 LLM Audit
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -1,7 +1,9 @@
 {
   "name": "@llmaudit/logship",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "Enterprise-grade, zero-dependency log shipping agent for @llmaudit. Lightweight, reliable, and performance-focused.",
+  "author": "LLM Audit Team",
+  "license": "MIT",
   "main": "src/index.js",
   "bin": {
     "logship": "bin/logship.js"
@@ -19,10 +21,6 @@
     "agent",
     "telemetry",
     "llmaudit",
-    "loki",
     "monitoring"
-  ],
-  "author": "Antigravity",
-  "license": "ISC",
-  "type": "commonjs"
+  ]
 }

package/src/batcher.js

@@ -1,10 +1,18 @@
-class Batcher {
+const EventEmitter = require('events');
+
+class Batcher extends EventEmitter {
     constructor(config, transport) {
+        super();
         this.config = config;
         this.transport = transport;
         this.queue = [];
         this.flushTimer = null;
         this.isFlushing = false;
+
+        // Backpressure limits
+        // If queue exceeds highWaterMark, we signal to stop reading
+        this.highWaterMark = this.config.maxBatchSize * 5;
+        this.isPaused = false;
     }
 
     start() {
@@ -25,6 +33,10 @@ class Batcher {
         return this.flush();
     }
 
+    isFull() {
+        return this.queue.length >= this.highWaterMark;
+    }
+
     /**
      * Adds a log line to the batch.
      * @param {string} line - The raw log line.
@@ -64,8 +76,11 @@ class Batcher {
             // For this implementation, we log the error. Retry logic is in Transport.
         } finally {
             this.isFlushing = false;
-            // If queue filled up while flushing, trigger another flush immediately? 
-            // Maybe not immediately to avoid storm, wait for next tick or interval.
+
+            // Check if we can resume reading
+            if (this.queue.length < this.highWaterMark / 2) {
+                this.emit('drain');
+            }
         }
     }
 }

package/src/index.js

@@ -12,9 +12,7 @@ class LogAgent {
         this.registry = new Registry();
         this.transport = new Transport(this.config);
         this.batcher = new Batcher(this.config, this.transport);
-        this.watcher = new WatchManager(this.config.logFiles, this.registry, (line) => {
-            this.batcher.push(line);
-        });
+        this.watcher = new WatchManager(this.config.logFiles, this.registry, this.batcher);
     }
 
     init() {

package/src/registry.js

@@ -6,6 +6,18 @@ class Registry {
         this.storagePath = storagePath || path.resolve(process.cwd(), '.agent-registry.json');
         this.data = {};
         this.saveTimer = null;
+
+        // Ensure we save on exit
+        process.on('exit', () => this.save());
+        process.on('SIGINT', () => {
+            this.save();
+            process.exit();
+        });
+        process.on('uncaughtException', (err) => {
+            console.error('Log Agent: Uncaught Exception', err);
+            this.save();
+            process.exit(1);
+        });
     }
 
     load() {
@@ -36,7 +48,7 @@ class Registry {
 
     scheduleSave() {
         if (this.saveTimer) return;
-        this.saveTimer = setTimeout(() => this.save(), 1000); // Debounce saves
+        this.saveTimer = setTimeout(() => this.save(), 500); // Faster debounce (500ms)
     }
 
     save() {

package/src/watcher.js

@@ -1,19 +1,31 @@
 const fs = require('fs');
 
 class FileWatcher {
-    constructor(filepath, registry, onLogLine) {
+    constructor(filepath, registry, batcher) {
         this.filepath = filepath;
         this.registry = registry;
-        this.onLogLine = onLogLine;
+        this.batcher = batcher; // Use Batcher instance directly (was onLogLine)
         this.currSize = 0;
         this.inode = null;
         this.signature = null;
         this.watcher = null;
         this.checkInterval = null;
         this.recreateTimer = null;
+        this.stream = null;
+        this.isProcessing = false;
+        this.debounceTimer = null;
+
+        // Listen for drain to resume reading if paused
+        this.batcher.on('drain', () => {
+            if (this.stream && this.stream.isPaused()) {
+                console.log(`Log Agent: Resuming read for ${this.filepath}`);
+                this.stream.resume();
+            }
+        });
     }
 
     start() {
+        this.stop(); // Prevent multiple instances
         this._initFile();
     }
 
@@ -30,6 +42,10 @@ class FileWatcher {
             clearTimeout(this.recreateTimer);
             this.recreateTimer = null;
         }
+        if (this.stream) {
+            this.stream.destroy();
+            this.stream = null;
+        }
     }
 
     _initFile() {
@@ -50,47 +66,125 @@ class FileWatcher {
                 const savedState = this.registry.get(this.filepath);
 
                 let isValid = false;
-                if (savedState && savedState.inode === this.inode) {
-                    if (savedState.signature && savedState.signature !== signature) {
-                        // Signature mismatch means content changed (reuse)
-                        console.log(`Log Agent: Signature Mismatch! Resetting.`);
-                        isValid = false;
-                    } else if (stats.birthtimeMs && stats.birthtimeMs > savedState.lastUpdated) {
-                        console.log(`Log Agent: Newer Birthtime Detected. Resetting.`);
-                        isValid = false;
-                    } else {
+                if (savedState) {
+                    // SIGNATURE-FIRST IDENTITY
+                    // If the first 256 bytes match, it is the same content stream.
+                    // We ignore Inode changes (common on Windows) and Birthtime (sensitive to edits).
+                    if (savedState.signature && savedState.signature === signature) {
+                        if (savedState.inode !== this.inode) {
+                            console.log(`Log Agent: Inode changed for ${this.filepath} but Signature matches. Resuming.`);
+                        }
                         isValid = true;
+                    } else if (savedState.inode === this.inode) {
+                        // Same inode but signature changed? This is a rotation or total overwrite.
+                        console.log(`Log Agent: Signature Mismatch for same Inode. Resetting.`);
+                        isValid = false;
                     }
                 }
 
                 if (isValid) {
-                    console.log(`Log Agent: Resuming ${this.filepath} from offset ${savedState.offset}`);
-                    this.currSize = savedState.offset;
-
-                    if (stats.size > this.currSize) {
-                        this._read(this.currSize, stats.size);
-                        this.currSize = stats.size;
-                        this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
-                    } else if (stats.size < this.currSize) {
-                        console.log('Log Agent: File truncated/rotated. Resetting.');
-                        this.currSize = 0;
-                        this._read(0, stats.size);
-                        this.currSize = stats.size;
-                        this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+                    if (savedState.offset > stats.size) {
+                        console.log(`Log Agent: Saved offset ${savedState.offset} exceeds file size ${stats.size}. Truncated?`);
+                        this._handleTruncation(stats);
+                        return; // HandleTruncation will call _afterInit
+                    } else {
+                        this.currSize = savedState.offset;
+                        console.log(`Log Agent: Resuming ${this.filepath} from offset ${this.currSize}`);
                     }
+                    this._afterInit(stats);
                 } else {
-                    console.log(`Log Agent: New file/rotation/mismatch. Starting at ${stats.size}`);
-                    this.currSize = stats.size;
-                    this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+                    // Start from last 100k lines
+                    const MAX_LINES = 100000;
+                    this._findTailOffset(this.filepath, MAX_LINES, (offset) => {
+                        this.currSize = offset;
+                        console.log(`Log Agent: New file. Starting at ${this.currSize} (Tail: last ${MAX_LINES} lines)`);
+
+                        // Force update registry so we don't re-read if crashed immediately
+                        this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+                        this._afterInit(stats);
+                    });
                 }
+            });
+        });
+    }
 
-                this._watch();
+    _afterInit(stats) {
+        // If file is ahead, read catchup
+        if (stats.size > this.currSize) {
+            this._read(this.currSize, stats.size);
+            this.currSize = stats.size;
+            this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+        } else if (stats.size < this.currSize) {
+            this._handleTruncation(stats);
+            return; // _handleTruncation will re-trigger what's needed
+        }
+
+        this._watch();
+    }
+
+    _findTailOffset(filepath, maxLines, callback) {
+        fs.open(filepath, 'r', (err, fd) => {
+            if (err) return callback(0);
+
+            fs.fstat(fd, (err, stats) => {
+                if (err) {
+                    fs.close(fd, () => { });
+                    return callback(0);
+                }
+
+                let size = stats.size;
+                if (size === 0) {
+                    fs.close(fd, () => { });
+                    return callback(0);
+                }
+
+                let linesCount = 0;
+                let offset = size;
+                const bufferSize = 64 * 1024; // 64KB
+                const buffer = Buffer.alloc(bufferSize);
+
+                const readNextChunk = () => {
+                    const toRead = Math.min(bufferSize, offset);
+                    const readAt = offset - toRead;
+
+                    fs.read(fd, buffer, 0, toRead, readAt, (err, bytesRead) => {
+                        if (err || bytesRead === 0) {
+                            fs.close(fd, () => { });
+                            return callback(0);
+                        }
+
+                        // Scan backwards for newlines
+                        for (let i = bytesRead - 1; i >= 0; i--) {
+                            if (buffer[i] === 10) { // \n
+                                linesCount++;
+                                if (linesCount > maxLines) {
+                                    // Found the line! Start offset is right after this \n
+                                    const foundOffset = readAt + i + 1;
+                                    fs.close(fd, () => { });
+                                    return callback(foundOffset);
+                                }
+                            }
+                        }
+
+                        offset -= bytesRead;
+                        if (offset > 0) {
+                            readNextChunk();
+                        } else {
+                            // Reached start of file without hitting maxLines
+                            fs.close(fd, () => { });
+                            callback(0);
+                        }
+                    });
+                };
+
+                readNextChunk();
             });
         });
     }
 
     _readSignature(filepath, callback) {
-        const stream = fs.createReadStream(filepath, { start: 0, end: 63 });
+        // Increased to 256 bytes to avoid collisions with long boilerplate headers
+        const stream = fs.createReadStream(filepath, { start: 0, end: 255 });
         let buf = Buffer.alloc(0);
         stream.on('data', chunk => {
             buf = Buffer.concat([buf, chunk]);
@@ -104,24 +198,25 @@ class FileWatcher {
     _watch() {
         try {
             this.watcher = fs.watch(this.filepath, (eventType, filename) => {
-                if (eventType === 'rename') {
-                    fs.stat(this.filepath, (err, stats) => {
-                        if (err && err.code === 'ENOENT') {
-                            this.stop();
-                            this.recreateTimer = setTimeout(() => this.start(), 100);
-                        } else {
-                            if (stats.ino !== this.inode) {
-                                this.inode = stats.ino;
-                                this.currSize = 0;
-                                this._processChange();
+                // Debounce rapidly firing events (common in high-traffic writes)
+                if (this.debounceTimer) clearTimeout(this.debounceTimer);
+
+                this.debounceTimer = setTimeout(() => {
+                    if (eventType === 'rename') {
+                        // On Windows, rename can mean a safe-write or a rotation.
+                        fs.stat(this.filepath, (err, stats) => {
+                            if (err && err.code === 'ENOENT') {
+                                console.log(`Log Agent: ${this.filepath} moved/deleted. Waiting for recreation...`);
+                                this.stop();
+                                this.recreateTimer = setTimeout(() => this.start(), 1000);
                             } else {
-                                this._processChange();
+                                this._initFile();
                             }
-                        }
-                    });
-                } else {
-                    this._processChange();
-                }
+                        });
+                    } else {
+                        this._processChange();
+                    }
+                }, 50); // 50ms pulse to group rapid writes
             });
         } catch (e) {
             console.error(`Log Agent: Watch failed for ${this.filepath}`, e);
@@ -129,18 +224,40 @@ class FileWatcher {
     }
 
     _processChange() {
+        if (this.isProcessing) return; // Prevent overlapping stat/read calls
+        this.isProcessing = true;
+
         fs.stat(this.filepath, (err, stats) => {
-            if (err) return;
+            if (err) {
+                this.isProcessing = false;
+                return;
+            }
 
             if (stats.size > this.currSize) {
                 this._read(this.currSize, stats.size);
                 this.currSize = stats.size;
                 this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
             } else if (stats.size < this.currSize) {
-                this.currSize = 0;
-                this._read(0, stats.size);
-                this.currSize = stats.size;
-                this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+                this._handleTruncation(stats);
+            }
+
+            this.isProcessing = false;
+        });
+    }
+
+    _handleTruncation(stats) {
+        console.log(`Log Agent: File ${this.filepath} truncated (${this.currSize} -> ${stats.size}). Re-tailing last 100k lines.`);
+        const MAX_LINES = 100000;
+        this._findTailOffset(this.filepath, MAX_LINES, (offset) => {
+            this.currSize = offset;
+            console.log(`Log Agent: Smart Resume after truncation at offset ${this.currSize}`);
+            this._read(this.currSize, stats.size);
+            this.currSize = stats.size;
+            this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+
+            // If this was called during init, we need to start watching
+            if (!this.watcher && !this.checkInterval) {
+                this._watch();
             }
         });
     }
@@ -149,42 +266,54 @@ class FileWatcher {
         if (start >= end) return;
         if (start < 0) start = 0;
 
-        const stream = fs.createReadStream(this.filepath, {
+        // Clean up previous stream if somehow open (though _read is usually sequential/triggered by serial events)
+        if (this.stream) {
+            this.stream.destroy();
+        }
+
+        this.stream = fs.createReadStream(this.filepath, {
             start: start,
             end: end - 1,
             encoding: 'utf8'
         });
 
         let buffer = '';
-        stream.on('data', (chunk) => {
+        this.stream.on('data', (chunk) => {
+            // Check Backpressure
+            if (this.batcher.isFull()) {
+                console.log(`Log Agent: Queue full, pausing read on ${this.filepath}`);
+                this.stream.pause();
+            }
+
             buffer += chunk;
             const lines = buffer.split('\n');
-            buffer = lines.pop();
+            buffer = lines.pop(); // Keep partial line
 
             for (const line of lines) {
-                if (line.trim()) this.onLogLine(line);
+                if (line.trim()) this.batcher.push(line);
             }
         });
 
-        stream.on('end', () => {
+        this.stream.on('end', () => {
             if (buffer && buffer.trim()) {
-                this.onLogLine(buffer);
+                this.batcher.push(buffer);
             }
+            this.stream = null;
         });
     }
 }
 
 class WatchManager {
-    constructor(logFiles, registry, onLogLine) {
+    constructor(logFiles, registry, batcher) {
         this.logFiles = logFiles;
         this.registry = registry;
-        this.onLogLine = onLogLine;
+        this.batcher = batcher; // Pass batcher instead of callback
         this.watchers = [];
     }
 
     start() {
         this.logFiles.forEach(file => {
-            const watcher = new FileWatcher(file, this.registry, this.onLogLine);
+            const watcher = new FileWatcher(file, this.registry, this.batcher);
             watcher.start();
             this.watchers.push(watcher);
         });