@llmaudit/logship 1.0.0

package/README.md

@@ -0,0 +1,65 @@
+# Logship (by @llmaudit)
+
+A high-performance, lightweight Node.js log shipping agent designed to tail files and pump logs to the LLM Audit Ingestion Server.
+
+## Features
+- **Zero Dependencies**: Built entirely with native Node.js modules for maximum stability and security.
+- **Smart Tailing**: Monitors multiple files simultaneously and resumes from where it left off.
+- **Resilient**: Automatic reconnection with exponential backoff and persistent registry for tracking file positions across restarts.
+- **Performance Focused**: Batches logs automatically to minimize HTTP overhead and server load.
+- **Zero Config Mode**: Run it entirely using CLI flags without creating any files.
+
+## Installation
+```bash
+npm install @llmaudit/logship
+```
+
+## CLI Usage (The easy way)
+
+### Option A: Using CLI Flags (Zero Config)
+Run Logship without creating a configuration file:
+
+```bash
+npx @llmaudit/logship -k your-key -f access.log,error.log
+```
+
+### Option B: Using a Config File
+If you have many logs or complex settings, use a JSON file:
+
+```bash
+npx @llmaudit/logship logship.config.json
+```
+
+### Option C: Default Lookup
+If you have a file named `logship.config.json` in your current folder, simply run:
+
+```bash
+npx @llmaudit/logship
+```
+
+## CLI Configuration Options
+
+| Flag | Short | Description |
+|------|-------|-------------|
+| `--api-key` | `-k` | Your unique API Key |
+| `--endpoint` | `-e` | Ingestion Server URL (Defaults to `https://log.llmaudit.ai`) |
+| `--files` | `-f` | Comma-separated list of log files |
+| `--env` | | Environment tag (`prod`, `staging`, `dev`) |
+| `--service` | `-s` | Name of the service |
+| `--interval` | | Flush interval in ms |
+| `--batch-size` | | Max logs per batch |
+
+## Library Usage
+```javascript
+const { initLogger } = require('@llmaudit/logship');
+
+const agent = initLogger({
+  apiKey: "your-api-key",
+  logFiles: ["./app.log"],
+  env: "prod",
+  service: "my-app"
+  // endpoint: "https://log.llmaudit.ai" (Optional: Defaulted for you)
+});
+
+// agent.shutdown();
+```

package/bin/logship.js

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+
+const { initLogger } = require('../src/index');
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * Poor man's CLI parser (Zero dependencies)
+ */
+function parseArgs(args) {
+    const config = {};
+    let configPath = null;
+
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+
+        if (arg === '--api-key' || arg === '-k') {
+            config.apiKey = args[++i];
+        } else if (arg === '--endpoint' || arg === '-e') {
+            config.endpoint = args[++i];
+        } else if (arg === '--env') {
+            config.env = args[++i];
+        } else if (arg === '--service' || arg === '-s') {
+            config.service = args[++i];
+        } else if (arg === '--files' || arg === '-f') {
+            const files = args[++i];
+            config.logFiles = files ? files.split(',') : [];
+        } else if (arg === '--interval') {
+            config.flushIntervalMs = parseInt(args[++i]);
+        } else if (arg === '--batch-size') {
+            config.maxBatchSize = parseInt(args[++i]);
+        } else if (arg === '--help' || arg === '-h') {
+            printHelp();
+            process.exit(0);
+        } else if (!arg.startsWith('-')) {
+            configPath = arg;
+        }
+    }
+
+    return { config, configPath };
+}
+
+function printHelp() {
+    console.log(`
+Logship - High-performance log shipping agent
+
+Usage:
+  logship [config-file.json] [options]
+
+Options:
+  -k, --api-key <key>       API Key for authentication
+  -f, --files <file1,file2> Comma-separated list of log files
+  -e, --endpoint <url>      Override Ingestion Server URL (Optional)
+  --env <env>               Environment (prod, staging, dev)
+  -s, --service <name>      Service name
+  --interval <ms>           Flush interval in milliseconds
+  --batch-size <num>        Maximum logs per batch
+  -h, --help                Show this help message
+
+Examples:
+  npx @llmaudit/logship --api-key k_123 --endpoint https://log.llmaudit.ai --files access.log
+  npx @llmaudit/logship logship.config.json --env staging
+`);
+}
+
+async function main() {
+    const rawArgs = process.argv.slice(2);
+    const { config: cliConfig, configPath: cliPath } = parseArgs(rawArgs);
+
+    // Determine which config file to load
+    const configFilePath = cliPath || process.env.LOG_AGENT_CONFIG || 'logship.config.json';
+    const absolutePath = path.resolve(process.cwd(), configFilePath);
+
+    let finalConfig = {};
+
+    // 1. Try to load config file if it exists or was explicitly requested
+    if (fs.existsSync(absolutePath)) {
+        try {
+            console.log('Loading config from:', absolutePath);
+            const fileContent = fs.readFileSync(absolutePath, 'utf8');
+            finalConfig = JSON.parse(fileContent);
+        } catch (error) {
+            console.error(`Error parsing config file: ${error.message}`);
+            process.exit(1);
+        }
+    } else if (cliPath) {
+        // User provided a path that doesn't exist
+        console.error(`Error: Config file not found at ${absolutePath}`);
+        process.exit(1);
+    }
+
+    // 2. Merge with CLI flags (Flags override file)
+    finalConfig = { ...finalConfig, ...cliConfig };
+
+    // 3. Last check for required fields before handing off to SDK
+    // SDK will also validate, but we want a nice error message here if no file AND no flags
+    if (!finalConfig.apiKey || !finalConfig.logFiles) {
+        if (!fs.existsSync(absolutePath)) {
+            console.error('Error: Required configuration missing. Provide a config file OR use CLI flags.');
+            console.error('Missing: ' + [!finalConfig.apiKey && 'api-key', !finalConfig.logFiles && 'logFiles'].filter(Boolean).join(', '));
+            printHelp();
+            process.exit(1);
+        }
+    }
+
+    try {
+        const logger = initLogger(finalConfig);
+
+        // Handle graceful shutdown
+        const stop = () => {
+            logger.shutdown();
+            process.exit(0);
+        };
+
+        process.on('SIGINT', stop);
+        process.on('SIGTERM', stop);
+
+    } catch (error) {
+        console.error('Error starting Logship:', error.message);
+        process.exit(1);
+    }
+}
+
+main();

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@llmaudit/logship",
+  "version": "1.0.0",
+  "description": "Enterprise-grade, zero-dependency log shipping agent for @llmaudit. Lightweight, reliable, and performance-focused.",
+  "main": "src/index.js",
+  "bin": {
+    "logship": "bin/logship.js"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "start": "node ./bin/logship.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [
+    "logging",
+    "log-shipper",
+    "agent",
+    "telemetry",
+    "llmaudit",
+    "loki",
+    "monitoring"
+  ],
+  "author": "Antigravity",
+  "license": "ISC",
+  "type": "commonjs"
+}

package/src/batcher.js

@@ -0,0 +1,73 @@
+class Batcher {
+    constructor(config, transport) {
+        this.config = config;
+        this.transport = transport;
+        this.queue = [];
+        this.flushTimer = null;
+        this.isFlushing = false;
+    }
+
+    start() {
+        // Ensure flush runs periodically
+        if (!this.flushTimer) {
+            this.flushTimer = setInterval(() => {
+                this.flush();
+            }, this.config.flushIntervalMs);
+        }
+    }
+
+    stop() {
+        if (this.flushTimer) {
+            clearInterval(this.flushTimer);
+            this.flushTimer = null;
+        }
+        // Attempt final flush
+        return this.flush();
+    }
+
+    /**
+     * Adds a log line to the batch.
+     * @param {string} line - The raw log line.
+     */
+    push(line) {
+        const timestamp = Date.now().toString() + '000000'; // Unix nano approximation (ms + 6 zeros)
+        this.queue.push({
+            timestamp,
+            line
+        });
+
+        if (this.queue.length >= this.config.maxBatchSize) {
+            this.flush();
+        }
+    }
+
+    async flush() {
+        if (this.queue.length === 0 || this.isFlushing) return;
+
+        this.isFlushing = true;
+
+        // Swap queue to process
+        const batch = [...this.queue];
+        this.queue = [];
+
+        const payload = {
+            env: this.config.env,
+            service: this.config.service,
+            logs: batch
+        };
+
+        try {
+            await this.transport.send(payload);
+        } catch (error) {
+            console.error('Logship: Failed to flush batch, logs lost.', error);
+            // In a more advanced system, we might re-enqueue or persist to disk here.
+            // For this implementation, we log the error. Retry logic is in Transport.
+        } finally {
+            this.isFlushing = false;
+            // If queue filled up while flushing, trigger another flush immediately? 
+            // Maybe not immediately to avoid storm, wait for next tick or interval.
+        }
+    }
+}
+
+module.exports = Batcher;

package/src/config.js

@@ -0,0 +1,50 @@
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * Validates the configuration object.
+ * @param {Object} config - The user provided config.
+ * @returns {Object} - The validated and defaulted config.
+ * @throws {Error} - If validation fails.
+ */
+function validateConfig(config) {
+    if (!config) {
+        throw new Error('Logship: Configuration object is required.');
+    }
+
+    const requiredFields = ['apiKey'];
+    for (const field of requiredFields) {
+        if (!config[field] || typeof config[field] !== 'string') {
+            throw new Error(`Logship: Missing or invalid required field: ${field}`);
+        }
+    }
+
+    if (config.env && !['prod', 'staging', 'dev'].includes(config.env)) {
+        throw new Error('Logship: Invalid env. Must be one of: prod, staging, dev');
+    }
+
+    if (!Array.isArray(config.logFiles) || config.logFiles.length === 0) {
+        throw new Error('Logship: logFiles must be a non-empty array of file paths.');
+    }
+
+    const endpoint = config.endpoint || 'https://log.llmaudit.ai';
+
+    // Validate endpoints starts with http/https
+    if (!endpoint.startsWith('http')) {
+        throw new Error('Logship: endpoint must start with http:// or https://');
+    }
+
+    // Set defaults
+    return {
+        apiKey: config.apiKey,
+        endpoint: endpoint.replace(/\/$/, ''), // Remove trailing slash
+        websiteId: config.websiteId, // Optional now, will be ignored by server anyway
+        env: config.env || 'dev',
+        service: config.service || 'default-service',
+        logFiles: config.logFiles.map(p => path.resolve(p)),
+        flushIntervalMs: config.flushIntervalMs || 5000,
+        maxBatchSize: config.maxBatchSize || 100,
+    };
+}
+
+module.exports = { validateConfig };

package/src/index.js

@@ -0,0 +1,56 @@
+const { validateConfig } = require('./config');
+const Transport = require('./transport');
+const Batcher = require('./batcher');
+const WatchManager = require('./watcher');
+const Registry = require('./registry');
+
+class LogAgent {
+    constructor(config) {
+        this.config = validateConfig(config);
+        this.isRunning = false;
+
+        this.registry = new Registry();
+        this.transport = new Transport(this.config);
+        this.batcher = new Batcher(this.config, this.transport);
+        this.watcher = new WatchManager(this.config.logFiles, this.registry, (line) => {
+            this.batcher.push(line);
+        });
+    }
+
+    init() {
+        if (this.isRunning) return;
+
+        console.log('Logship: Initializing...');
+
+        this.registry.load();
+        this.batcher.start();
+        this.watcher.start();
+
+        this.isRunning = true;
+        console.log(`Logship: Started for service ${this.config.service} on ${this.config.env}`);
+        console.log(`Logship: Watching ${this.config.logFiles} files.`);
+    }
+
+    shutdown() {
+        this.isRunning = false;
+        console.log('Logship: Shutting down...');
+        this.watcher.stop();
+        this.batcher.stop().then(() => {
+            this.registry.close();
+            console.log('Logship: Shutdown complete.');
+        });
+    }
+}
+
+/**
+ * Initializes the global log agent instance.
+ * @param {Object} config 
+ * @returns {LogAgent}
+ */
+function initLogger(config) {
+    const agent = new LogAgent(config);
+    agent.init();
+    return agent;
+}
+
+module.exports = { initLogger, LogAgent };

package/src/registry.js

@@ -0,0 +1,60 @@
+const fs = require('fs');
+const path = require('path');
+
+class Registry {
+    constructor(storagePath) {
+        this.storagePath = storagePath || path.resolve(process.cwd(), '.agent-registry.json');
+        this.data = {};
+        this.saveTimer = null;
+    }
+
+    load() {
+        try {
+            if (fs.existsSync(this.storagePath)) {
+                const content = fs.readFileSync(this.storagePath, 'utf8');
+                this.data = JSON.parse(content);
+            }
+        } catch (e) {
+            console.error('Log Agent: Failed to load registry', e);
+            this.data = {};
+        }
+    }
+
+    get(filepath) {
+        return this.data[filepath];
+    }
+
+    set(filepath, offset, inode, signature) {
+        this.data[filepath] = { offset, inode, signature, lastUpdated: Date.now() };
+        this.scheduleSave();
+    }
+
+    remove(filepath) {
+        delete this.data[filepath];
+        this.scheduleSave();
+    }
+
+    scheduleSave() {
+        if (this.saveTimer) return;
+        this.saveTimer = setTimeout(() => this.save(), 1000); // Debounce saves
+    }
+
+    save() {
+        try {
+            console.log(`Log Agent: Saving registry to ${this.storagePath}`);
+            fs.writeFileSync(this.storagePath, JSON.stringify(this.data, null, 2));
+            this.saveTimer = null;
+        } catch (e) {
+            console.error('Log Agent: Failed to save registry', e);
+        }
+    }
+
+    close() {
+        if (this.saveTimer) {
+            clearTimeout(this.saveTimer);
+            this.save();
+        }
+    }
+}
+
+module.exports = Registry;

package/src/transport.js

@@ -0,0 +1,90 @@
+const https = require('https');
+const http = require('http');
+const paramURL = require('url');
+
+class Transport {
+    constructor(config) {
+        this.config = config;
+        this.maxRetries = 5;
+        this.baseDelay = 1000; // 1 second
+    }
+
+    /**
+     * Sends a batch of logs to the ingestion server.
+     * @param {Object} payload - The complete payload to send.
+     * @returns {Promise<void>}
+     */
+    async send(payload) {
+        let attempt = 0;
+
+        while (attempt <= this.maxRetries) {
+            try {
+                await this._post(payload);
+                return; // Success
+            } catch (error) {
+                attempt++;
+                if (attempt > this.maxRetries) {
+                    console.error(`Logship: Failed to send logs after ${this.maxRetries} attempts. Error: ${error.message}`);
+                    throw error; // Give up
+                }
+
+                const delay = this.baseDelay * Math.pow(2, attempt - 1); // Exponential backoff
+                // Add some jitter
+                const jitter = Math.random() * 200;
+                const totalDelay = delay + jitter;
+
+                console.warn(`Logship: Send failed, retrying in ${Math.round(totalDelay)}ms. Attempt ${attempt}/${this.maxRetries}. Error: ${error.message}`);
+                await new Promise(resolve => setTimeout(resolve, totalDelay));
+            }
+        }
+    }
+
+    _post(payload) {
+        return new Promise((resolve, reject) => {
+            const data = JSON.stringify(payload);
+            const urlObj = paramURL.parse(this.config.endpoint);
+
+            const options = {
+                hostname: urlObj.hostname,
+                port: urlObj.port || (urlObj.protocol === 'https:' ? 443 : 80),
+                path: '/ingest/logs', // Use fixed path as per req, or could be part of endpoint config. 
+                // Based on prompt: "endpoint: string, // central server URL", and server has "POST /ingest/logs"
+                // I'll assume endpoint is the BASE URL e.g. "https://api.logger.com".
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Content-Length': Buffer.byteLength(data),
+                    'x-api-key': this.config.apiKey
+                },
+                timeout: 10000 // 10s timeout
+            };
+
+            const lib = urlObj.protocol === 'https:' ? https : http;
+
+            const req = lib.request(options, (res) => {
+                let body = '';
+                res.on('data', (chunk) => body += chunk);
+                res.on('end', () => {
+                    if (res.statusCode >= 200 && res.statusCode < 300) {
+                        resolve(body);
+                    } else {
+                        reject(new Error(`HTTP Status ${res.statusCode}: ${body}`));
+                    }
+                });
+            });
+
+            req.on('error', (e) => {
+                reject(e);
+            });
+
+            req.on('timeout', () => {
+                req.destroy(new Error('Request Timeout'));
+            });
+
+            req.write(data);
+            req.end();
+        });
+    }
+}
+
+module.exports = Transport;

package/src/watcher.js

@@ -0,0 +1,199 @@
+const fs = require('fs');
+
+class FileWatcher {
+    constructor(filepath, registry, onLogLine) {
+        this.filepath = filepath;
+        this.registry = registry;
+        this.onLogLine = onLogLine;
+        this.currSize = 0;
+        this.inode = null;
+        this.signature = null;
+        this.watcher = null;
+        this.checkInterval = null;
+        this.recreateTimer = null;
+    }
+
+    start() {
+        this._initFile();
+    }
+
+    stop() {
+        if (this.watcher) {
+            this.watcher.close();
+            this.watcher = null;
+        }
+        if (this.checkInterval) {
+            clearInterval(this.checkInterval);
+            this.checkInterval = null;
+        }
+        if (this.recreateTimer) {
+            clearTimeout(this.recreateTimer);
+            this.recreateTimer = null;
+        }
+    }
+
+    _initFile() {
+        fs.stat(this.filepath, (err, stats) => {
+            if (err) {
+                if (err.code === 'ENOENT') {
+                    console.log(`Log Agent: File ${this.filepath} not found, waiting...`);
+                    this.recreateTimer = setTimeout(() => this._initFile(), 1000);
+                    return;
+                }
+                return;
+            }
+
+            this.inode = stats.ino;
+
+            this._readSignature(this.filepath, (signature) => {
+                this.signature = signature;
+                const savedState = this.registry.get(this.filepath);
+
+                let isValid = false;
+                if (savedState && savedState.inode === this.inode) {
+                    if (savedState.signature && savedState.signature !== signature) {
+                        // Signature mismatch means content changed (reuse)
+                        console.log(`Log Agent: Signature Mismatch! Resetting.`);
+                        isValid = false;
+                    } else if (stats.birthtimeMs && stats.birthtimeMs > savedState.lastUpdated) {
+                        console.log(`Log Agent: Newer Birthtime Detected. Resetting.`);
+                        isValid = false;
+                    } else {
+                        isValid = true;
+                    }
+                }
+
+                if (isValid) {
+                    console.log(`Log Agent: Resuming ${this.filepath} from offset ${savedState.offset}`);
+                    this.currSize = savedState.offset;
+
+                    if (stats.size > this.currSize) {
+                        this._read(this.currSize, stats.size);
+                        this.currSize = stats.size;
+                        this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+                    } else if (stats.size < this.currSize) {
+                        console.log('Log Agent: File truncated/rotated. Resetting.');
+                        this.currSize = 0;
+                        this._read(0, stats.size);
+                        this.currSize = stats.size;
+                        this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+                    }
+                } else {
+                    console.log(`Log Agent: New file/rotation/mismatch. Starting at ${stats.size}`);
+                    this.currSize = stats.size;
+                    this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+                }
+
+                this._watch();
+            });
+        });
+    }
+
+    _readSignature(filepath, callback) {
+        const stream = fs.createReadStream(filepath, { start: 0, end: 63 });
+        let buf = Buffer.alloc(0);
+        stream.on('data', chunk => {
+            buf = Buffer.concat([buf, chunk]);
+        });
+        stream.on('end', () => {
+            callback(buf.toString('hex'));
+        });
+        stream.on('error', () => callback(null));
+    }
+
+    _watch() {
+        try {
+            this.watcher = fs.watch(this.filepath, (eventType, filename) => {
+                if (eventType === 'rename') {
+                    fs.stat(this.filepath, (err, stats) => {
+                        if (err && err.code === 'ENOENT') {
+                            this.stop();
+                            this.recreateTimer = setTimeout(() => this.start(), 100);
+                        } else {
+                            if (stats.ino !== this.inode) {
+                                this.inode = stats.ino;
+                                this.currSize = 0;
+                                this._processChange();
+                            } else {
+                                this._processChange();
+                            }
+                        }
+                    });
+                } else {
+                    this._processChange();
+                }
+            });
+        } catch (e) {
+            console.error(`Log Agent: Watch failed for ${this.filepath}`, e);
+        }
+    }
+
+    _processChange() {
+        fs.stat(this.filepath, (err, stats) => {
+            if (err) return;
+
+            if (stats.size > this.currSize) {
+                this._read(this.currSize, stats.size);
+                this.currSize = stats.size;
+                this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+            } else if (stats.size < this.currSize) {
+                this.currSize = 0;
+                this._read(0, stats.size);
+                this.currSize = stats.size;
+                this.registry.set(this.filepath, this.currSize, this.inode, this.signature);
+            }
+        });
+    }
+
+    _read(start, end) {
+        if (start >= end) return;
+        if (start < 0) start = 0;
+
+        const stream = fs.createReadStream(this.filepath, {
+            start: start,
+            end: end - 1,
+            encoding: 'utf8'
+        });
+
+        let buffer = '';
+        stream.on('data', (chunk) => {
+            buffer += chunk;
+            const lines = buffer.split('\n');
+            buffer = lines.pop();
+
+            for (const line of lines) {
+                if (line.trim()) this.onLogLine(line);
+            }
+        });
+
+        stream.on('end', () => {
+            if (buffer && buffer.trim()) {
+                this.onLogLine(buffer);
+            }
+        });
+    }
+}
+
+class WatchManager {
+    constructor(logFiles, registry, onLogLine) {
+        this.logFiles = logFiles;
+        this.registry = registry;
+        this.onLogLine = onLogLine;
+        this.watchers = [];
+    }
+
+    start() {
+        this.logFiles.forEach(file => {
+            const watcher = new FileWatcher(file, this.registry, this.onLogLine);
+            watcher.start();
+            this.watchers.push(watcher);
+        });
+    }
+
+    stop() {
+        this.watchers.forEach(w => w.stop());
+        this.watchers = [];
+    }
+}
+
+module.exports = WatchManager;