logstrip 1.0.0

package/dist/cli/index.d.ts

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+import { Writable } from 'node:stream';
+import { type Aggressiveness, type LogStripResult } from '../core/logstrip-parser';
+export declare const CLI_VERSION = "1.0.0";
+export declare const HELP_TEXT = "Usage: logstrip [INPUT] [options]\n\nStream-based log compression that trims noisy server logs, build\npipelines, vulnerability scanners, and container workloads down to the\ndiagnostic context an LLM actually needs.\n\nArguments:\n  INPUT                    Path to the raw log. When omitted, reads from stdin.\n\nOptions:\n  -o, --output <path>      Write the compressed log to <path>. Defaults to stdout.\n  -a, --aggressiveness <l> Compression preset: low | medium | high | aggressive.\n                           Default: high.\n  -s, --stats              Print compression statistics to stderr.\n  -j, --json               Print LogStripResult as JSON to stdout. Requires --output.\n      --config <path>      Path to .logstrip.yml config file. Auto-detects from cwd.\n  -h, --help               Show this help text and exit.\n  -v, --version            Print the CLI version and exit.\n\nExamples:\n  logstrip raw.log -o clean.log\n  cat raw.log | logstrip > clean.log\n  logstrip raw.log --stats > clean.log\n  logstrip raw.log -o clean.log --json\n";
+export interface CliOptions {
+    input?: string;
+    output?: string;
+    aggressiveness: Aggressiveness;
+    stats: boolean;
+    json: boolean;
+    config?: string;
+    help: boolean;
+    version: boolean;
+}
+export interface CliIo {
+    stdin: NodeJS.ReadableStream;
+    stdout: NodeJS.WritableStream;
+    stderr: NodeJS.WritableStream;
+    stdinIsTTY: boolean;
+}
+export declare class CliError extends Error {
+    readonly exitCode: number;
+    constructor(message: string, exitCode?: number);
+}
+export declare function messageOf(error: unknown): string;
+export declare function parseCliOptions(argv: readonly string[]): CliOptions;
+export declare function formatStats(result: LogStripResult): string;
+export declare function writeAll(stream: NodeJS.WritableStream, chunk: string): Promise<void>;
+export declare function endStream(stream: Writable): Promise<void>;
+export declare function runCli(argv: readonly string[], io: CliIo): Promise<number>;

package/dist/cli/index.js

@@ -0,0 +1,224 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CliError = exports.HELP_TEXT = exports.CLI_VERSION = void 0;
+exports.messageOf = messageOf;
+exports.parseCliOptions = parseCliOptions;
+exports.formatStats = formatStats;
+exports.writeAll = writeAll;
+exports.endStream = endStream;
+exports.runCli = runCli;
+const node_fs_1 = require("node:fs");
+const node_util_1 = require("node:util");
+const logstrip_parser_1 = require("../core/logstrip-parser");
+exports.CLI_VERSION = '1.0.0'; // x-release-please-version
+exports.HELP_TEXT = `Usage: logstrip [INPUT] [options]
+
+Stream-based log compression that trims noisy server logs, build
+pipelines, vulnerability scanners, and container workloads down to the
+diagnostic context an LLM actually needs.
+
+Arguments:
+  INPUT                    Path to the raw log. When omitted, reads from stdin.
+
+Options:
+  -o, --output <path>      Write the compressed log to <path>. Defaults to stdout.
+  -a, --aggressiveness <l> Compression preset: low | medium | high | aggressive.
+                           Default: high.
+  -s, --stats              Print compression statistics to stderr.
+  -j, --json               Print LogStripResult as JSON to stdout. Requires --output.
+      --config <path>      Path to .logstrip.yml config file. Auto-detects from cwd.
+  -h, --help               Show this help text and exit.
+  -v, --version            Print the CLI version and exit.
+
+Examples:
+  logstrip raw.log -o clean.log
+  cat raw.log | logstrip > clean.log
+  logstrip raw.log --stats > clean.log
+  logstrip raw.log -o clean.log --json
+`;
+class CliError extends Error {
+    exitCode;
+    constructor(message, exitCode = 1) {
+        super(message);
+        this.name = 'CliError';
+        this.exitCode = exitCode;
+    }
+}
+exports.CliError = CliError;
+function messageOf(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function parseCliOptions(argv) {
+    let parsed;
+    try {
+        parsed = (0, node_util_1.parseArgs)({
+            args: argv,
+            allowPositionals: true,
+            strict: true,
+            options: {
+                output: { type: 'string', short: 'o' },
+                aggressiveness: { type: 'string', short: 'a' },
+                stats: { type: 'boolean', short: 's', default: false },
+                json: { type: 'boolean', short: 'j', default: false },
+                config: { type: 'string' },
+                help: { type: 'boolean', short: 'h', default: false },
+                version: { type: 'boolean', short: 'v', default: false },
+            },
+        });
+    }
+    catch (error) {
+        throw new CliError(messageOf(error), 2);
+    }
+    if (parsed.positionals.length > 1) {
+        throw new CliError(`Unexpected positional argument: ${parsed.positionals[1]}`, 2);
+    }
+    const aggressivenessInput = typeof parsed.values.aggressiveness === 'string'
+        ? parsed.values.aggressiveness
+        : 'high';
+    let aggressiveness;
+    try {
+        aggressiveness = (0, logstrip_parser_1.parseAggressiveness)(aggressivenessInput);
+    }
+    catch (error) {
+        throw new CliError(messageOf(error), 2);
+    }
+    return {
+        input: parsed.positionals[0],
+        output: typeof parsed.values.output === 'string' ? parsed.values.output : undefined,
+        aggressiveness,
+        stats: parsed.values.stats === true,
+        json: parsed.values.json === true,
+        config: typeof parsed.values.config === 'string'
+            ? parsed.values.config
+            : undefined,
+        help: parsed.values.help === true,
+        version: parsed.values.version === true,
+    };
+}
+function formatStats(result) {
+    const lines = [
+        'LogStrip compression report',
+        `  input lines     : ${result.stats.inputLines}`,
+        `  output lines    : ${result.stats.outputLines}`,
+        `  dropped lines   : ${result.stats.droppedLines}`,
+        `  duplicate lines : ${result.stats.duplicateLines}`,
+        `  hidden internal : ${result.stats.hiddenInternalStackLines}`,
+        `  input tokens    : ${result.inputTokens}`,
+        `  output tokens   : ${result.outputTokens}`,
+        `  saved tokens    : ${result.savedTokens}`,
+        `  savings         : ${result.savingsPercent.toFixed(2)}%`,
+    ];
+    if (result.outputPath !== undefined) {
+        lines.push(`  output path     : ${result.outputPath}`);
+    }
+    return `${lines.join('\n')}\n`;
+}
+function writeAll(stream, chunk) {
+    return new Promise((resolve, reject) => {
+        stream.write(chunk, (error) => {
+            if (error) {
+                reject(error);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+}
+function endStream(stream) {
+    return new Promise((resolve, reject) => {
+        stream.end((error) => {
+            if (error) {
+                reject(error);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+}
+async function runCli(argv, io) {
+    let options;
+    try {
+        options = parseCliOptions(argv);
+    }
+    catch (error) {
+        const cliError = error;
+        await writeAll(io.stderr, `logstrip: ${cliError.message}\n`);
+        return cliError.exitCode;
+    }
+    if (options.help) {
+        await writeAll(io.stdout, exports.HELP_TEXT);
+        return 0;
+    }
+    if (options.version) {
+        await writeAll(io.stdout, `${exports.CLI_VERSION}\n`);
+        return 0;
+    }
+    if (options.json && options.output === undefined) {
+        await writeAll(io.stderr, 'logstrip: --json requires --output so the compressed log does not collide with the JSON report on stdout\n');
+        return 2;
+    }
+    if (options.input === undefined && io.stdinIsTTY) {
+        await writeAll(io.stderr, 'logstrip: no INPUT given and stdin is a terminal. Pass a file path or pipe a log.\n');
+        return 2;
+    }
+    if (options.input !== undefined &&
+        options.output !== undefined &&
+        (0, logstrip_parser_1.pathsReferToSameFile)(options.input, options.output)) {
+        await writeAll(io.stderr, 'logstrip: INPUT and --output must be different paths; refusing to overwrite the input log\n');
+        return 2;
+    }
+    const input = options.input !== undefined
+        ? (0, node_fs_1.createReadStream)(options.input, { encoding: 'utf8' })
+        : io.stdin;
+    const output = options.output !== undefined
+        ? (0, node_fs_1.createWriteStream)(options.output, { encoding: 'utf8' })
+        : io.stdout;
+    let result;
+    try {
+        result = await (0, logstrip_parser_1.processLogStream)(input, output, {
+            aggressiveness: options.aggressiveness,
+            configPath: options.config,
+        });
+    }
+    catch (error) {
+        if (options.input !== undefined) {
+            input.destroy();
+        }
+        if (options.output !== undefined) {
+            output.destroy();
+        }
+        await writeAll(io.stderr, `logstrip: ${messageOf(error)}\n`);
+        return 1;
+    }
+    if (options.output !== undefined) {
+        result = { ...result, outputPath: options.output };
+        await endStream(output);
+    }
+    if (options.json) {
+        await writeAll(io.stdout, `${JSON.stringify(result, null, 2)}\n`);
+    }
+    else if (options.stats) {
+        await writeAll(io.stderr, formatStats(result));
+    }
+    return 0;
+}
+/* v8 ignore start */
+if (require.main === module) {
+    runCli(process.argv.slice(2), {
+        stdin: process.stdin,
+        stdout: process.stdout,
+        stderr: process.stderr,
+        stdinIsTTY: Boolean(process.stdin.isTTY),
+    })
+        .then((code) => {
+        process.exitCode = code;
+    })
+        .catch((error) => {
+        process.stderr.write(`logstrip: ${error instanceof Error ? error.stack ?? error.message : String(error)}\n`);
+        process.exitCode = 1;
+    });
+}
+/* v8 ignore stop */

package/dist/core/logstrip-config.d.ts

@@ -0,0 +1,19 @@
+export interface LogStripSourceSignature {
+    name: string;
+    markers: readonly string[];
+}
+export interface LogStripCustomConfig {
+    sources: readonly LogStripSourceSignature[];
+    diagnosticPatterns: readonly string[];
+    ignorePatterns: readonly string[];
+    sanitizePatterns: readonly SanitizeRule[];
+    internalStackPatterns: readonly string[];
+}
+export interface SanitizeRule {
+    pattern: string;
+    replacement: string;
+    flags?: string;
+}
+export declare function parseLogStripConfig(content: string): LogStripCustomConfig;
+export declare function loadLogStripConfig(explicitPath?: string, startDir?: string): LogStripCustomConfig;
+export declare function resolveConfigPath(explicitPath?: string, startDir?: string): string | undefined;

package/dist/core/logstrip-config.js

@@ -0,0 +1,268 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseLogStripConfig = parseLogStripConfig;
+exports.loadLogStripConfig = loadLogStripConfig;
+exports.resolveConfigPath = resolveConfigPath;
+const node_fs_1 = require("node:fs");
+const node_path_1 = __importDefault(require("node:path"));
+const CONFIG_FILENAME = '.logstrip.yml';
+const EMPTY_CONFIG = {
+    sources: [],
+    diagnosticPatterns: [],
+    ignorePatterns: [],
+    sanitizePatterns: [],
+    internalStackPatterns: [],
+};
+function parseLogStripConfig(content) {
+    const parsed = parseMinimalYaml(content);
+    return normalizeConfig(parsed);
+}
+function loadLogStripConfig(explicitPath, startDir) {
+    const configPath = resolveConfigPath(explicitPath, startDir);
+    if (configPath === undefined)
+        return EMPTY_CONFIG;
+    const content = (0, node_fs_1.readFileSync)(configPath, 'utf8');
+    return parseLogStripConfig(content);
+}
+function resolveConfigPath(explicitPath, startDir) {
+    if (explicitPath !== undefined) {
+        if (!(0, node_fs_1.existsSync)(explicitPath))
+            return undefined;
+        return explicitPath;
+    }
+    const dir = startDir ?? process.cwd();
+    const candidate = node_path_1.default.join(dir, CONFIG_FILENAME);
+    if ((0, node_fs_1.existsSync)(candidate))
+        return candidate;
+    return undefined;
+}
+function normalizeConfig(raw) {
+    return {
+        sources: normalizeSources(raw.sources),
+        diagnosticPatterns: normalizeStringArray(raw.diagnosticPatterns),
+        ignorePatterns: normalizeStringArray(raw.ignorePatterns),
+        sanitizePatterns: normalizeSanitizeRules(raw.sanitizePatterns),
+        internalStackPatterns: normalizeStringArray(raw.internalStackPatterns),
+    };
+}
+function normalizeSources(raw) {
+    if (!Array.isArray(raw))
+        return [];
+    return raw
+        .filter((entry) => typeof entry === 'object' && entry !== null)
+        .map((entry) => ({
+        name: String(entry.name ?? ''),
+        markers: normalizeStringArray(entry.markers),
+    }))
+        .filter((entry) => entry.name !== '' && entry.markers.length > 0);
+}
+function normalizeStringArray(raw) {
+    if (!Array.isArray(raw))
+        return [];
+    return raw.filter((v) => typeof v === 'string');
+}
+function normalizeSanitizeRules(raw) {
+    if (!Array.isArray(raw))
+        return [];
+    return raw
+        .filter((entry) => typeof entry === 'object' && entry !== null)
+        .map((entry) => {
+        const flagsRaw = entry.flags;
+        const flags = flagsRaw !== undefined ? String(flagsRaw) : undefined;
+        return {
+            pattern: String(entry.pattern ?? ''),
+            replacement: String(entry.replacement ?? ''),
+            flags,
+        };
+    })
+        .filter((entry) => entry.pattern !== '');
+}
+function parseMinimalYaml(content) {
+    const result = {};
+    const lines = content.split(/\r?\n/u);
+    // State machine: track which block we are building
+    let topKey = '';
+    let topValue = null;
+    // For array-of-objects (like sources:)
+    let objList = null;
+    let currentObj = null;
+    // For plain arrays (like diagnosticPatterns:)
+    let scalarList = null;
+    // For sub-array inside currentObj (like markers: inside a source)
+    let subArrayKey = '';
+    let subArray = null;
+    const flushTop = () => {
+        if (topKey === '')
+            return;
+        // Flush any pending sub-array into the current object
+        if (subArrayKey !== '' && subArray !== null && currentObj !== null) {
+            currentObj[subArrayKey] = subArray;
+        }
+        if (objList !== null) {
+            result[topKey] = objList;
+        }
+        else if (scalarList !== null) {
+            result[topKey] = scalarList;
+        }
+        else if (topValue !== null) {
+            result[topKey] = topValue;
+        }
+        topKey = '';
+        topValue = null;
+        objList = null;
+        currentObj = null;
+        scalarList = null;
+        subArrayKey = '';
+        subArray = null;
+    };
+    for (const rawLine of lines) {
+        const line = rawLine.replace(/#.*$/u, '').trimEnd();
+        if (line.trim().length === 0)
+            continue;
+        const indent = rawLine.match(/^(\s*)/u)[1].length;
+        // ── Top-level key ──
+        const topMatch = line.match(/^([\w-]+):\s*(.*)$/u);
+        if (topMatch && indent === 0) {
+            flushTop();
+            topKey = topMatch[1];
+            const inlineVal = topMatch[2].trim();
+            if (inlineVal !== '') {
+                topValue = parseYamlValue(inlineVal);
+            }
+            continue;
+        }
+        // ── Array item: "- value" or "- name: foo" at indent 2 ──
+        const arrMatch = line.match(/^(\s{2})-\s+(.*)$/u);
+        if (arrMatch) {
+            const rest = arrMatch[2];
+            // Is this "- name: value" (object in array)?
+            const objPropMatch = rest.match(/^([\w-]+):\s*(.*)$/u);
+            if (objPropMatch) {
+                // Flush sub-array if we were building one
+                if (subArrayKey !== '' && subArray !== null && currentObj !== null) {
+                    currentObj[subArrayKey] = subArray;
+                    subArrayKey = '';
+                    subArray = null;
+                }
+                // Start a new object in the array
+                if (objList === null)
+                    objList = [];
+                currentObj = { [objPropMatch[1]]: parseYamlValue(objPropMatch[2]) };
+                objList.push(currentObj);
+            }
+            else {
+                // Plain scalar array item
+                if (scalarList === null)
+                    scalarList = [];
+                scalarList.push(parseYamlValue(rest));
+            }
+            continue;
+        }
+        // ── Sub-array item: "    - value" at indent 4+ ──
+        const subArrMatch = line.match(/^(\s{4,})-\s+(.*)$/u);
+        if (subArrMatch && currentObj !== null) {
+            if (subArray === null)
+                subArray = [];
+            subArray.push(parseYamlValue(subArrMatch[2]));
+            continue;
+        }
+        // ── Property inside currentObj: "    key: value" at indent 4+ ──
+        const propMatch = line.match(/^(\s{4,})([\w-]+):\s*(.*)$/u);
+        if (propMatch && currentObj !== null) {
+            // Flush previous sub-array
+            if (subArrayKey !== '') {
+                if (subArray !== null) {
+                    currentObj[subArrayKey] = subArray;
+                }
+            }
+            const propKey = propMatch[2];
+            const propVal = propMatch[3].trim();
+            if (propVal === '') {
+                // Property with no inline value → next lines will be sub-array
+                subArrayKey = propKey;
+                subArray = null;
+            }
+            else {
+                currentObj[propKey] = parseYamlValue(propVal);
+                subArrayKey = '';
+                subArray = null;
+            }
+            continue;
+        }
+    }
+    // Flush final state
+    if (subArrayKey !== '' && subArray !== null && currentObj !== null) {
+        currentObj[subArrayKey] = subArray;
+    }
+    flushTop();
+    return result;
+}
+function parseYamlValue(value) {
+    if (value === 'true')
+        return true;
+    if (value === 'false')
+        return false;
+    if (value === 'null' || value === '~')
+        return null;
+    // Quoted string
+    if ((value.startsWith('"') && value.endsWith('"')) ||
+        (value.startsWith("'") && value.endsWith("'"))) {
+        const inner = value.slice(1, -1);
+        // Double-quoted YAML strings unescape \\ → \ and \" → "
+        if (value.startsWith('"')) {
+            return inner.replace(/\\\\/gu, '\\').replace(/\\"/gu, '"');
+        }
+        return inner;
+    }
+    // Inline array: [a, b, c]
+    if (value.startsWith('[') && value.endsWith(']')) {
+        const inner = value.slice(1, -1).trim();
+        if (inner === '')
+            return [];
+        return splitInlineArray(inner).map((s) => parseYamlValue(s.trim()));
+    }
+    // Number
+    if (/^-?\d+(\.\d+)?$/u.test(value)) {
+        return Number(value);
+    }
+    return value;
+}
+function splitInlineArray(inner) {
+    const result = [];
+    let current = '';
+    let inQuote = null;
+    let depth = 0;
+    for (let i = 0; i < inner.length; i++) {
+        const ch = inner[i];
+        if (inQuote !== null) {
+            current += ch;
+            if (ch === inQuote)
+                inQuote = null;
+        }
+        else if (ch === '"' || ch === "'") {
+            inQuote = ch;
+            current += ch;
+        }
+        else if (ch === '[') {
+            depth++;
+            current += ch;
+        }
+        else if (ch === ']') {
+            depth--;
+            current += ch;
+        }
+        else if (ch === ',' && depth === 0) {
+            result.push(current);
+            current = '';
+        }
+        else {
+            current += ch;
+        }
+    }
+    if (current.trim() !== '')
+        result.push(current);
+    return result;
+}

package/dist/core/logstrip-parser.d.ts

@@ -0,0 +1,58 @@
+import { Writable } from 'node:stream';
+import { type LogStripCustomConfig } from './logstrip-config.js';
+export type Aggressiveness = 'low' | 'medium' | 'high' | 'aggressive';
+export interface LogStripOptions {
+    aggressiveness?: Aggressiveness;
+    configPath?: string;
+}
+export interface LogStripStats {
+    inputLines: number;
+    outputLines: number;
+    inputWords: number;
+    outputWords: number;
+    inputBytes: number;
+    outputBytes: number;
+    droppedLines: number;
+    duplicateLines: number;
+    hiddenInternalStackLines: number;
+}
+export interface LogStripResult {
+    stats: LogStripStats;
+    inputTokens: number;
+    outputTokens: number;
+    savedTokens: number;
+    savingsPercent: number;
+    detectedSources?: readonly string[];
+    outputPath?: string;
+}
+export declare const INTERNAL_STACK_MARKER = "[... hidden internal library frames ...]";
+export declare const CONTEXT_WINDOW_BEFORE = 3;
+export declare const CONTEXT_WINDOW_AFTER = 2;
+export declare const SCORE_KEEP_THRESHOLD = 40;
+export declare const TFIDF_REPEAT_THRESHOLD = 3;
+export declare const TFIDF_PENALTY = 8;
+export declare const TFIDF_MAP_LIMIT = 50000;
+export declare const MAX_REPEAT_DELTA_VALUES = 3;
+export declare const LOG_SOURCE_SIGNATURES: ReadonlyArray<readonly [source: string, markers: readonly string[]]>;
+export declare const KNOWN_LOG_SOURCES: readonly string[];
+export declare function detectLogSources(input: string | readonly string[], limit?: number): readonly string[];
+export declare function parseAggressiveness(value: string | undefined): Aggressiveness;
+export declare function sanitizeLine(line: string): string;
+export declare function createRepeatSignature(line: string): string;
+export declare function shouldKeepLine(line: string): boolean;
+export declare function looksLikeDiagnosticLine(line: string): boolean;
+export declare function isInternalStackTraceLine(line: string): boolean;
+export declare function estimateTokens(wordCount: number): number;
+/**
+ * Multi-signal relevance score for a single (sanitized) log line.
+ * >= SCORE_KEEP_THRESHOLD  → emit immediately
+ * 0 to <SCORE_KEEP_THRESHOLD → buffer in context window
+ * < 0 or -Infinity         → drop
+ */
+export declare function scoreLineRelevance(line: string, aggressiveness: Aggressiveness, seenCount?: number): number;
+export declare function buildMergedConfig(options?: LogStripOptions): LogStripCustomConfig & {
+    mergedSources: readonly (readonly [string, readonly string[]])[];
+};
+export declare function processLogStream(input: NodeJS.ReadableStream, output: Writable, options?: LogStripOptions): Promise<LogStripResult>;
+export declare function processLogFile(inputPath: string, outputPath: string, options?: LogStripOptions): Promise<LogStripResult>;
+export declare function pathsReferToSameFile(inputPath: string, outputPath: string): boolean;