logpare 0.0.1 → 0.0.3

package/dist/cli.cjs

@@ -0,0 +1,729 @@
+#!/usr/bin/env node
+"use strict";
+
+// src/cli.ts
+var import_node_util = require("util");
+var import_node_fs = require("fs");
+var import_node_url = require("url");
+var import_node_path = require("path");
+
+// src/preprocessing/patterns.ts
+var DEFAULT_PATTERNS = {
+  // Timestamps (most specific - must run before port to avoid fragmentation)
+  isoTimestamp: /\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?/g,
+  unixTimestamp: /\b\d{10,13}\b/g,
+  // Network addresses
+  ipv4: /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g,
+  ipv6: /\b[0-9a-fA-F:]{7,39}\b/g,
+  port: /:\d{2,5}\b/g,
+  // Identifiers
+  uuid: /\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g,
+  hexId: /\b0x[0-9a-fA-F]+\b/g,
+  blockId: /\bblk_-?\d+\b/g,
+  // Paths and URLs
+  filePath: /(?:\/[\w.-]+)+/g,
+  url: /https?:\/\/[^\s]+/g,
+  // Numbers (applied last - most aggressive)
+  // Matches standalone numbers and numbers with units (e.g., 250ms, 1024KB)
+  numbers: /\b\d+(?:\.\d+)?[a-zA-Z]*\b/g
+};
+var WILDCARD = "<*>";
+function applyPatterns(line, patterns, wildcard = WILDCARD) {
+  let result = line;
+  for (const pattern of Object.values(patterns)) {
+    const regex = new RegExp(pattern.source, pattern.flags);
+    result = result.replace(regex, wildcard);
+  }
+  return result;
+}
+
+// src/preprocessing/default.ts
+var defaultStrategy = {
+  /**
+   * Preprocess a log line by masking common variable patterns.
+   */
+  preprocess(line) {
+    return applyPatterns(line, DEFAULT_PATTERNS, WILDCARD);
+  },
+  /**
+   * Tokenize a line by splitting on whitespace.
+   */
+  tokenize(line) {
+    return line.split(/\s+/).filter((token) => token.length > 0);
+  },
+  /**
+   * Get similarity threshold for a given tree depth.
+   * Uses a constant threshold of 0.4.
+   */
+  getSimThreshold(_depth) {
+    return 0.4;
+  }
+};
+
+// src/drain/node.ts
+var DrainNode = class _DrainNode {
+  /** Depth of this node in the tree (0 = root) */
+  depth;
+  /**
+   * Child nodes keyed by token value.
+   * Using Map instead of Object for V8 optimization.
+   */
+  children;
+  /**
+   * Clusters (templates) stored at this node.
+   * Only leaf nodes contain clusters.
+   */
+  clusters;
+  constructor(depth) {
+    this.depth = depth;
+    this.children = /* @__PURE__ */ new Map();
+    this.clusters = [];
+  }
+  /**
+   * Get or create a child node for the given key.
+   */
+  getOrCreateChild(key) {
+    let child = this.children.get(key);
+    if (child === void 0) {
+      child = new _DrainNode(this.depth + 1);
+      this.children.set(key, child);
+    }
+    return child;
+  }
+  /**
+   * Check if this node has a child for the given key.
+   */
+  hasChild(key) {
+    return this.children.has(key);
+  }
+  /**
+   * Get a child node by key, or undefined if not found.
+   */
+  getChild(key) {
+    return this.children.get(key);
+  }
+  /**
+   * Add a cluster to this node.
+   */
+  addCluster(cluster) {
+    this.clusters.push(cluster);
+  }
+  /**
+   * Get the number of children.
+   */
+  get childCount() {
+    return this.children.size;
+  }
+  /**
+   * Get the number of clusters.
+   */
+  get clusterCount() {
+    return this.clusters.length;
+  }
+};
+
+// src/drain/cluster.ts
+var LogCluster = class {
+  /** Unique identifier for this cluster */
+  id;
+  /** Template tokens (with wildcards for variable positions) */
+  tokens;
+  /** Number of log lines matching this template */
+  count;
+  /** Sample variable values from first N matches */
+  sampleVariables;
+  /** Line index of first occurrence */
+  firstSeen;
+  /** Line index of most recent occurrence */
+  lastSeen;
+  /** Maximum number of sample variables to store */
+  maxSamples;
+  constructor(id, tokens, lineIndex, maxSamples = 3) {
+    this.id = id;
+    this.tokens = tokens.slice();
+    this.count = 1;
+    this.sampleVariables = [];
+    this.firstSeen = lineIndex;
+    this.lastSeen = lineIndex;
+    this.maxSamples = maxSamples;
+  }
+  /**
+   * Update the cluster with a new matching log line.
+   * Returns the variables extracted from this match.
+   */
+  update(tokens, lineIndex) {
+    this.count++;
+    this.lastSeen = lineIndex;
+    const variables = [];
+    for (let i = 0; i < this.tokens.length && i < tokens.length; i++) {
+      if (this.tokens[i] === WILDCARD) {
+        variables.push(tokens[i] ?? "");
+      }
+    }
+    if (this.sampleVariables.length < this.maxSamples) {
+      this.sampleVariables.push(variables);
+    }
+    return variables;
+  }
+  /**
+   * Get the template pattern as a string.
+   */
+  getPattern() {
+    return this.tokens.join(" ");
+  }
+  /**
+   * Compute similarity between this cluster's template and a set of tokens.
+   * Returns a value between 0.0 and 1.0.
+   */
+  computeSimilarity(tokens) {
+    if (this.tokens.length === 0) {
+      return 0;
+    }
+    if (tokens.length !== this.tokens.length) {
+      return 0;
+    }
+    let matchCount = 0;
+    for (let i = 0; i < this.tokens.length; i++) {
+      const templateToken = this.tokens[i];
+      const inputToken = tokens[i];
+      if (templateToken === WILDCARD) {
+        matchCount++;
+      } else if (templateToken === inputToken) {
+        matchCount++;
+      }
+    }
+    return matchCount / this.tokens.length;
+  }
+  /**
+   * Merge tokens into the template, converting differing positions to wildcards.
+   * Mutates the template tokens in place.
+   */
+  mergeTokens(tokens) {
+    for (let i = 0; i < this.tokens.length && i < tokens.length; i++) {
+      if (this.tokens[i] !== WILDCARD && this.tokens[i] !== tokens[i]) {
+        this.tokens[i] = WILDCARD;
+      }
+    }
+  }
+};
+
+// src/output/formatter.ts
+function formatSummary(templates, stats) {
+  const lines = [];
+  lines.push("=== Log Compression Summary ===");
+  lines.push(
+    `Input: ${stats.inputLines.toLocaleString()} lines \u2192 ${stats.uniqueTemplates} templates (${(stats.compressionRatio * 100).toFixed(1)}% reduction)`
+  );
+  lines.push("");
+  if (templates.length === 0) {
+    lines.push("No templates discovered.");
+    return lines.join("\n");
+  }
+  lines.push("Top templates by frequency:");
+  const topTemplates = templates.slice(0, 20);
+  topTemplates.forEach((template, index) => {
+    const count = template.occurrences.toLocaleString();
+    lines.push(`${index + 1}. [${count}x] ${template.pattern}`);
+  });
+  if (templates.length > 20) {
+    lines.push(`... and ${templates.length - 20} more templates`);
+  }
+  const rareTemplates = templates.filter((t) => t.occurrences <= 5);
+  if (rareTemplates.length > 0) {
+    lines.push("");
+    lines.push(`Rare events (\u22645 occurrences): ${rareTemplates.length} templates`);
+    const shownRare = rareTemplates.slice(0, 5);
+    for (const template of shownRare) {
+      lines.push(`- [${template.occurrences}x] ${template.pattern}`);
+    }
+    if (rareTemplates.length > 5) {
+      lines.push(`... and ${rareTemplates.length - 5} more rare templates`);
+    }
+  }
+  return lines.join("\n");
+}
+function formatDetailed(templates, stats) {
+  const lines = [];
+  lines.push("=== Log Compression Details ===");
+  lines.push(
+    `Input: ${stats.inputLines.toLocaleString()} lines \u2192 ${stats.uniqueTemplates} templates (${(stats.compressionRatio * 100).toFixed(1)}% reduction)`
+  );
+  lines.push(`Estimated token reduction: ${(stats.estimatedTokenReduction * 100).toFixed(1)}%`);
+  lines.push("");
+  if (templates.length === 0) {
+    lines.push("No templates discovered.");
+    return lines.join("\n");
+  }
+  for (const template of templates) {
+    lines.push(`=== Template ${template.id} (${template.occurrences.toLocaleString()} occurrences) ===`);
+    lines.push(`Pattern: ${template.pattern}`);
+    lines.push(`First seen: line ${template.firstSeen + 1}`);
+    lines.push(`Last seen: line ${template.lastSeen + 1}`);
+    if (template.sampleVariables.length > 0) {
+      lines.push("Sample variables:");
+      for (const vars of template.sampleVariables) {
+        if (vars.length > 0) {
+          lines.push(`  - ${vars.join(", ")}`);
+        }
+      }
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+function formatJson(templates, stats) {
+  const output = {
+    version: "1.0",
+    stats: {
+      inputLines: stats.inputLines,
+      uniqueTemplates: stats.uniqueTemplates,
+      compressionRatio: Math.round(stats.compressionRatio * 1e3) / 1e3,
+      estimatedTokenReduction: Math.round(stats.estimatedTokenReduction * 1e3) / 1e3
+    },
+    templates: templates.map((t) => ({
+      id: t.id,
+      pattern: t.pattern,
+      occurrences: t.occurrences,
+      samples: t.sampleVariables,
+      firstSeen: t.firstSeen,
+      lastSeen: t.lastSeen
+    }))
+  };
+  return JSON.stringify(output, null, 2);
+}
+
+// src/drain/drain.ts
+var DEFAULTS = {
+  depth: 4,
+  simThreshold: 0.4,
+  maxChildren: 100,
+  maxClusters: 1e3,
+  maxSamples: 3
+};
+var WILDCARD_KEY = "<WILDCARD>";
+var Drain = class {
+  root;
+  clusters;
+  strategy;
+  depth;
+  maxChildren;
+  maxClusters;
+  maxSamples;
+  lineCount;
+  nextClusterId;
+  constructor(options = {}) {
+    this.root = new DrainNode(0);
+    this.clusters = [];
+    this.strategy = options.preprocessing ?? defaultStrategy;
+    this.depth = options.depth ?? DEFAULTS.depth;
+    this.maxChildren = options.maxChildren ?? DEFAULTS.maxChildren;
+    this.maxClusters = options.maxClusters ?? DEFAULTS.maxClusters;
+    this.maxSamples = options.maxSamples ?? DEFAULTS.maxSamples;
+    this.lineCount = 0;
+    this.nextClusterId = 1;
+  }
+  /**
+   * Process a single log line.
+   */
+  addLogLine(line) {
+    const lineIndex = this.lineCount++;
+    const trimmed = line.trim();
+    if (trimmed.length === 0) {
+      return null;
+    }
+    const preprocessed = this.strategy.preprocess(trimmed);
+    const tokens = this.strategy.tokenize(preprocessed);
+    if (tokens.length === 0) {
+      return null;
+    }
+    const matchedCluster = this.treeSearch(tokens);
+    if (matchedCluster !== null) {
+      matchedCluster.update(tokens, lineIndex);
+      matchedCluster.mergeTokens(tokens);
+      return matchedCluster;
+    }
+    if (this.clusters.length >= this.maxClusters) {
+      return null;
+    }
+    return this.createCluster(tokens, lineIndex);
+  }
+  /**
+   * Process multiple log lines.
+   */
+  addLogLines(lines) {
+    for (const line of lines) {
+      this.addLogLine(line);
+    }
+  }
+  /**
+   * Search the parse tree for a matching cluster.
+   */
+  treeSearch(tokens) {
+    const tokenCount = tokens.length;
+    const tokenCountKey = String(tokenCount);
+    const lengthNode = this.root.getChild(tokenCountKey);
+    if (lengthNode === void 0) {
+      return null;
+    }
+    const firstToken = tokens[0];
+    if (firstToken === void 0) {
+      return null;
+    }
+    let currentNode = lengthNode.getChild(firstToken);
+    if (currentNode === void 0) {
+      currentNode = lengthNode.getChild(WILDCARD_KEY);
+    }
+    if (currentNode === void 0) {
+      return null;
+    }
+    let searchNode = currentNode;
+    for (let i = 1; i < Math.min(tokens.length, this.depth); i++) {
+      const token = tokens[i];
+      if (token === void 0) {
+        break;
+      }
+      let nextNode = searchNode.getChild(token);
+      if (nextNode === void 0) {
+        nextNode = searchNode.getChild(WILDCARD_KEY);
+      }
+      if (nextNode === void 0) {
+        break;
+      }
+      searchNode = nextNode;
+    }
+    return this.findBestMatch(searchNode, tokens);
+  }
+  /**
+   * Find the best matching cluster at a node.
+   */
+  findBestMatch(node, tokens) {
+    let bestCluster = null;
+    let bestSimilarity = 0;
+    const threshold = this.strategy.getSimThreshold(node.depth);
+    for (const cluster of node.clusters) {
+      const similarity = cluster.computeSimilarity(tokens);
+      if (similarity >= threshold && similarity > bestSimilarity) {
+        bestSimilarity = similarity;
+        bestCluster = cluster;
+      }
+    }
+    return bestCluster;
+  }
+  /**
+   * Create a new cluster and add it to the tree.
+   */
+  createCluster(tokens, lineIndex) {
+    const clusterId = `t${String(this.nextClusterId++).padStart(3, "0")}`;
+    const cluster = new LogCluster(clusterId, tokens, lineIndex, this.maxSamples);
+    const tokenCount = tokens.length;
+    const tokenCountKey = String(tokenCount);
+    const lengthNode = this.root.getOrCreateChild(tokenCountKey);
+    const firstToken = tokens[0];
+    if (firstToken === void 0) {
+      this.clusters.push(cluster);
+      return cluster;
+    }
+    const firstKey = this.shouldUseWildcard(lengthNode, firstToken) ? WILDCARD_KEY : firstToken;
+    let currentNode = lengthNode.getOrCreateChild(firstKey);
+    for (let i = 1; i < Math.min(tokens.length, this.depth); i++) {
+      const token = tokens[i];
+      if (token === void 0) {
+        break;
+      }
+      const key = this.shouldUseWildcard(currentNode, token) ? WILDCARD_KEY : token;
+      currentNode = currentNode.getOrCreateChild(key);
+    }
+    currentNode.addCluster(cluster);
+    this.clusters.push(cluster);
+    return cluster;
+  }
+  /**
+   * Determine if we should use a wildcard key for a token.
+   * Uses maxChildren limit to prevent tree explosion.
+   */
+  shouldUseWildcard(node, token) {
+    if (node.hasChild(token)) {
+      return false;
+    }
+    if (node.hasChild(WILDCARD_KEY) && node.childCount >= this.maxChildren) {
+      return true;
+    }
+    if (this.looksLikeVariable(token)) {
+      return true;
+    }
+    return false;
+  }
+  /**
+   * Heuristic to detect if a token looks like a variable value.
+   */
+  looksLikeVariable(token) {
+    if (token === WILDCARD) {
+      return true;
+    }
+    const firstChar = token.charAt(0);
+    if (firstChar >= "0" && firstChar <= "9") {
+      return true;
+    }
+    if (/^[0-9a-fA-F]+$/.test(token) && token.length > 8) {
+      return true;
+    }
+    return false;
+  }
+  /**
+   * Get all discovered templates.
+   */
+  getTemplates() {
+    return this.clusters.map((cluster) => ({
+      id: cluster.id,
+      pattern: cluster.getPattern(),
+      occurrences: cluster.count,
+      sampleVariables: cluster.sampleVariables,
+      firstSeen: cluster.firstSeen,
+      lastSeen: cluster.lastSeen
+    }));
+  }
+  /**
+   * Get compression result with formatted output.
+   */
+  getResult(format = "summary", maxTemplates = 50) {
+    const templates = this.getTemplates();
+    templates.sort((a, b) => b.occurrences - a.occurrences);
+    const limitedTemplates = templates.slice(0, maxTemplates);
+    const stats = this.calculateStats(templates);
+    let formatted;
+    switch (format) {
+      case "detailed":
+        formatted = formatDetailed(limitedTemplates, stats);
+        break;
+      case "json":
+        formatted = formatJson(limitedTemplates, stats);
+        break;
+      case "summary":
+      default:
+        formatted = formatSummary(limitedTemplates, stats);
+        break;
+    }
+    return {
+      templates: limitedTemplates,
+      stats,
+      formatted
+    };
+  }
+  /**
+   * Calculate compression statistics.
+   */
+  calculateStats(templates) {
+    const inputLines = this.lineCount;
+    const uniqueTemplates = templates.length;
+    const compressionRatio = inputLines > 0 ? 1 - uniqueTemplates / inputLines : 0;
+    let originalChars = 0;
+    let compressedChars = 0;
+    for (const template of templates) {
+      const patternLength = template.pattern.length;
+      originalChars += patternLength * template.occurrences;
+      compressedChars += patternLength + 20;
+    }
+    const estimatedTokenReduction = originalChars > 0 ? 1 - compressedChars / originalChars : 0;
+    return {
+      inputLines,
+      uniqueTemplates,
+      compressionRatio: Math.max(0, Math.min(1, compressionRatio)),
+      estimatedTokenReduction: Math.max(0, Math.min(1, estimatedTokenReduction))
+    };
+  }
+  /**
+   * Get the number of lines processed.
+   */
+  get totalLines() {
+    return this.lineCount;
+  }
+  /**
+   * Get the number of clusters (templates) discovered.
+   */
+  get totalClusters() {
+    return this.clusters.length;
+  }
+};
+function createDrain(options) {
+  return new Drain(options);
+}
+
+// src/api.ts
+function compress(lines, options = {}) {
+  const { format = "summary", maxTemplates = 50, drain: drainOptions } = options;
+  const drain = createDrain(drainOptions);
+  drain.addLogLines(lines);
+  return drain.getResult(format, maxTemplates);
+}
+function compressText(text, options = {}) {
+  const lines = text.split(/\r?\n/);
+  return compress(lines, options);
+}
+
+// src/cli.ts
+var import_meta = {};
+var __dirname = (0, import_node_path.dirname)((0, import_node_url.fileURLToPath)(import_meta.url));
+var pkg = JSON.parse((0, import_node_fs.readFileSync)((0, import_node_path.join)(__dirname, "..", "package.json"), "utf-8"));
+var VERSION = pkg.version;
+var HELP = `
+logpare - Semantic log compression for LLM context windows
+
+USAGE:
+  logpare [options] [file...]
+  cat logs.txt | logpare [options]
+
+OPTIONS:
+  -f, --format <fmt>      Output format: summary, detailed, json (default: summary)
+  -o, --output <file>     Write output to file instead of stdout
+  -d, --depth <n>         Parse tree depth (default: 4)
+  -t, --threshold <n>     Similarity threshold 0.0-1.0 (default: 0.4)
+  -c, --max-children <n>  Max children per node (default: 100)
+  -m, --max-clusters <n>  Max total clusters (default: 1000)
+  -n, --max-templates <n> Max templates in output (default: 50)
+  -h, --help              Show this help message
+  -v, --version           Show version number
+
+EXAMPLES:
+  logpare server.log
+  cat /var/log/syslog | logpare --format json
+  logpare --depth 5 --threshold 0.5 app.log -o templates.txt
+  logpare access.log error.log --format detailed
+
+DESCRIPTION:
+  logpare uses the Drain algorithm to extract templates from repetitive log
+  data, achieving 60-90% token reduction while preserving diagnostic information.
+  This is useful for fitting more log context into LLM prompts.
+
+  For more information: https://github.com/logpare/logpare
+`;
+function parseCliArgs() {
+  const { values, positionals } = (0, import_node_util.parseArgs)({
+    options: {
+      format: { type: "string", short: "f", default: "summary" },
+      output: { type: "string", short: "o" },
+      depth: { type: "string", short: "d", default: "4" },
+      threshold: { type: "string", short: "t", default: "0.4" },
+      "max-children": { type: "string", short: "c", default: "100" },
+      "max-clusters": { type: "string", short: "m", default: "1000" },
+      "max-templates": { type: "string", short: "n", default: "50" },
+      help: { type: "boolean", short: "h", default: false },
+      version: { type: "boolean", short: "v", default: false }
+    },
+    allowPositionals: true
+  });
+  const format = values.format;
+  if (!["summary", "detailed", "json"].includes(format)) {
+    console.error(
+      `Error: Invalid format "${format}". Use: summary, detailed, json`
+    );
+    process.exit(1);
+  }
+  const depth = parseInt(values.depth, 10);
+  const threshold = parseFloat(values.threshold);
+  const maxChildren = parseInt(values["max-children"], 10);
+  const maxClusters = parseInt(values["max-clusters"], 10);
+  const maxTemplates = parseInt(values["max-templates"], 10);
+  if (isNaN(depth) || depth < 1) {
+    console.error(
+      `Error: Invalid depth "${values.depth}". Must be a positive integer.`
+    );
+    process.exit(1);
+  }
+  if (isNaN(threshold) || threshold < 0 || threshold > 1) {
+    console.error(
+      `Error: Invalid threshold "${values.threshold}". Must be a number between 0.0 and 1.0.`
+    );
+    process.exit(1);
+  }
+  if (isNaN(maxChildren) || maxChildren < 1) {
+    console.error(
+      `Error: Invalid max-children "${values["max-children"]}". Must be a positive integer.`
+    );
+    process.exit(1);
+  }
+  if (isNaN(maxClusters) || maxClusters < 1) {
+    console.error(
+      `Error: Invalid max-clusters "${values["max-clusters"]}". Must be a positive integer.`
+    );
+    process.exit(1);
+  }
+  if (isNaN(maxTemplates) || maxTemplates < 1) {
+    console.error(
+      `Error: Invalid max-templates "${values["max-templates"]}". Must be a positive integer.`
+    );
+    process.exit(1);
+  }
+  return {
+    format,
+    output: values.output,
+    depth,
+    threshold,
+    maxChildren,
+    maxClusters,
+    maxTemplates,
+    files: positionals,
+    help: values.help,
+    version: values.version
+  };
+}
+function readInput(files) {
+  if (files.length > 0) {
+    const contents = [];
+    for (const file of files) {
+      if (!(0, import_node_fs.existsSync)(file)) {
+        console.error(`Error: File not found: ${file}`);
+        process.exit(1);
+      }
+      contents.push((0, import_node_fs.readFileSync)(file, "utf-8"));
+    }
+    return contents.join("\n");
+  }
+  if (process.stdin.isTTY) {
+    console.error(
+      "Error: No input provided. Provide file(s) or pipe input via stdin."
+    );
+    console.error('Run "logpare --help" for usage information.');
+    process.exit(1);
+  }
+  return (0, import_node_fs.readFileSync)(0, "utf-8");
+}
+function main() {
+  const args = parseCliArgs();
+  if (args.help) {
+    console.log(HELP);
+    process.exit(0);
+  }
+  if (args.version) {
+    console.log(`logpare v${VERSION}`);
+    process.exit(0);
+  }
+  const input = readInput(args.files);
+  if (!input.trim()) {
+    console.error("Error: Empty input");
+    process.exit(1);
+  }
+  const options = {
+    format: args.format,
+    maxTemplates: args.maxTemplates,
+    drain: {
+      depth: args.depth,
+      simThreshold: args.threshold,
+      maxChildren: args.maxChildren,
+      maxClusters: args.maxClusters
+    }
+  };
+  const result = compressText(input, options);
+  const output = args.format === "json" ? JSON.stringify(
+    { templates: result.templates, stats: result.stats },
+    null,
+    2
+  ) : result.formatted;
+  if (args.output) {
+    (0, import_node_fs.writeFileSync)(args.output, output, "utf-8");
+    console.error(`Output written to ${args.output}`);
+  } else {
+    console.log(output);
+  }
+}
+main();
+//# sourceMappingURL=cli.cjs.map