logtunnel 0.3.0 → 1.0.0

package/README.md

@@ -0,0 +1,61 @@
+# Logtunnel
+
+CLI tool that allows you to format, filter and search your log output
+
+## Requirements
+
+- [NodeJS >= 12](https://nodejs.org/en/download/)
+- [NPM](https://docs.npmjs.com/cli/v7/configuring-npm/install)
+
+## Installation
+
+``sudo npm i -g logtunnel``
+
+## Usage
+
+Find logs that contain "alice":
+``curl -s https://cdn.codetunnel.net/lt/text.log | lt alice``
+
+Find logs that contain "alice" and "purchase":
+``curl -s https://cdn.codetunnel.net/lt/text.log | lt -f alice -f purchase``
+
+Find logs that contain "alice" and ignore the ones that contain "info":
+``curl -s https://cdn.codetunnel.net/lt/text.log | lt -f alice -i info``
+
+Parse logs as JSON and output them with that template:
+``curl -s https://cdn.codetunnel.net/lt/json.log | lt -p json -o '[{{lvl}}] {{log}}'``
+
+Parse logs as JSON, apply template and find the ones containing "alice":
+``curl -s https://cdn.codetunnel.net/lt/json.log | lt -p json -o '[{{lvl}}] {{log}}' -f alice``
+
+Parse logs as JSON, apply template and show the ones with "delay > 200":
+``curl -s https://cdn.codetunnel.net/lt/json.log | lt -p json -o '[{{lvl}} in {{delay}}ms] {{log}}' -F 'delay > 200'``
+
+Parse logs as JSON, apply template and show the ones with "log" containing "Alice":
+``curl -s https://cdn.codetunnel.net/lt/json.log | lt -p json -o '[{{lvl}}] {{log}}' -F 'log.toLowerCase().includes("alice")'``
+
+Parse logs as logfmt, show the ones with "delay > 200" and show their original line (as if no parsing happened):
+``curl -s https://cdn.codetunnel.net/lt/logfmt.log | lt -p logfmt -o original -F 'delay > 200'``
+
+Parse logs as JSON and output them as a table:
+``curl -s https://cdn.codetunnel.net/lt/json.log | lt -p json -o table``
+
+Parse logs with regex, and output in logfmt:
+``curl -s https://cdn.codetunnel.net/lt/text.log | lt -p '\[(?<lvl>\S*) in\s*(?<delay>\d*)ms\] (?<log>.*)' -o logfmt``
+
+Parse logs with regex, and show the ones with "delay > 200":
+``curl -s https://cdn.codetunnel.net/lt/text.log | lt -p '(?<delay>\d+)ms' -o original -F 'delay > 200'``
+
+Parse table and show rows containing "cilium":
+``curl -s https://cdn.codetunnel.net/lt/table.log | lt -p table -o original -f cilium``
+
+Parse table, show rows containing "cilium" and the first headers row:
+``curl -s https://cdn.codetunnel.net/lt/table.log | lt -p table -o original -f cilium -H``
+
+Parse table, show rows with RESTARTS > 0:
+``curl -s https://cdn.codetunnel.net/lt/table.log | lt -p table -o original -F 'RESTARTS > 0' -H``
+
+Show rows that are not ready:
+``curl -s https://cdn.codetunnel.net/lt/table.log | lt -p '(?<up>\d)/(?<total>\d)' -o original -F 'up < total' -H``
+
+For more information ``lt --help``

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "logtunnel",
-  "version": "0.3.0",
+  "version": "1.0.0",
   "description": "CLI tool that allows you to format, filter and search your log output",
   "main": "src/main.js",
   "bin": {
     "lt": "src/main.js"
   },
   "scripts": {
-    "test": "lab -v -m 4000 -I require -a @hapi/code"
+    "test": "lab -v -t 100 -I require -a @hapi/code"
   },
   "repository": {
     "type": "git",
@@ -34,10 +34,10 @@
     "@hapi/bossy": "^5.1.0",
     "@hapi/hoek": "^9.2.0",
     "@hapi/podium": "^4.1.3",
+    "bigodon": "^2.3.0",
     "colors": "^1.4.0",
     "debug": "^4.3.2",
     "logfmt": "^1.3.2",
-    "mustache": "^4.2.0",
     "semver": "^7.3.5"
   }
 }

package/src/definition.js

@@ -29,7 +29,7 @@ const definition = {
         type: 'string',
     },
     o: {
-        description: 'Formats the output using this template. Allowed: json, logfmt, original or a mustache template.',
+        description: 'Formats the output using this template. Allowed: json, logfmt, inspect, original, table or a bigodon template.',
         alias: 'output',
         type: 'string',
     },
@@ -65,6 +65,8 @@ const examples = [
     $ + 'curl -s https://cdn.codetunnel.net/lt/json.log | lt -p json -o \'[{{lvl}}] {{log}}\' -F \'log.toLowerCase().includes("alice")\'',
     'Parse logs as logfmt, show the ones with "delay > 200" and show their original line (as if no parsing happened)'.dim,
     $ + 'curl -s https://cdn.codetunnel.net/lt/logfmt.log | lt -p logfmt -o original -F \'delay > 200\'',
+    'Parse logs as JSON and output them as a table'.dim,
+    $ + 'curl -s https://cdn.codetunnel.net/lt/json.log | lt -p json -o table',
     'Parse logs with regex, and output in logfmt'.dim,
     $ + 'curl -s https://cdn.codetunnel.net/lt/text.log | lt -p \'\\[(?<lvl>\\S*) in\\s*(?<delay>\\d*)ms\\] (?<log>.*)\' -o logfmt',
     'Parse logs with regex, and show the ones with "delay > 200"'.dim,

package/src/log-source.js

@@ -0,0 +1,26 @@
+const Podium = require("@hapi/podium");
+
+module.exports.logSource = emitter => {
+    const source = new Podium(['log-line', 'end']);
+    let incompleteLine = '';
+
+    emitter.on('data', data => {
+        const lines = data.toString()
+            .split(/[\r\n|\n]/);
+
+        lines[0] = incompleteLine + lines[0];
+        incompleteLine = lines.pop(); // Either an incomplete line or an empty string due to the last \n
+
+        lines.filter(line => line.length > 0)
+            .forEach(line => source.emit('log-line', line));
+    });
+
+    emitter.on('end', () => {
+        if(incompleteLine.length > 0) {
+            source.emit('log-line', incompleteLine);
+        }
+        source.emit('end');
+    });
+
+    return source;
+};

package/src/main.js

@@ -9,15 +9,9 @@ require('colors');
 const Bossy = require('@hapi/bossy');
 const pkg = require('../package.json');
 const { definition, usage } = require('./definition');
-const { stdin } = require('./stdin');
+const { logSource } = require('./log-source');
 const { LogPipeline } = require('./pipeline');
 
-const filter = require('./transformers/filter');
-const ignore = require('./transformers/ignore');
-const parse = require('./transformers/parse');
-const field = require('./transformers/field');
-const output = require('./transformers/output');
-
 const debug = require('debug')('logtunnel:main');
 
 function run() {
@@ -36,17 +30,11 @@ function run() {
 
     try {
         debug('building pipeline');
-        const pipeline = new LogPipeline([
-            args._ ? filter(args._) : null,
-            ...args.filter.map(filter),
-            ...args.ignore.map(ignore),
-            parse(args.parser),
-            ...args.field.map(field),
-            output(args.output),
-        ], args);
-
+        const pipeline = new LogPipeline(args, process.stdout);
         debug('registering stdin');
-        stdin.on('log-line', l => pipeline.onLogLine(l));
+        logSource(process.stdin)
+            .on('log-line', l => pipeline.onLogLine(l))
+            .on('end', () => pipeline.onEnd());
     } catch(e) {
         console.error('Error:', e.message);
         process.exit(1);

package/src/pipeline.js

@@ -1,10 +1,22 @@
+'use strict';
+
+const { FieldFilter } = require('./transformers/filters/field');
+const { FindFilter } = require('./transformers/filters/find');
+const { IgnoreFilter } = require('./transformers/filters/ignore');
+const { outputFactory } = require('./transformers/outputs/factory');
+const { parseFactory } = require('./transformers/parsers/factory');
+
 const debug = require('debug')('logtunnel:pipeline');
 
 class LogPipeline {
-    constructor(transformers, args) {
-        this.transformers = transformers.filter(t => t !== null);
+    constructor(args, stdout) {
         this.firstLine = null;
         this.args = args;
+        this.stdout = stdout;
+        this.outputTransformer = outputFactory(this.args.output);
+        this.isOutputBuffered = Boolean(this.outputTransformer.flush);
+        this.transformers = this._buildTransformers()
+                .filter(t => t !== null);
     }
 
     onLogLine(line) {
@@ -13,16 +25,19 @@ class LogPipeline {
             this._logLine(line);
             this._updateFirstLine(line);
         } catch (e) {
+            // Covering this would kill the process
+            /* $lab:coverage:off$ */
             console.error('Error:', e.message);
             process.exit(1);
+            /* $lab:coverage:on$ */
         }
     }
 
-    _logLine(line) {
+    async _logLine(line) {
         let output = line;
 
         for (let transformer of this.transformers) {
-            const result = transformer(output, line, this);
+            const result = await transformer.run(output, line, this);
 
             // Transformer accepted the line
             if(result === true) {
@@ -38,10 +53,14 @@ class LogPipeline {
 
             // Transformer modified the line
             output = result;
-            debug('line transformed: ' + JSON.stringify(output)); 
+            debug('line transformed: ' + JSON.stringify(output));
+        }
+
+        if (this.isOutputBuffered) {
+            return;
         }
 
-        process.stdout.write(output + '\n');
+        this.stdout.write(output + '\n');
     }
 
     _updateFirstLine(line) {
@@ -50,9 +69,33 @@ class LogPipeline {
         }
 
         this.firstLine = line;
-        if(this.args.headers) {
-            process.stdout.write(this.firstLine + '\n');
+        if(this.args.headers && !this.isOutputBuffered) {
+            this.stdout.write(this.firstLine + '\n');
+        }
+    }
+
+    onEnd() {
+        if (!this.isOutputBuffered) {
+            return;
         }
+
+        const lines = this.outputTransformer.flush();
+        lines.forEach(line => this.stdout.write(line + '\n'));
+    }
+
+    _buildTransformers() {
+        return [
+            // First of all, filter which lines to accept
+            this.args._ ? new FindFilter(this.args._) : null,
+            ...this.args.filter.map(f => new FindFilter(f)),
+            ...this.args.ignore.map(i => new IgnoreFilter(i)),
+            // Parse them...
+            parseFactory(this.args.parser),
+            // ...and apply field filters
+            ...this.args.field.map(f => new FieldFilter(f)),
+            // And finally format the output
+            this.outputTransformer,
+        ];
     }
 }
 

package/src/transformers/filters/field.js

@@ -0,0 +1,18 @@
+const { compileExpression } = require('bigodon');
+
+class FieldFilter {
+    constructor(expression) {
+        this.expression = compileExpression(expression);
+    }
+
+    async run(line) {
+        if(typeof line !== 'object') {
+            throw new Error("To use a field filter, you need to specify a parser like '-p json' ");
+        }
+
+        const result = await this.expression(line);
+        return Boolean(result);
+    }
+}
+
+module.exports = { FieldFilter };

package/src/transformers/filters/find.js

@@ -0,0 +1,11 @@
+class FindFilter {
+    constructor(regexStr) {
+        this.regex = new RegExp(regexStr, 'i');
+    }
+
+    run(line) {
+        return this.regex.test(line);
+    }
+}
+
+module.exports = { FindFilter };

package/src/transformers/filters/ignore.js

@@ -0,0 +1,11 @@
+class IgnoreFilter {
+    constructor(regexStr) {
+        this.regex = new RegExp(regexStr, 'i');
+    }
+
+    run(line) {
+        return !this.regex.test(line);
+    }
+}
+
+module.exports = { IgnoreFilter };

package/src/transformers/outputs/bigodon.js

@@ -0,0 +1,17 @@
+const { compile } = require('bigodon');
+
+class BigodonOutput {
+    constructor(template) {
+        this.template = compile(template);
+    }
+
+    async run(line) {
+        if (typeof line !== 'object') {
+            throw new Error("To use an output transformer, you need to specify a parser like '-p json' ");
+        }
+
+        return this.template(line);
+    }
+}
+
+module.exports = { BigodonOutput };

package/src/transformers/outputs/factory.js

@@ -0,0 +1,21 @@
+const { JsonOutput } = require('./json');
+const { LogfmtOutput } = require('./logfmt');
+const { BigodonOutput } = require('./bigodon');
+const { OriginalOutput } = require('./original');
+const { InspectOutput } = require('./inspect');
+const { TableOutput } = require('./table');
+
+
+function outputFactory(format) {
+    switch(format?.toLowerCase()) {
+        case  undefined: return new InspectOutput(false);
+        case     'json': return new JsonOutput();
+        case    'table': return new TableOutput();
+        case   'logfmt': return new LogfmtOutput();
+        case  'inspect': return new InspectOutput(true);
+        case 'original': return new OriginalOutput();
+        default:         return new BigodonOutput(format);
+    }
+}
+
+module.exports = { outputFactory };

package/src/transformers/outputs/inspect.js

@@ -0,0 +1,22 @@
+const util = require('util');
+
+class InspectOutput {
+    constructor(breakLines = false) {
+        this.options = {
+            colors: true,
+            depth: null,
+            breakLength: breakLines ? 80 : Infinity,
+            compact: !breakLines,
+        };
+    }
+
+    run(line) {
+        if (typeof line === 'string') {
+            return true;
+        }
+
+        return util.inspect({ ...line }, this.options);
+    }
+}
+
+module.exports = { InspectOutput };

package/src/transformers/outputs/json.js

@@ -0,0 +1,11 @@
+class JsonOutput {
+    run(line) {
+        if (typeof line !== 'object') {
+            throw new Error("To use an output transformer, you need to specify a parser like '-p json'");
+        }
+
+        return JSON.stringify(line);
+    }
+}
+
+module.exports = { JsonOutput };

package/src/transformers/outputs/logfmt.js

@@ -0,0 +1,13 @@
+const logfmt = require('logfmt');
+
+class LogfmtOutput {
+    run(line) {
+        if (typeof line !== 'object') {
+            throw new Error("To use an output transformer, you need to specify a parser like '-p json'");
+        }
+
+        return logfmt.stringify(line);
+    }
+}
+
+module.exports = { LogfmtOutput };

package/src/transformers/outputs/original.js

@@ -0,0 +1,7 @@
+class OriginalOutput {
+    run(_line, original) {
+        return original;
+    }
+}
+
+module.exports = { OriginalOutput };

package/src/transformers/outputs/table.js

@@ -0,0 +1,67 @@
+class TableOutput {
+    constructor() {
+        this.rows = [];
+        this.columns = [];
+        this.columnWidths = new Map();
+    }
+
+    run(line) {
+        if (typeof line !== 'object') {
+            throw new Error("To use an output transformer, you need to specify a parser like '-p json'");
+        }
+
+        this.rows.push(line);
+
+        Object.keys(line).forEach(column => {
+            if (!this.columnWidths.has(column)) {
+                this.columns.push(column);
+                this.columnWidths.set(column, column.length);
+            }
+
+            const cell = this._toCell(line[column]);
+            const width = this.columnWidths.get(column);
+            if (cell.length > width) {
+                this.columnWidths.set(column, cell.length);
+            }
+        });
+        return true;
+    }
+
+    flush() {
+        if (this.rows.length === 0) {
+            return [];
+        }
+
+        const columns = this.columns;
+        const widths = columns.map(column => this.columnWidths.get(column));
+
+        const lines = [];
+        lines.push(this._formatRow(columns, widths));
+        this.rows.forEach(row => {
+            const cells = columns.map(column => this._toCell(row[column]));
+            lines.push(this._formatRow(cells, widths));
+        });
+
+        return lines;
+    }
+
+    _toCell(value) {
+        if (value === null || value === undefined) {
+            return '';
+        }
+        if (typeof value === 'object') {
+            return JSON.stringify(value);
+        }
+
+        return String(value);
+    }
+
+    _formatRow(cells, widths) {
+        return cells
+            .map((cell, index) => cell.padEnd(widths[index]))
+            .join('  ')
+            .trimEnd();
+    }
+}
+
+module.exports = { TableOutput };

package/src/transformers/parsers/factory.js

@@ -0,0 +1,16 @@
+const { JsonParse } = require("./json");
+const { LogfmtParse } = require("./logfmt");
+const { TableParse } = require("./table");
+const { RegexParse } = require("./regex");
+
+function parseFactory(format) {
+    switch(format?.toLowerCase()) {
+        case   void 0: return null;
+        case   'json': return new JsonParse();
+        case 'logfmt': return new LogfmtParse();
+        case  'table': return new TableParse();
+        default:       return new RegexParse(format);
+    }
+}
+
+module.exports = { parseFactory };

package/src/transformers/parsers/json.js

@@ -0,0 +1,16 @@
+class JsonParse {
+    run(line) {
+        try {
+            const obj = JSON.parse(line);
+            if (typeof obj !== 'object') {
+                return false;
+            }
+
+            return obj;
+        } catch {
+            return false;
+        }
+    }
+}
+
+module.exports = { JsonParse };

package/src/transformers/parsers/logfmt.js

@@ -0,0 +1,13 @@
+const logfmt = require('logfmt');
+
+class LogfmtParse {
+    run(line) {
+        try {
+            return logfmt.parse(line);
+        } catch {
+            return false;
+        }
+    }
+}
+
+module.exports = { LogfmtParse };

package/src/transformers/parsers/regex.js

@@ -0,0 +1,11 @@
+class RegexParse {
+    constructor(regexStr) {
+        this.regex = new RegExp(regexStr, 'i');
+    }
+
+    run(line) {
+        return this.regex.exec(line)?.groups ?? false;
+    }
+}
+
+module.exports = { RegexParse };

package/src/transformers/parsers/table.js

@@ -0,0 +1,30 @@
+class TableParse {
+    constructor() {
+        this.headers = null;
+    }
+
+    run(line, _original, pipeline) {
+        if (!pipeline.firstLine) {
+            // Ignore first line, it's the headers
+            return false;
+        }
+        if (!this.headers) {
+            this.headers = this._splitColumns(pipeline.firstLine);
+        }
+
+        const columns = this._splitColumns(line);
+        const obj = {};
+
+        this.headers.forEach((header, i) => {
+            obj[header] = columns[i];
+        });
+
+        return obj;
+    }
+
+    _splitColumns(line) {
+        return line.replace(/\s+/g, ' ').split(' ');
+    }
+}
+
+module.exports = { TableParse };

package/test/filters.spec.js

@@ -0,0 +1,86 @@
+const Lab = require('@hapi/lab');
+const Code = require('@hapi/code');
+
+const { FindFilter } = require('../src/transformers/filters/find');
+const { IgnoreFilter } = require('../src/transformers/filters/ignore');
+const { FieldFilter } = require('../src/transformers/filters/field');
+
+const { describe, it } = exports.lab = Lab.script();
+const { expect } = Code;
+
+describe('filters', () => {
+    describe('--filter', () => {
+        it('should accept matching strings', () => {
+            const transformer = new FindFilter('psu');
+            const result = transformer.run('lorem ipsum dolor sit amet');
+            expect(result).to.be.true();
+        });
+
+        it('should reject non-matching strings', () => {
+            const transformer = new FindFilter('aaa');
+            const result = transformer.run('lorem ipsum dolor sit amet');
+            expect(result).to.be.false();
+        });
+
+        it('should be case insensitive', () => {
+            const transformer = new FindFilter('PsU');
+            const result = transformer.run('lorem ipSUm dolor sit amet');
+            expect(result).to.be.true();
+        });
+
+        it('should support regex', () => {
+            const transformer = new FindFilter('i.*m');
+            const result = transformer.run('lorem ipsum dolor sit amet');
+            expect(result).to.be.true();
+        });
+    });
+    describe('--ignore', () => {
+        it('should reject matching strings', () => {
+            const transformer = new IgnoreFilter('psu');
+            const result = transformer.run('lorem ipsum dolor sit amet');
+            expect(result).to.be.false();
+        });
+
+        it('should accept non-matching strings', () => {
+            const transformer = new IgnoreFilter('aaa');
+            const result = transformer.run('lorem ipsum dolor sit amet');
+            expect(result).to.be.true();
+        });
+
+        it('should be case insensitive', () => {
+            const transformer = new IgnoreFilter('PsU');
+            const result = transformer.run('lorem ipSUm dolor sit amet');
+            expect(result).to.be.false();
+        });
+
+        it('should support regex', () => {
+            const transformer = new IgnoreFilter('i.*m');
+            const result = transformer.run('lorem ipsum dolor sit amet');
+            expect(result).to.be.false();
+        });
+    });
+    describe('--field', () => {
+        it('should error when strings wasn\'t parsed into object', async () => {
+            const transformer = new FieldFilter('a');
+            expect(transformer.run('a')).to.reject();
+        });
+
+        it('should compare equality', async () => {
+            const transformer = new FieldFilter('eq foo "bar"');
+            expect(await transformer.run({ foo: 'bar' })).to.be.true();
+            expect(await transformer.run({ foo: 'baz' })).to.be.false();
+        });
+
+        it('should compare inequality with strings', async () => {
+            const transformer = new FieldFilter('gte foo 3');
+            expect(await transformer.run({ foo: '3' })).to.be.true();
+            expect(await transformer.run({ foo: '2' })).to.be.false();
+        });
+
+        it('should accept javascript', async () => {
+            const transformer = new FieldFilter('includes (lower foo) "bar"');
+            expect(await transformer.run({ foo: 'ABARA' })).to.be.true();
+            expect(await transformer.run({ foo: 'ABADA' })).to.be.false();
+        });
+    });
+});