logtunnel 0.3.0 → 1.0.0

package/test/log-source.spec.js

@@ -0,0 +1,71 @@
+const Lab = require('@hapi/lab');
+const Code = require('@hapi/code');
+const Hoek = require('@hapi/hoek');
+const { logSource } = require('../src/log-source');
+const { pod, slowPod } = require('./utils');
+
+const { describe, it } = exports.lab = Lab.script();
+const { expect } = Code;
+
+describe('log-source', () => {
+
+    it('should split into lines with LF', async () => {
+        const source = pod('hello\nworld\n');
+        const output = logSource(source);
+        const lines = [];
+
+        output.on('log-line', l => lines.push(l));
+        await output.once('end');
+
+        expect(lines).to.equal(['hello', 'world']);
+    });
+
+    it('should split into lines with CRLF', async () => {
+        const source = pod('hello\r\nworld\r\n');
+        const output = logSource(source);
+        const lines = [];
+
+        output.on('log-line', l => lines.push(l));
+        await output.once('end');
+
+        expect(lines).to.equal(['hello', 'world']);
+    });
+
+    it('should join incomplete lines', async () => {
+        const source = pod('hello\nwor', 'ld\n:)\n');
+        const output = logSource(source);
+        const lines = [];
+
+        output.on('log-line', l => lines.push(l));
+        await output.once('end');
+
+        expect(lines).to.equal(['hello', 'world', ':)']);
+    });
+
+    it('should flush incomplete lines on end', async () => {
+        const source = pod('hello\nworld');
+        const output = logSource(source);
+        const lines = [];
+
+        output.on('log-line', l => lines.push(l));
+        await output.once('end');
+
+        expect(lines).to.equal(['hello', 'world']);
+    });
+
+    it('should not wait for end or line completion to send lines', async () => {
+        const source = slowPod('hello\nwo', 'rld\n:)');
+        const output = logSource(source);
+        const lines = [];
+        output.on('log-line', l => lines.push(l));
+
+        await Hoek.wait(50);
+        expect(lines).to.equal(['hello']);
+
+        await Hoek.wait(100);
+        expect(lines).to.equal(['hello', 'world']);
+
+        await Hoek.wait(100);
+        expect(lines).to.equal(['hello', 'world', ':)']);
+    });
+});

package/test/output.spec.js

@@ -0,0 +1,96 @@
+const Lab = require('@hapi/lab');
+const Code = require('@hapi/code');
+
+const { outputFactory } = require('../src/transformers/outputs/factory');
+
+const { describe, it } = exports.lab = Lab.script();
+const { expect } = Code;
+
+describe('outputs', () => {
+    it('should should not modify unparsed when unset', () => {
+        const format = outputFactory();
+        expect(format.run('foo')).to.be.true();
+    });
+
+    it('should inspect parsed lines single-line when unset', () => {
+        const format = outputFactory();
+        const line = format.run({ foo: 'bar' });
+        expect(line).to.be.a.string();
+        expect(line).to.match(/\{.*foo.*:.*bar.*\}/);
+        expect(line.split('\n').length).to.equal(1);
+    });
+
+    it('should inspect parsed lines multi-line', () => {
+        const format = outputFactory('inspect');
+        const line = format.run({ foo: 'bar' });
+        expect(line).to.be.a.string();
+        expect(line.split('\n').join(' ')).to.match(/\{.*foo.*:.*bar.*\}/);
+        expect(line.split('\n').length).to.equal(3);
+    });
+
+    it('should format as json', () => {
+        const format = outputFactory('json');
+        const line = format.run({ foo: 'bar' });
+        expect(line).to.equal('{"foo":"bar"}');
+    });
+
+    it('should format as logfmt', () => {
+        const format = outputFactory('logfmt');
+        const line = format.run({ foo: 'bar' });
+        expect(line).to.equal('foo=bar');
+    });
+
+    it('should format as original', () => {
+        const format = outputFactory('original');
+        const line = format.run({ foo: 'bar' }, 'fubá');
+        expect(line).to.equal('fubá');
+    });
+
+    it('should format as table', () => {
+        const format = outputFactory('table');
+        format.run({ foo: 'bar', baz: 'qux' });
+        format.run({ foo: 'longer' });
+        const lines = format.flush();
+
+        expect(lines).to.equal([
+            'foo     baz',
+            'bar     qux',
+            'longer',
+        ]);
+    });
+
+    it('should return empty table output when no rows were provided', () => {
+        const format = outputFactory('table');
+        expect(format.flush()).to.equal([]);
+    });
+
+    it('should stringify object and null table cells', () => {
+        const format = outputFactory('table');
+        format.run({ foo: null, bar: { baz: 1 } });
+        const lines = format.flush();
+
+        expect(lines).to.equal([
+            'foo  bar',
+            '     {"baz":1}',
+        ]);
+    });
+
+    it('should format with bigodon', async () => {
+        const format = outputFactory('[{{foo}}]');
+        const line = await format.run({ foo: 'bar' });
+        expect(line).to.equal('[bar]');
+    });
+
+    it('should allow bigodon helpers', async () => {
+        const format = outputFactory('[{{upper foo }}]');
+        const line = await format.run({ foo: 'bar' });
+        expect(line).to.equal('[BAR]');
+    });
+
+    it('should refuse to format unparsed lines', async () => {
+        expect(() => outputFactory('json').run('foo')).to.throw();
+        expect(() => outputFactory('logfmt').run('foo')).to.throw();
+        expect(() => outputFactory('table').run('foo')).to.throw();
+        await expect(outputFactory('{{foo}}').run('foo')).to.reject();
+    });
+});

package/test/parse.spec.js

@@ -0,0 +1,72 @@
+const Lab = require('@hapi/lab');
+const Code = require('@hapi/code');
+
+const { parseFactory } = require('../src/transformers/parsers/factory');
+
+const { describe, it } = exports.lab = Lab.script();
+const { expect } = Code;
+
+describe('parsers', () => {
+    it('should return null when no format is specified', () => {
+        expect(parseFactory(void 0)).to.be.null();
+    });
+
+    it('should parse json', () => {
+        const parser = parseFactory('json');
+        expect(parser.run('{"foo": "bar"}')).to.equal({ foo: 'bar' });
+    });
+
+    it('should ignore invalid json lines when parsing json', () => {
+        const parser = parseFactory('json');
+        expect(parser.run('Oy')).to.be.false();
+        expect(parser.run('"Oy"')).to.be.false();
+    });
+
+    it('should parse logfmt', () => {
+        const parser = parseFactory('logfmt');
+        expect(parser.run('foo=bar')).to.equal({ foo: 'bar' });
+    });
+
+    it('should ignore invalid logfmt lines when parsing logfmt', () => {
+        const parser = parseFactory('logfmt');
+        expect(parser.run(void 0)).to.be.false();
+    });
+
+    it('should parse regex', () => {
+        const parser = parseFactory('\\[(?<severity>\\S+)\\s*(?<delay>\\d+)ms\\] (?<message>.*)');
+        expect(parser.run('[info 20ms] lorem ipsum dolor')).to.equal({
+            severity: 'info',
+            delay: '20',
+            message: 'lorem ipsum dolor',
+        });
+
+        expect(parser.run('foo')).to.be.false();
+    });
+
+    describe('tables', () => {
+        const tableRows = [
+            'NAME     TYPE',
+            'foo      bar',
+            'baz      qux',
+        ];
+
+        it('should ignore headers', () => {
+            const pipeline = { firstLine: null };
+            const parser = parseFactory('table');
+            const line = tableRows[0];
+
+            expect(parser.run(line, line, pipeline)).to.be.false();
+        });
+
+        it('should parse subsequent rows', () => {
+            const pipeline = { firstLine: tableRows[0] };
+            const parser = parseFactory('table');
+            let line;
+
+            line = tableRows[1];
+            expect(parser.run(line, line, pipeline)).to.equal({ NAME: 'foo', TYPE: 'bar' });
+            line = tableRows[2];
+            expect(parser.run(line, line, pipeline)).to.equal({ NAME: 'baz', TYPE: 'qux' });
+        });
+    });
+});

package/test/pipeline.spec.js

@@ -0,0 +1,199 @@
+const Lab = require('@hapi/lab');
+const Code = require('@hapi/code');
+const { runPipeline, _, f, i, F, p, o, H } = require('./utils');
+
+const { describe, it } = exports.lab = Lab.script();
+const { expect } = Code;
+
+describe('pipeline', () => {
+    it('should remove rejected lines and keep accepted ones', async () => {
+        const args = [
+            f('potato')
+        ];
+        const actual = await runPipeline([
+            'yada yada yada',
+            'lorem ipsum POTATO dolor',
+            'ablueblue POTATO',
+            'foo bar',
+        ], args);
+        const expected = [
+            'lorem ipsum POTATO dolor',
+            'ablueblue POTATO',
+        ];
+
+        expect(actual).to.equal(expected);
+    });
+
+    it('should apply multiple filters', async () => {
+        const args = [
+            f('foo'),
+            f('bar'),
+            i('nope'),
+        ];
+        const actual = await runPipeline([
+            '1. foo',
+            '2. foo bar nope',
+            '3. foo bar',
+            '4. bar',
+        ], args);
+        const expected = [
+            '3. foo bar',
+        ];
+
+        expect(actual).to.equal(expected);
+    });
+
+    it('should apply default argument as filter', async () => {
+        const args = [
+            _('foo'),
+            i('bar'),
+        ];
+        const actual = await runPipeline([
+            '1. foo',
+            '2. foo bar',
+            '3. baz',
+        ], args);
+        const expected = [
+            '1. foo',
+        ];
+
+        expect(actual).to.equal(expected);
+    });
+
+    it('should parse and format correctly', async () => {
+        const args = [
+            p('logfmt'),
+            o('json'),
+        ];
+        const actual = await runPipeline([
+            'foo=bar baz=qux',
+        ], args);
+        const expected = [
+            '{"foo":"bar","baz":"qux"}',
+        ];
+
+        expect(actual).to.equal(expected);
+    });
+
+    it('should buffer and format tables when requested', async () => {
+        const args = [
+            p('logfmt'),
+            o('table'),
+        ];
+        const actual = await runPipeline([
+            'foo=bar baz=qux',
+            'foo=longer',
+        ], args);
+        const expected = [
+            'foo     baz',
+            'bar     qux',
+            'longer',
+        ];
+
+        expect(actual).to.equal(expected);
+    });
+
+    it('should filter by the original line even when parsers and outputs are provided', async () => {
+        const args = [
+            p('logfmt'),
+            o('json'),
+            i('o=n'),
+        ];
+        const actual = await runPipeline([
+            'foo=bar baz=qux',
+            'foo=nope baz=qux',
+        ], args);
+        const expected = [
+            '{"foo":"bar","baz":"qux"}',
+        ];
+
+        expect(actual).to.equal(expected);
+    });
+
+    it('should apply field filters after parse', async () => {
+        const args = [
+            p('logfmt'),
+            o('json'),
+            F('startsWith foo "b"'),
+        ];
+        const actual = await runPipeline([
+            'foo=bar baz=qux',
+            'foo=nope baz=qux',
+        ], args);
+        const expected = [
+            '{"foo":"bar","baz":"qux"}',
+        ];
+
+        expect(actual).to.equal(expected);
+    });
+
+    it('should apply multiple field filters', async () => {
+        const args = [
+            p('logfmt'),
+            o('json'),
+            F('startsWith foo "b"'),
+            F('endsWith foo "r"'),
+        ];
+        const actual = await runPipeline([
+            'foo=baz baz=qux',
+            'foo=bar baz=qux',
+            'foo=tar baz=qux',
+        ], args);
+        const expected = [
+            '{"foo":"bar","baz":"qux"}',
+        ];
+
+        expect(actual).to.equal(expected);
+    });
+
+    it('should log headers when -H is provided', async () => {
+        const args = [
+            p('table'),
+            o('original'),
+            F('eq NAME "foo"'),
+            H(),
+        ];
+        const actual = await runPipeline([
+            'NAME',
+            'foo',
+            'bar',
+        ], args);
+        const expected = [
+            'NAME',
+            'foo',
+        ];
+
+        expect(actual).to.equal(expected);
+    });
+
+    it('should log headers only once when both -H and -o table is provided', async () => {
+        const args = [
+            p('table'),
+            o('table'),
+            F('eq NAME "foo"'),
+            H(),
+        ];
+        const actual = await runPipeline([
+            'NAME',
+            'foo',
+            'bar',
+        ], args);
+        const expected = [
+            'NAME',
+            'foo',
+        ];
+
+        expect(actual).to.equal(expected);
+    });
+
+    it('should not specify null prototype of regex parser', async () => {
+        const args = [
+            p('(?<num>\\d*)'),
+        ];
+        const [actual] = await runPipeline([
+            '12',
+        ], args);
+
+        expect(actual).not.to.include('null prototype');
+    });
+});

package/test/utils.js

@@ -0,0 +1,71 @@
+const Podium = require('@hapi/podium');
+const Hoek = require('@hapi/hoek');
+const { LogPipeline } = require('../src/pipeline');
+
+function pod(...strs) {
+    const emitter = new Podium(['data', 'end']);
+
+    (async () => {
+        await Hoek.wait(0);
+        for(const str of strs) {
+            emitter.emit('data', str);
+        }
+        emitter.emit('end');
+    })();
+
+    return emitter;
+}
+
+function slowPod(...strs) {
+    const emitter = new Podium(['data', 'end']);
+
+    (async () => {
+        await Hoek.wait(0);
+        for(const str of strs) {
+            emitter.emit('data', str);
+            await Hoek.wait(100);
+        }
+        emitter.emit('end');
+    })();
+
+    return emitter;
+}
+
+async function runPipeline(inputLines, args) {
+    const lines = [];
+
+    const stdout = { write: (line) => lines.push(line) };
+    const pipeline = new LogPipeline(mergeArgs(args), stdout);
+
+    inputLines.forEach(line => pipeline.onLogLine(line));
+
+    // Waiting for nextTick promises to resolve
+    await Hoek.wait(0);
+
+    pipeline.onEnd();
+
+    // Removing \n from the end of each line
+    return lines.map(line => line.slice(0, -1));
+}
+
+function mergeArgs(args) {
+    let argsObj = {
+        filter: [],
+        ignore: [],
+        field: [],
+    };
+
+    args.forEach(arg => argsObj = Hoek.merge(argsObj, arg));
+
+    return argsObj;
+}
+
+const _ = str => ({ _: str });
+const f = str => ({ filter: [str] });
+const i = str => ({ ignore: [str] });
+const F = str => ({ field: [str] });
+const p = str => ({ parser: str });
+const o = str => ({ output: str });
+const H = () => ({ headers: true });
+
+module.exports = { pod, slowPod, runPipeline, _, f, i, F, p, o, H };

package/src/stdin.js

@@ -1,17 +0,0 @@
-const Podium = require("@hapi/podium");
-
-const stdin = new Podium('log-line');
-let incompleteLine = '';
-
-process.stdin.on('data', data => {
-    const lines = data.toString()
-        .split(/[\r\n|\n]/);
-    
-    lines[0] = incompleteLine + lines[0];
-    incompleteLine = lines.pop(); // Either an incomplete line or an empty string due to the last \n
-
-    lines.filter(line => line.length > 0)
-        .forEach(line => stdin.emit('log-line', line));
-});
-
-module.exports.stdin = stdin;

package/src/transformers/field.js

@@ -1,13 +0,0 @@
-const vm = require('vm');
-
-module.exports = filter => {
-    return line => {
-        if(typeof line !== 'object') {
-            throw new Error("To use a field filter, you need to specify a parser like '-p json' ");
-        }
-
-        vm.createContext(line);
-        const result = vm.runInContext(filter, line);
-        return Boolean(result);
-    };
-};

package/src/transformers/filter.js

@@ -1,4 +0,0 @@
-module.exports = regexStr => line => {
-    const regex = new RegExp(regexStr, 'i');
-    return regex.test(line);
-};

package/src/transformers/ignore.js

@@ -1,4 +0,0 @@
-module.exports = regexStr => line => {
-    const regex = new RegExp(regexStr, 'i');
-    return !regex.test(line);
-};

package/src/transformers/output-json.js

@@ -1,7 +0,0 @@
-module.exports = () => line => {
-    if(typeof line !== 'object') {
-        throw new Error("To use an output transformer, you need to specify a parser like '-p json'");
-    }
-
-    return JSON.stringify(line);
-};

package/src/transformers/output-logfmt.js

@@ -1,9 +0,0 @@
-const logfmt = require('logfmt');
-
-module.exports = () => line => {
-    if(typeof line !== 'object') {
-        throw new Error("To use an output transformer, you need to specify a parser like '-p json'");
-    }
-
-    return logfmt.stringify(line);
-};

package/src/transformers/output-mustache.js

@@ -1,10 +0,0 @@
-const Mustache = require('mustache');
-Mustache.escape = text => text;
-
-module.exports = format => line => {
-    if(typeof line !== 'object') {
-        throw new Error("To use an output transformer, you need to specify a parser like '-p json' ");
-    }
-
-    return Mustache.render(format, line);
-};

package/src/transformers/output-original.js

@@ -1,6 +0,0 @@
-const Mustache = require('mustache');
-Mustache.escape = text => text;
-
-module.exports = () => (_line, original) => {
-    return original;
-};

package/src/transformers/output-unset.js

@@ -1,12 +0,0 @@
-const util = require('util');
-
-module.exports = () => line => {
-    if(typeof line === 'string') {
-        return true;
-    }
-
-    return util.inspect({ ...line }, {
-        colors: true,
-        depth: null,
-    });
-};

package/src/transformers/output.js

@@ -1,15 +0,0 @@
-const json = require('./output-json');
-const logfmt = require('./output-logfmt');
-const mustache = require('./output-mustache');
-const original = require('./output-original');
-const unset = require('./output-unset');
-
-module.exports = format => {
-    switch(format?.toLowerCase()) {
-        case     void 0: return unset();
-        case     'json': return json();
-        case   'logfmt': return logfmt();
-        case 'original': return original();
-        default:         return mustache(format);
-    }
-};

package/src/transformers/parse-json.js

@@ -1,7 +0,0 @@
-module.exports = () => line => {
-    try {
-        return JSON.parse(line);
-    } catch {
-        return false;
-    }
-};

package/src/transformers/parse-logfmt.js

@@ -1,9 +0,0 @@
-const logfmt = require('logfmt');
-
-module.exports = () => line => {
-    try {
-        return logfmt.parse(line);
-    } catch {
-        return false;
-    }
-};

package/src/transformers/parse-regex.js

@@ -1,11 +0,0 @@
-module.exports = regexStr => {
-    const regex = new RegExp(regexStr, 'i');
-
-    return line => {
-        try {
-            return regex.exec(line)?.groups ?? false;
-        } catch {
-            return false;
-        }
-    };
-};

package/src/transformers/parse-table.js

@@ -1,24 +0,0 @@
-let headers;
-
-function splitColumns(line) {
-    return line.replace(/\s+/g, ' ').split(' ');
-}
-
-module.exports = () => (line, _original, pipeline) => {
-    if(!pipeline.firstLine) {
-        // Ignore first line, it's the headers
-        return false;
-    }
-    if(!headers) {
-        headers = splitColumns(pipeline.firstLine);
-    }
-
-    const columns = splitColumns(line);
-    const obj = {};
-
-    headers.forEach((header, i) => {
-        obj[header] = columns[i];
-    });
-
-    return obj;
-};