@peter_marklund/json 0.0.3 → 0.0.5

package/README.md

@@ -16,17 +16,71 @@ npm install @peter_marklund/json -g
 
 ```sh
 echo '{"foo": "1"}' | json .foo
+
+# Colorized pretty printing is the default
+cat test/input/array.json | json           
+# [
+#   {
+#     "id": 1,
+#     "name": "Item 1",
+#     "value": 100
+#   },
+#   {
+#     "id": 2,
+#     "name": "Item 2",
+#     "value": 200
+#   },
+#   {
+#     "id": 3,
+#     "name": "Item 3",
+#     "value": 300
+#   }
+# ]
+
+# Without pretty printing (single line):
+cat test/input/basic.json | PRETTY=false json
+# {"bar":"Hello world","baz":false,"data":[{"id":1,"name":"Item 1","value":100},{"id":2,"name":"Item 2","value":200},{"id":3,"name":"Item 3","value":300}],"foo":1,"nested":{"foo":{"bar":"nested value"}}}
+
+# JSONL output (for an array with one JSON object per line)
+cat test/input/array.json | JSONL=true json 
+# {"id":1,"name":"Item 1","value":100}
+# {"id":2,"name":"Item 2","value":200}
+# {"id":3,"name":"Item 3","value":300}
+
+# The json command can take JSONL as input as well
+cat test/input/array.jsonl | json
+
+# The json command can also parse JSON data at the end of log lines:
+cat test/input/log-with-json.log | json
+# [
+#   {
+#     "_line": "192.168.1.1 - - [21/Feb/2026:10:00:01 +0000] \"GET /api/users HTTP/1.1\" 200 ",
+#     "cache": "hit",
+#     "duration_ms": 42,
+#     "user_id": 1021
+#   },
+#   {
+#     "_line": "192.168.1.2 - - [21/Feb/2026:10:00:03 +0000] \"POST /api/orders HTTP/1.1\" 201 ",
+#     "cache": "miss",
+#     "duration_ms": 87,
+#     "user_id": 4432
+#   },
+#   ...
+# ]
 ```
 
-## Developing the Library Locally
+## Running the Tests
 
 ```sh
+npm install
 npm link
+npm test
 ```
 
 ## Publishing a new Version
 
 ```sh
+npm login
 npm publish --access public
 ```
 

package/bin/json.js

@@ -42,8 +42,10 @@ const fs = require("fs");
 const readline = require("readline");
 const _ = require("lodash");
 Object.assign(global, require("lodash"));
+Object.assign(global, require("../src/helpers.js"));
 const { diff } = require("object-diffy");
 const { colorize } = require("json-colorizer");
+const stringify = require('fast-json-stable-stringify')
 
 function getCodeArg() {
     let code = process.argv[2] || "data"
@@ -59,12 +61,43 @@ function getCodeArg() {
 function readStdIn() {
     return fs.readFileSync(0).toString();
 }
-  
+
+function parseLine(line, openLines) {
+  let result = { openLines: [...openLines] };
+  try {
+    const openIndex = line.startsWith("[") ? 0 : line.indexOf("{");
+    if (
+      openLines.length === 0 &&
+      ((line.indexOf("{") >= 0 && line.endsWith("}")) ||
+        (line.startsWith("[") && line.endsWith("]")))
+    ) {
+      const doc = JSON.parse(line.substring(openIndex));
+      if (openIndex > 0 && !doc._line) doc._line = line.substring(0, openIndex);
+      result.parsedLine = doc;
+    } else if (line === "{") {
+      result.openLines = [line];
+    } else if (openLines.length > 0) {
+      result.openLines.push(line);
+      if (line === "}") {
+        result.parsedLine = JSON.parse(openLines.join("\n"));
+        result.openLines = [];
+      }
+    } else {
+      result.parsedLine = { _line: line };
+    }
+  } catch (err) {
+    result.error = `Error thrown parsing line: ${line} - ${err.stack}`;
+    result.openLines = [];
+    result.parsedLine = { _line: line };
+  }
+  return result;
+}
+
 async function jsonIn(filePath) {
     let textInput
     try {
       if (filePath) {
-        return require(filePath)
+        return JSON.parse(fs.readFileSync(filePath, 'utf8'))
       } else {
         textInput = readStdIn()
         // NOTE: I've found JSON.parse intermittently errors out for data sizes around 15 MB but require(filePath) can handle more?
@@ -119,12 +152,18 @@ async function jsonIn(filePath) {
     }
 }
   
+function printJsonLines(data) {
+  for (const line of data) {
+    console.log(stringify(line));
+  }
+}
+
 function printJson(data) {
-    console.log(JSON.stringify(data))
+    console.log(stringify(data))
 }
-  
+
 function printPrettyJson(data) {
-    console.log(colorize(JSON.stringify(data, null, 4)))
+    console.log(colorize(stringify(data, null, 4)))
 }
 
 async function main() {

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@peter_marklund/json",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "description": "A convenient way to work with JSON using JavaScript in the terminal",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "./test/run.js"
   },
   "repository": {
     "type": "git",
@@ -22,6 +22,7 @@
   },
   "homepage": "https://github.com/peter/json#readme",
   "dependencies": {
+    "fast-json-stable-stringify": "^2.1.0",
     "json-colorizer": "^3.0.1",
     "lodash": "^4.17.23",
     "object-diffy": "^1.0.4"

package/src/helpers.js

@@ -0,0 +1,120 @@
+function flattenJson(data, path = []) {
+    if (Array.isArray(data)) {
+        return data.reduce((acc, value, index) => {
+            const valueJson = flattenJson(value, [...path, index]);
+            for (const [key, value] of Object.entries(valueJson)) {
+                acc[key] = value;
+            }
+            return acc;
+        }, {});
+    } else if (typeof data === "object" && data !== null) {
+        return Object.keys(data).reduce((acc, key) => {
+            const keyJson = flattenJson(data[key], [...path, key]);
+            for (const [key, value] of Object.entries(keyJson)) {
+                acc[key] = value;
+            }
+            return acc;
+        }, {});
+    } else {
+        return { [path.join(".")]: data };
+    }
+}
+
+// https://stackoverflow.com/questions/1248302/how-to-get-the-size-of-a-javascript-object
+function sizeOfObject(object) {
+    var objectList = [];
+    var stack = [object];
+    var bytes = 0;
+    
+    while (stack.length) {
+        var value = stack.pop();
+        
+        if (typeof value === "boolean") {
+            bytes += 4;
+        } else if (typeof value === "string") {
+            bytes += value.length * 2;
+        } else if (typeof value === "number") {
+            bytes += 8;
+        } else if (typeof value === "object" && objectList.indexOf(value) === -1) {
+            objectList.push(value);
+            
+            for (var i in value) {
+                stack.push(value[i]);
+            }
+        }
+    }
+    return bytes;
+}
+
+function base64Decode(data) {
+    return data && Buffer.from(data, "base64").toString();
+}
+
+function base64Encode(data) {
+    return data && Buffer.from(data).toString("base64");
+}
+
+function percentile(arr, p) {
+    if (typeof p !== 'number' || p <= 0 || p >= 1.0) {
+        throw new Error('Percentile must be a number between 0 and 1');
+    }
+    const sorted = [...arr].sort((a, b) => a - b);
+    const index = p * (sorted.length - 1);
+    const lower = Math.floor(index);
+    const upper = Math.ceil(index);
+    const weight = index - lower;
+  
+    return sorted[lower] * (1 - weight) + sorted[upper] * weight;
+}
+
+function p50(values) {
+    return percentile(values, 0.5);
+}
+
+function p90(values) {
+    return percentile(values, 0.9);
+}
+
+function p99(values) {
+    return percentile(values, 0.99);
+}
+
+function stdDev(values, avg) {
+    const sum = values.reduce((acc, v) => acc + (v - avg) ** 2, 0);
+    return Math.sqrt(sum / values.length);
+}
+
+function stats(values) {
+    const sum = values.reduce((acc, v) => acc + v, 0);
+    const avg = sum / values.length;
+    return {
+        count: values.length,
+        min: Math.min(...values),
+        max: Math.max(...values),
+        stdDev: stdDev(values, avg),
+        sum,
+        avg,
+        p1: percentile(values, 0.01),
+        p5: percentile(values, 0.05),
+        p10: percentile(values, 0.1),
+        p20: percentile(values, 0.2),
+        p30: percentile(values, 0.3),
+        p40: percentile(values, 0.4),
+        p50: percentile(values, 0.5),
+        p60: percentile(values, 0.6),
+        p70: percentile(values, 0.7),
+        p80: percentile(values, 0.8),
+        p90: percentile(values, 0.9),
+        p95: percentile(values, 0.95),
+        p99: percentile(values, 0.99),
+        p999: percentile(values, 0.999),
+    };
+}
+
+module.exports = {
+    flattenJson,
+    sizeOfObject,
+    base64Decode,
+    base64Encode,
+    stats,
+}

package/test/input/array.json

@@ -0,0 +1,17 @@
+[
+    {
+        "id": 1,
+        "name": "Item 1",
+        "value": 100
+    },
+    {
+        "id": 2,
+        "name": "Item 2",
+        "value": 200
+    },
+    {
+        "id": 3,
+        "name": "Item 3",
+        "value": 300
+    }
+]

package/test/input/array.jsonl

@@ -0,0 +1,3 @@
+{"id":1,"name":"Item 1","value":100}
+{"id":2,"name":"Item 2","value":200}
+{"id":3,"name":"Item 3","value":300}

package/test/input/basic.json

@@ -0,0 +1,27 @@
+{
+    "foo": 1,
+    "bar": "Hello world",
+    "baz": false,
+    "nested": {
+        "foo": {
+            "bar": "nested value"
+        }
+    },
+    "data": [
+        {
+            "id": 1,
+            "name": "Item 1",
+            "value": 100
+        },
+        {
+            "id": 2,
+            "name": "Item 2",
+            "value": 200
+        },
+        {
+            "id": 3,
+            "name": "Item 3",
+            "value": 300
+        }
+    ]
+}

package/test/run.js

@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+
+const fs = require('fs')
+const path = require('path')
+const childProcess = require('child_process')
+const { promisify } = require('util')
+const assert = require('node:assert')
+const stringify = require('fast-json-stable-stringify')
+
+const exec = (cmd) => promisify(childProcess.exec)(cmd).then(result => result.stdout.trim())
+
+// Is the input a JSON string, number, boolean, or null (i.e. not an object or array)
+function isJsonScalarValue(text) {
+    // return text && (text === 'null' || text === 'true' || text === 'false' || text.match(/^\d+$/) || (text.startsWith('"') && text.endsWith('"')))
+    return !(text.startsWith('{') || text.startsWith('['))
+}
+
+async function runTest(test) {
+    const startTime = Date.now()
+    console.log('\n--------------------------------------')
+    console.log(`Test: ${test.name}`)
+    console.log('--------------------------------------\n')
+    console.log(`command: ${test.command}`)
+    const output = await exec(test.command)
+    console.log(`output: ${output}`)
+    const elapsedTime = Date.now() - startTime
+    console.log(`elapsed: ${elapsedTime}`)
+    if (isJsonScalarValue(test.expected) || test.command.includes('JSONL=true')) {
+        assert.strictEqual(output, test.expected)
+    } else {
+        assert.strictEqual(stringify(JSON.parse(output)), stringify(JSON.parse(test.expected)))
+    }
+}
+
+async function run() {
+    try {
+        const startTime = Date.now()
+        const spec = JSON.parse(fs.readFileSync(path.join(__dirname, 'test_spec.json'), 'utf8'))
+        for (const test of spec.tests) {
+            const result = await runTest(test)
+        }
+        const elapsedTime = Date.now() - startTime
+        console.log(`\nTotal elapsed: ${elapsedTime}`)
+        console.log(`Total tests run: ${spec.tests.length}`)
+        console.log('SUCCESS!')
+        process.exit(0)
+    } catch (error) {
+        console.log(error.stack || error)
+        console.log('FAILURE!')
+        process.exit(1)
+    }
+}
+
+run()

package/test/test_spec.json

@@ -0,0 +1,69 @@
+{
+    "tests": [
+        {
+            "name": "echo_no_arg",
+            "command": "echo '{\"foo\": 1}' | json",
+            "expected": "{\"foo\": 1}"
+        },
+        {
+            "name": "echo_dot",
+            "command": "echo '{\"foo\": 1}' | json .",
+            "expected": "{\"foo\": 1}"
+        },
+        {
+            "name": "echo_data",
+            "command": "echo '{\"foo\": 1}' | json data",
+            "expected": "{\"foo\": 1}"
+        },
+        {
+            "name": "get_path",
+            "command": "cat test/input/basic.json | json '.nested?.foo?.bar'",
+            "expected": "nested value"
+        },
+        {
+            "name": "file_path_arg",
+            "command": "json .foo test/input/basic.json",
+            "expected": "1"
+        },
+        {
+            "name": "lodash_sum",
+            "command": "cat test/input/basic.json | json 'sum(data.data.map(i => i.value))'",
+            "expected": "600"
+        },
+        {
+            "name": "object_keys",
+            "command": "cat test/input/basic.json | json 'Object.keys(data)'",
+            "expected": "[\"foo\", \"bar\", \"baz\", \"nested\", \"data\"]"
+        },
+        {
+            "name": "array_length",
+            "command": "cat test/input/basic.json | json '.data.length'",
+            "expected": "3"
+        },
+        {
+            "name": "flatten_json",
+            "command": "cat test/input/basic.json | json 'flattenJson(data)'",
+            "expected": "{\"foo\": 1, \"bar\": \"Hello world\", \"baz\": false, \"nested.foo.bar\": \"nested value\", \"data.0.id\": 1, \"data.0.name\": \"Item 1\", \"data.0.value\": 100, \"data.1.id\": 2, \"data.1.name\": \"Item 2\", \"data.1.value\": 200, \"data.2.id\": 3, \"data.2.name\": \"Item 3\", \"data.2.value\": 300}"
+        },
+        {
+            "name": "stats",
+            "command": "cat test/input/basic.json | json 'pick(stats(data.data.map(i => i.value)), [\"min\", \"max\", \"avg\", \"p50\", \"p90\"])'",
+            "expected": "{\"min\": 100, \"max\": 300, \"avg\": 200, \"p50\": 200, \"p90\": 280}"
+        },
+        {
+            "name": "jsonl_input",
+            "command": "cat test/input/array.jsonl | json",
+            "expected": "[{\"id\":1,\"name\":\"Item 1\",\"value\":100},{\"id\":2,\"name\":\"Item 2\",\"value\":200},{\"id\":3,\"name\":\"Item 3\",\"value\":300}]"
+        },
+        {
+            "name": "jsonl_output",
+            "command": "cat test/input/array.jsonl | JSONL=true json",
+            "expected": "{\"id\":1,\"name\":\"Item 1\",\"value\":100}\n{\"id\":2,\"name\":\"Item 2\",\"value\":200}\n{\"id\":3,\"name\":\"Item 3\",\"value\":300}"
+        },
+        {
+            "name": "log_with_json_input",
+            "command": "cat test/input/log-with-json.log | json 'sum(data.map(d => d.duration_ms))'",
+            "expected": "141"
+        }
+    ]
+}