@peter_marklund/json 0.0.3 → 0.0.4

package/README.md

@@ -18,15 +18,18 @@ npm install @peter_marklund/json -g
 echo '{"foo": "1"}' | json .foo
 ```
 
-## Developing the Library Locally
+## Running the Tests
 
 ```sh
+npm install
 npm link
+npm test
 ```
 
 ## Publishing a new Version
 
 ```sh
+npm login
 npm publish --access public
 ```
 

package/bin/json.js

@@ -42,8 +42,10 @@ const fs = require("fs");
 const readline = require("readline");
 const _ = require("lodash");
 Object.assign(global, require("lodash"));
+Object.assign(global, require("../src/helpers.js"));
 const { diff } = require("object-diffy");
 const { colorize } = require("json-colorizer");
+const stringify = require('fast-json-stable-stringify')
 
 function getCodeArg() {
     let code = process.argv[2] || "data"
@@ -64,7 +66,7 @@ async function jsonIn(filePath) {
     let textInput
     try {
       if (filePath) {
-        return require(filePath)
+        return JSON.parse(fs.readFileSync(filePath, 'utf8'))
       } else {
         textInput = readStdIn()
         // NOTE: I've found JSON.parse intermittently errors out for data sizes around 15 MB but require(filePath) can handle more?
@@ -119,12 +121,18 @@ async function jsonIn(filePath) {
     }
 }
   
+function printJsonLines(data) {
+  for (const line of data) {
+    console.log(stringify(line));
+  }
+}
+
 function printJson(data) {
-    console.log(JSON.stringify(data))
+    console.log(stringify(data))
 }
-  
+
 function printPrettyJson(data) {
-    console.log(colorize(JSON.stringify(data, null, 4)))
+    console.log(colorize(stringify(data, null, 4)))
 }
 
 async function main() {

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@peter_marklund/json",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "A convenient way to work with JSON using JavaScript in the terminal",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "./test/run.js"
   },
   "repository": {
     "type": "git",
@@ -22,6 +22,7 @@
   },
   "homepage": "https://github.com/peter/json#readme",
   "dependencies": {
+    "fast-json-stable-stringify": "^2.1.0",
     "json-colorizer": "^3.0.1",
     "lodash": "^4.17.23",
     "object-diffy": "^1.0.4"

package/src/helpers.js

@@ -0,0 +1,113 @@
+function flattenJson(data, path = []) {
+    if (Array.isArray(data)) {
+        return data.reduce((acc, value, index) => {
+            const valueJson = flattenJson(value, [...path, index]);
+            for (const [key, value] of Object.entries(valueJson)) {
+                acc[key] = value;
+            }
+            return acc;
+        }, {});
+    } else if (typeof data === "object" && data !== null) {
+        return Object.keys(data).reduce((acc, key) => {
+            const keyJson = flattenJson(data[key], [...path, key]);
+            for (const [key, value] of Object.entries(keyJson)) {
+                acc[key] = value;
+            }
+            return acc;
+        }, {});
+    } else {
+        return { [path.join(".")]: data };
+    }
+}
+
+// https://stackoverflow.com/questions/1248302/how-to-get-the-size-of-a-javascript-object
+function sizeOfObject(object) {
+    var objectList = [];
+    var stack = [object];
+    var bytes = 0;
+    
+    while (stack.length) {
+        var value = stack.pop();
+        
+        if (typeof value === "boolean") {
+            bytes += 4;
+        } else if (typeof value === "string") {
+            bytes += value.length * 2;
+        } else if (typeof value === "number") {
+            bytes += 8;
+        } else if (typeof value === "object" && objectList.indexOf(value) === -1) {
+            objectList.push(value);
+            
+            for (var i in value) {
+                stack.push(value[i]);
+            }
+        }
+    }
+    return bytes;
+}
+
+function base64Decode(data) {
+    return data && Buffer.from(data, "base64").toString();
+}
+
+function base64Encode(data) {
+    return data && Buffer.from(data).toString("base64");
+}
+
+function percentile(values, pct) {
+    const sorted = values.sort((a, b) => a - b);
+    const index = Math.floor(sorted.length * pct);
+    return sorted[index];
+}
+
+function p50(values) {
+    return percentile(values, 0.5);
+}
+
+function p90(values) {
+    return percentile(values, 0.9);
+}
+
+function p99(values) {
+    return percentile(values, 0.99);
+}
+
+function stdDev(values, avg) {
+    const sum = values.reduce((acc, v) => acc + (v - avg) ** 2, 0);
+    return Math.sqrt(sum / values.length);
+}
+
+function stats(values) {
+    const sum = values.reduce((acc, v) => acc + v, 0);
+    const avg = sum / values.length;
+    return {
+        count: values.length,
+        min: Math.min(...values),
+        max: Math.max(...values),
+        stdDev: stdDev(values, avg),
+        sum,
+        avg,
+        p1: percentile(values, 0.01),
+        p5: percentile(values, 0.05),
+        p10: percentile(values, 0.1),
+        p20: percentile(values, 0.2),
+        p30: percentile(values, 0.3),
+        p40: percentile(values, 0.4),
+        p50: percentile(values, 0.5),
+        p60: percentile(values, 0.6),
+        p70: percentile(values, 0.7),
+        p80: percentile(values, 0.8),
+        p90: percentile(values, 0.9),
+        p95: percentile(values, 0.95),
+        p99: percentile(values, 0.99),
+        p999: percentile(values, 0.999),
+    };
+}
+
+module.exports = {
+    flattenJson,
+    sizeOfObject,
+    base64Decode,
+    base64Encode,
+    stats,
+}

package/test/input/basic.json

@@ -0,0 +1,27 @@
+{
+    "foo": 1,
+    "bar": "Hello world",
+    "baz": false,
+    "nested": {
+        "foo": {
+            "bar": "nested value"
+        }
+    },
+    "data": [
+        {
+            "id": 1,
+            "name": "Item 1",
+            "value": 100
+        },
+        {
+            "id": 2,
+            "name": "Item 2",
+            "value": 200
+        },
+        {
+            "id": 3,
+            "name": "Item 3",
+            "value": 300
+        }
+    ]
+}

package/test/run.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+const fs = require('fs')
+const path = require('path')
+const childProcess = require('child_process')
+const { promisify } = require('util')
+const assert = require('node:assert')
+const stringify = require('fast-json-stable-stringify')
+
+const exec = (cmd) => promisify(childProcess.exec)(cmd).then(result => result.stdout.trim())
+
+// Is the input a JSON string, number, boolean, or null (i.e. not an object or array)
+function isJsonScalarValue(text) {
+    // return text && (text === 'null' || text === 'true' || text === 'false' || text.match(/^\d+$/) || (text.startsWith('"') && text.endsWith('"')))
+    return !(text.startsWith('{') || text.startsWith('['))
+}
+
+async function runTest(test) {
+    const startTime = Date.now()
+    console.log('\n--------------------------------------')
+    console.log(`Test: ${test.name}`)
+    console.log('--------------------------------------\n')
+    console.log(`command: ${test.command}`)
+    const output = await exec(test.command)
+    console.log(`output: ${output}`)
+    const elapsedTime = Date.now() - startTime
+    console.log(`elapsed: ${elapsedTime}`)
+    if (isJsonScalarValue(test.expected)) {
+        assert.strictEqual(output, test.expected)
+    } else {
+        assert.strictEqual(stringify(JSON.parse(output)), stringify(JSON.parse(test.expected)))
+    }
+}
+
+async function run() {
+    try {
+        const startTime = Date.now()
+        const spec = JSON.parse(fs.readFileSync(path.join(__dirname, 'test_spec.json'), 'utf8'))
+        for (const test of spec.tests) {
+            const result = await runTest(test)
+        }
+        const elapsedTime = Date.now() - startTime
+        console.log(`\nTotal elapsed: ${elapsedTime}`)
+        console.log(`Total tests run: ${spec.tests.length}`)
+        process.exit(0)
+    } catch (error) {
+        console.log(error.stack || error)
+        console.log('FAILURE!')
+        process.exit(1)
+    }
+}
+
+run()

package/test/test_spec.json

@@ -0,0 +1,54 @@
+{
+    "tests": [
+        {
+            "name": "echo_no_arg",
+            "command": "echo '{\"foo\": 1}' | json",
+            "expected": "{\"foo\": 1}"
+        },
+        {
+            "name": "echo_dot",
+            "command": "echo '{\"foo\": 1}' | json .",
+            "expected": "{\"foo\": 1}"
+        },
+        {
+            "name": "echo_data",
+            "command": "echo '{\"foo\": 1}' | json data",
+            "expected": "{\"foo\": 1}"
+        },
+        {
+            "name": "get_path",
+            "command": "cat test/input/basic.json | json '.nested?.foo?.bar'",
+            "expected": "nested value"
+        },
+        {
+            "name": "file_path_arg",
+            "command": "json .foo test/input/basic.json",
+            "expected": "1"
+        },
+        {
+            "name": "lodash_sum",
+            "command": "cat test/input/basic.json | json 'sum(data.data.map(i => i.value))'",
+            "expected": "600"
+        },
+        {
+            "name": "object_keys",
+            "command": "cat test/input/basic.json | json 'Object.keys(data)'",
+            "expected": "[\"foo\", \"bar\", \"baz\", \"nested\", \"data\"]"
+        },
+        {
+            "name": "array_length",
+            "command": "cat test/input/basic.json | json '.data.length'",
+            "expected": "3"
+        },
+        {
+            "name": "flatten_json",
+            "command": "cat test/input/basic.json | json 'flattenJson(data)'",
+            "expected": "{\"foo\": 1, \"bar\": \"Hello world\", \"baz\": false, \"nested.foo.bar\": \"nested value\", \"data.0.id\": 1, \"data.0.name\": \"Item 1\", \"data.0.value\": 100, \"data.1.id\": 2, \"data.1.name\": \"Item 2\", \"data.1.value\": 200, \"data.2.id\": 3, \"data.2.name\": \"Item 3\", \"data.2.value\": 300}"
+        },
+        {
+            "name": "stats",
+            "command": "cat test/input/basic.json | json 'pick(stats(data.data.map(i => i.value)), [\"min\", \"max\", \"avg\", \"p50\", \"p90\"])'",
+            "expected": "{\"min\": 100, \"max\": 300, \"avg\": 200, \"p50\": 200, \"p90\": 300}"
+        }
+    ]
+}