@peter_marklund/json 0.0.4 → 0.0.6

package/README.md

@@ -1,10 +1,6 @@
 # @peter_marklund/json
 
-An npm package that provides a convenient way to work with JSON using JavaScript in the terminal.
-
-## TODO
-
-* Use stable/sorted JSON stringify
+An npm package that provides a convenient way to use JavaScript to work with JSON in the terminal. This tool is for those of us who like `jq` but prefer JavaScript over `jq` syntax.
 
 ## Installation
 
@@ -14,8 +10,138 @@ npm install @peter_marklund/json -g
 
 ## Usage
 
+The JSON data is typically passed to the `json` command via stdin but can also be passed as a file path via the second argument. The first argument to the `json` command is a string with JavaScript code to be evaluated. All [lodash](https://lodash.com/docs/4.17.23) functions (i.e. `pick`, `pickBy`, `mapValues`, `sum` etc.) are available as are a number of [helper functions](src/helpers.js). It is also possible to provide custom JavaScript helper functions via the `JSON_HELPERS_PATH` environment variable.
+
 ```sh
+# Get the value at a path
 echo '{"foo": "1"}' | json .foo
+
+# Get the keys of the JSON data
+cat test/input/basic.json | json 'Object.keys(data)'                               
+# [
+#   "foo",
+#   "bar",
+#   "baz",
+#   "nested",
+#   "data"
+# ]
+
+# Get the length of an array:
+cat test/input/basic.json | json '.data.length'     
+# 3
+
+# Use lodash functions
+cat test/input/basic.json | json '.data.map(d => pick(d, ["value"]))'
+# [
+#   {
+#     "value": 100
+#   },
+#   {
+#     "value": 200
+#   },
+#   {
+#     "value": 300
+#   }
+# ]
+
+# Use the flattenJson helper to find the path of a deeply nested value:
+cat test/input/basic.json | json 'flattenJson(data)'
+# {
+#   "bar": "Hello world",
+#   "baz": false,
+#   "data.0.id": 1,
+#   "data.0.name": "Item 1",
+#   "data.0.value": 100,
+#   "data.1.id": 2,
+#   "data.1.name": "Item 2",
+#   "data.1.value": 200,
+#   "data.2.id": 3,
+#   "data.2.name": "Item 3",
+#   "data.2.value": 300,
+#   "foo": 1,
+#   "nested.foo.bar": "nested value"
+# }
+
+# Use the stats helper function to get min/max/avg/median/p90 etc. for numerical values
+cat test/input/basic.json | json 'data.data.map(d => d.value)' | json 'stats(data)'
+# {
+#   "avg": 200,
+#   "count": 3,
+#   "max": 300,
+#   "min": 100,
+#   "p1": 102,
+#   "p10": 120,
+#   "p20": 140,
+#   "p30": 160,
+#   "p40": 180,
+#   "p5": 110,
+#   "p50": 200,
+#   "p60": 220,
+#   "p70": 240,
+#   "p80": 260,
+#   "p90": 280,
+#   "p95": 290,
+#   "p99": 298,
+#   "p999": 299.79999999999995,
+#   "stdDev": 81.64965809277261,
+#   "sum": 600
+# }
+
+
+# Colorized pretty printing is the default
+cat test/input/array.json | json           
+# [
+#   {
+#     "id": 1,
+#     "name": "Item 1",
+#     "value": 100
+#   },
+#   {
+#     "id": 2,
+#     "name": "Item 2",
+#     "value": 200
+#   },
+#   {
+#     "id": 3,
+#     "name": "Item 3",
+#     "value": 300
+#   }
+# ]
+
+# Without pretty printing (single line):
+cat test/input/basic.json | PRETTY=false json
+# {"bar":"Hello world","baz":false,"data":[{"id":1,"name":"Item 1","value":100},{"id":2,"name":"Item 2","value":200},{"id":3,"name":"Item 3","value":300}],"foo":1,"nested":{"foo":{"bar":"nested value"}}}
+
+# JSONL output (for an array with one JSON object per line)
+cat test/input/array.json | JSONL=true json 
+# {"id":1,"name":"Item 1","value":100}
+# {"id":2,"name":"Item 2","value":200}
+# {"id":3,"name":"Item 3","value":300}
+
+# The json command can take JSONL as input as well
+cat test/input/array.jsonl | json
+
+# The json command can also parse JSON data at the end of log lines:
+cat test/input/log-with-json.log | json
+# [
+#   {
+#     "_line": "192.168.1.1 - - [21/Feb/2026:10:00:01 +0000] \"GET /api/users HTTP/1.1\" 200 ",
+#     "cache": "hit",
+#     "duration_ms": 42,
+#     "user_id": 1021
+#   },
+#   {
+#     "_line": "192.168.1.2 - - [21/Feb/2026:10:00:03 +0000] \"POST /api/orders HTTP/1.1\" 201 ",
+#     "cache": "miss",
+#     "duration_ms": 87,
+#     "user_id": 4432
+#   },
+#   ...
+# ]
+
+# Using custom helper functions via the JSON_HELPERS_PATH env var and a javascript module with exported functions
+echo '{"values1": [1, 2, 3, 4], "values2": [3, 5, 1, 11]}' | JSON_HELPERS_PATH="$(pwd)/test/custom-helpers.js" json 'correlation(data.values1, data.values2)'
+# 0.5976143046671968
 ```
 
 ## Running the Tests
@@ -35,4 +161,5 @@ npm publish --access public
 
 # Prior Art
 
+* [jq](https://github.com/jqlang/jq) - the standard for processing JSON in the terminal
 * [trentm/json](https://github.com/trentm/json) - nice library with [very good documentation](https://trentm.com/json/)

package/bin/json.js

@@ -40,9 +40,14 @@
 
 const fs = require("fs");
 const readline = require("readline");
+
 const _ = require("lodash");
 Object.assign(global, require("lodash"));
 Object.assign(global, require("../src/helpers.js"));
+if (process.env.JSON_HELPERS_PATH) {
+    Object.assign(global, require(process.env.JSON_HELPERS_PATH));
+}
+
 const { diff } = require("object-diffy");
 const { colorize } = require("json-colorizer");
 const stringify = require('fast-json-stable-stringify')
@@ -61,7 +66,38 @@ function getCodeArg() {
 function readStdIn() {
     return fs.readFileSync(0).toString();
 }
-  
+
+function parseLine(line, openLines) {
+  let result = { openLines: [...openLines] };
+  try {
+    const openIndex = line.startsWith("[") ? 0 : line.indexOf("{");
+    if (
+      openLines.length === 0 &&
+      ((line.indexOf("{") >= 0 && line.endsWith("}")) ||
+        (line.startsWith("[") && line.endsWith("]")))
+    ) {
+      const doc = JSON.parse(line.substring(openIndex));
+      if (openIndex > 0 && !doc._line) doc._line = line.substring(0, openIndex);
+      result.parsedLine = doc;
+    } else if (line === "{") {
+      result.openLines = [line];
+    } else if (openLines.length > 0) {
+      result.openLines.push(line);
+      if (line === "}") {
+        result.parsedLine = JSON.parse(openLines.join("\n"));
+        result.openLines = [];
+      }
+    } else {
+      result.parsedLine = { _line: line };
+    }
+  } catch (err) {
+    result.error = `Error thrown parsing line: ${line} - ${err.stack}`;
+    result.openLines = [];
+    result.parsedLine = { _line: line };
+  }
+  return result;
+}
+
 async function jsonIn(filePath) {
     let textInput
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peter_marklund/json",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "A convenient way to work with JSON using JavaScript in the terminal",
   "main": "index.js",
   "scripts": {

package/src/helpers.js

@@ -54,10 +54,17 @@ function base64Encode(data) {
     return data && Buffer.from(data).toString("base64");
 }
 
-function percentile(values, pct) {
-    const sorted = values.sort((a, b) => a - b);
-    const index = Math.floor(sorted.length * pct);
-    return sorted[index];
+function percentile(arr, p) {
+    if (typeof p !== 'number' || p <= 0 || p >= 1.0) {
+        throw new Error('Percentile must be a number between 0 and 1');
+    }
+    const sorted = [...arr].sort((a, b) => a - b);
+    const index = p * (sorted.length - 1);
+    const lower = Math.floor(index);
+    const upper = Math.ceil(index);
+    const weight = index - lower;
+  
+    return sorted[lower] * (1 - weight) + sorted[upper] * weight;
 }
 
 function p50(values) {

package/test/custom-helpers.js

@@ -0,0 +1,20 @@
+function correlation(x, y) {
+    const n = x.length;
+    const meanX = x.reduce((a, b) => a + b, 0) / n;
+    const meanY = y.reduce((a, b) => a + b, 0) / n;
+
+    let num = 0, denomX = 0, denomY = 0;
+    for (let i = 0; i < n; i++) {
+        const dx = x[i] - meanX;
+        const dy = y[i] - meanY;
+        num += dx * dy;
+        denomX += dx * dx;
+        denomY += dy * dy;
+    }
+
+    return num / Math.sqrt(denomX * denomY);
+}
+
+module.exports = {
+    correlation,
+}

package/test/input/array.json

@@ -0,0 +1,17 @@
+[
+    {
+        "id": 1,
+        "name": "Item 1",
+        "value": 100
+    },
+    {
+        "id": 2,
+        "name": "Item 2",
+        "value": 200
+    },
+    {
+        "id": 3,
+        "name": "Item 3",
+        "value": 300
+    }
+]

package/test/input/array.jsonl

@@ -0,0 +1,3 @@
+{"id":1,"name":"Item 1","value":100}
+{"id":2,"name":"Item 2","value":200}
+{"id":3,"name":"Item 3","value":300}

package/test/run.js

@@ -25,7 +25,7 @@ async function runTest(test) {
     console.log(`output: ${output}`)
     const elapsedTime = Date.now() - startTime
     console.log(`elapsed: ${elapsedTime}`)
-    if (isJsonScalarValue(test.expected)) {
+    if (isJsonScalarValue(test.expected) || test.command.includes('JSONL=true')) {
         assert.strictEqual(output, test.expected)
     } else {
         assert.strictEqual(stringify(JSON.parse(output)), stringify(JSON.parse(test.expected)))
@@ -42,6 +42,7 @@ async function run() {
         const elapsedTime = Date.now() - startTime
         console.log(`\nTotal elapsed: ${elapsedTime}`)
         console.log(`Total tests run: ${spec.tests.length}`)
+        console.log('SUCCESS!')
         process.exit(0)
     } catch (error) {
         console.log(error.stack || error)

package/test/test_spec.json

@@ -48,7 +48,27 @@
         {
             "name": "stats",
             "command": "cat test/input/basic.json | json 'pick(stats(data.data.map(i => i.value)), [\"min\", \"max\", \"avg\", \"p50\", \"p90\"])'",
-            "expected": "{\"min\": 100, \"max\": 300, \"avg\": 200, \"p50\": 200, \"p90\": 300}"
+            "expected": "{\"min\": 100, \"max\": 300, \"avg\": 200, \"p50\": 200, \"p90\": 280}"
+        },
+        {
+            "name": "jsonl_input",
+            "command": "cat test/input/array.jsonl | json",
+            "expected": "[{\"id\":1,\"name\":\"Item 1\",\"value\":100},{\"id\":2,\"name\":\"Item 2\",\"value\":200},{\"id\":3,\"name\":\"Item 3\",\"value\":300}]"
+        },
+        {
+            "name": "jsonl_output",
+            "command": "cat test/input/array.jsonl | JSONL=true json",
+            "expected": "{\"id\":1,\"name\":\"Item 1\",\"value\":100}\n{\"id\":2,\"name\":\"Item 2\",\"value\":200}\n{\"id\":3,\"name\":\"Item 3\",\"value\":300}"
+        },
+        {
+            "name": "log_with_json_input",
+            "command": "cat test/input/log-with-json.log | json 'sum(data.map(d => d.duration_ms))'",
+            "expected": "141"
+        },
+        {
+            "name": "custom_helper_functions",
+            "command": "echo '{\"values1\": [1, 2, 3, 4], \"values2\": [3, 5, 1, 11]}' | JSON_HELPERS_PATH=\"$(pwd)/test/custom-helpers.js\" json 'round(correlation(data.values1, data.values2), 3)'",
+            "expected": "0.598"
         }
     ]
 }