full-json-extractor 1.0.2 → 1.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "full-json-extractor",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "main": "dist/index.js",
   "scripts": {
     "clean": "rimraf dist build .cache",
@@ -8,7 +8,7 @@
     "format:fix": "eslint . --fix",
     "build": "tsc",
     "test": "jest",
-    "release": "npm run test && npm run build"
+    "release": "npm run format && npm run test && npm run build"
   },
   "keywords": [
     "json",

package/.vscode/launch.json

@@ -1,21 +0,0 @@
-{
-  // Use IntelliSense to learn about possible attributes.
-  // Hover to view descriptions of existing attributes.
-  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-  "version": "0.2.0",
-  "configurations": [
-    
-    {
-      "type": "node",
-      "request": "launch",
-      "name": "Launch Program",
-      "skipFiles": [
-        "<node_internals>/**"
-      ],
-      "program": "${workspaceFolder}\\dist\\index.js",
-      "outFiles": [
-        "${workspaceFolder}/**/*.js"
-      ]
-    }
-  ]
-}

package/eslint.config.js

@@ -1,39 +0,0 @@
-// eslint.config.mjs
-import js from "@eslint/js";
-import tseslint from "typescript-eslint";
-import tsParser from "@typescript-eslint/parser";
-import prettier from "eslint-plugin-prettier";
-import { fileURLToPath } from "node:url";
-import { dirname } from "node:path";
-import prettierConfig from "eslint-config-prettier";
-import { defineConfig } from "eslint/config";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-export default defineConfig([
-  { ignores: ["node_modules", "dist", "eslint.config.*", "jest.config.*"] },
-
-  {
-    files: ["**/*.{js,cjs,mjs}"],
-    extends: [js.configs.recommended],
-  },
-  {
-    files: ["**/*.{ts,tsx}"],
-    languageOptions: {
-      parser: tsParser,
-      parserOptions: {
-        projectService: true,
-        tsconfigRootDir: __dirname,
-      },
-    },
-    plugins: {
-      "@typescript-eslint": tseslint.plugin,
-      prettier,
-    },
-    rules: {
-      "prettier/prettier": ["error", { tabWidth: 4}],
-    }
-  },
-  prettierConfig,
-  tseslint.configs.stylisticTypeChecked,
-]);

package/jest.config.js

@@ -1,10 +0,0 @@
-/** @type {import("jest").Config} **/
-export default {
-  preset: 'ts-jest',
-  testEnvironment: "node",
-  transform: {
-    "^.+\\.m?ts?$": ["ts-jest", {
-        useESM: true
-    }]
-  }
-};

package/src/__tests__/json-parser.test.ts

@@ -1,171 +0,0 @@
-import { extractJsons } from "../extractor";
-
-describe("extractJsonIntervals.test", () => {
-    it("only json, valid json, 1-depth", () => {
-        const testData = `{"sample_id": 1, "data": { "key": "xdsc" }}`;
-        const expectedData = [
-            {
-                sample_id: 1,
-                data: {
-                    key: "xdsc",
-                },
-            },
-        ];
-
-        expect(extractJsons(testData)[0]).toEqual(expectedData[0]);
-    });
-
-    it("only json, invalid json, 0-depth", () => {
-        const testData = "{key: 1}";
-        expect(extractJsons(testData)).toEqual([]);
-    });
-
-    it("raw string, invalid json, 2-depth", () => {
-        const testData = '[hi] { "outer": { "inner": { key: 1 } } }';
-        expect(extractJsons(testData)).toEqual([]);
-    });
-
-    it("raw string, valid json, 1-depth", () => {
-        const testData = `[hi] {"sample_id": 1, "data": { "key": "xdsc" }}`;
-        const expectedData = [
-            {
-                sample_id: 1,
-                data: {
-                    key: "xdsc",
-                },
-            },
-        ];
-
-        expect(extractJsons(testData)[0]).toEqual(expectedData[0]);
-    });
-
-    it("raw string, valid escaped json, 1-depth", () => {
-        const testData = `[hi] "{\"sample_id\": 1, \"data\": { \"key\": \"xdsc\" }}"`;
-        const expectedData = [
-            {
-                sample_id: 1,
-                data: {
-                    key: "xdsc",
-                },
-            },
-        ];
-
-        expect(extractJsons(testData)[0]).toEqual(expectedData[0]);
-    });
-
-    it("raw string, valid json, 1-depth, 3 objects", () => {
-        const testData = `[hi] {"sample_id": 1, "data": { "key": "xdsc" }} {"sample_id": 2, "data": { "key": "xdsc" }} {"sample_id": 3, "data": { "key": "xdsc" }}`;
-        const expectedData = [
-            {
-                sample_id: 1,
-                data: {
-                    key: "xdsc",
-                },
-            },
-            {
-                sample_id: 2,
-                data: {
-                    key: "xdsc",
-                },
-            },
-            {
-                sample_id: 3,
-                data: {
-                    key: "xdsc",
-                },
-            },
-        ];
-
-        const result = extractJsons(testData);
-        for (const json of expectedData) {
-            expect(result).toContainEqual(json);
-        }
-    });
-
-    it("raw string, valid json, 1-depth, 3 objects, extra {", () => {
-        const testData = `[hi] {{"sample_id": 1, "data": { "key": "xdsc" }} {"sample_id": 2, "data": { "key": "xdsc" }} {"sample_id": 3, "data": { "key": "xdsc" }}`;
-        const expectedData = [
-            {
-                sample_id: 1,
-                data: {
-                    key: "xdsc",
-                },
-            },
-            {
-                sample_id: 2,
-                data: {
-                    key: "xdsc",
-                },
-            },
-            {
-                sample_id: 3,
-                data: {
-                    key: "xdsc",
-                },
-            },
-        ];
-
-        const result = extractJsons(testData);
-        for (const json of expectedData) {
-            expect(result).toContainEqual(json);
-        }
-    });
-
-    it("raw string, valid json, 1-depth, 3 objects, extra {, {} in string", () => {
-        const testData = `[hi] {{"sample_id": 1, "data": { "key": "x}dsc" }} {"sample_id": 2, "data": { "key": "xd{sc" }} {"sample_id": 3, "data": { "key": "xdsc" }}`;
-        const expectedData = [
-            {
-                sample_id: 1,
-                data: {
-                    key: "x}dsc",
-                },
-            },
-            {
-                sample_id: 2,
-                data: {
-                    key: "xd{sc",
-                },
-            },
-            {
-                sample_id: 3,
-                data: {
-                    key: "xdsc",
-                },
-            },
-        ];
-
-        const result = extractJsons(testData);
-        for (const json of expectedData) {
-            expect(result).toContainEqual(json);
-        }
-    });
-
-    it("raw string, valid json, 2-depth, 150 objects, extra {, {} in string", () => {
-        const testData: string[] = [];
-        for (let i = 0; i < 150; i++) {
-            testData.push(
-                `[hi] {{"sample_id": ${i}, "data": { "key": "x}ds{c", "subdata": { "key": 2 }}}`,
-            );
-        }
-
-        const expectedData: object[] = [];
-        for (let i = 0; i < 150; i++) {
-            expectedData.push({
-                sample_id: i,
-                data: {
-                    key: "x}ds{c",
-                    subdata: {
-                        key: 2,
-                    },
-                },
-            });
-        }
-
-        const testDataString = testData.join("");
-        const result = extractJsons(testDataString);
-        expect(result.length).toEqual(150);
-        for (const json of expectedData) {
-            expect(result).toContainEqual(json);
-        }
-    });
-});

package/src/benchmark.ts

@@ -1,60 +0,0 @@
-import { performance } from "node:perf_hooks";
-import process from "node:process";
-
-import { extractJsons } from "./extractor.js";
-
-function generateNestedJSON(depth: number): object {
-    let obj: any = { value: "test" };
-    for (let i = 0; i < depth; i++) {
-        obj = { nested: obj };
-    }
-    return obj;
-}
-
-function generateRawString(
-    depth: number,
-    count: number,
-    sizeFactor: number,
-): string {
-    const jsons = [];
-    for (let i = 0; i < count; i++) {
-        const base = generateNestedJSON(depth);
-        (base as any).padding = "x".repeat(sizeFactor);
-        jsons.push(JSON.stringify(base));
-    }
-    return jsons.join(" some text in between ");
-}
-
-function getMemoryUsageMB(): number {
-    const used = process.memoryUsage().heapUsed / 1024 / 1024;
-    return Math.round(used * 100) / 100;
-}
-
-async function runBenchmark() {
-    const depths = [1, 3, 5, 10, 20];
-    const sizes = [0, 1000, 10000, 50_000]; // padding sizes
-    const counts = [1, 5, 10, 20]; // number of JSONs per string
-
-    for (const depth of depths) {
-        for (const size of sizes) {
-            for (const count of counts) {
-                const input = generateRawString(depth, count, size);
-
-                const memBefore = getMemoryUsageMB();
-                const start = performance.now();
-
-                const result = extractJsons(input);
-
-                const end = performance.now();
-                const memAfter = getMemoryUsageMB();
-
-                console.log(
-                    `Depth=${depth}, Size=${size}, Count=${count} | Time=${(end - start).toFixed(3)} ms | ` +
-                        `MemΔ=${(memAfter - memBefore).toFixed(3)} MB | OutputLen=${JSON.stringify(result).length}`,
-                );
-            }
-        }
-    }
-}
-
-runBenchmark();

package/src/extractor.ts

@@ -1,228 +0,0 @@
-import IntervalTree from "interval-tree-1d";
-
-import {
-    BraceLocationInfo,
-    IntervalTreeType,
-    Limit,
-    MemoPosition,
-} from "./interfaces";
-import Denque from "denque";
-
-const LBRACE = "{";
-const RBRACE = "}";
-
-class JsonExtractError extends Error {}
-
-/**
- * Extracts json objects from a given input string
- * @param input input string
- * @param limit Sets pre-check behavior. If set to 'log2', method will terminate pre-check after reaching log2(n) characters. Useful for extremely large malformed data i.e. many {} + non-json text
- * Else, will do a O(n) pre-check scan to coarsely validate brace matches. Useful for many json objects (i.e. early termination)
- * @returns array of JSON objects
- */
-export function extractJsons<T = unknown>(
-    input: string,
-    limit: Limit = "none",
-): T[] {
-    if (!input.length) {
-        return [];
-    }
-
-    const locations = generateBracesPrefixAndSufix(input);
-    if (!locations.prefix.length || !locations.suffix.length) {
-        return [];
-    }
-
-    return findValidJsons<T>(locations, input, limit);
-}
-
-function setMapValue(
-    memo: Map<number, Set<number>>,
-    leftIndex: number,
-    rightIndex: number,
-): boolean {
-    let set = memo.get(leftIndex);
-    if (!set) {
-        set = new Set<number>();
-        memo.set(leftIndex, set);
-    }
-    if (set.has(rightIndex)) {
-        return false;
-    }
-    set.add(rightIndex);
-    return true;
-}
-
-function findValidJsons<T = unknown>(
-    { prefix, suffix }: BraceLocationInfo,
-    input: string,
-    limit: Limit,
-): T[] {
-    const tree = IntervalTree();
-    const queue = new Denque<MemoPosition>([[0, suffix.length - 1]]);
-    const memo = new Map<number, Set<number>>([
-        [0, new Set([suffix.length - 1])],
-    ]);
-    const jsons: T[] = [];
-
-    while (!queue.isEmpty()) {
-        const [leftIndex, rightIndex] = queue.shift()!;
-        const leftPosition: number = prefix[leftIndex]!;
-        const rightPosition: number = suffix[rightIndex]!;
-
-        if (
-            rightPosition < leftPosition ||
-            queryIntervalSync(tree, leftPosition, rightPosition)
-        ) {
-            continue;
-        }
-
-        const set = memo.get(leftIndex)!;
-        set.delete(rightIndex);
-
-        try {
-            if (
-                isBalancedWithOneJson(input, leftPosition, rightPosition, limit)
-            ) {
-                jsons.push(
-                    JSON.parse(input.slice(leftPosition, rightPosition + 1)),
-                );
-                tree.insert([leftPosition, rightPosition]);
-                continue;
-            }
-        } catch (error) {
-            if (!(error instanceof SyntaxError)) {
-                throw error as JsonExtractError;
-            }
-        }
-        if (
-            rightIndex - 1 >= 0 &&
-            setMapValue(memo, leftIndex, rightIndex - 1)
-        ) {
-            queue.push([leftIndex, rightIndex - 1]);
-        }
-
-        if (
-            leftIndex + 1 < prefix.length &&
-            setMapValue(memo, leftIndex + 1, rightIndex)
-        ) {
-            queue.push([leftIndex + 1, rightIndex]);
-        }
-
-        if (!set.size) {
-            memo.delete(leftIndex);
-        }
-    }
-    return jsons;
-}
-
-function generateBracesPrefixAndSufix(input: string): BraceLocationInfo {
-    const prefix: number[] = [];
-    const suffix: number[] = [];
-
-    for (let i = 0; i < input.length; i++) {
-        if (input[i] == LBRACE) {
-            prefix.push(i);
-        } else if (input[i] == RBRACE) {
-            suffix.push(i);
-        }
-    }
-
-    return {
-        prefix,
-        suffix,
-    };
-}
-
-function generateLimit(input: string, left: number, limit: Limit) {
-    switch (limit) {
-        case "log2":
-            return (
-                left +
-                Math.min(input.length, Math.ceil(Math.log2(input.length)))
-            );
-        case "none":
-            return input.length;
-        default:
-            throw new JsonExtractError("unknown limit type provided");
-    }
-}
-
-/**
- * Coarse pre-check to filter out invalid json candidates. Short circuits if >1 json candidates exist in slice
- * @param input
- * @param left
- * @param right
- * @returns
- */
-function isBalancedWithOneJson(
-    input: string,
-    left: number,
-    right: number,
-    limit: Limit,
-): boolean {
-    const terminationThreshold = generateLimit(input, left, limit);
-    let braceCount = 0;
-    let inString = false;
-    let escapeNext = false;
-    let firstJsonObj = true;
-
-    for (let i = left; i <= right; i++) {
-        if (i >= terminationThreshold) {
-            return true;
-        }
-
-        const char = input[i];
-
-        if (escapeNext) {
-            escapeNext = false;
-            continue;
-        }
-
-        if (char === "\\") {
-            escapeNext = true;
-            continue;
-        }
-
-        if (char === '"') {
-            inString = !inString;
-            continue;
-        }
-
-        if (inString) continue;
-
-        if (char === LBRACE) {
-            braceCount++;
-        } else if (char === RBRACE) {
-            braceCount--;
-        }
-
-        if (braceCount < 0) {
-            return false;
-        }
-
-        if (braceCount === 0) {
-            if (!firstJsonObj) {
-                return false;
-            }
-            firstJsonObj = !firstJsonObj;
-        }
-    }
-
-    return braceCount === 0;
-}
-
-function queryIntervalSync(
-    tree: IntervalTreeType,
-    low: number,
-    high: number,
-): boolean {
-    let intervalExists = false;
-    tree.queryInterval(low, high, ([left, right]: [number, number]) => {
-        if (left < low && high < right) {
-            intervalExists = true;
-            return;
-        }
-    });
-    return intervalExists;
-}

package/src/index.ts

@@ -1 +0,0 @@
-export * from "./extractor";

package/src/interfaces.ts

@@ -1,37 +0,0 @@
-export interface BraceLocationInfo {
-    readonly prefix: number[];
-    readonly suffix: number[];
-}
-
-export type Interval = [number, number];
-
-export interface IntervalTreeType {
-    readonly count: number;
-    insert(interval: Interval): void;
-    readonly intervals: Interval[];
-    queryInterval(lo: number, hi: number, cb: (interval: Interval) => any): any;
-    queryPoint(x: number, cb: (interval: Interval) => any): any;
-    remove(interval: Interval): boolean;
-    root: IntervalTreeNode | null;
-}
-
-export type Limit = "log2" | "none";
-
-/**
- * [left, right]: [number, number]
- */
-export type MemoPosition = [number, number];
-
-interface IntervalTreeNode {
-    count: number;
-    insert(interval: Interval): void;
-    intervals(result?: Interval[]): Interval[];
-    left: IntervalTreeNode | null;
-    leftPoints: Interval[];
-    mid: number;
-    queryInterval(lo: number, hi: number, cb: (interval: Interval) => any): any;
-    queryPoint(x: number, cb: (interval: Interval) => any): any;
-    remove(interval: Interval): number;
-    right: IntervalTreeNode | null;
-    rightPoints: Interval[];
-}

package/src/types/interval-tree-1d.d.ts

@@ -1,39 +0,0 @@
-declare module "interval-tree-1d" {
-    type Interval = [number, number];
-
-    interface IntervalTree {
-        readonly count: number;
-        insert(interval: Interval): void;
-        readonly intervals: Interval[];
-        queryInterval(
-            lo: number,
-            hi: number,
-            cb: (interval: Interval) => any,
-        ): any;
-        queryPoint(x: number, cb: (interval: Interval) => any): any;
-        remove(interval: Interval): boolean;
-        root: IntervalTreeNode | null;
-    }
-
-    interface IntervalTreeNode {
-        count: number;
-        insert(interval: Interval): void;
-        intervals(result?: Interval[]): Interval[];
-        left: IntervalTreeNode | null;
-        leftPoints: Interval[];
-        mid: number;
-        queryInterval(
-            lo: number,
-            hi: number,
-            cb: (interval: Interval) => any,
-        ): any;
-        queryPoint(x: number, cb: (interval: Interval) => any): any;
-        remove(interval: Interval): number;
-        right: IntervalTreeNode | null;
-        rightPoints: Interval[];
-    }
-
-    function createWrapper(intervals?: Interval[]): IntervalTree;
-
-    export = createWrapper;
-}