json-web-streams 0.0.1 → 1.0.0

package/README.md

@@ -1,5 +1,11 @@
 # json-web-streams
 
+[![Build Status](https://github.com/zengm-games/json-web-streams/actions/workflows/test.yaml/badge.svg)](https://github.com/zengm-games/json-web-streams/actions/workflows/test.yaml)
+[![GitHub Repo stars](https://img.shields.io/github/stars/zengm-games/json-web-streams)](https://github.com/zengm-games/json-web-streams)
+[![GitHub](https://img.shields.io/github/license/zengm-games/json-web-streams)](https://github.com/zengm-games/json-web-streams)
+[![npm](https://img.shields.io/npm/v/json-web-streams)](https://www.npmjs.com/package/json-web-streams)
+[![npm](https://img.shields.io/npm/dm/json-web-streams)](https://www.npmjs.com/package/json-web-streams)
+
 - **Stream large JSON files** without loading everything into memory
 - Built on the **Web Streams API** so it runs in web browsers, Node.js, and more
 - Query with **JSONPath** to extract only the data you need

package/dist/JSONParseStream.d.ts

@@ -0,0 +1,28 @@
+import { JSONParseStreamRaw } from "./JSONParseStreamRaw.js";
+import { JSONPath } from "./jsonPathToPathArray.js";
+import { StandardSchemaV1 } from "@standard-schema/spec";
+
+//#region src/JSONParseStream.d.ts
+type JSONParseStreamOutput<T> = T extends {
+  path: infer P extends JSONPath;
+  schema: infer S extends StandardSchemaV1;
+} ? {
+  path: P;
+  value: StandardSchemaV1.InferOutput<S>;
+  wildcardKeys?: string[];
+} : T extends JSONPath ? {
+  path: T;
+  value: unknown;
+  wildcardKeys?: string[];
+} : never;
+declare class JSONParseStream<T extends readonly (JSONPath | {
+  path: JSONPath;
+  schema: StandardSchemaV1;
+})[]> extends TransformStream<string, JSONParseStreamOutput<T[number]>> {
+  _parser: JSONParseStreamRaw;
+  constructor(jsonPaths: T, options?: {
+    multi?: boolean;
+  });
+}
+//#endregion
+export { JSONParseStream };

package/dist/JSONParseStream.js

@@ -0,0 +1,116 @@
+import { JSONParseStreamRaw } from "./JSONParseStreamRaw.js";
+import { jsonPathToPathArray } from "./jsonPathToPathArray.js";
+
+//#region src/JSONParseStream.ts
+const stackToPathComponent = (stackComponent) => {
+	if (stackComponent.mode === "OBJECT" && stackComponent.key !== void 0) return {
+		type: "key",
+		value: stackComponent.key
+	};
+	if (stackComponent.mode === "ARRAY") return { type: "wildcard" };
+	throw new Error(`Unexpected mode "${stackComponent.mode}"`);
+};
+const isEqual = (x, y) => {
+	if (!y) return false;
+	if (x.type === y?.type) {
+		if (x.type === "wildcard") return true;
+		if (x.value === y.value) return true;
+		return false;
+	}
+	return x.type === "wildcard";
+};
+var JSONParseStream = class extends TransformStream {
+	_parser;
+	constructor(jsonPaths, options) {
+		let parser;
+		const multi = options?.multi ?? false;
+		const jsonPathInfos = jsonPaths.map((row) => {
+			let path;
+			let schema;
+			if (typeof row === "string") path = row;
+			else {
+				path = row.path;
+				schema = row.schema;
+			}
+			const pathArray = jsonPathToPathArray(path);
+			let wildcardIndexes;
+			for (const [i, component] of pathArray.entries()) if (component.type === "wildcard") {
+				if (wildcardIndexes === void 0) wildcardIndexes = [];
+				wildcardIndexes.push(i);
+			}
+			return {
+				path,
+				pathArray,
+				schema,
+				wildcardIndexes
+			};
+		});
+		super({
+			start(controller) {
+				parser = new JSONParseStreamRaw({
+					multi,
+					onValue: (value) => {
+						const { key: parserKey, mode: parserMode, stack: parserStack, value: parserValue } = parser;
+						const stackLength = parserStack.length;
+						const stackPathArray = new Array(stackLength);
+						for (let i = 1; i < stackLength; i++) stackPathArray[i - 1] = stackToPathComponent(parserStack[i]);
+						if (parserStack.length > 0) stackPathArray[stackLength - 1] = stackToPathComponent({
+							value: parserValue,
+							key: parserKey,
+							mode: parserMode
+						});
+						let keep = false;
+						for (const { path, pathArray, schema, wildcardIndexes } of jsonPathInfos) {
+							if (stackPathArray.length < pathArray.length) continue;
+							if (pathArray.every((x, j) => isEqual(x, stackPathArray[j]))) if (stackPathArray.length === pathArray.length) {
+								let valueToEmit;
+								if (schema) {
+									const result = schema["~standard"].validate(value);
+									if (result instanceof Promise) throw new TypeError("Schema validation must be synchronous");
+									if (result.issues) throw new Error(JSON.stringify(result.issues, null, 2));
+									valueToEmit = result.value;
+								} else valueToEmit = value;
+								let wildcardKeys;
+								if (wildcardIndexes) for (const index of wildcardIndexes) {
+									const pathComponent = stackPathArray[index];
+									if (pathComponent?.type === "key") {
+										if (!wildcardKeys) wildcardKeys = [];
+										wildcardKeys.push(pathComponent.value);
+									}
+								}
+								if (wildcardKeys) controller.enqueue({
+									path,
+									value: valueToEmit,
+									wildcardKeys
+								});
+								else controller.enqueue({
+									path,
+									value: valueToEmit
+								});
+							} else keep = true;
+							else {
+								const type = typeof value;
+								if (type === "string" || type === "number" || type === "boolean" || value === null) keep = true;
+							}
+						}
+						if (!keep) {
+							for (const row of parserStack) row.value = void 0;
+							if (typeof parserValue === "object" && parserValue !== null && parserKey !== void 0) parserValue[parserKey] = void 0;
+						}
+					}
+				});
+			},
+			transform(chunk) {
+				parser.write(chunk);
+			},
+			flush(controller) {
+				parser.checkEnd();
+				controller.terminate();
+			}
+		});
+		this._parser = parser;
+	}
+};
+
+//#endregion
+export { JSONParseStream };

package/dist/{JSONParserText.d.ts → JSONParseStreamRaw.d.ts}

@@ -1,4 +1,4 @@
-//#region src/JSONParserText.d.ts
+//#region src/JSONParseStreamRaw.d.ts
 type Token = "LEFT_BRACE" | "RIGHT_BRACE" | "LEFT_BRACKET" | "RIGHT_BRACKET" | "COLON" | "COMMA" | "TRUE" | "FALSE" | "NULL" | "STRING" | "NUMBER";
 type ParserState = "VALUE" | "KEY" | "VALUE_AFTER_COMMA" | "KEY_AFTER_COMMA";
 type TokenizerState = "START" | "TRUE1" | "TRUE2" | "TRUE3" | "FALSE1" | "FALSE2" | "FALSE3" | "FALSE4" | "NULL1" | "NULL2" | "NULL3" | "NUMBER-" | "NUMBER0" | "NUMBER" | "STRING1" | "STRING2" | "STRING3" | "STRING4" | "STRING5" | "STRING6";
@@ -10,8 +10,8 @@ type Stack = {
   value: Value | undefined;
   mode: Mode | undefined;
 }[];
-type OnValue = (value: Value, stack: Stack) => void;
-declare class JSONParserText {
+type OnValue = (value: Value) => void;
+declare class JSONParseStreamRaw {
   tokenizerState: TokenizerState;
   state: Token | ParserState;
   mode: Mode | undefined;
@@ -44,4 +44,4 @@ declare class JSONParserText {
   checkEnd(): void;
 }
 //#endregion
-export { JSONParserText };
+export { JSONParseStreamRaw };

package/dist/{JSONParserText.js → JSONParseStreamRaw.js}

@@ -1,11 +1,11 @@
-//#region src/JSONParserText.ts
+//#region src/JSONParseStreamRaw.ts
 const WHITESPACE = new Set([
 	" ",
 	"	",
 	"\n",
 	"\r"
 ]);
-var JSONParserText = class {
+var JSONParseStreamRaw = class {
 	tokenizerState = "START";
 	state = "VALUE";
 	mode;
@@ -34,7 +34,6 @@ var JSONParserText = class {
 		for (let i = 0, l = text.length; i < l; i++) {
 			const n = text[i];
 			if (!this.multi && this.stack.length === 0 && this.seenRootObject && !WHITESPACE.has(n)) return this.charError(n, i);
-			if (this.multi && this.stack.length === 0 && this.seenRootObject) {}
 			if (this.tokenizerState === "START") if (n === "{") this.onToken("LEFT_BRACE", "{", i);
 			else if (n === "}") this.onToken("RIGHT_BRACE", "}", i);
 			else if (n === "[") this.onToken("LEFT_BRACKET", "[", i);
@@ -194,11 +193,8 @@ var JSONParserText = class {
 	}
 	emit(value) {
 		if (this.mode) this.state = "COMMA";
-		this.onValue(value, [...this.stack, {
-			value: this.value,
-			key: this.key,
-			mode: this.mode
-		}]);
+		if (value === void 0) return;
+		this.onValue(value);
 	}
 	onToken(token, value, i) {
 		if (this.stack.length === 0) {
@@ -255,7 +251,6 @@ var JSONParserText = class {
 		} else if (!this.seenRootObject) throw new Error("No data in input");
 	}
 };
-var JSONParserText_default = JSONParserText;
 
 //#endregion
-export { JSONParserText_default as default };
+export { JSONParseStreamRaw };

package/dist/index.d.ts

@@ -1,3 +1,4 @@
-import { JSONPath } from "./jsonPathToQueryPath.js";
-import { createJSONParserStream } from "./JSONParserStream.js";
-export { type JSONPath, createJSONParserStream };
+import { JSONParseStreamRaw } from "./JSONParseStreamRaw.js";
+import { JSONPath } from "./jsonPathToPathArray.js";
+import { JSONParseStream } from "./JSONParseStream.js";
+export { JSONParseStream, JSONParseStreamRaw, type JSONPath };

package/dist/index.js

@@ -1,3 +1,4 @@
-import { createJSONParserStream } from "./JSONParserStream.js";
+import { JSONParseStreamRaw } from "./JSONParseStreamRaw.js";
+import { JSONParseStream } from "./JSONParseStream.js";
 
-export { createJSONParserStream };
+export { JSONParseStream, JSONParseStreamRaw };

package/dist/{jsonPathToQueryPath.d.ts → jsonPathToPathArray.d.ts}

@@ -1,4 +1,4 @@
-//#region src/jsonPathToQueryPath.d.ts
+//#region src/jsonPathToPathArray.d.ts
 
 type JSONPath = "$" | `$${"." | "["}${string}`;
 //#endregion

package/dist/{jsonPathToQueryPath.js → jsonPathToPathArray.js}

@@ -1,12 +1,12 @@
 import parser from "jsonpath-rfc9535/parser";
 
-//#region src/jsonPathToQueryPath.ts
-const jsonPathToQueryPath = (jsonPath) => {
+//#region src/jsonPathToPathArray.ts
+const jsonPathToPathArray = (path) => {
 	let parsed;
 	try {
-		parsed = parser(jsonPath);
+		parsed = parser(path);
 	} catch (error) {
-		throw new Error(`Error parsing JSONPath "${jsonPath}"`, { cause: error });
+		throw new Error(`Error parsing JSONPath "${path}"`, { cause: error });
 	}
 	return parsed.segments.flatMap((segment) => {
 		if (segment.type === "ChildSegment") {
@@ -30,4 +30,4 @@ const jsonPathToQueryPath = (jsonPath) => {
 };
 
 //#endregion
-export { jsonPathToQueryPath };
+export { jsonPathToPathArray };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "json-web-streams",
-	"version": "0.0.1",
+	"version": "1.0.0",
 	"description": "Streaming JSON parser built on top of the Web Streams API, so it works in web browsers, Node.js, and many other environments",
 	"keywords": [
 		"web streams",
@@ -26,6 +26,7 @@
 		"lint": "npm-run-all --parallel lint:oxlint lint:tsc",
 		"lint:oxlint": "oxlint --type-aware",
 		"lint:tsc": "tsc",
+		"prepack": "node --run build",
 		"prepare": "husky",
 		"test": "vitest"
 	},

package/dist/JSONParserStream.d.ts

@@ -1,18 +0,0 @@
-import { JSONParserText } from "./JSONParserText.js";
-import { JSONPath } from "./jsonPathToQueryPath.js";
-import { StandardSchemaV1 } from "@standard-schema/spec";
-
-//#region src/JSONParserStream.d.ts
-declare const createJSONParserStream: <JSONPathsObject extends Partial<Record<JSONPath, StandardSchemaV1 | null | undefined>> = Record<JSONPath, never>>(jsonPaths: JSONPathsObject) => JSONParserStream<JSONPathsObject>;
-declare class JSONParserStream<JSONPathsObject extends Partial<Record<JSONPath, StandardSchemaV1 | null | undefined>>> extends TransformStream<string, {
-  jsonPath: JSONPath;
-  value: unknown;
-  wildcardKeys?: string[];
-}> {
-  _parser: JSONParserText;
-  constructor(jsonPaths: JSONPathsObject, options?: {
-    multi?: boolean;
-  });
-}
-//#endregion
-export { createJSONParserStream };

package/dist/JSONParserStream.js

@@ -1,108 +0,0 @@
-import JSONParserText_default from "./JSONParserText.js";
-import { jsonPathToQueryPath } from "./jsonPathToQueryPath.js";
-
-//#region src/JSONParserStream.ts
-const stackToQueryPath = (stack) => {
-	return stack.slice(1).map((row) => {
-		if (row.mode === "OBJECT" && row.key !== void 0) return {
-			type: "key",
-			value: row.key
-		};
-		if (row.mode === "ARRAY") return { type: "wildcard" };
-		throw new Error(`Unexpected mode "${row.mode}"`);
-	});
-};
-const isEqual = (x, y) => {
-	if (!y) return false;
-	if (x.type === y?.type) {
-		if (x.type === "wildcard") return true;
-		if (x.value === y.value) return true;
-		return false;
-	}
-	return x.type === "wildcard";
-};
-const createJSONParserStream = (jsonPaths) => {
-	return new JSONParserStream(jsonPaths);
-};
-var JSONParserStream = class extends TransformStream {
-	_parser;
-	constructor(jsonPaths, options) {
-		let parser;
-		const multi = options?.multi ?? false;
-		const queryInfos = /* @__PURE__ */ new Map();
-		for (const [key, schema] of Object.entries(jsonPaths)) {
-			const jsonPath = key;
-			const queryPath = jsonPathToQueryPath(jsonPath);
-			let wildcardIndexes;
-			for (const [i, component] of queryPath.entries()) if (component.type === "wildcard") {
-				if (wildcardIndexes === void 0) wildcardIndexes = [];
-				wildcardIndexes.push(i);
-			}
-			queryInfos.set(jsonPath, {
-				jsonPath,
-				queryPath,
-				schema,
-				wildcardIndexes
-			});
-		}
-		super({
-			start(controller) {
-				parser = new JSONParserText_default({
-					multi,
-					onValue: (value, stack) => {
-						const path = stackToQueryPath(stack);
-						let keep = false;
-						for (const { jsonPath, queryPath, schema, wildcardIndexes } of queryInfos.values()) if (queryPath.every((x, j) => isEqual(x, path[j]))) if (path.length === queryPath.length) {
-							let valueToEmit;
-							if (schema) {
-								const result = schema["~standard"].validate(value);
-								if (result instanceof Promise) throw new Error("async schema validation is not supported");
-								if (result.issues) throw new Error(JSON.stringify(result.issues, null, 2));
-								valueToEmit = result.value;
-							} else valueToEmit = queryInfos.size === 1 ? value : structuredClone(value);
-							console.log("valueToEmit", value, valueToEmit);
-							let wildcardKeys;
-							if (wildcardIndexes) {
-								if (wildcardIndexes) for (const index of wildcardIndexes) {
-									const pathComponent = path[index];
-									if (pathComponent?.type === "key") {
-										if (!wildcardKeys) wildcardKeys = [];
-										wildcardKeys.push(pathComponent.value);
-									}
-								}
-							}
-							if (wildcardKeys) controller.enqueue({
-								jsonPath,
-								value: valueToEmit,
-								wildcardKeys
-							});
-							else controller.enqueue({
-								jsonPath,
-								value: valueToEmit
-							});
-						} else keep = true;
-						else {
-							const type = typeof value;
-							if (type === "string" || type === "number" || type === "boolean" || value === null) keep = true;
-						}
-						if (!keep) {
-							for (const row of parser.stack) row.value = void 0;
-							if (typeof parser.value === "object" && parser.value !== null && parser.key !== void 0) delete parser.value[parser.key];
-						}
-					}
-				});
-			},
-			transform(chunk) {
-				parser.write(chunk);
-			},
-			flush(controller) {
-				parser.checkEnd();
-				controller.terminate();
-			}
-		});
-		this._parser = parser;
-	}
-};
-
-//#endregion
-export { createJSONParserStream };