@depup/stream-json 1.9.1-depup.0

package/jsonl/Parser.js

@@ -0,0 +1,131 @@
+'use strict';
+
+const Utf8Stream = require('../utils/Utf8Stream');
+
+class JsonlParser extends Utf8Stream {
+  static make(options) {
+    return new JsonlParser(options);
+  }
+
+  static checkedParse(input, reviver, errorIndicator) {
+    try {
+      return JSON.parse(input, reviver);
+    } catch (error) {
+      if (typeof errorIndicator == 'function') return errorIndicator(error, input, reviver);
+    }
+    return errorIndicator;
+  }
+
+  constructor(options) {
+    super(Object.assign({}, options, {readableObjectMode: true}));
+    this._rest = '';
+    this._counter = 0;
+    this._reviver = options && options.reviver;
+    this._errorIndicator = options && options.errorIndicator;
+    if (options && options.checkErrors) {
+      this._processBuffer = this._checked_processBuffer;
+      this._flush = this._checked_flush;
+    }
+    if (options && 'errorIndicator' in options) {
+      this._processBuffer = this._suppressed_processBuffer;
+      this._flush = this._suppressed_flush;
+    }
+  }
+
+  _processBuffer(callback) {
+    const lines = this._buffer.split('\n');
+    this._rest += lines[0];
+    if (lines.length > 1) {
+      this._rest && this.push({key: this._counter++, value: JSON.parse(this._rest, this._reviver)});
+      this._rest = lines.pop();
+      for (let i = 1; i < lines.length; ++i) {
+        lines[i] && this.push({key: this._counter++, value: JSON.parse(lines[i], this._reviver)});
+      }
+    }
+    this._buffer = '';
+    callback(null);
+  }
+
+  _flush(callback) {
+    super._flush(error => {
+      if (error) return callback(error);
+      if (this._rest) {
+        this.push({key: this._counter++, value: JSON.parse(this._rest, this._reviver)});
+        this._rest = '';
+      }
+      callback(null);
+    });
+  }
+
+  _suppressed_processBuffer(callback) {
+    const lines = this._buffer.split('\n');
+    this._rest += lines[0];
+    if (lines.length > 1) {
+      if (this._rest) {
+        const value = JsonlParser.checkedParse(this._rest, this._reviver, this._errorIndicator);
+        value !== undefined && this.push({key: this._counter++, value});
+      }
+      this._rest = lines.pop();
+      for (let i = 1; i < lines.length; ++i) {
+        if (!lines[i]) continue;
+        const value = JsonlParser.checkedParse(lines[i], this._reviver, this._errorIndicator);
+        value !== undefined && this.push({key: this._counter++, value});
+      }
+    }
+    this._buffer = '';
+    callback(null);
+  }
+
+  _suppressed_flush(callback) {
+    super._flush(error => {
+      if (error) return callback(error);
+      if (this._rest) {
+        const value = JsonlParser.checkedParse(this._rest, this._reviver, this._errorIndicator);
+        value !== undefined && this.push({key: this._counter++, value});
+        this._rest = '';
+      }
+      callback(null);
+    });
+  }
+
+  _checked_processBuffer(callback) {
+    const lines = this._buffer.split('\n');
+    this._rest += lines[0];
+    if (lines.length > 1) {
+      try {
+        this._rest && this.push({key: this._counter++, value: JSON.parse(this._rest, this._reviver)});
+        this._rest = lines.pop();
+        for (let i = 1; i < lines.length; ++i) {
+          lines[i] && this.push({key: this._counter++, value: JSON.parse(lines[i], this._reviver)});
+        }
+      } catch (cbErr) {
+        this._buffer = '';
+        callback(cbErr);
+        return;
+      }
+    }
+    this._buffer = '';
+    callback(null);
+  }
+
+  _checked_flush(callback) {
+    super._flush(error => {
+      if (error) return callback(error);
+      if (this._rest) {
+        try {
+          this.push({key: this._counter++, value: JSON.parse(this._rest, this._reviver)});
+        } catch (cbErr) {
+          this._rest = '';
+          callback(cbErr);
+          return;
+        }
+        this._rest = '';
+      }
+      callback(null);
+    });
+  }
+}
+JsonlParser.parser = JsonlParser.make;
+JsonlParser.make.Constructor = JsonlParser;
+
+module.exports = JsonlParser;

package/jsonl/Stringer.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const {Transform} = require('stream');
+
+class JsonlStringer extends Transform {
+  static make(options) {
+    return new JsonlStringer(options);
+  }
+
+  constructor(options) {
+    super(Object.assign({}, options, {writableObjectMode: true, readableObjectMode: false}));
+    this._replacer = options && options.replacer;
+  }
+
+  _transform(chunk, _, callback) {
+    this.push(JSON.stringify(chunk, this._replacer));
+    this._transform = this._nextTransform;
+    callback(null);
+  }
+
+  _nextTransform(chunk, _, callback) {
+    this.push('\n' + JSON.stringify(chunk, this._replacer));
+    callback(null);
+  }
+}
+JsonlStringer.stringer = JsonlStringer.make;
+JsonlStringer.make.Constructor = JsonlStringer;
+
+module.exports = JsonlStringer;

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "@depup/stream-json",
+  "version": "1.9.1-depup.0",
+  "description": "[DepUp] stream-json is the micro-library of Node.js stream components for creating custom JSON processing pipelines with a minimal memory footprint. It can parse JSON files far exceeding available memory streaming individual primitives using a SAX-inspired API. Includes utilities to stream JSON database dumps.",
+  "homepage": "http://github.com/uhop/stream-json",
+  "bugs": "http://github.com/uhop/stream-json/issues",
+  "main": "index.js",
+  "directories": {
+    "test": "tests"
+  },
+  "dependencies": {
+    "stream-chain": "^3.4.1"
+  },
+  "devDependencies": {
+    "heya-unit": "^0.3.0"
+  },
+  "scripts": {
+    "test": "node tests/tests.js",
+    "debug": "node --inspect-brk tests/tests.js"
+  },
+  "github": "http://github.com/uhop/stream-json",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/uhop/stream-json.git"
+  },
+  "keywords": [
+    "depup",
+    "dependency-bumped",
+    "updated-deps",
+    "stream-json",
+    "scanner",
+    "lexer",
+    "tokenizer",
+    "parser",
+    "django",
+    "stream",
+    "streaming",
+    "json"
+  ],
+  "author": "Eugene Lazutkin <eugene.lazutkin@gmail.com> (http://lazutkin.com/)",
+  "license": "BSD-3-Clause",
+  "files": [
+    "/*.js",
+    "/filters",
+    "/jsonl",
+    "/streamers",
+    "/utils",
+    "changes.json",
+    "README.md"
+  ],
+  "depup": {
+    "changes": {
+      "stream-chain": {
+        "from": "^2.2.5",
+        "to": "^3.4.1"
+      }
+    },
+    "depsUpdated": 1,
+    "originalPackage": "stream-json",
+    "originalVersion": "1.9.1",
+    "processedAt": "2026-03-17T16:39:26.281Z",
+    "smokeTest": "passed"
+  }
+}

package/streamers/StreamArray.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const StreamBase = require('./StreamBase');
+const withParser = require('../utils/withParser');
+
+class StreamArray extends StreamBase {
+  static make(options) {
+    return new StreamArray(options);
+  }
+
+  static withParser(options) {
+    return withParser(StreamArray.make, options);
+  }
+
+  constructor(options) {
+    super(options);
+    this._level = 1;
+    this._counter = 0;
+  }
+
+  _wait(chunk, _, callback) {
+    // first chunk should open an array
+    if (chunk.name !== 'startArray') {
+      return callback(new Error('Top-level object should be an array.'));
+    }
+    this._transform = this._filter;
+    return this._transform(chunk, _, callback);
+  }
+
+  _push(discard) {
+    if (this._assembler.current.length) {
+      if (discard) {
+        ++this._counter;
+        this._assembler.current.pop();
+      } else {
+        this.push({key: this._counter++, value: this._assembler.current.pop()});
+      }
+    }
+  }
+}
+StreamArray.streamArray = StreamArray.make;
+StreamArray.make.Constructor = StreamArray;
+
+module.exports = StreamArray;

package/streamers/StreamBase.js

@@ -0,0 +1,101 @@
+'use strict';
+
+const {Transform} = require('stream');
+const Assembler = require('../Assembler');
+
+class Counter {
+  constructor(initialDepth) {
+    this.depth = initialDepth;
+  }
+  startObject() {
+    ++this.depth;
+  }
+  endObject() {
+    --this.depth;
+  }
+  startArray() {
+    ++this.depth;
+  }
+  endArray() {
+    --this.depth;
+  }
+}
+
+class StreamBase extends Transform {
+  constructor(options) {
+    super(Object.assign({}, options, {writableObjectMode: true, readableObjectMode: true}));
+    if (options) {
+      this.objectFilter = options.objectFilter;
+      this.includeUndecided = options.includeUndecided;
+    }
+    if (typeof this.objectFilter != 'function') {
+      this._filter = this._transform;
+    }
+    this._transform = this._wait || this._filter;
+    this._assembler = new Assembler(options);
+  }
+
+  _transform(chunk, encoding, callback) {
+    if (this._assembler[chunk.name]) {
+      this._assembler[chunk.name](chunk.value);
+      if (this._assembler.depth === this._level) {
+        this._push();
+      }
+    }
+    callback(null);
+  }
+
+  _filter(chunk, encoding, callback) {
+    if (this._assembler[chunk.name]) {
+      this._assembler[chunk.name](chunk.value);
+      const result = this.objectFilter(this._assembler);
+      if (result) {
+        if (this._assembler.depth === this._level) {
+          this._push();
+          this._transform = this._filter;
+        }
+        this._transform = this._accept;
+        return callback(null);
+      }
+      if (result === false) {
+        this._saved_assembler = this._assembler;
+        this._assembler = new Counter(this._saved_assembler.depth);
+        this._saved_assembler.dropToLevel(this._level);
+        if (this._assembler.depth === this._level) {
+          this._assembler = this._saved_assembler;
+          this._transform = this._filter;
+        }
+        this._transform = this._reject;
+        return callback(null);
+      }
+      if (this._assembler.depth === this._level) {
+        this._push(!this.includeUndecided);
+      }
+    }
+    callback(null);
+  }
+
+  _accept(chunk, encoding, callback) {
+    if (this._assembler[chunk.name]) {
+      this._assembler[chunk.name](chunk.value);
+      if (this._assembler.depth === this._level) {
+        this._push();
+        this._transform = this._filter;
+      }
+    }
+    callback(null);
+  }
+
+  _reject(chunk, encoding, callback) {
+    if (this._assembler[chunk.name]) {
+      this._assembler[chunk.name](chunk.value);
+      if (this._assembler.depth === this._level) {
+        this._assembler = this._saved_assembler;
+        this._transform = this._filter;
+      }
+    }
+    callback(null);
+  }
+}
+
+module.exports = StreamBase;

package/streamers/StreamObject.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const StreamBase = require('./StreamBase');
+const withParser = require('../utils/withParser');
+
+class StreamObject extends StreamBase {
+  static make(options) {
+    return new StreamObject(options);
+  }
+
+  static withParser(options) {
+    return withParser(StreamObject.make, options);
+  }
+
+  constructor(options) {
+    super(options);
+    this._level = 1;
+    this._lastKey = null;
+  }
+
+  _wait(chunk, _, callback) {
+    // first chunk should open an array
+    if (chunk.name !== 'startObject') {
+      return callback(new Error('Top-level object should be an object.'));
+    }
+    this._transform = this._filter;
+    return this._transform(chunk, _, callback);
+  }
+
+  _push(discard) {
+    if (this._lastKey === null) {
+      this._lastKey = this._assembler.key;
+    } else {
+      !discard && this.push({key: this._lastKey, value: this._assembler.current[this._lastKey]});
+      this._assembler.current = {};
+      this._lastKey = null;
+    }
+  }
+}
+StreamObject.streamObject = StreamObject.make;
+StreamObject.make.Constructor = StreamObject;
+
+module.exports = StreamObject;

package/streamers/StreamValues.js

@@ -0,0 +1,33 @@
+'use strict';
+
+const StreamBase = require('./StreamBase');
+const withParser = require('../utils/withParser');
+
+class StreamValues extends StreamBase {
+  static make(options) {
+    return new StreamValues(options);
+  }
+
+  static withParser(options) {
+    return withParser(StreamValues.make, Object.assign({}, options, {jsonStreaming: true}));
+  }
+
+  constructor(options) {
+    super(options);
+    this._counter = 0;
+    this._level = 0;
+  }
+
+  _push(discard) {
+    if (discard) {
+      ++this._counter;
+    } else {
+      this.push({key: this._counter++, value: this._assembler.current});
+    }
+    this._assembler.current = this._assembler.key = null;
+  }
+}
+StreamValues.streamValues = StreamValues.make;
+StreamValues.make.Constructor = StreamValues;
+
+module.exports = StreamValues;

package/utils/Batch.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const {Transform} = require('stream');
+const withParser = require('./withParser');
+
+class Batch extends Transform {
+  static make(options) {
+    return new Batch(options);
+  }
+
+  static withParser(options) {
+    return withParser(Batch.make, options);
+  }
+
+  constructor(options) {
+    super(Object.assign({}, options, {writableObjectMode: true, readableObjectMode: true}));
+    this._batchSize = 1000;
+    if (options && typeof options.batchSize == 'number' && options.batchSize > 0) {
+      this._batchSize = options.batchSize;
+    }
+    this._accumulator = [];
+  }
+
+  _transform(chunk, _, callback) {
+    this._accumulator.push(chunk);
+    if (this._accumulator.length >= this._batchSize) {
+      this.push(this._accumulator);
+      this._accumulator = [];
+    }
+    callback(null);
+  }
+
+  _flush(callback) {
+    if (this._accumulator.length) {
+      this.push(this._accumulator);
+      this._accumulator = null;
+    }
+    callback(null);
+  }
+}
+Batch.batch = Batch.make;
+Batch.make.Constructor = Batch;
+
+module.exports = Batch;

package/utils/Utf8Stream.js

@@ -0,0 +1,53 @@
+'use strict';
+
+const {Transform} = require('stream');
+const {StringDecoder} = require('string_decoder');
+
+class Utf8Stream extends Transform {
+  constructor(options) {
+    super(Object.assign({}, options, {writableObjectMode: false}));
+    this._buffer = '';
+  }
+
+  _transform(chunk, encoding, callback) {
+    if (typeof chunk == 'string') {
+      this._transform = this._transformString;
+    } else {
+      this._stringDecoder = new StringDecoder();
+      this._transform = this._transformBuffer;
+    }
+    this._transform(chunk, encoding, callback);
+  }
+
+  _transformBuffer(chunk, _, callback) {
+    this._buffer += this._stringDecoder.write(chunk);
+    this._processBuffer(callback);
+  }
+
+  _transformString(chunk, _, callback) {
+    this._buffer += chunk.toString();
+    this._processBuffer(callback);
+  }
+
+  _processBuffer(callback) {
+    if (this._buffer) {
+      this.push(this._buffer, 'utf8');
+      this._buffer = '';
+    }
+    callback(null);
+  }
+
+  _flushInput() {
+    // meant to be called from _flush()
+    if (this._stringDecoder) {
+      this._buffer += this._stringDecoder.end();
+    }
+  }
+
+  _flush(callback) {
+    this._flushInput();
+    this._processBuffer(callback);
+  }
+}
+
+module.exports = Utf8Stream;