json-seq-stream 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/InputJsonLinesStream.d.ts +38 -0
- package/InputJsonSequenceParseStream.d.ts +4 -9
- package/InputJsonSequenceStream.d.ts +27 -5
- package/InputLineFeedSeparattedSequenceStream.d.ts +8 -0
- package/InputRecordSequenceStream.d.ts +29 -0
- package/OutputJsonLinesStream.d.ts +39 -0
- package/OutputJsonSequenceStream.d.ts +29 -5
- package/OutputJsonSequenceStringifyStream.d.ts +9 -0
- package/OutputRecordSequenceStream.d.ts +19 -0
- package/OutputTextJoinLineFeedSequenceStream.d.ts +9 -0
- package/ParseStream.d.ts +43 -0
- package/README.md +62 -11
- package/RecordToSequenceStream.d.ts +21 -0
- package/SequenceToRecordStream.d.ts +25 -0
- package/StringifyStream.d.ts +44 -0
- package/TextJoinStream.d.ts +31 -0
- package/TextSplitStream.d.ts +32 -0
- package/index.d.ts +14 -4
- package/index.js +253 -82
- package/index.umd.cjs +4 -2
- package/jsonlines.d.ts +23 -0
- package/package.json +2 -1
- package/rfc7464.d.ts +6 -1
- package/InputSequenceStream.d.ts +0 -20
- package/OutputJsonSequenceStfingifyStream.d.ts +0 -14
- package/OutputSequenceStream.d.ts +0 -10
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { InputJsonSequenceParseStreamOptions } from "./InputJsonSequenceParseStream";
|
|
2
|
+
import type { InputLineFeedSeparattedSequenceStreamOptions } from "./InputLineFeedSeparattedSequenceStream";
|
|
3
|
+
export type InputJsonLinesStreamOptions<T> = {
|
|
4
|
+
label?: string;
|
|
5
|
+
} & TextDecoderOptions & InputLineFeedSeparattedSequenceStreamOptions & InputJsonSequenceParseStreamOptions<T>;
|
|
6
|
+
/**
|
|
7
|
+
* A composite stream that parses a binary NDJSON / JSON Lines stream
|
|
8
|
+
* (`application/jsonl`) into a stream of typed JSON values (`T`).
|
|
9
|
+
*
|
|
10
|
+
* This stream is designed to work with binary sources such as `fetch().body`,
|
|
11
|
+
* decoding line-delimited JSON records into structured JavaScript objects.
|
|
12
|
+
*
|
|
13
|
+
* Internally, it composes:
|
|
14
|
+
* - `TextDecoderStream`: Decodes binary data (e.g. UTF-8) into text.
|
|
15
|
+
* - `InputLineFeedSeparattedSequenceStream`: Splits the text stream into individual lines
|
|
16
|
+
* using newline (`\n`) or custom delimiters.
|
|
17
|
+
* - `InputJsonSequenceParseStream`: Parses each line as a JSON object of type `T`.
|
|
18
|
+
*
|
|
19
|
+
* Options:
|
|
20
|
+
* - `label`, `fatal`: Text decoding behavior (passed to `TextDecoderStream`).
|
|
21
|
+
* - `splitter`, `chunkEndSplit`: Controls how lines are detected and split.
|
|
22
|
+
* - `parse`: Custom JSON parse function for each line (defaults to `JSON.parse`).
|
|
23
|
+
* - `errorFallback`: Optional handler for parse errors to gracefully handle invalid lines.
|
|
24
|
+
*
|
|
25
|
+
* Useful for:
|
|
26
|
+
* - Streaming NDJSON (newline-delimited JSON) over HTTP, WebSockets, or other binary transport.
|
|
27
|
+
* - Processing server-sent logs, analytics, or event streams in real-time.
|
|
28
|
+
*
|
|
29
|
+
* @example ```ts
|
|
30
|
+
* const stream = response.body.pipeThrough(new InputJsonLinesStream<MyType>());
|
|
31
|
+
* ```
|
|
32
|
+
*/
|
|
33
|
+
export declare class InputJsonLinesStream<T> implements ReadableWritablePair<T, BufferSource> {
|
|
34
|
+
#private;
|
|
35
|
+
get writable(): WritableStream<BufferSource>;
|
|
36
|
+
get readable(): ReadableStream<T>;
|
|
37
|
+
constructor(options?: InputJsonLinesStreamOptions<T>);
|
|
38
|
+
}
|
|
@@ -1,15 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
error: (reason: unknown) => void;
|
|
5
|
-
chunk: string;
|
|
6
|
-
enqueue: (arg: T) => void;
|
|
7
|
-
}) => (void | Promise<void>));
|
|
8
|
-
};
|
|
1
|
+
import { ParseStream } from "./ParseStream";
|
|
2
|
+
import type { ParseStreamOptions } from "./ParseStream";
|
|
3
|
+
export type InputJsonSequenceParseStreamOptions<T> = Partial<ParseStreamOptions<T>>;
|
|
9
4
|
/**
|
|
10
5
|
* string の sequence を 連続した `T` の sequence に変換する TransformStream
|
|
11
6
|
*/
|
|
12
|
-
export declare class InputJsonSequenceParseStream<T> extends
|
|
7
|
+
export declare class InputJsonSequenceParseStream<T> extends ParseStream<T> {
|
|
13
8
|
constructor(options?: InputJsonSequenceParseStreamOptions<T>);
|
|
14
9
|
}
|
|
15
10
|
/**
|
|
@@ -1,18 +1,40 @@
|
|
|
1
1
|
import type { InputJsonSequenceParseStreamOptions } from "./InputJsonSequenceParseStream";
|
|
2
|
-
import type {
|
|
2
|
+
import type { InputRecordSequenceStreamOptions } from "./InputRecordSequenceStream";
|
|
3
3
|
export type InputJsonSequenceStreamOptions<T> = {
|
|
4
4
|
label?: string;
|
|
5
|
-
} & TextDecoderOptions & InputJsonSequenceParseStreamOptions<T> &
|
|
5
|
+
} & TextDecoderOptions & InputJsonSequenceParseStreamOptions<T> & InputRecordSequenceStreamOptions;
|
|
6
6
|
/**
|
|
7
|
-
*
|
|
7
|
+
* A composite stream that parses a binary stream in `application/json-seq` format
|
|
8
|
+
* into a stream of typed JSON values (`T`).
|
|
9
|
+
*
|
|
10
|
+
* This stream is intended for use with binary sources such as `Response.body` from `fetch()`,
|
|
11
|
+
* where the content is encoded as a sequence of JSON objects, delimited by custom start/end markers.
|
|
12
|
+
*
|
|
13
|
+
* It internally combines the following components:
|
|
14
|
+
* - `TextDecoderStream`: Decodes binary data (e.g., UTF-8) into text.
|
|
15
|
+
* - `InputRecordSequenceStream`: Splits the text into logical record boundaries using `begin` and `end` delimiters.
|
|
16
|
+
* - `InputJsonSequenceParseStream`: Parses each record as a JSON object of type `T`.
|
|
17
|
+
*
|
|
18
|
+
* Options:
|
|
19
|
+
* - `label`, `fatal`: Passed to the `TextDecoderStream` for text decoding behavior.
|
|
20
|
+
* - `begin`, `end`: Record delimiters used by `InputRecordSequenceStream`.
|
|
21
|
+
* - `parse`: Custom JSON parse function (defaults to `JSON.parse`).
|
|
22
|
+
* - `errorFallback`: Error recovery handler if JSON parsing fails.
|
|
23
|
+
*
|
|
24
|
+
* Useful for:
|
|
25
|
+
* - Processing streaming NDJSON or JSON-seq (RFC 7464) over Fetch API or other binary streams.
|
|
26
|
+
* - Handling structured streaming protocols over HTTP or WebSocket.
|
|
27
|
+
*
|
|
28
|
+
* @example ```ts
|
|
29
|
+
* const stream = response.body.pipeThrough(new InputJsonSequenceStream<MyType>());
|
|
30
|
+
* ```
|
|
8
31
|
*/
|
|
9
|
-
declare class InputJsonSequenceStream<T>
|
|
32
|
+
export declare class InputJsonSequenceStream<T> implements ReadableWritablePair<T, BufferSource> {
|
|
10
33
|
#private;
|
|
11
34
|
get writable(): WritableStream<BufferSource>;
|
|
12
35
|
get readable(): ReadableStream<T>;
|
|
13
36
|
constructor(options?: InputJsonSequenceStreamOptions<T>);
|
|
14
37
|
}
|
|
15
|
-
export { InputJsonSequenceStream, InputJsonSequenceStream as default, };
|
|
16
38
|
/**
|
|
17
39
|
* @deprecated rename to InputJsonSequenceStream
|
|
18
40
|
* @see InputJsonSequenceStream
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { TextSplitStream, TextSplitStreamOptions } from "./TextSplitStream";
|
|
2
|
+
export type InputLineFeedSeparattedSequenceStreamOptions = Partial<TextSplitStreamOptions>;
|
|
3
|
+
/**
|
|
4
|
+
* Stream to convert to LF or CRLF delimited sequence
|
|
5
|
+
*/
|
|
6
|
+
export declare class InputLineFeedSeparattedSequenceStream extends TextSplitStream {
|
|
7
|
+
constructor(options?: InputLineFeedSeparattedSequenceStreamOptions);
|
|
8
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { RecordToSequenceStream } from "./RecordToSequenceStream";
|
|
2
|
+
import type { RecordToSequenceStreamOptions } from "./RecordToSequenceStream";
|
|
3
|
+
export type InputRecordSequenceStreamOptions = Partial<RecordToSequenceStreamOptions>;
|
|
4
|
+
/**
|
|
5
|
+
* RS で始まって LF で1行を表す 連続した文字列データを 行データとして整形して返す
|
|
6
|
+
*/
|
|
7
|
+
export declare class InputRecordSequenceStream extends RecordToSequenceStream {
|
|
8
|
+
constructor(options?: InputRecordSequenceStreamOptions);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* @deprecated rename to InputSequenceStreamOptions
|
|
12
|
+
* @see InputRecordSequenceStreamOptions
|
|
13
|
+
*/
|
|
14
|
+
export type SequenceStreamOptions = InputRecordSequenceStreamOptions;
|
|
15
|
+
/**
|
|
16
|
+
* @deprecated rename to InputSequenceStream
|
|
17
|
+
* @see InputRecordSequenceStream
|
|
18
|
+
*/
|
|
19
|
+
export declare const SequenceStream: typeof InputRecordSequenceStream;
|
|
20
|
+
/**
|
|
21
|
+
* @deprecated rename to InputRecordSequenceStreamOptions
|
|
22
|
+
* @see InputRecordSequenceStreamOptions
|
|
23
|
+
*/
|
|
24
|
+
export type InputSequenceStreamOptions = InputRecordSequenceStreamOptions;
|
|
25
|
+
/**
|
|
26
|
+
* @deprecated rename to InputRecordSequenceStream
|
|
27
|
+
* @see InputRecordSequenceStream
|
|
28
|
+
*/
|
|
29
|
+
export declare const InputSequenceStream: typeof InputRecordSequenceStream;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { OutputJsonSequenceStringifyStreamOptions } from "./OutputJsonSequenceStringifyStream";
|
|
2
|
+
import type { OutputTextJoinLineFeedSequenceStreamOptions } from "./OutputTextJoinLineFeedSequenceStream";
|
|
3
|
+
export type OutputJsonLinesStreamOptions<T> = OutputJsonSequenceStringifyStreamOptions<T> & OutputTextJoinLineFeedSequenceStreamOptions;
|
|
4
|
+
/**
|
|
5
|
+
* A composite stream that serializes typed values (`T`) into the NDJSON / JSON Lines format
|
|
6
|
+
* and outputs them as a binary stream (`BufferSource`).
|
|
7
|
+
*
|
|
8
|
+
* This stream is designed for producing newline-delimited JSON (`application/jsonl`) from
|
|
9
|
+
* typed application data, for use in streaming APIs, server-sent logs, or line-based protocols.
|
|
10
|
+
*
|
|
11
|
+
* Internally, it composes:
|
|
12
|
+
* - `OutputJsonSequenceStringifyStream`: Converts each `T` value to a JSON string.
|
|
13
|
+
* - `OutputTextJoinLineFeedSequenceStream`: Appends a line delimiter (default: `\n`) between records,
|
|
14
|
+
* with options to control start/end behavior and skip empty values.
|
|
15
|
+
* - `TextEncoderStream`: Encodes the final joined string into a binary stream.
|
|
16
|
+
*
|
|
17
|
+
* Options:
|
|
18
|
+
* - `stringify`: Function to convert a `T` value into a string (defaults to `JSON.stringify`).
|
|
19
|
+
* - `errorFallback`: Optional error handler for stringification failures.
|
|
20
|
+
* - `delimiter`: Character(s) used to separate lines (default is usually `\n`).
|
|
21
|
+
* - `start`, `end`: Whether to insert delimiters at the start or end of the stream.
|
|
22
|
+
* - `skip`: Whether to skip empty or undefined string outputs.
|
|
23
|
+
*
|
|
24
|
+
* Useful for:
|
|
25
|
+
* - Streaming NDJSON to HTTP request bodies or WebSockets.
|
|
26
|
+
* - Logging structured data line-by-line.
|
|
27
|
+
* - Interfacing with systems that consume line-delimited JSON.
|
|
28
|
+
*
|
|
29
|
+
* @example ```ts
|
|
30
|
+
* const stream = new OutputJsonLinesStream<MyType>({ delimiter: "\n" });
|
|
31
|
+
* stream.writable.getWriter().write({ id: 1 });
|
|
32
|
+
* ```
|
|
33
|
+
*/
|
|
34
|
+
export declare class OutputJsonLinesStream<T> implements ReadableWritablePair<BufferSource, T> {
|
|
35
|
+
#private;
|
|
36
|
+
get writable(): WritableStream<T>;
|
|
37
|
+
get readable(): ReadableStream<BufferSource>;
|
|
38
|
+
constructor(options?: OutputJsonLinesStreamOptions<T>);
|
|
39
|
+
}
|
|
@@ -1,10 +1,34 @@
|
|
|
1
|
-
import type { OutputJsonSequenceStringifyStreamOptions } from "./
|
|
2
|
-
import type {
|
|
3
|
-
export type OutputJsonSequenceStreamOptions<T> = OutputJsonSequenceStringifyStreamOptions<T> &
|
|
1
|
+
import type { OutputJsonSequenceStringifyStreamOptions } from "./OutputJsonSequenceStringifyStream";
|
|
2
|
+
import type { OutputRecordSequenceStreamOptions } from "./OutputRecordSequenceStream";
|
|
3
|
+
export type OutputJsonSequenceStreamOptions<T> = OutputJsonSequenceStringifyStreamOptions<T> & OutputRecordSequenceStreamOptions;
|
|
4
4
|
/**
|
|
5
|
-
* T
|
|
5
|
+
* A composite stream that serializes typed values (`T`) into the `application/json-seq` format
|
|
6
|
+
* and outputs them as a binary stream (`BufferSource`).
|
|
7
|
+
*
|
|
8
|
+
* This stream is intended for use in scenarios where JSON values must be streamed
|
|
9
|
+
* over binary-compatible channels (e.g., HTTP request bodies, WebSocket streams),
|
|
10
|
+
* using a record-based format like JSON Text Sequences (RFC 7464) or custom delimiters.
|
|
11
|
+
*
|
|
12
|
+
* Internally, this stream is composed of:
|
|
13
|
+
* - `OutputJsonSequenceStringifyStream`: Converts `T` values into JSON strings.
|
|
14
|
+
* - `OutputRecordSequenceStream`: Wraps each JSON string with custom `begin` and `end` delimiters.
|
|
15
|
+
* - `TextEncoderStream`: Encodes the final delimited text into a binary stream.
|
|
16
|
+
*
|
|
17
|
+
* Options:
|
|
18
|
+
* - `stringify`: Custom function to convert a `T` value into a JSON string (defaults to `JSON.stringify`).
|
|
19
|
+
* - `errorFallback`: Optional handler for recoverable stringify errors.
|
|
20
|
+
* - `begin`, `end`: Delimiters inserted before and after each JSON record.
|
|
21
|
+
*
|
|
22
|
+
* Useful for:
|
|
23
|
+
* - Writing streaming JSON data over `fetch()` or `WebSocket`.
|
|
24
|
+
* - Generating NDJSON or JSON Text Sequences (RFC 7464) from typed application data.
|
|
25
|
+
*
|
|
26
|
+
* @example ```ts
|
|
27
|
+
* const stream = new OutputJsonSequenceStream<MyType>();
|
|
28
|
+
* stream.writable.getWriter().write({ id: 1 });
|
|
29
|
+
* ```
|
|
6
30
|
*/
|
|
7
|
-
export declare class OutputJsonSequenceStream<T>
|
|
31
|
+
export declare class OutputJsonSequenceStream<T> implements ReadableWritablePair<BufferSource, T> {
|
|
8
32
|
#private;
|
|
9
33
|
get writable(): WritableStream<T>;
|
|
10
34
|
get readable(): ReadableStream<BufferSource>;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { StringifyStream } from "./StringifyStream";
|
|
2
|
+
import type { StringifyStreamOptions } from "./StringifyStream";
|
|
3
|
+
export type OutputJsonSequenceStringifyStreamOptions<T> = Partial<StringifyStreamOptions<T>>;
|
|
4
|
+
/**
|
|
5
|
+
* 連続した `T` の sequence を string の sequence に変換する TransformStream
|
|
6
|
+
*/
|
|
7
|
+
export declare class OutputJsonSequenceStringifyStream<T> extends StringifyStream<T> {
|
|
8
|
+
constructor(options?: OutputJsonSequenceStringifyStreamOptions<T>);
|
|
9
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { SequenceToRecordStream } from "./SequenceToRecordStream";
|
|
2
|
+
import type { SequenceToRecordStreamOptions } from "./SequenceToRecordStream";
|
|
3
|
+
export type OutputRecordSequenceStreamOptions = Partial<SequenceToRecordStreamOptions>;
|
|
4
|
+
/**
|
|
5
|
+
* Converts line data into continuous string data starting with RS and each LF represents one line.
|
|
6
|
+
*/
|
|
7
|
+
export declare class OutputRecordSequenceStream extends SequenceToRecordStream {
|
|
8
|
+
constructor(options?: OutputRecordSequenceStreamOptions);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* @deprecated rename to OutputRecordSequenceStreamOptions
|
|
12
|
+
* @see SequenceToRecordStreamOptions
|
|
13
|
+
*/
|
|
14
|
+
export type OutputSequenceStreamOptions = SequenceToRecordStreamOptions;
|
|
15
|
+
/**
|
|
16
|
+
* @deprecated rename to OutputRecordSequenceStream
|
|
17
|
+
* @see OutputRecordSequenceStream
|
|
18
|
+
*/
|
|
19
|
+
export declare const OutputSequenceStream: typeof OutputRecordSequenceStream;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { TextJoinStream } from "./TextJoinStream";
|
|
2
|
+
import type { TextJoinStreamOptions } from "./TextJoinStream";
|
|
3
|
+
export type OutputTextJoinLineFeedSequenceStreamOptions = Partial<TextJoinStreamOptions>;
|
|
4
|
+
/**
|
|
5
|
+
* Put LF between sequences Convert to a stream
|
|
6
|
+
*/
|
|
7
|
+
export declare class OutputTextJoinLineFeedSequenceStream extends TextJoinStream {
|
|
8
|
+
constructor(options?: OutputTextJoinLineFeedSequenceStreamOptions);
|
|
9
|
+
}
|
package/ParseStream.d.ts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
export type ParseStreamOptions<T> = {
|
|
2
|
+
/** A function that converts a string to type `T`. */
|
|
3
|
+
parse: (text: string) => (T | undefined);
|
|
4
|
+
/**
|
|
5
|
+
* A function that handles parse errors. It receives the error,
|
|
6
|
+
* the original chunk, and helper methods (`enqueue`, `error`) to control the stream behavior.
|
|
7
|
+
* If set to `false`, parse errors are silently ignored.
|
|
8
|
+
*/
|
|
9
|
+
errorFallback: false | ((error: unknown, args: {
|
|
10
|
+
error: (reason: unknown) => void;
|
|
11
|
+
chunk: string;
|
|
12
|
+
enqueue: (arg: T) => void;
|
|
13
|
+
}) => (void | Promise<void>));
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* A TransformStream that parses incoming string chunks into typed values of type `T`.
|
|
17
|
+
*
|
|
18
|
+
* This stream attempts to apply a user-provided `parse` function to each input chunk.
|
|
19
|
+
* If the parsing succeeds and returns a value (not `undefined`), it is enqueued.
|
|
20
|
+
* If parsing fails (throws an error), an optional `errorFallback` handler can be used
|
|
21
|
+
* to recover or handle the error.
|
|
22
|
+
*
|
|
23
|
+
* Options:
|
|
24
|
+
* - `parse`: A function that converts a string to type `T`.
|
|
25
|
+
* - `errorFallback`: A function that handles parse errors. It receives the error,
|
|
26
|
+
* the original chunk, and helper methods (`enqueue`, `error`) to control the stream behavior.
|
|
27
|
+
* If set to `false`, parse errors are silently ignored.
|
|
28
|
+
*
|
|
29
|
+
* Example use case:
|
|
30
|
+
* - Parsing newline-delimited JSON objects from a text stream.
|
|
31
|
+
* - Custom deserialization of structured text messages.
|
|
32
|
+
*
|
|
33
|
+
* Example:
|
|
34
|
+
* ```text
|
|
35
|
+
* Input chunks: ['{"id":1}', '{"id":2}', 'INVALID']
|
|
36
|
+
* parse: JSON.parse
|
|
37
|
+
* errorFallback: (err, { chunk }) => console.warn("Failed to parse:", chunk)
|
|
38
|
+
* Output: [{ id: 1 }, { id: 2 }]
|
|
39
|
+
* ```
|
|
40
|
+
*/
|
|
41
|
+
export declare class ParseStream<T> extends TransformStream<string, T> {
|
|
42
|
+
constructor(options: ParseStreamOptions<T>);
|
|
43
|
+
}
|
package/README.md
CHANGED
|
@@ -1,27 +1,78 @@
|
|
|
1
1
|
# json-seq-stream
|
|
2
2
|
|
|
3
|
-
application/json-seq support stream / async iterator library
|
|
3
|
+
application/json-seq / jsonl (NDJSON) support stream / async iterator library
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
This library provides composable `TransformStream` / `ReadableWritablePair` utilities to work with streaming JSON data formats.
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- [x] Supports [RFC7464](https://datatracker.ietf.org/doc/html/rfc7464) (application/json-seq)
|
|
10
|
+
- [x] Supports JSON Lines / NDJSON (application/jsonl, application/x-ndjson)
|
|
11
|
+
- [x] Fully stream-compatible (Fetch API, File API, Bun/Deno, etc.)
|
|
12
|
+
- [x] Composable with standard Web Streams (`pipeThrough()`)
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
## Usage Examples
|
|
17
|
+
|
|
18
|
+
### JSON Sequence (RFC 7464)
|
|
6
19
|
|
|
7
20
|
```ts
|
|
8
|
-
import { InputJsonSequenceStream
|
|
21
|
+
import { InputJsonSequenceStream } from "json-seq-stream";
|
|
9
22
|
|
|
10
|
-
type Value = {
|
|
11
|
-
value: number;
|
|
12
|
-
};
|
|
23
|
+
type Value = { value: number };
|
|
13
24
|
|
|
14
|
-
const sequenceStream = new
|
|
25
|
+
const sequenceStream = new InputJsonSequenceStream<Value>();
|
|
15
26
|
|
|
16
27
|
const response = await fetch(url);
|
|
17
|
-
if (!response.ok) throw new Error();
|
|
18
28
|
const readable = response.body!.pipeThrough(sequenceStream);
|
|
29
|
+
|
|
19
30
|
for await (const value of readable) {
|
|
20
|
-
|
|
31
|
+
console.log(value);
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## JSON Lines / NDJSON
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
import { InputJsonLinesStream } from "json-seq-stream";
|
|
39
|
+
|
|
40
|
+
type Value = { value: number };
|
|
41
|
+
|
|
42
|
+
const linesStream = new InputJsonLinesStream<Value>();
|
|
43
|
+
|
|
44
|
+
const response = await fetch("https://example.com/data.jsonl");
|
|
45
|
+
const readable = response.body!.pipeThrough(linesStream);
|
|
46
|
+
|
|
47
|
+
for await (const value of readable) {
|
|
48
|
+
console.log(value);
|
|
49
|
+
}
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Output Example
|
|
53
|
+
|
|
54
|
+
```ts
|
|
55
|
+
import { OutputJsonLinesStream } from "json-seq-stream";
|
|
56
|
+
|
|
57
|
+
type Value = { value: number };
|
|
58
|
+
|
|
59
|
+
const { readable, writable } = new OutputJsonLinesStream<Value>();
|
|
60
|
+
|
|
61
|
+
const writer = writable.getWriter();
|
|
62
|
+
writer.write({ value: 1 });
|
|
63
|
+
writer.write({ value: 2 });
|
|
64
|
+
writer.close();
|
|
65
|
+
|
|
66
|
+
for await (const chunk of readable) {
|
|
67
|
+
console.log(new TextDecoder().decode(chunk)); // NDJSON text lines
|
|
21
68
|
}
|
|
22
69
|
```
|
|
23
70
|
|
|
24
|
-
##
|
|
71
|
+
## See also
|
|
25
72
|
|
|
26
|
-
-
|
|
73
|
+
- RFC 7464 - JavaScript Object Notation (JSON) Text Sequences \
|
|
27
74
|
[https://datatracker.ietf.org/doc/html/rfc7464](https://datatracker.ietf.org/doc/html/rfc7464)
|
|
75
|
+
- JSON Lines \
|
|
76
|
+
[https://jsonlines.org](https://jsonlines.org)
|
|
77
|
+
- JSON streaming - Wikipedia (en) \
|
|
78
|
+
[https://en.wikipedia.org/wiki/JSON_streaming](https://en.wikipedia.org/wiki/JSON_streaming)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
declare const modes: {
|
|
2
|
+
readonly begin: "begin";
|
|
3
|
+
readonly end: "end";
|
|
4
|
+
readonly record: "record";
|
|
5
|
+
readonly init: "init";
|
|
6
|
+
readonly flush: "flush";
|
|
7
|
+
};
|
|
8
|
+
type Mode = typeof modes[keyof typeof modes];
|
|
9
|
+
export type RecordToSequenceStreamOptions = {
|
|
10
|
+
begin: string;
|
|
11
|
+
end: string;
|
|
12
|
+
chunkEndSplit: boolean;
|
|
13
|
+
fallbackSkip: true | false | ((minChunk: string, mode: Mode, next: Mode) => boolean);
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* RS で始まって LF で1行を表す 連続した文字列データを 行データとして整形して返す
|
|
17
|
+
*/
|
|
18
|
+
export declare class RecordToSequenceStream extends TransformStream<string, string> {
|
|
19
|
+
constructor(options: RecordToSequenceStreamOptions);
|
|
20
|
+
}
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export type SequenceToRecordStreamOptions = {
|
|
2
|
+
/** record begin character */
|
|
3
|
+
begin: string;
|
|
4
|
+
/** record end character */
|
|
5
|
+
end: string;
|
|
6
|
+
/** empty chunk skip enable */
|
|
7
|
+
skip: boolean;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* A TransformStream that wraps each input string chunk with a specified begin and end delimiter,
|
|
11
|
+
* optionally skipping empty chunks.
|
|
12
|
+
*
|
|
13
|
+
* This stream is useful for converting plain string sequences into delimited records, such as
|
|
14
|
+
* preparing data for structured streaming protocols or framing messages.
|
|
15
|
+
*
|
|
16
|
+
* Example:
|
|
17
|
+
* ```example:text
|
|
18
|
+
* Input chunks: ["foo", "", "bar"]
|
|
19
|
+
* With begin = "<", end = ">", and skip = true
|
|
20
|
+
* Output chunks: ["<", "foo", ">", "<", "bar", ">"]
|
|
21
|
+
* ```
|
|
22
|
+
*/
|
|
23
|
+
export declare class SequenceToRecordStream extends TransformStream<string, string> {
|
|
24
|
+
constructor(options: SequenceToRecordStreamOptions);
|
|
25
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export type StringifyStreamOptions<T> = {
|
|
2
|
+
/** A function that converts a `T` value to a string. */
|
|
3
|
+
stringify: (value: T) => (string | undefined);
|
|
4
|
+
/**
|
|
5
|
+
* A function that handles errors during stringification. It receives the error,
|
|
6
|
+
* the original value, and helper methods (`enqueue`, `error`) to control the stream behavior.
|
|
7
|
+
* If set to `false`, errors are silently ignored.
|
|
8
|
+
*/
|
|
9
|
+
errorFallback: false | ((error: unknown, args: {
|
|
10
|
+
error: (reason: unknown) => void;
|
|
11
|
+
chunk: T;
|
|
12
|
+
enqueue: (arg: string) => void;
|
|
13
|
+
}) => (void | Promise<void>));
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* A TransformStream that converts typed values of type `T` into string chunks using a
|
|
17
|
+
* customizable stringifier function.
|
|
18
|
+
*
|
|
19
|
+
* This stream applies a user-provided `stringify` function to each incoming `T` value.
|
|
20
|
+
* If the function returns a non-empty string, it is emitted as output.
|
|
21
|
+
* If the stringification process throws an error, an optional `errorFallback` handler
|
|
22
|
+
* can be used to recover or handle the error.
|
|
23
|
+
*
|
|
24
|
+
* Options:
|
|
25
|
+
* - `stringify`: A function that converts a `T` value to a string.
|
|
26
|
+
* - `errorFallback`: A function that handles errors during stringification. It receives the error,
|
|
27
|
+
* the original value, and helper methods (`enqueue`, `error`) to control the stream behavior.
|
|
28
|
+
* If set to `false`, errors are silently ignored.
|
|
29
|
+
*
|
|
30
|
+
* Example use case:
|
|
31
|
+
* - Serializing structured data into newline-delimited JSON (NDJSON)
|
|
32
|
+
* - Converting typed messages into protocol-specific text formats
|
|
33
|
+
*
|
|
34
|
+
* Example:
|
|
35
|
+
* ```text
|
|
36
|
+
* Input chunks: [{ id: 1 }, { id: 2 }, CircularRefObject]
|
|
37
|
+
* stringify: JSON.stringify
|
|
38
|
+
* errorFallback: (err, { chunk }) => console.warn("Failed to stringify:", chunk)
|
|
39
|
+
* Output: ['{"id":1}', '{"id":2}']
|
|
40
|
+
* ```
|
|
41
|
+
*/
|
|
42
|
+
export declare class StringifyStream<T> extends TransformStream<T, string> {
|
|
43
|
+
constructor(options: StringifyStreamOptions<T>);
|
|
44
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export type TextJoinStreamOptions = {
|
|
2
|
+
/** delimiter character */
|
|
3
|
+
delimiter: string;
|
|
4
|
+
/** insert a delimiter at the start */
|
|
5
|
+
start: boolean;
|
|
6
|
+
/** insert a delimiter at the end */
|
|
7
|
+
end: boolean;
|
|
8
|
+
/** skip empty string chunks */
|
|
9
|
+
skip: boolean;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* A TransformStream that joins incoming string chunks using a specified delimiter.
|
|
13
|
+
*
|
|
14
|
+
* This stream inserts the delimiter between chunks, and can optionally:
|
|
15
|
+
* - insert a delimiter at the start (`start: true`)
|
|
16
|
+
* - insert a delimiter at the end (`end: true`)
|
|
17
|
+
* - skip empty string chunks (`skip: true`)
|
|
18
|
+
*
|
|
19
|
+
* This is useful for constructing joined strings from a stream of segments,
|
|
20
|
+
* such as building CSV lines, JSON arrays, or custom formatted output.
|
|
21
|
+
*
|
|
22
|
+
* Example:
|
|
23
|
+
* ```text
|
|
24
|
+
* Input chunks: ["foo", "bar", "baz"]
|
|
25
|
+
* Options: { delimiter: ",", start: false, end: true, skip: false }
|
|
26
|
+
* Output chunks: ["foo", ",", "bar", ",", "baz", ","]
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
29
|
+
export declare class TextJoinStream extends TransformStream<string, string> {
|
|
30
|
+
constructor(options: TextJoinStreamOptions);
|
|
31
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export type TextSplitStreamOptions = {
|
|
2
|
+
/** separator character */
|
|
3
|
+
splitter: string | string[];
|
|
4
|
+
/** chunk end split force */
|
|
5
|
+
chunkEndSplit: boolean;
|
|
6
|
+
};
|
|
7
|
+
/**
|
|
8
|
+
* A TransformStream that splits incoming string chunks by specified delimiters
|
|
9
|
+
* and emits each segment as a separate chunk.
|
|
10
|
+
*
|
|
11
|
+
* This stream allows splitting based on one or more delimiters (characters or strings),
|
|
12
|
+
* and supports optional flushing at the end of each input chunk.
|
|
13
|
+
*
|
|
14
|
+
* Options:
|
|
15
|
+
* - `splitter`: A string or array of strings used as delimiters.
|
|
16
|
+
* - `chunkEndSplit`: If `true`, forces split and flush at the end of each input chunk.
|
|
17
|
+
*
|
|
18
|
+
* Empty segments resulting from splitting are automatically skipped.
|
|
19
|
+
*
|
|
20
|
+
* Example:
|
|
21
|
+
* ```text
|
|
22
|
+
* Input chunk: "foo,bar|baz"
|
|
23
|
+
* Options: { splitter: [",", "|"], chunkEndSplit: true }
|
|
24
|
+
* Output chunks: ["foo", "bar", "baz"]
|
|
25
|
+
* ```
|
|
26
|
+
*
|
|
27
|
+
* Useful for processing delimited text data such as CSV rows, log lines,
|
|
28
|
+
* or command sequences.
|
|
29
|
+
*/
|
|
30
|
+
export declare class TextSplitStream extends TransformStream<string, string> {
|
|
31
|
+
constructor(options: TextSplitStreamOptions);
|
|
32
|
+
}
|
package/index.d.ts
CHANGED
|
@@ -1,6 +1,16 @@
|
|
|
1
|
-
export * from "./
|
|
2
|
-
export * from "./
|
|
1
|
+
export * from "./TextSplitStream";
|
|
2
|
+
export * from "./TextJoinStream";
|
|
3
3
|
export * from "./InputJsonSequenceStream";
|
|
4
|
-
export * from "./OutputSequenceStream";
|
|
5
|
-
export * from "./OutputJsonSequenceStfingifyStream";
|
|
6
4
|
export * from "./OutputJsonSequenceStream";
|
|
5
|
+
export * from "./InputJsonSequenceParseStream";
|
|
6
|
+
export * from "./OutputJsonSequenceStringifyStream";
|
|
7
|
+
export * from "./InputRecordSequenceStream";
|
|
8
|
+
export * from "./OutputRecordSequenceStream";
|
|
9
|
+
export * from "./InputJsonLinesStream";
|
|
10
|
+
export * from "./OutputJsonLinesStream";
|
|
11
|
+
export * from "./InputLineFeedSeparattedSequenceStream";
|
|
12
|
+
export * from "./OutputTextJoinLineFeedSequenceStream";
|
|
13
|
+
export * from "./SequenceToRecordStream";
|
|
14
|
+
export * from "./RecordToSequenceStream";
|
|
15
|
+
export * from "./StringifyStream";
|
|
16
|
+
export * from "./ParseStream";
|
package/index.js
CHANGED
|
@@ -1,50 +1,77 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
function l({ lineBegin: t, lineEnd: e } = {}) {
|
|
4
|
-
t ??= d, e ??= h;
|
|
5
|
-
const n = [], c = new RegExp(`(${RegExp.escape(t)}|${RegExp.escape(e)})`, "u");
|
|
1
|
+
function E({ splitter: r, chunkEndSplit: e }) {
|
|
2
|
+
const { separator: t, checker: s } = R(r), n = [];
|
|
6
3
|
return {
|
|
7
4
|
args: [
|
|
8
5
|
{
|
|
9
6
|
transform(u, a) {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
if (
|
|
14
|
-
|
|
15
|
-
|
|
7
|
+
e || (u = [...n.splice(0, n.length), u].join(""));
|
|
8
|
+
for (const o of u.split(t))
|
|
9
|
+
if (!(o.length <= 0)) {
|
|
10
|
+
if (s.test(o)) {
|
|
11
|
+
g(n, a);
|
|
12
|
+
continue;
|
|
13
|
+
}
|
|
14
|
+
n.push(o);
|
|
16
15
|
}
|
|
17
|
-
|
|
18
|
-
}
|
|
16
|
+
e && g(n, a);
|
|
19
17
|
},
|
|
20
18
|
flush(u) {
|
|
21
|
-
|
|
22
|
-
a.length <= 0 || u.enqueue(a);
|
|
19
|
+
g(n, u);
|
|
23
20
|
}
|
|
24
21
|
}
|
|
25
22
|
]
|
|
26
23
|
};
|
|
27
|
-
|
|
28
|
-
|
|
24
|
+
}
|
|
25
|
+
function R(r) {
|
|
26
|
+
typeof r == "string" && (r = [r]);
|
|
27
|
+
const e = r.map((t) => RegExp.escape(t)).join("|");
|
|
28
|
+
return {
|
|
29
|
+
separator: new RegExp(`(${e})`, "ug"),
|
|
30
|
+
checker: new RegExp(`^(${e})$`)
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
function g(r, e) {
|
|
34
|
+
if (r.length <= 0) return;
|
|
35
|
+
const t = r.splice(0, r.length).join("");
|
|
36
|
+
e.enqueue(t);
|
|
37
|
+
}
|
|
38
|
+
class J extends TransformStream {
|
|
39
|
+
constructor(e) {
|
|
40
|
+
const { args: t } = E(e);
|
|
41
|
+
super(...t);
|
|
29
42
|
}
|
|
30
43
|
}
|
|
31
|
-
|
|
44
|
+
function k({ delimiter: r, start: e, end: t, skip: s }) {
|
|
45
|
+
let n = !0;
|
|
46
|
+
return {
|
|
47
|
+
args: [
|
|
48
|
+
{
|
|
49
|
+
transform(u, a) {
|
|
50
|
+
s && u.length <= 0 || (n ? (e && a.enqueue(r), n = !1) : a.enqueue(r), a.enqueue(u));
|
|
51
|
+
},
|
|
52
|
+
flush(u) {
|
|
53
|
+
n || t && u.enqueue(r);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
]
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
class O extends TransformStream {
|
|
32
60
|
constructor(e) {
|
|
33
|
-
const { args:
|
|
34
|
-
super(...
|
|
61
|
+
const { args: t } = k(e);
|
|
62
|
+
super(...t);
|
|
35
63
|
}
|
|
36
64
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
return t ??= JSON.parse, e ??= !1, { args: [
|
|
65
|
+
function L({ parse: r, errorFallback: e }) {
|
|
66
|
+
return { args: [
|
|
40
67
|
{
|
|
41
|
-
transform(
|
|
68
|
+
transform(s, n) {
|
|
42
69
|
try {
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
} catch (
|
|
70
|
+
const c = r(s);
|
|
71
|
+
c && n.enqueue(c);
|
|
72
|
+
} catch (c) {
|
|
46
73
|
if (!e) return;
|
|
47
|
-
const u =
|
|
74
|
+
const u = n.error.bind(n), a = n.enqueue.bind(n), o = e(c, { chunk: s, error: u, enqueue: a });
|
|
48
75
|
if (o?.then)
|
|
49
76
|
return o.catch(() => {
|
|
50
77
|
});
|
|
@@ -53,18 +80,81 @@ function b({ parse: t, errorFallback: e } = {}) {
|
|
|
53
80
|
}
|
|
54
81
|
] };
|
|
55
82
|
}
|
|
56
|
-
class
|
|
57
|
-
constructor(e
|
|
58
|
-
const { args:
|
|
59
|
-
super(...
|
|
83
|
+
class F extends TransformStream {
|
|
84
|
+
constructor(e) {
|
|
85
|
+
const { args: t } = L(e);
|
|
86
|
+
super(...t);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
class m extends F {
|
|
90
|
+
constructor(e) {
|
|
91
|
+
let { errorFallback: t, parse: s } = e ?? {};
|
|
92
|
+
s ??= JSON.parse, t ??= !1, super({ parse: s, errorFallback: t });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const X = m, f = {
|
|
96
|
+
begin: "begin",
|
|
97
|
+
end: "end",
|
|
98
|
+
record: "record",
|
|
99
|
+
init: "init",
|
|
100
|
+
flush: "flush"
|
|
101
|
+
};
|
|
102
|
+
function y({ begin: r, end: e, chunkEndSplit: t, fallbackSkip: s }) {
|
|
103
|
+
const n = [], c = new RegExp(`(${RegExp.escape(r)}|${RegExp.escape(e)})`, "u"), u = s === !0 ? () => !0 : s === !1 ? () => !1 : s;
|
|
104
|
+
let a = f.init;
|
|
105
|
+
return {
|
|
106
|
+
args: [
|
|
107
|
+
{
|
|
108
|
+
transform(i, d) {
|
|
109
|
+
t || (i = [...n.splice(0, n.length), i].join(""));
|
|
110
|
+
for (const l of i.split(c)) {
|
|
111
|
+
if (l === r) {
|
|
112
|
+
const p = f.begin;
|
|
113
|
+
a === f.end || a === void 0 ? (S(n, d), a = p) : u(l, a, p) || (S(n, d), a = p);
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
if (l === e) {
|
|
117
|
+
const p = f.end;
|
|
118
|
+
a === f.record || a === f.begin ? (S(n, d), a = p) : u(l, a, p) || (S(n, d), a = p);
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const h = f.record;
|
|
122
|
+
a === f.begin || a === f.record ? (n.push(l), a = h) : u(l, a, f.record) || (n.push(l), a = h);
|
|
123
|
+
}
|
|
124
|
+
t && (S(n, d), a = f.end);
|
|
125
|
+
},
|
|
126
|
+
flush(i) {
|
|
127
|
+
u("", a, f.flush) || S(n, i);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
]
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
function S(r, e) {
|
|
134
|
+
if (r.length <= 0) return;
|
|
135
|
+
const t = r.splice(0, r.length).join("");
|
|
136
|
+
t.length <= 0 || e.enqueue(t);
|
|
137
|
+
}
|
|
138
|
+
class j extends TransformStream {
|
|
139
|
+
constructor(e) {
|
|
140
|
+
const { args: t } = y(e);
|
|
141
|
+
super(...t);
|
|
60
142
|
}
|
|
61
143
|
}
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
144
|
+
const q = "", w = `
|
|
145
|
+
`, N = q, D = w;
|
|
146
|
+
class b extends j {
|
|
147
|
+
constructor(e) {
|
|
148
|
+
let { begin: t, end: s, chunkEndSplit: n, fallbackSkip: c } = e ?? {};
|
|
149
|
+
t ??= N, s ??= D, n ??= !1, c ??= !1, super({ begin: t, end: s, chunkEndSplit: n, fallbackSkip: c });
|
|
150
|
+
}
|
|
66
151
|
}
|
|
67
|
-
|
|
152
|
+
const Y = b, Z = b;
|
|
153
|
+
function $(r) {
|
|
154
|
+
const { label: e, fatal: t, begin: s, end: n, parse: c, errorFallback: u } = r, a = new TextDecoderStream(e, { fatal: t }), o = new b({ begin: s, end: n }), i = new m({ parse: c, errorFallback: u }), { writable: d } = a, l = a.readable.pipeThrough(o).pipeThrough(i);
|
|
155
|
+
return { writable: d, readable: l };
|
|
156
|
+
}
|
|
157
|
+
class P {
|
|
68
158
|
#e;
|
|
69
159
|
#t;
|
|
70
160
|
get writable() {
|
|
@@ -74,40 +164,23 @@ class w extends TransformStream {
|
|
|
74
164
|
return this.#t;
|
|
75
165
|
}
|
|
76
166
|
constructor(e = {}) {
|
|
77
|
-
|
|
167
|
+
({
|
|
78
168
|
readable: this.#t,
|
|
79
169
|
writable: this.#e
|
|
80
|
-
} =
|
|
170
|
+
} = $(e));
|
|
81
171
|
}
|
|
82
172
|
}
|
|
83
|
-
const
|
|
84
|
-
function
|
|
85
|
-
return
|
|
86
|
-
args: [
|
|
87
|
-
{
|
|
88
|
-
transform(c, r) {
|
|
89
|
-
r.enqueue(`${t}${c}${e}`);
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
]
|
|
93
|
-
};
|
|
94
|
-
}
|
|
95
|
-
class E extends TransformStream {
|
|
96
|
-
constructor(e) {
|
|
97
|
-
const { args: n } = T(e);
|
|
98
|
-
super(...n);
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
function N({ stringify: t, errorFallback: e } = {}) {
|
|
102
|
-
return t ??= JSON.stringify, e ??= !1, { args: [
|
|
173
|
+
const ee = P;
|
|
174
|
+
function _({ stringify: r, errorFallback: e }) {
|
|
175
|
+
return { args: [
|
|
103
176
|
{
|
|
104
|
-
transform(
|
|
177
|
+
transform(s, n) {
|
|
105
178
|
try {
|
|
106
|
-
const
|
|
107
|
-
(
|
|
108
|
-
} catch (
|
|
179
|
+
const c = r(s);
|
|
180
|
+
(c?.length ?? 0) > 0 && n.enqueue(c);
|
|
181
|
+
} catch (c) {
|
|
109
182
|
if (!e) return;
|
|
110
|
-
const u =
|
|
183
|
+
const u = n.error.bind(n), a = n.enqueue.bind(n), o = e(c, { chunk: s, error: u, enqueue: a });
|
|
111
184
|
if (o?.then)
|
|
112
185
|
return o.catch(() => {
|
|
113
186
|
});
|
|
@@ -116,17 +189,103 @@ function N({ stringify: t, errorFallback: e } = {}) {
|
|
|
116
189
|
}
|
|
117
190
|
] };
|
|
118
191
|
}
|
|
119
|
-
class
|
|
120
|
-
constructor(e
|
|
121
|
-
const { args:
|
|
122
|
-
super(...
|
|
192
|
+
class C extends TransformStream {
|
|
193
|
+
constructor(e) {
|
|
194
|
+
const { args: t } = _(e);
|
|
195
|
+
super(...t);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
class x extends C {
|
|
199
|
+
constructor(e) {
|
|
200
|
+
let { errorFallback: t, stringify: s } = e ?? {};
|
|
201
|
+
t ??= !1, s ??= JSON.stringify, super({ stringify: s, errorFallback: t });
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
function v({ begin: r, end: e, skip: t }) {
|
|
205
|
+
return {
|
|
206
|
+
args: [
|
|
207
|
+
{
|
|
208
|
+
transform(n, c) {
|
|
209
|
+
t && n.length <= 0 || (c.enqueue(r), c.enqueue(n), c.enqueue(e));
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
]
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
class A extends TransformStream {
|
|
216
|
+
constructor(e) {
|
|
217
|
+
const { args: t } = v(e);
|
|
218
|
+
super(...t);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
const B = q, G = w, K = !1;
|
|
222
|
+
class T extends A {
|
|
223
|
+
constructor(e) {
|
|
224
|
+
let { begin: t, end: s, skip: n, ...c } = e ?? {};
|
|
225
|
+
t ??= B, s ??= G, n ??= K, super({ begin: t, end: s, skip: n, ...c });
|
|
123
226
|
}
|
|
124
227
|
}
|
|
125
|
-
|
|
126
|
-
|
|
228
|
+
const te = T;
|
|
229
|
+
function z(r) {
|
|
230
|
+
const { begin: e, end: t, stringify: s, errorFallback: n } = r, c = new TextEncoderStream(), u = new T({ begin: e, end: t }), a = new x({ stringify: s, errorFallback: n }), { writable: o } = a, i = a.readable.pipeThrough(u).pipeThrough(c);
|
|
127
231
|
return { writable: o, readable: i };
|
|
128
232
|
}
|
|
129
|
-
class
|
|
233
|
+
class ne {
|
|
234
|
+
#e;
|
|
235
|
+
#t;
|
|
236
|
+
get writable() {
|
|
237
|
+
return this.#e;
|
|
238
|
+
}
|
|
239
|
+
get readable() {
|
|
240
|
+
return this.#t;
|
|
241
|
+
}
|
|
242
|
+
constructor(e = {}) {
|
|
243
|
+
({
|
|
244
|
+
readable: this.#t,
|
|
245
|
+
writable: this.#e
|
|
246
|
+
} = z(e));
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
const I = `
|
|
250
|
+
`, H = `\r
|
|
251
|
+
`, M = [I, H];
|
|
252
|
+
class Q extends J {
|
|
253
|
+
constructor(e) {
|
|
254
|
+
let { splitter: t, chunkEndSplit: s } = e ?? {};
|
|
255
|
+
t ??= M, s ??= !1, super({ splitter: t, chunkEndSplit: s });
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
function U(r) {
|
|
259
|
+
const { label: e, fatal: t, splitter: s, chunkEndSplit: n, parse: c, errorFallback: u } = r ?? {}, a = new TextDecoderStream(e, { fatal: t }), o = new Q({ splitter: s, chunkEndSplit: n }), i = new m({ parse: c, errorFallback: u }), { writable: d } = a, l = a.readable.pipeThrough(o).pipeThrough(i);
|
|
260
|
+
return { writable: d, readable: l };
|
|
261
|
+
}
|
|
262
|
+
class re {
|
|
263
|
+
#e;
|
|
264
|
+
#t;
|
|
265
|
+
get writable() {
|
|
266
|
+
return this.#e;
|
|
267
|
+
}
|
|
268
|
+
get readable() {
|
|
269
|
+
return this.#t;
|
|
270
|
+
}
|
|
271
|
+
constructor(e) {
|
|
272
|
+
({
|
|
273
|
+
readable: this.#t,
|
|
274
|
+
writable: this.#e
|
|
275
|
+
} = U(e));
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
class V extends O {
|
|
279
|
+
constructor(e) {
|
|
280
|
+
let { delimiter: t, start: s, end: n, skip: c } = e ?? {};
|
|
281
|
+
t ??= I, s ??= !1, n ??= !1, c ??= !0, super({ delimiter: t, start: s, end: n, skip: c });
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
function W(r) {
|
|
285
|
+
const { delimiter: e, skip: t, start: s, end: n, stringify: c, errorFallback: u } = r, a = new TextEncoderStream(), o = new V({ delimiter: e, end: n, skip: t, start: s }), i = new x({ stringify: c, errorFallback: u }), { writable: d } = i, l = i.readable.pipeThrough(o).pipeThrough(a);
|
|
286
|
+
return { writable: d, readable: l };
|
|
287
|
+
}
|
|
288
|
+
class se {
|
|
130
289
|
#e;
|
|
131
290
|
#t;
|
|
132
291
|
get writable() {
|
|
@@ -136,20 +295,32 @@ class _ extends TransformStream {
|
|
|
136
295
|
return this.#t;
|
|
137
296
|
}
|
|
138
297
|
constructor(e = {}) {
|
|
139
|
-
|
|
298
|
+
({
|
|
140
299
|
readable: this.#t,
|
|
141
300
|
writable: this.#e
|
|
142
|
-
} =
|
|
301
|
+
} = W(e));
|
|
143
302
|
}
|
|
144
303
|
}
|
|
145
304
|
export {
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
305
|
+
re as InputJsonLinesStream,
|
|
306
|
+
m as InputJsonSequenceParseStream,
|
|
307
|
+
P as InputJsonSequenceStream,
|
|
308
|
+
Q as InputLineFeedSeparattedSequenceStream,
|
|
309
|
+
b as InputRecordSequenceStream,
|
|
310
|
+
Z as InputSequenceStream,
|
|
311
|
+
X as JsonSequenceFormatStream,
|
|
312
|
+
ee as JsonSequenceStream,
|
|
313
|
+
se as OutputJsonLinesStream,
|
|
314
|
+
ne as OutputJsonSequenceStream,
|
|
315
|
+
x as OutputJsonSequenceStringifyStream,
|
|
316
|
+
T as OutputRecordSequenceStream,
|
|
317
|
+
te as OutputSequenceStream,
|
|
318
|
+
V as OutputTextJoinLineFeedSequenceStream,
|
|
319
|
+
F as ParseStream,
|
|
320
|
+
j as RecordToSequenceStream,
|
|
321
|
+
Y as SequenceStream,
|
|
322
|
+
A as SequenceToRecordStream,
|
|
323
|
+
C as StringifyStream,
|
|
324
|
+
O as TextJoinStream,
|
|
325
|
+
J as TextSplitStream
|
|
155
326
|
};
|
package/index.umd.cjs
CHANGED
|
@@ -1,2 +1,4 @@
|
|
|
1
|
-
(function(
|
|
2
|
-
`,
|
|
1
|
+
(function(o,g){typeof exports=="object"&&typeof module<"u"?g(exports):typeof define=="function"&&define.amd?define(["exports"],g):(o=typeof globalThis<"u"?globalThis:o||self,g(o.index={}))})(this,function(o){"use strict";function g({splitter:r,chunkEndSplit:e}){const{separator:t,checker:s}=D(r),n=[];return{args:[{transform(c,a){e||(c=[...n.splice(0,n.length),c].join(""));for(const i of c.split(t))if(!(i.length<=0)){if(s.test(i)){h(n,a);continue}n.push(i)}e&&h(n,a)},flush(c){h(n,c)}}]}}function D(r){typeof r=="string"&&(r=[r]);const e=r.map(t=>RegExp.escape(t)).join("|");return{separator:new RegExp(`(${e})`,"ug"),checker:new RegExp(`^(${e})$`)}}function h(r,e){if(r.length<=0)return;const t=r.splice(0,r.length).join("");e.enqueue(t)}class I extends TransformStream{constructor(e){const{args:t}=g(e);super(...t)}}function $({delimiter:r,start:e,end:t,skip:s}){let n=!0;return{args:[{transform(c,a){s&&c.length<=0||(n?(e&&a.enqueue(r),n=!1):a.enqueue(r),a.enqueue(c))},flush(c){n||t&&c.enqueue(r)}}]}}class J extends TransformStream{constructor(e){const{args:t}=$(e);super(...t)}}function C({parse:r,errorFallback:e}){return{args:[{transform(s,n){try{const u=r(s);u&&n.enqueue(u)}catch(u){if(!e)return;const c=n.error.bind(n),a=n.enqueue.bind(n),i=e(u,{chunk:s,error:c,enqueue:a});if(i?.then)return i.catch(()=>{})}}}]}}class R extends TransformStream{constructor(e){const{args:t}=C(e);super(...t)}}class b extends R{constructor(e){let{errorFallback:t,parse:s}=e??{};s??=JSON.parse,t??=!1,super({parse:s,errorFallback:t})}}const _=b,d={begin:"begin",end:"end",record:"record",init:"init",flush:"flush"};function v({begin:r,end:e,chunkEndSplit:t,fallbackSkip:s}){const n=[],u=new RegExp(`(${RegExp.escape(r)}|${RegExp.escape(e)})`,"u"),c=s===!0?()=>!0:s===!1?()=>!1:s;let a=d.init;return{args:[{transform(S,f){t||(S=[...n.splice(0,n.length),S].join(""));for(const l of S.split(u)){if(l===r){const m=d.begin;a===d.end||a===void 0?(p(n,f),a=m):c(l,a,m)||(p(n,f),a=m);continue}if(l===e){const m=d.end;a===d.record||a===d.begin?(p(n,f),a=m):c(l,a,m)||(p(n,f),a=m);continue}const P=d.record;a===d.begin||a===d.record?(n.push(l),a=P):c(l,a,d.record)||(n.push(l),a=P)}t&&(p(n,f),a=d.end)},flush(S){c("",a,d.flush)||p(n,S)}}]}}function p(r,e){if(r.length<=0)return;const t=r.splice(0,r.length).join("");t.length<=0||e.enqueue(t)}class E extends TransformStream{constructor(e){const{args:t}=v(e);super(...t)}}const O="",L=`
|
|
2
|
+
`,A=O,B=L;class q extends E{constructor(e){let{begin:t,end:s,chunkEndSplit:n,fallbackSkip:u}=e??{};t??=A,s??=B,n??=!1,u??=!1,super({begin:t,end:s,chunkEndSplit:n,fallbackSkip:u})}}const G=q,K=q;function M(r){const{label:e,fatal:t,begin:s,end:n,parse:u,errorFallback:c}=r,a=new TextDecoderStream(e,{fatal:t}),i=new q({begin:s,end:n}),S=new b({parse:u,errorFallback:c}),{writable:f}=a,l=a.readable.pipeThrough(i).pipeThrough(S);return{writable:f,readable:l}}class x{#e;#t;get writable(){return this.#e}get readable(){return this.#t}constructor(e={}){({readable:this.#t,writable:this.#e}=M(e))}}const z=x;function H({stringify:r,errorFallback:e}){return{args:[{transform(s,n){try{const u=r(s);(u?.length??0)>0&&n.enqueue(u)}catch(u){if(!e)return;const c=n.error.bind(n),a=n.enqueue.bind(n),i=e(u,{chunk:s,error:c,enqueue:a});if(i?.then)return i.catch(()=>{})}}}]}}class y extends TransformStream{constructor(e){const{args:t}=H(e);super(...t)}}class T extends y{constructor(e){let{errorFallback:t,stringify:s}=e??{};t??=!1,s??=JSON.stringify,super({stringify:s,errorFallback:t})}}function Q({begin:r,end:e,skip:t}){return{args:[{transform(n,u){t&&n.length<=0||(u.enqueue(r),u.enqueue(n),u.enqueue(e))}}]}}class k extends TransformStream{constructor(e){const{args:t}=Q(e);super(...t)}}const U=O,V=L,W=!1;class w extends k{constructor(e){let{begin:t,end:s,skip:n,...u}=e??{};t??=U,s??=V,n??=W,super({begin:t,end:s,skip:n,...u})}}const X=w;function Y(r){const{begin:e,end:t,stringify:s,errorFallback:n}=r,u=new TextEncoderStream,c=new w({begin:e,end:t}),a=new T({stringify:s,errorFallback:n}),{writable:i}=a,S=a.readable.pipeThrough(c).pipeThrough(u);return{writable:i,readable:S}}class Z{#e;#t;get writable(){return this.#e}get readable(){return this.#t}constructor(e={}){({readable:this.#t,writable:this.#e}=Y(e))}}const F=`
|
|
3
|
+
`,ee=[F,`\r
|
|
4
|
+
`];class j extends I{constructor(e){let{splitter:t,chunkEndSplit:s}=e??{};t??=ee,s??=!1,super({splitter:t,chunkEndSplit:s})}}function te(r){const{label:e,fatal:t,splitter:s,chunkEndSplit:n,parse:u,errorFallback:c}=r??{},a=new TextDecoderStream(e,{fatal:t}),i=new j({splitter:s,chunkEndSplit:n}),S=new b({parse:u,errorFallback:c}),{writable:f}=a,l=a.readable.pipeThrough(i).pipeThrough(S);return{writable:f,readable:l}}class ne{#e;#t;get writable(){return this.#e}get readable(){return this.#t}constructor(e){({readable:this.#t,writable:this.#e}=te(e))}}class N extends J{constructor(e){let{delimiter:t,start:s,end:n,skip:u}=e??{};t??=F,s??=!1,n??=!1,u??=!0,super({delimiter:t,start:s,end:n,skip:u})}}function re(r){const{delimiter:e,skip:t,start:s,end:n,stringify:u,errorFallback:c}=r,a=new TextEncoderStream,i=new N({delimiter:e,end:n,skip:t,start:s}),S=new T({stringify:u,errorFallback:c}),{writable:f}=S,l=S.readable.pipeThrough(i).pipeThrough(a);return{writable:f,readable:l}}class se{#e;#t;get writable(){return this.#e}get readable(){return this.#t}constructor(e={}){({readable:this.#t,writable:this.#e}=re(e))}}o.InputJsonLinesStream=ne,o.InputJsonSequenceParseStream=b,o.InputJsonSequenceStream=x,o.InputLineFeedSeparattedSequenceStream=j,o.InputRecordSequenceStream=q,o.InputSequenceStream=K,o.JsonSequenceFormatStream=_,o.JsonSequenceStream=z,o.OutputJsonLinesStream=se,o.OutputJsonSequenceStream=Z,o.OutputJsonSequenceStringifyStream=T,o.OutputRecordSequenceStream=w,o.OutputSequenceStream=X,o.OutputTextJoinLineFeedSequenceStream=N,o.ParseStream=R,o.RecordToSequenceStream=E,o.SequenceStream=G,o.SequenceToRecordStream=k,o.StringifyStream=y,o.TextJoinStream=J,o.TextSplitStream=I,Object.defineProperty(o,Symbol.toStringTag,{value:"Module"})});
|
package/jsonlines.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @see https://jsonlines.org/
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* 3. Line Terminator is '\n'
|
|
6
|
+
* @see https://jsonlines.org/#line-terminator-is-n
|
|
7
|
+
*/
|
|
8
|
+
export declare const LF = "\n";
|
|
9
|
+
/**
|
|
10
|
+
* This means '\r\n' is also supported because surrounding white space is implicitly ignored when parsing JSON values.
|
|
11
|
+
* @see https://jsonlines.org/#line-terminator-is-n
|
|
12
|
+
*/
|
|
13
|
+
export declare const CRLF = "\r\n";
|
|
14
|
+
/**
|
|
15
|
+
* MIME type may be `application/jsonl`, but this is not yet standardized; any help writing the RFC would be greatly appreciated (see [issue](https://github.com/wardi/jsonlines/issues/19)).
|
|
16
|
+
* @see https://jsonlines.org/#conventions
|
|
17
|
+
*/
|
|
18
|
+
export declare const MIME_TYPE = "application/jsonl";
|
|
19
|
+
/**
|
|
20
|
+
* JSON Lines files may be saved with the file extension `.jsonl.`
|
|
21
|
+
* @see https://jsonlines.org/#conventions
|
|
22
|
+
*/
|
|
23
|
+
export declare const EXTENSION = ".jsonl";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "json-seq-stream",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.8",
|
|
4
4
|
"description": "application/json-seq support stream / async iterator library",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./index.js",
|
|
@@ -49,6 +49,7 @@
|
|
|
49
49
|
"@eslint/js": "^9.30.1",
|
|
50
50
|
"@eslint/json": "^0.12.0",
|
|
51
51
|
"@eslint/markdown": "^6.6.0",
|
|
52
|
+
"@stylistic/eslint-plugin": "^5.2.0",
|
|
52
53
|
"@vitest/coverage-v8": "^3.2.4",
|
|
53
54
|
"cpx": "^1.2.1",
|
|
54
55
|
"eslint": "^9.30.1",
|
package/rfc7464.d.ts
CHANGED
|
@@ -7,10 +7,15 @@
|
|
|
7
7
|
* @see https://datatracker.ietf.org/doc/html/rfc7464#section-2.2
|
|
8
8
|
* @see https://datatracker.ietf.org/doc/html/rfc20
|
|
9
9
|
*/
|
|
10
|
-
export declare const RS = "\
|
|
10
|
+
export declare const RS = "\u001E";
|
|
11
11
|
/**
|
|
12
12
|
* LF = %x0A; "line feed" (LF), see RFC 20
|
|
13
13
|
* @see https://datatracker.ietf.org/doc/html/rfc7464#section-2.2
|
|
14
14
|
* @see https://datatracker.ietf.org/doc/html/rfc20
|
|
15
15
|
*/
|
|
16
16
|
export declare const LF = "\n";
|
|
17
|
+
/**
|
|
18
|
+
* The MIME media type for JSON text sequences is application/json-seq.
|
|
19
|
+
* @see https://datatracker.ietf.org/doc/html/rfc7464#section-4
|
|
20
|
+
*/
|
|
21
|
+
export declare const MIME_TYPE = "application/json-seq";
|
package/InputSequenceStream.d.ts
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
export type InputSequenceStreamOptions = {
|
|
2
|
-
lineBegin?: string;
|
|
3
|
-
lineEnd?: string;
|
|
4
|
-
};
|
|
5
|
-
/**
|
|
6
|
-
* RS で始まって LF で1行を表す 連続した文字列データを 行データとして整形して返す
|
|
7
|
-
*/
|
|
8
|
-
export declare class InputSequenceStream extends TransformStream<string, string> {
|
|
9
|
-
constructor(options?: InputSequenceStreamOptions);
|
|
10
|
-
}
|
|
11
|
-
/**
|
|
12
|
-
* @deprecated rename to InputSequenceStreamOptions
|
|
13
|
-
* @see InputSequenceStreamOptions
|
|
14
|
-
*/
|
|
15
|
-
export type SequenceStreamOptions = InputSequenceStreamOptions;
|
|
16
|
-
/**
|
|
17
|
-
* @deprecated rename to InputSequenceStream
|
|
18
|
-
* @see InputSequenceStream
|
|
19
|
-
*/
|
|
20
|
-
export declare const SequenceStream: typeof InputSequenceStream;
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
export type OutputJsonSequenceStringifyStreamOptions<T> = {
|
|
2
|
-
stringify?: (value: T) => (string | undefined);
|
|
3
|
-
errorFallback?: false | ((error: unknown, args: {
|
|
4
|
-
error: (reason: unknown) => void;
|
|
5
|
-
chunk: T;
|
|
6
|
-
enqueue: (arg: string) => void;
|
|
7
|
-
}) => (void | Promise<void>));
|
|
8
|
-
};
|
|
9
|
-
/**
|
|
10
|
-
* 連続した `T` の sequence を string の sequence に変換する TransformStream
|
|
11
|
-
*/
|
|
12
|
-
export declare class OutputJsonSequenceStringifyStream<T> extends TransformStream<T, string> {
|
|
13
|
-
constructor(options?: OutputJsonSequenceStringifyStreamOptions<T>);
|
|
14
|
-
}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
export type OutputSequenceStreamOptions = {
|
|
2
|
-
lineBegin?: string;
|
|
3
|
-
lineEnd?: string;
|
|
4
|
-
};
|
|
5
|
-
/**
|
|
6
|
-
* 行データを RS で始まって LF で1行を表す 連続した文字列データに変換する
|
|
7
|
-
*/
|
|
8
|
-
export declare class OutputSequenceStream extends TransformStream<string, string> {
|
|
9
|
-
constructor(options?: OutputSequenceStreamOptions);
|
|
10
|
-
}
|