links-notation 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -0
- package/README.ru.md +21 -0
- package/dist/index.js +218 -0
- package/index.d.ts +76 -0
- package/package.json +1 -1
- package/src/StreamParser.js +284 -0
- package/src/index.js +1 -0
- package/tests/StreamParser.test.js +165 -0
package/README.md
CHANGED
|
@@ -147,6 +147,45 @@ const group = new LinksGroup(parsed);
|
|
|
147
147
|
console.log(group.format());
|
|
148
148
|
```
|
|
149
149
|
|
|
150
|
+
### Streaming Parser (for Large Messages)
|
|
151
|
+
|
|
152
|
+
The `StreamParser` allows you to parse Links Notation incrementally, processing data as it arrives without loading the entire message into memory. This is ideal for:
|
|
153
|
+
|
|
154
|
+
- Large files that don't fit in memory
|
|
155
|
+
- Network streaming (TCP/HTTP)
|
|
156
|
+
- Real-time processing of incoming data
|
|
157
|
+
|
|
158
|
+
```javascript
|
|
159
|
+
import { StreamParser } from 'links-notation';
|
|
160
|
+
|
|
161
|
+
const parser = new StreamParser({ collect: false, maxBufferSize: 1024 * 1024 });
|
|
162
|
+
|
|
163
|
+
// Listen for parsed links
|
|
164
|
+
parser.on('link', (link) => {
|
|
165
|
+
console.log('Parsed link:', link.toString());
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
// Listen for errors with location information
|
|
169
|
+
parser.on('error', (error) => {
|
|
170
|
+
console.error(
|
|
171
|
+
`Error at line ${error.line}, col ${error.column}: ${error.message}`
|
|
172
|
+
);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
// Feed data incrementally (e.g., from network chunks)
|
|
176
|
+
parser.write('papa (lovesMama: loves mama)\n');
|
|
177
|
+
parser.write('son lovesMama\n');
|
|
178
|
+
parser.write('daughter lovesMama\n');
|
|
179
|
+
|
|
180
|
+
// Finish the last record. Links are not retained when collection is disabled.
|
|
181
|
+
parser.end();
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
`StreamParser.parse(chunks)` and `StreamParser.parseAsync(chunks)` expose native
|
|
185
|
+
sync and async iterables. The parser also provides `position()`, `drain()`,
|
|
186
|
+
`reset()`, and a maximum unresolved-record size. See the
|
|
187
|
+
[streaming parser example](../examples/js_streaming_parser.js).
|
|
188
|
+
|
|
150
189
|
### TypeScript Usage Examples
|
|
151
190
|
|
|
152
191
|
```typescript
|
|
@@ -314,6 +353,19 @@ Container for grouping related links.
|
|
|
314
353
|
- `constructor(links)` - Create a new group
|
|
315
354
|
- `format()` - Format the group as a string
|
|
316
355
|
|
|
356
|
+
#### `StreamParser`
|
|
357
|
+
|
|
358
|
+
EventEmitter-based streaming parser for incremental parsing.
|
|
359
|
+
|
|
360
|
+
- `constructor(options)` - Create a new streaming parser
|
|
361
|
+
- `options.maxInputSize` - Maximum input size in bytes (default: 10MB)
|
|
362
|
+
- `options.maxDepth` - Maximum nesting depth (default: 1000)
|
|
363
|
+
- `write(chunk)` - Feed a chunk of data to the parser
|
|
364
|
+
- `end()` - Finish parsing and return all parsed links
|
|
365
|
+
- `position()` - Get current parsing position (line, column, offset)
|
|
366
|
+
- Event: `'link'` - Emitted when a link is parsed (callback receives Link object)
|
|
367
|
+
- Event: `'error'` - Emitted on parse error (callback receives Error with line/column info)
|
|
368
|
+
|
|
317
369
|
#### `ParseError`
|
|
318
370
|
|
|
319
371
|
Thrown by `parse(input)` when the document does not parse. The message says
|
package/README.ru.md
CHANGED
|
@@ -105,6 +105,27 @@ const group = new LinksGroup(parsed);
|
|
|
105
105
|
console.log(group.format());
|
|
106
106
|
```
|
|
107
107
|
|
|
108
|
+
### Потоковый парсер
|
|
109
|
+
|
|
110
|
+
`StreamParser` принимает произвольные порции данных и отправляет события только
|
|
111
|
+
для завершённых связей верхнего уровня. Отключите накопление для обработки с
|
|
112
|
+
ограниченной памятью.
|
|
113
|
+
|
|
114
|
+
```javascript
|
|
115
|
+
import { StreamParser } from 'links-notation';
|
|
116
|
+
|
|
117
|
+
const stream = new StreamParser({ collect: false, maxBufferSize: 1024 * 1024 });
|
|
118
|
+
stream.on('link', (link) => console.log(link.toString()));
|
|
119
|
+
stream.on('error', (error) => console.error(error));
|
|
120
|
+
stream.write('профиль:\n имя Ада\n');
|
|
121
|
+
stream.end('следующая связь');
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
`StreamParser.parse(chunks)` и `StreamParser.parseAsync(chunks)` предоставляют
|
|
125
|
+
нативные синхронные и асинхронные итераторы. Также доступны `position()`,
|
|
126
|
+
`drain()`, `reset()` и ограничение размера незавершённой записи. См.
|
|
127
|
+
[исполняемый пример](../examples/js_streaming_parser.js).
|
|
128
|
+
|
|
108
129
|
## Примеры синтаксиса
|
|
109
130
|
|
|
110
131
|
### Дуплеты (2-кортежи)
|
package/dist/index.js
CHANGED
|
@@ -2419,6 +2419,222 @@ class Parser {
|
|
|
2419
2419
|
return new Link(item.id ?? null, []);
|
|
2420
2420
|
}
|
|
2421
2421
|
}
|
|
2422
|
+
// src/StreamParser.js
|
|
2423
|
+
import { EventEmitter } from "events";
|
|
2424
|
+
var DEFAULT_MAX_BUFFER_SIZE = 10 * 1024 * 1024;
|
|
2425
|
+
|
|
2426
|
+
class StreamParseError extends Error {
|
|
2427
|
+
constructor(error, startOffset, startLine) {
|
|
2428
|
+
const localOffset = error?.offset ?? 0;
|
|
2429
|
+
const localLine = error?.line ?? 1;
|
|
2430
|
+
const column = error?.column ?? 1;
|
|
2431
|
+
const line = startLine + localLine - 1;
|
|
2432
|
+
super(`Stream parse error at line ${line}, column ${column}: ${error.message}`);
|
|
2433
|
+
this.name = "StreamParseError";
|
|
2434
|
+
this.cause = error;
|
|
2435
|
+
this.offset = startOffset + localOffset;
|
|
2436
|
+
this.line = line;
|
|
2437
|
+
this.column = column;
|
|
2438
|
+
this.found = error?.found ?? null;
|
|
2439
|
+
this.lineText = error?.lineText ?? "";
|
|
2440
|
+
this.snippet = error?.snippet ?? "";
|
|
2441
|
+
}
|
|
2442
|
+
}
|
|
2443
|
+
|
|
2444
|
+
class StreamParser extends EventEmitter {
|
|
2445
|
+
constructor(options = {}) {
|
|
2446
|
+
super();
|
|
2447
|
+
this.parser = options.parser ?? new Parser(options);
|
|
2448
|
+
this.comments = options.comments ?? this.parser.comments ?? true;
|
|
2449
|
+
this.maxBufferSize = options.maxBufferSize ?? options.maxInputSize ?? DEFAULT_MAX_BUFFER_SIZE;
|
|
2450
|
+
this.collect = options.collect ?? true;
|
|
2451
|
+
this.reset();
|
|
2452
|
+
}
|
|
2453
|
+
write(chunk) {
|
|
2454
|
+
if (typeof chunk !== "string") {
|
|
2455
|
+
return this._fail(new TypeError("Input must be a string"));
|
|
2456
|
+
}
|
|
2457
|
+
if (this.ended) {
|
|
2458
|
+
return this._fail(new Error("Cannot write after end()"));
|
|
2459
|
+
}
|
|
2460
|
+
const emitted = [];
|
|
2461
|
+
for (let index = 0;index < chunk.length; index++) {
|
|
2462
|
+
const character = chunk[index];
|
|
2463
|
+
this.currentLine += character;
|
|
2464
|
+
this.offset += 1;
|
|
2465
|
+
if (character === `
|
|
2466
|
+
`) {
|
|
2467
|
+
this.buffer += this.currentLine;
|
|
2468
|
+
this.currentLine = "";
|
|
2469
|
+
this.lineClassified = false;
|
|
2470
|
+
this.line += 1;
|
|
2471
|
+
this.column = 1;
|
|
2472
|
+
} else {
|
|
2473
|
+
if (!this.lineClassified && character !== " " && character !== "\t" && character !== "\r") {
|
|
2474
|
+
this.lineClassified = true;
|
|
2475
|
+
const isComment = this.comments && character === "#";
|
|
2476
|
+
if (!isComment) {
|
|
2477
|
+
const indentation = leadingSpaces(this.currentLine);
|
|
2478
|
+
this._startContentLine(indentation, emitted);
|
|
2479
|
+
}
|
|
2480
|
+
}
|
|
2481
|
+
this.column += 1;
|
|
2482
|
+
}
|
|
2483
|
+
if (this.buffer.length + this.currentLine.length > this.maxBufferSize) {
|
|
2484
|
+
return this._fail(new RangeError(`Buffered record exceeds maximum size of ${this.maxBufferSize} characters`));
|
|
2485
|
+
}
|
|
2486
|
+
}
|
|
2487
|
+
return emitted;
|
|
2488
|
+
}
|
|
2489
|
+
end(chunk = "") {
|
|
2490
|
+
const emitted = chunk === "" ? [] : this.write(chunk);
|
|
2491
|
+
if (this.ended) {
|
|
2492
|
+
return this.collect ? [...this.links] : emitted;
|
|
2493
|
+
}
|
|
2494
|
+
const document = this.buffer + this.currentLine;
|
|
2495
|
+
if (document.length > 0) {
|
|
2496
|
+
let links;
|
|
2497
|
+
try {
|
|
2498
|
+
links = this.parser.parse(document);
|
|
2499
|
+
} catch (error) {
|
|
2500
|
+
const streamError = new StreamParseError(error, this.segmentOffset, this.segmentLine);
|
|
2501
|
+
return this._fail(streamError);
|
|
2502
|
+
}
|
|
2503
|
+
this._publish(links, emitted);
|
|
2504
|
+
this._advanceSegment(document);
|
|
2505
|
+
}
|
|
2506
|
+
this.buffer = "";
|
|
2507
|
+
this.currentLine = "";
|
|
2508
|
+
this.baseIndentation = null;
|
|
2509
|
+
this.ended = true;
|
|
2510
|
+
const result = this.collect ? [...this.links] : emitted;
|
|
2511
|
+
this.emit("end", result);
|
|
2512
|
+
return result;
|
|
2513
|
+
}
|
|
2514
|
+
drain() {
|
|
2515
|
+
const links = this.links;
|
|
2516
|
+
this.links = [];
|
|
2517
|
+
return links;
|
|
2518
|
+
}
|
|
2519
|
+
reset() {
|
|
2520
|
+
this.buffer = "";
|
|
2521
|
+
this.currentLine = "";
|
|
2522
|
+
this.baseIndentation = null;
|
|
2523
|
+
this.lineClassified = false;
|
|
2524
|
+
this.links = [];
|
|
2525
|
+
this.offset = 0;
|
|
2526
|
+
this.line = 1;
|
|
2527
|
+
this.column = 1;
|
|
2528
|
+
this.segmentOffset = 0;
|
|
2529
|
+
this.segmentLine = 1;
|
|
2530
|
+
this.ended = false;
|
|
2531
|
+
return this;
|
|
2532
|
+
}
|
|
2533
|
+
position() {
|
|
2534
|
+
return {
|
|
2535
|
+
offset: this.offset,
|
|
2536
|
+
line: this.line,
|
|
2537
|
+
column: this.column,
|
|
2538
|
+
buffered: this.buffer.length + this.currentLine.length
|
|
2539
|
+
};
|
|
2540
|
+
}
|
|
2541
|
+
static *parse(chunks, options = {}) {
|
|
2542
|
+
const parser = new StreamParser({ ...options, collect: false });
|
|
2543
|
+
for (const chunk of chunks) {
|
|
2544
|
+
yield* parser.write(chunk);
|
|
2545
|
+
}
|
|
2546
|
+
yield* parser.end();
|
|
2547
|
+
}
|
|
2548
|
+
static async* parseAsync(chunks, options = {}) {
|
|
2549
|
+
const parser = new StreamParser({ ...options, collect: false });
|
|
2550
|
+
for await (const chunk of chunks) {
|
|
2551
|
+
yield* parser.write(chunk);
|
|
2552
|
+
}
|
|
2553
|
+
yield* parser.end();
|
|
2554
|
+
}
|
|
2555
|
+
_startContentLine(indentation, emitted) {
|
|
2556
|
+
if (this.buffer.length > 0 && this.baseIndentation !== null && indentation <= this.baseIndentation && structurallyComplete(this.buffer, this.comments)) {
|
|
2557
|
+
let links;
|
|
2558
|
+
try {
|
|
2559
|
+
links = this.parser.parse(this.buffer);
|
|
2560
|
+
} catch {
|
|
2561
|
+
links = null;
|
|
2562
|
+
}
|
|
2563
|
+
if (links !== null) {
|
|
2564
|
+
this._publish(links, emitted);
|
|
2565
|
+
this._advanceSegment(this.buffer);
|
|
2566
|
+
this.buffer = "";
|
|
2567
|
+
this.baseIndentation = null;
|
|
2568
|
+
}
|
|
2569
|
+
}
|
|
2570
|
+
if (this.baseIndentation === null) {
|
|
2571
|
+
this.baseIndentation = indentation;
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
2574
|
+
_publish(links, emitted) {
|
|
2575
|
+
for (const link of links) {
|
|
2576
|
+
emitted.push(link);
|
|
2577
|
+
if (this.collect)
|
|
2578
|
+
this.links.push(link);
|
|
2579
|
+
this.emit("link", link);
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2582
|
+
_advanceSegment(document) {
|
|
2583
|
+
this.segmentOffset += document.length;
|
|
2584
|
+
this.segmentLine += countNewlines(document);
|
|
2585
|
+
}
|
|
2586
|
+
_fail(error) {
|
|
2587
|
+
this.emit("error", error);
|
|
2588
|
+
throw error;
|
|
2589
|
+
}
|
|
2590
|
+
}
|
|
2591
|
+
function leadingSpaces(line) {
|
|
2592
|
+
let indentation = 0;
|
|
2593
|
+
while (line[indentation] === " ")
|
|
2594
|
+
indentation += 1;
|
|
2595
|
+
return indentation;
|
|
2596
|
+
}
|
|
2597
|
+
function countNewlines(value) {
|
|
2598
|
+
let count = 0;
|
|
2599
|
+
for (const character of value) {
|
|
2600
|
+
if (character === `
|
|
2601
|
+
`)
|
|
2602
|
+
count += 1;
|
|
2603
|
+
}
|
|
2604
|
+
return count;
|
|
2605
|
+
}
|
|
2606
|
+
function structurallyComplete(document, comments) {
|
|
2607
|
+
const quotes = ['"', "'", "`"];
|
|
2608
|
+
const beforeReference = [" ", "\t", `
|
|
2609
|
+
`, "\r", "(", ":"];
|
|
2610
|
+
const beforeComment = [" ", "\t", `
|
|
2611
|
+
`, "\r"];
|
|
2612
|
+
let depth = 0;
|
|
2613
|
+
for (let position = 0;position < document.length; position++) {
|
|
2614
|
+
const character = document[position];
|
|
2615
|
+
const previous = position === 0 ? null : document[position - 1];
|
|
2616
|
+
if (quotes.includes(character) && (previous === null || beforeReference.includes(previous))) {
|
|
2617
|
+
const end = quotedReferenceEnd(document, position);
|
|
2618
|
+
if (end === null)
|
|
2619
|
+
return false;
|
|
2620
|
+
position = end - 1;
|
|
2621
|
+
continue;
|
|
2622
|
+
}
|
|
2623
|
+
if (comments && character === "#" && (previous === null || beforeComment.includes(previous))) {
|
|
2624
|
+
const newline = document.indexOf(`
|
|
2625
|
+
`, position);
|
|
2626
|
+
if (newline === -1)
|
|
2627
|
+
return true;
|
|
2628
|
+
position = newline;
|
|
2629
|
+
continue;
|
|
2630
|
+
}
|
|
2631
|
+
if (character === "(")
|
|
2632
|
+
depth += 1;
|
|
2633
|
+
if (character === ")")
|
|
2634
|
+
depth -= 1;
|
|
2635
|
+
}
|
|
2636
|
+
return depth === 0;
|
|
2637
|
+
}
|
|
2422
2638
|
// src/FormatOptions.js
|
|
2423
2639
|
class FormatOptions {
|
|
2424
2640
|
constructor(options = {}) {
|
|
@@ -2454,6 +2670,8 @@ export {
|
|
|
2454
2670
|
LinksGroup,
|
|
2455
2671
|
ParseError,
|
|
2456
2672
|
Parser,
|
|
2673
|
+
StreamParseError,
|
|
2674
|
+
StreamParser,
|
|
2457
2675
|
formatLinks,
|
|
2458
2676
|
stripComments
|
|
2459
2677
|
};
|
package/index.d.ts
CHANGED
|
@@ -160,6 +160,82 @@ export class Parser {
|
|
|
160
160
|
parse(input: string): Link[];
|
|
161
161
|
}
|
|
162
162
|
|
|
163
|
+
/** Options for incremental parsing. */
|
|
164
|
+
export interface StreamParserOptions extends ParserOptions {
|
|
165
|
+
/** Canonical parser instance to use for completed records. */
|
|
166
|
+
parser?: Parser;
|
|
167
|
+
|
|
168
|
+
/** Largest unresolved record retained in memory (default: 10MB). */
|
|
169
|
+
maxBufferSize?: number;
|
|
170
|
+
|
|
171
|
+
/** Retain emitted links for end() and drain() (default: true). */
|
|
172
|
+
collect?: boolean;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Position immediately after the last character written. */
|
|
176
|
+
export interface StreamPosition {
|
|
177
|
+
offset: number;
|
|
178
|
+
line: number;
|
|
179
|
+
column: number;
|
|
180
|
+
buffered: number;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Canonical parse error translated to coordinates in the complete stream. */
|
|
184
|
+
export class StreamParseError extends Error {
|
|
185
|
+
cause: Error;
|
|
186
|
+
offset: number;
|
|
187
|
+
line: number;
|
|
188
|
+
column: number;
|
|
189
|
+
found: string | null;
|
|
190
|
+
lineText: string;
|
|
191
|
+
snippet: string;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Incrementally emits complete top-level Links Notation records. */
|
|
195
|
+
export class StreamParser {
|
|
196
|
+
parser: Parser;
|
|
197
|
+
comments: boolean;
|
|
198
|
+
maxBufferSize: number;
|
|
199
|
+
collect: boolean;
|
|
200
|
+
|
|
201
|
+
constructor(options?: StreamParserOptions);
|
|
202
|
+
|
|
203
|
+
/** Feed a chunk and return links made complete by it. */
|
|
204
|
+
write(chunk: string): Link[];
|
|
205
|
+
|
|
206
|
+
/** Finish the stream, optionally after consuming one final chunk. */
|
|
207
|
+
end(chunk?: string): Link[];
|
|
208
|
+
|
|
209
|
+
/** Return and forget retained links. */
|
|
210
|
+
drain(): Link[];
|
|
211
|
+
|
|
212
|
+
/** Reuse this parser while preserving its options and listeners. */
|
|
213
|
+
reset(): this;
|
|
214
|
+
|
|
215
|
+
/** Return the absolute stream position and unresolved buffer size. */
|
|
216
|
+
position(): StreamPosition;
|
|
217
|
+
|
|
218
|
+
on(event: 'link', listener: (link: Link) => void): this;
|
|
219
|
+
on(event: 'end', listener: (links: Link[]) => void): this;
|
|
220
|
+
on(event: 'error', listener: (error: Error) => void): this;
|
|
221
|
+
once(event: 'link', listener: (link: Link) => void): this;
|
|
222
|
+
once(event: 'end', listener: (links: Link[]) => void): this;
|
|
223
|
+
once(event: 'error', listener: (error: Error) => void): this;
|
|
224
|
+
off(event: string, listener: (...arguments_: any[]) => void): this;
|
|
225
|
+
|
|
226
|
+
/** Lazily parse a synchronous iterable of chunks. */
|
|
227
|
+
static parse(
|
|
228
|
+
chunks: Iterable<string>,
|
|
229
|
+
options?: StreamParserOptions
|
|
230
|
+
): IterableIterator<Link>;
|
|
231
|
+
|
|
232
|
+
/** Lazily parse an asynchronous iterable of chunks. */
|
|
233
|
+
static parseAsync(
|
|
234
|
+
chunks: AsyncIterable<string> | Iterable<string>,
|
|
235
|
+
options?: StreamParserOptions
|
|
236
|
+
): AsyncIterableIterator<Link>;
|
|
237
|
+
}
|
|
238
|
+
|
|
163
239
|
/**
|
|
164
240
|
* Blank out every comment in a document, keeping its length so that every
|
|
165
241
|
* character stays at the offset it was written at.
|
package/package.json
CHANGED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
import { EventEmitter } from 'events';
|
|
2
|
+
import { Parser } from './Parser.js';
|
|
3
|
+
import { quotedReferenceEnd } from './comments.js';
|
|
4
|
+
|
|
5
|
+
const DEFAULT_MAX_BUFFER_SIZE = 10 * 1024 * 1024;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* An error raised while finishing a streamed document.
|
|
9
|
+
*
|
|
10
|
+
* The canonical parser reports positions relative to the currently buffered
|
|
11
|
+
* record. This wrapper translates them back to the complete stream.
|
|
12
|
+
*/
|
|
13
|
+
export class StreamParseError extends Error {
|
|
14
|
+
constructor(error, startOffset, startLine) {
|
|
15
|
+
const localOffset = error?.offset ?? 0;
|
|
16
|
+
const localLine = error?.line ?? 1;
|
|
17
|
+
const column = error?.column ?? 1;
|
|
18
|
+
const line = startLine + localLine - 1;
|
|
19
|
+
super(
|
|
20
|
+
`Stream parse error at line ${line}, column ${column}: ${error.message}`
|
|
21
|
+
);
|
|
22
|
+
this.name = 'StreamParseError';
|
|
23
|
+
this.cause = error;
|
|
24
|
+
this.offset = startOffset + localOffset;
|
|
25
|
+
this.line = line;
|
|
26
|
+
this.column = column;
|
|
27
|
+
this.found = error?.found ?? null;
|
|
28
|
+
this.lineText = error?.lineText ?? '';
|
|
29
|
+
this.snippet = error?.snippet ?? '';
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Incrementally parses complete top-level Links Notation records.
|
|
35
|
+
*
|
|
36
|
+
* A line can acquire indented children, so its terminating newline alone does
|
|
37
|
+
* not prove that the record is complete. The parser commits a buffered record
|
|
38
|
+
* as soon as the first non-indented character of the next record arrives. It
|
|
39
|
+
* uses the regular Parser as the source of truth, which keeps quoted strings,
|
|
40
|
+
* nested contexts, comments, and future grammar changes in one place.
|
|
41
|
+
*/
|
|
42
|
+
export class StreamParser extends EventEmitter {
|
|
43
|
+
constructor(options = {}) {
|
|
44
|
+
super();
|
|
45
|
+
this.parser = options.parser ?? new Parser(options);
|
|
46
|
+
this.comments = options.comments ?? this.parser.comments ?? true;
|
|
47
|
+
this.maxBufferSize =
|
|
48
|
+
options.maxBufferSize ?? options.maxInputSize ?? DEFAULT_MAX_BUFFER_SIZE;
|
|
49
|
+
this.collect = options.collect ?? true;
|
|
50
|
+
this.reset();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Feed a string chunk and return the links made complete by that chunk. */
|
|
54
|
+
write(chunk) {
|
|
55
|
+
if (typeof chunk !== 'string') {
|
|
56
|
+
return this._fail(new TypeError('Input must be a string'));
|
|
57
|
+
}
|
|
58
|
+
if (this.ended) {
|
|
59
|
+
return this._fail(new Error('Cannot write after end()'));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const emitted = [];
|
|
63
|
+
for (let index = 0; index < chunk.length; index++) {
|
|
64
|
+
const character = chunk[index];
|
|
65
|
+
this.currentLine += character;
|
|
66
|
+
this.offset += 1;
|
|
67
|
+
|
|
68
|
+
if (character === '\n') {
|
|
69
|
+
this.buffer += this.currentLine;
|
|
70
|
+
this.currentLine = '';
|
|
71
|
+
this.lineClassified = false;
|
|
72
|
+
this.line += 1;
|
|
73
|
+
this.column = 1;
|
|
74
|
+
} else {
|
|
75
|
+
if (
|
|
76
|
+
!this.lineClassified &&
|
|
77
|
+
character !== ' ' &&
|
|
78
|
+
character !== '\t' &&
|
|
79
|
+
character !== '\r'
|
|
80
|
+
) {
|
|
81
|
+
this.lineClassified = true;
|
|
82
|
+
const isComment = this.comments && character === '#';
|
|
83
|
+
if (!isComment) {
|
|
84
|
+
const indentation = leadingSpaces(this.currentLine);
|
|
85
|
+
this._startContentLine(indentation, emitted);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
this.column += 1;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (this.buffer.length + this.currentLine.length > this.maxBufferSize) {
|
|
92
|
+
return this._fail(
|
|
93
|
+
new RangeError(
|
|
94
|
+
`Buffered record exceeds maximum size of ${this.maxBufferSize} characters`
|
|
95
|
+
)
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return emitted;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Finish the stream. With collection enabled (the default), returns every
|
|
104
|
+
* undrained link; otherwise returns only links completed by this call.
|
|
105
|
+
*/
|
|
106
|
+
end(chunk = '') {
|
|
107
|
+
const emitted = chunk === '' ? [] : this.write(chunk);
|
|
108
|
+
if (this.ended) {
|
|
109
|
+
return this.collect ? [...this.links] : emitted;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const document = this.buffer + this.currentLine;
|
|
113
|
+
if (document.length > 0) {
|
|
114
|
+
let links;
|
|
115
|
+
try {
|
|
116
|
+
links = this.parser.parse(document);
|
|
117
|
+
} catch (error) {
|
|
118
|
+
const streamError = new StreamParseError(
|
|
119
|
+
error,
|
|
120
|
+
this.segmentOffset,
|
|
121
|
+
this.segmentLine
|
|
122
|
+
);
|
|
123
|
+
return this._fail(streamError);
|
|
124
|
+
}
|
|
125
|
+
this._publish(links, emitted);
|
|
126
|
+
this._advanceSegment(document);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
this.buffer = '';
|
|
130
|
+
this.currentLine = '';
|
|
131
|
+
this.baseIndentation = null;
|
|
132
|
+
this.ended = true;
|
|
133
|
+
const result = this.collect ? [...this.links] : emitted;
|
|
134
|
+
this.emit('end', result);
|
|
135
|
+
return result;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Return and forget links retained since the previous drain. */
|
|
139
|
+
drain() {
|
|
140
|
+
const links = this.links;
|
|
141
|
+
this.links = [];
|
|
142
|
+
return links;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Reuse this parser for a new stream while keeping listeners and options. */
|
|
146
|
+
reset() {
|
|
147
|
+
this.buffer = '';
|
|
148
|
+
this.currentLine = '';
|
|
149
|
+
this.baseIndentation = null;
|
|
150
|
+
this.lineClassified = false;
|
|
151
|
+
this.links = [];
|
|
152
|
+
this.offset = 0;
|
|
153
|
+
this.line = 1;
|
|
154
|
+
this.column = 1;
|
|
155
|
+
this.segmentOffset = 0;
|
|
156
|
+
this.segmentLine = 1;
|
|
157
|
+
this.ended = false;
|
|
158
|
+
return this;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** Position immediately after the last character written. */
|
|
162
|
+
position() {
|
|
163
|
+
return {
|
|
164
|
+
offset: this.offset,
|
|
165
|
+
line: this.line,
|
|
166
|
+
column: this.column,
|
|
167
|
+
buffered: this.buffer.length + this.currentLine.length,
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Lazily parse a synchronous iterable of chunks. */
|
|
172
|
+
static *parse(chunks, options = {}) {
|
|
173
|
+
const parser = new StreamParser({ ...options, collect: false });
|
|
174
|
+
for (const chunk of chunks) {
|
|
175
|
+
yield* parser.write(chunk);
|
|
176
|
+
}
|
|
177
|
+
yield* parser.end();
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Lazily parse an async iterable of chunks. */
|
|
181
|
+
static async *parseAsync(chunks, options = {}) {
|
|
182
|
+
const parser = new StreamParser({ ...options, collect: false });
|
|
183
|
+
for await (const chunk of chunks) {
|
|
184
|
+
yield* parser.write(chunk);
|
|
185
|
+
}
|
|
186
|
+
yield* parser.end();
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
_startContentLine(indentation, emitted) {
|
|
190
|
+
if (
|
|
191
|
+
this.buffer.length > 0 &&
|
|
192
|
+
this.baseIndentation !== null &&
|
|
193
|
+
indentation <= this.baseIndentation &&
|
|
194
|
+
structurallyComplete(this.buffer, this.comments)
|
|
195
|
+
) {
|
|
196
|
+
let links;
|
|
197
|
+
try {
|
|
198
|
+
links = this.parser.parse(this.buffer);
|
|
199
|
+
} catch {
|
|
200
|
+
links = null;
|
|
201
|
+
}
|
|
202
|
+
if (links !== null) {
|
|
203
|
+
this._publish(links, emitted);
|
|
204
|
+
this._advanceSegment(this.buffer);
|
|
205
|
+
this.buffer = '';
|
|
206
|
+
this.baseIndentation = null;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
if (this.baseIndentation === null) {
|
|
211
|
+
this.baseIndentation = indentation;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
_publish(links, emitted) {
|
|
216
|
+
for (const link of links) {
|
|
217
|
+
emitted.push(link);
|
|
218
|
+
if (this.collect) this.links.push(link);
|
|
219
|
+
this.emit('link', link);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
_advanceSegment(document) {
|
|
224
|
+
this.segmentOffset += document.length;
|
|
225
|
+
this.segmentLine += countNewlines(document);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
_fail(error) {
|
|
229
|
+
this.emit('error', error);
|
|
230
|
+
throw error;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function leadingSpaces(line) {
|
|
235
|
+
let indentation = 0;
|
|
236
|
+
while (line[indentation] === ' ') indentation += 1;
|
|
237
|
+
return indentation;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function countNewlines(value) {
|
|
241
|
+
let count = 0;
|
|
242
|
+
for (const character of value) {
|
|
243
|
+
if (character === '\n') count += 1;
|
|
244
|
+
}
|
|
245
|
+
return count;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function structurallyComplete(document, comments) {
|
|
249
|
+
const quotes = ['"', "'", '`'];
|
|
250
|
+
const beforeReference = [' ', '\t', '\n', '\r', '(', ':'];
|
|
251
|
+
const beforeComment = [' ', '\t', '\n', '\r'];
|
|
252
|
+
let depth = 0;
|
|
253
|
+
|
|
254
|
+
for (let position = 0; position < document.length; position++) {
|
|
255
|
+
const character = document[position];
|
|
256
|
+
const previous = position === 0 ? null : document[position - 1];
|
|
257
|
+
|
|
258
|
+
if (
|
|
259
|
+
quotes.includes(character) &&
|
|
260
|
+
(previous === null || beforeReference.includes(previous))
|
|
261
|
+
) {
|
|
262
|
+
const end = quotedReferenceEnd(document, position);
|
|
263
|
+
if (end === null) return false;
|
|
264
|
+
position = end - 1;
|
|
265
|
+
continue;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
if (
|
|
269
|
+
comments &&
|
|
270
|
+
character === '#' &&
|
|
271
|
+
(previous === null || beforeComment.includes(previous))
|
|
272
|
+
) {
|
|
273
|
+
const newline = document.indexOf('\n', position);
|
|
274
|
+
if (newline === -1) return true;
|
|
275
|
+
position = newline;
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (character === '(') depth += 1;
|
|
280
|
+
if (character === ')') depth -= 1;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
return depth === 0;
|
|
284
|
+
}
|
package/src/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export { Link, formatLinks } from './Link.js';
|
|
2
2
|
export { LinksGroup } from './LinksGroup.js';
|
|
3
3
|
export { Parser } from './Parser.js';
|
|
4
|
+
export { StreamParseError, StreamParser } from './StreamParser.js';
|
|
4
5
|
export { ParseError } from './ParseError.js';
|
|
5
6
|
export { FormatConfig } from './FormatConfig.js';
|
|
6
7
|
export { FormatOptions } from './FormatOptions.js';
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import {
|
|
3
|
+
formatLinks,
|
|
4
|
+
Parser,
|
|
5
|
+
StreamParseError,
|
|
6
|
+
StreamParser,
|
|
7
|
+
} from '../src/index.js';
|
|
8
|
+
|
|
9
|
+
const document = `first loves data
|
|
10
|
+
# streamed comment
|
|
11
|
+
profile:
|
|
12
|
+
name Ada
|
|
13
|
+
note "line one
|
|
14
|
+
line two"
|
|
15
|
+
(nested:
|
|
16
|
+
child value)
|
|
17
|
+
last sees first`;
|
|
18
|
+
|
|
19
|
+
const formatted = (links) => formatLinks(links);
|
|
20
|
+
|
|
21
|
+
describe('StreamParser', () => {
|
|
22
|
+
test('matches canonical parser one symbol at a time', () => {
|
|
23
|
+
const stream = new StreamParser();
|
|
24
|
+
for (const character of document) stream.write(character);
|
|
25
|
+
|
|
26
|
+
expect(formatted(stream.end())).toBe(
|
|
27
|
+
formatted(new Parser().parse(document))
|
|
28
|
+
);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test('supports line chunks final record and position', () => {
|
|
32
|
+
const stream = new StreamParser();
|
|
33
|
+
for (const line of document.match(/.*(?:\n|$)/g).filter(Boolean)) {
|
|
34
|
+
stream.write(line);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
expect(formatted(stream.end())).toBe(
|
|
38
|
+
formatted(new Parser().parse(document))
|
|
39
|
+
);
|
|
40
|
+
expect(stream.position().offset).toBe(document.length);
|
|
41
|
+
expect(stream.position().buffered).toBe(0);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test('emits a record as soon as the next top-level record starts', () => {
|
|
45
|
+
const stream = new StreamParser();
|
|
46
|
+
const seen = [];
|
|
47
|
+
stream.on('link', (link) => seen.push(link.toString()));
|
|
48
|
+
|
|
49
|
+
expect(stream.write('first loves data\n')).toEqual([]);
|
|
50
|
+
expect(stream.write('s').map(String)).toEqual(['(first loves data)']);
|
|
51
|
+
expect(seen).toEqual(['(first loves data)']);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test('emits only complete records', () => {
|
|
55
|
+
const stream = new StreamParser();
|
|
56
|
+
const seen = [];
|
|
57
|
+
stream.on('link', (link) => seen.push(link));
|
|
58
|
+
|
|
59
|
+
for (const character of 'profile:\n name Ada\n role engineer\n') {
|
|
60
|
+
stream.write(character);
|
|
61
|
+
}
|
|
62
|
+
expect(seen).toEqual([]);
|
|
63
|
+
|
|
64
|
+
stream.write('n');
|
|
65
|
+
expect(formatted(seen)).toBe('(profile: (name Ada) (role engineer))');
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test('does not split multiline quotes or parenthesized contexts', () => {
|
|
69
|
+
const stream = new StreamParser();
|
|
70
|
+
const seen = [];
|
|
71
|
+
stream.on('link', (link) => seen.push(link));
|
|
72
|
+
|
|
73
|
+
stream.write('"first\nsecond"\n(nested:\n value one)\n');
|
|
74
|
+
expect(seen).toHaveLength(1);
|
|
75
|
+
stream.write('n');
|
|
76
|
+
expect(seen).toHaveLength(2);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
test('accepts a final record without a newline', () => {
|
|
80
|
+
const stream = new StreamParser();
|
|
81
|
+
stream.write('only one record');
|
|
82
|
+
expect(formatted(stream.end())).toBe('(only one record)');
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test('reports positions relative to the whole stream', () => {
|
|
86
|
+
const stream = new StreamParser();
|
|
87
|
+
stream.on('error', () => {});
|
|
88
|
+
stream.write('good link\nnext link\nb');
|
|
89
|
+
|
|
90
|
+
let caught;
|
|
91
|
+
try {
|
|
92
|
+
stream.end('ad: value: nope');
|
|
93
|
+
} catch (error) {
|
|
94
|
+
caught = error;
|
|
95
|
+
}
|
|
96
|
+
expect(caught).toBeInstanceOf(StreamParseError);
|
|
97
|
+
expect(caught.line).toBe(3);
|
|
98
|
+
expect(caught.offset).toBeGreaterThan('good link\nnext link\n'.length);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test('can avoid retaining output for bounded-memory event processing', () => {
|
|
102
|
+
const stream = new StreamParser({ collect: false });
|
|
103
|
+
const seen = [];
|
|
104
|
+
stream.on('link', (link) => seen.push(link.toString()));
|
|
105
|
+
stream.write('one link\nt');
|
|
106
|
+
expect(stream.drain()).toEqual([]);
|
|
107
|
+
expect(stream.end('wo link').map(String)).toEqual(['(two link)']);
|
|
108
|
+
expect(seen).toEqual(['(one link)', '(two link)']);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
test('supports drain reset and bounded memory', () => {
|
|
112
|
+
const stream = new StreamParser();
|
|
113
|
+
stream.write('one link\nt');
|
|
114
|
+
expect(stream.drain().map(String)).toEqual(['(one link)']);
|
|
115
|
+
expect(stream.end('wo link').map(String)).toEqual(['(two link)']);
|
|
116
|
+
|
|
117
|
+
stream.reset().write('fresh record');
|
|
118
|
+
expect(stream.end().map(String)).toEqual(['(fresh record)']);
|
|
119
|
+
|
|
120
|
+
const bounded = new StreamParser({ maxBufferSize: 4 });
|
|
121
|
+
bounded.on('error', () => {});
|
|
122
|
+
expect(() => bounded.write('12345')).toThrow(RangeError);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test('provides lazy adapters', async () => {
|
|
126
|
+
expect(
|
|
127
|
+
[...StreamParser.parse(['one link\n', 'two link'])].map(String)
|
|
128
|
+
).toEqual(['(one link)', '(two link)']);
|
|
129
|
+
|
|
130
|
+
async function* chunks() {
|
|
131
|
+
yield 'one link\n';
|
|
132
|
+
yield 'two link';
|
|
133
|
+
}
|
|
134
|
+
const links = [];
|
|
135
|
+
for await (const link of StreamParser.parseAsync(chunks())) {
|
|
136
|
+
links.push(link.toString());
|
|
137
|
+
}
|
|
138
|
+
expect(links).toEqual(['(one link)', '(two link)']);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
test('limits only the unresolved record, not total stream size', () => {
|
|
142
|
+
const stream = new StreamParser({ maxBufferSize: 12, collect: false });
|
|
143
|
+
stream.on('error', () => {});
|
|
144
|
+
for (let index = 0; index < 100; index++) stream.write(`a${index}\n`);
|
|
145
|
+
expect(stream.end().map(String)).toEqual(['(a99)']);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test('rejects writes after finish', () => {
|
|
149
|
+
const stream = new StreamParser();
|
|
150
|
+
stream.end('one');
|
|
151
|
+
stream.on('error', () => {});
|
|
152
|
+
expect(() => stream.write('two')).toThrow('after end');
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test('tracks the absolute stream position', () => {
|
|
156
|
+
const stream = new StreamParser();
|
|
157
|
+
stream.write('one\ntw');
|
|
158
|
+
expect(stream.position()).toEqual({
|
|
159
|
+
offset: 6,
|
|
160
|
+
line: 2,
|
|
161
|
+
column: 3,
|
|
162
|
+
buffered: 2,
|
|
163
|
+
});
|
|
164
|
+
});
|
|
165
|
+
});
|