@stxt-lang/core 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -3
- package/out/all.d.ts +3 -1
- package/out/all.js +3 -1
- package/out/core/Constants.d.ts +15 -0
- package/out/core/Constants.js +15 -0
- package/out/core/Parser.d.ts +68 -3
- package/out/core/Parser.js +128 -29
- package/out/exceptions/LimitException.d.ts +17 -0
- package/out/exceptions/LimitException.js +26 -0
- package/out/processors/StreamObserver.d.ts +35 -0
- package/out/processors/StreamObserver.js +3 -0
- package/out/runtime/NodeWriter.d.ts +1 -1
- package/out/runtime/NodeWriter.js +1 -1
- package/package.json +3 -2
- package/out/runtime/ConditionalValidator.d.ts +0 -28
- package/out/runtime/ConditionalValidator.js +0 -36
package/README.md
CHANGED
|
@@ -281,6 +281,33 @@ parser.registerObserver(new LoggingObserver());
|
|
|
281
281
|
parser.parseResult(text);
|
|
282
282
|
```
|
|
283
283
|
|
|
284
|
+
`StreamObserver` watches the results instead of the process: each completed root node and each
|
|
285
|
+
error, in every mode. With `parseStream` the parser retains nothing — no nodes, no errors — so a
|
|
286
|
+
file larger than memory can be processed one root tree at a time:
|
|
287
|
+
|
|
288
|
+
```ts
|
|
289
|
+
import { Parser, StreamObserver, Node, ParseException } from '@stxt-lang/core';
|
|
290
|
+
|
|
291
|
+
const parser = new Parser();
|
|
292
|
+
parser.registerStreamObserver({
|
|
293
|
+
onRootNode(node: Node): void { handle(node); }, // one complete root at a time
|
|
294
|
+
onError(error: ParseException): void { report(error); },
|
|
295
|
+
} satisfies StreamObserver);
|
|
296
|
+
parser.parseStream(readLinesLazily(file)); // any Iterable<string> of lines
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
## Parser limits
|
|
300
|
+
|
|
301
|
+
The parser rejects hostile or runaway inputs by default (STXT-SPEC §11.2): documents nesting
|
|
302
|
+
more than 100 levels, lines longer than 10 000 characters, or inputs over 10 000 000
|
|
303
|
+
characters. A limit error is a `LimitException` (`LIMIT_NESTING_EXCEEDED`,
|
|
304
|
+
`LIMIT_LINE_LENGTH_EXCEEDED`, `LIMIT_INPUT_SIZE_EXCEEDED`) and aborts the parse: it is always
|
|
305
|
+
the last error reported. Each limit is configurable per parser; `-1` disables it:
|
|
306
|
+
|
|
307
|
+
```ts
|
|
308
|
+
const parser = new Parser({ maxNesting: 500, maxInputSize: -1 });
|
|
309
|
+
```
|
|
310
|
+
|
|
284
311
|
## Writing STXT back out
|
|
285
312
|
|
|
286
313
|
```ts
|
|
@@ -311,14 +338,18 @@ if (errors.length === 0) {
|
|
|
311
338
|
|
|
312
339
|
Everything importable from the package:
|
|
313
340
|
|
|
314
|
-
- **Parsing** — `Node`, `InlineNode`, `TextNode`, `Parser`, `ParseResult`, `Line`, `Constants`, `parseLine`, `StringUtils`
|
|
315
|
-
- **Exceptions** — `ParseException`, `ValidationException`, `RuntimeException`
|
|
316
|
-
- **Extension points** — `Observer`, `Validator`
|
|
341
|
+
- **Parsing** — `Node`, `InlineNode`, `TextNode`, `Parser`, `ParserOptions`, `ParseResult`, `Line`, `Constants`, `parseLine`, `StringUtils`
|
|
342
|
+
- **Exceptions** — `ParseException`, `ValidationException`, `LimitException`, `RuntimeException`
|
|
343
|
+
- **Extension points** — `Observer`, `StreamObserver`, `Validator`
|
|
317
344
|
- **Schemas** — `Schema`, `SchemaValidator`, `SchemaProvider`, `SchemaProviderMemory`, `SchemaProviderMeta`, `NodeDefinition`, `ChildDefinition`, `TypeRegistry`, `Type`, `transformNodeToSchema`
|
|
318
345
|
- **Templates** — `transformTemplateNodeToSchema`, `TEMPLATE_NAMESPACE`, `TemplateSchemaProviderMemory`, `MetaTemplateSchemaProvider`
|
|
319
346
|
- **Runtime** — `UnifiedSchemaProvider`, `NodeWriter`, `IndentStyle`, `Formatter`, `FormatResult`, `toCanonicalTree`, `toCanonicalJson`
|
|
320
347
|
- **Discovery** — `DiscoveryResolver`, `DiscoveryOptions`, `DiscoveryResult`, `DiscoveryDefinition`, `DiscoveryLevel`, `DiscoveryError`, `DiscoveryFileSystem`, `DiscoveryEntry`, `DiscoveryEnvironment`
|
|
321
348
|
|
|
349
|
+
## Conformance
|
|
350
|
+
|
|
351
|
+
`@stxt-lang/core` implements the five STXT specifications at `SPEC_VERSION` (exposed by the package; the package version is independent) and passes every case of the official conformance kit, [`stxt-lang/conformance`](https://github.com/stxt-lang/stxt-lang/tree/master/conformance), across all its profiles: `core`, `schema`, `template`, `discovery` and `text`. The kit is the same one any other implementation can run, which is what makes the three ports interchangeable. What the 1.0 line freezes, and what it does not, is stated at <https://stxt.dev/lang-stability>.
|
|
352
|
+
|
|
322
353
|
## License
|
|
323
354
|
|
|
324
355
|
MIT — see [LICENSE](LICENSE).
|
package/out/all.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export { Node } from "./core/Node";
|
|
2
2
|
export { InlineNode } from "./core/InlineNode";
|
|
3
3
|
export { TextNode } from "./core/TextNode";
|
|
4
|
-
export { Parser } from "./core/Parser";
|
|
4
|
+
export { Parser, ParserOptions } from "./core/Parser";
|
|
5
5
|
export { ParseResult } from "./core/ParseResult";
|
|
6
6
|
export { Line } from "./core/Line";
|
|
7
7
|
export { Constants } from "./core/Constants";
|
|
@@ -15,8 +15,10 @@ export { parseLine } from "./core/LineParser";
|
|
|
15
15
|
export { StringUtils } from "./core/StringUtils";
|
|
16
16
|
export { ParseException } from "./exceptions/ParseException";
|
|
17
17
|
export { ValidationException } from "./exceptions/ValidationException";
|
|
18
|
+
export { LimitException } from "./exceptions/LimitException";
|
|
18
19
|
export { RuntimeException } from "./exceptions/RuntimeException";
|
|
19
20
|
export { Observer } from "./processors/Observer";
|
|
21
|
+
export { StreamObserver } from "./processors/StreamObserver";
|
|
20
22
|
export { Validator } from "./processors/Validator";
|
|
21
23
|
export { Schema } from "./schema/Schema";
|
|
22
24
|
export { SchemaValidator } from "./schema/SchemaValidator";
|
package/out/all.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Anything that should be consumable by third parties (e.g. the VSCode extension)
|
|
4
4
|
// has to be re-exported from here.
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.DiscoveryError = exports.DiscoveryResult = exports.DiscoveryResolver = exports.MetaTemplateSchemaProvider = exports.TemplateSchemaProviderMemory = exports.TEMPLATE_NAMESPACE = exports.transformTemplateNodeToSchema = exports.toCanonicalJson = exports.toCanonicalTree = exports.Formatter = exports.IndentStyle = exports.NodeWriter = exports.UnifiedSchemaProvider = exports.transformNodeToSchema = exports.ChildDefinition = exports.NodeDefinition = exports.TypeRegistry = exports.SchemaProviderMeta = exports.SchemaProviderMemory = exports.SchemaValidator = exports.Schema = exports.RuntimeException = exports.ValidationException = exports.ParseException = exports.StringUtils = exports.parseLine = exports.SPEC_VERSION = exports.Constants = exports.Line = exports.ParseResult = exports.Parser = exports.TextNode = exports.InlineNode = exports.Node = void 0;
|
|
6
|
+
exports.DiscoveryError = exports.DiscoveryResult = exports.DiscoveryResolver = exports.MetaTemplateSchemaProvider = exports.TemplateSchemaProviderMemory = exports.TEMPLATE_NAMESPACE = exports.transformTemplateNodeToSchema = exports.toCanonicalJson = exports.toCanonicalTree = exports.Formatter = exports.IndentStyle = exports.NodeWriter = exports.UnifiedSchemaProvider = exports.transformNodeToSchema = exports.ChildDefinition = exports.NodeDefinition = exports.TypeRegistry = exports.SchemaProviderMeta = exports.SchemaProviderMemory = exports.SchemaValidator = exports.Schema = exports.RuntimeException = exports.LimitException = exports.ValidationException = exports.ParseException = exports.StringUtils = exports.parseLine = exports.SPEC_VERSION = exports.Constants = exports.Line = exports.ParseResult = exports.Parser = exports.TextNode = exports.InlineNode = exports.Node = void 0;
|
|
7
7
|
var Node_1 = require("./core/Node");
|
|
8
8
|
Object.defineProperty(exports, "Node", { enumerable: true, get: function () { return Node_1.Node; } });
|
|
9
9
|
var InlineNode_1 = require("./core/InlineNode");
|
|
@@ -33,6 +33,8 @@ var ParseException_1 = require("./exceptions/ParseException");
|
|
|
33
33
|
Object.defineProperty(exports, "ParseException", { enumerable: true, get: function () { return ParseException_1.ParseException; } });
|
|
34
34
|
var ValidationException_1 = require("./exceptions/ValidationException");
|
|
35
35
|
Object.defineProperty(exports, "ValidationException", { enumerable: true, get: function () { return ValidationException_1.ValidationException; } });
|
|
36
|
+
var LimitException_1 = require("./exceptions/LimitException");
|
|
37
|
+
Object.defineProperty(exports, "LimitException", { enumerable: true, get: function () { return LimitException_1.LimitException; } });
|
|
36
38
|
var RuntimeException_1 = require("./exceptions/RuntimeException");
|
|
37
39
|
Object.defineProperty(exports, "RuntimeException", { enumerable: true, get: function () { return RuntimeException_1.RuntimeException; } });
|
|
38
40
|
var Schema_1 = require("./schema/Schema");
|
package/out/core/Constants.d.ts
CHANGED
|
@@ -22,4 +22,19 @@ export declare class Constants {
|
|
|
22
22
|
static readonly SEP_TEXT_NODE: string;
|
|
23
23
|
/** Namespace of a node that declares none and inherits none. */
|
|
24
24
|
static readonly EMPTY_NAMESPACE: string;
|
|
25
|
+
/**
|
|
26
|
+
* Default maximum open nesting levels (STXT-SPEC §11.2); level 0 is the first. Configure it
|
|
27
|
+
* per parser with {@link ParserOptions.maxNesting}; -1 disables the limit.
|
|
28
|
+
*/
|
|
29
|
+
static readonly DEFAULT_MAX_NESTING: number;
|
|
30
|
+
/**
|
|
31
|
+
* Default maximum length of one input line, indentation included (STXT-SPEC §11.2).
|
|
32
|
+
* Configure it per parser with {@link ParserOptions.maxLineLength}; -1 disables the limit.
|
|
33
|
+
*/
|
|
34
|
+
static readonly DEFAULT_MAX_LINE_LENGTH: number;
|
|
35
|
+
/**
|
|
36
|
+
* Default maximum total input consumed (STXT-SPEC §11.2). Configure it per parser with
|
|
37
|
+
* {@link ParserOptions.maxInputSize}; -1 disables the limit.
|
|
38
|
+
*/
|
|
39
|
+
static readonly DEFAULT_MAX_INPUT_SIZE: number;
|
|
25
40
|
}
|
package/out/core/Constants.js
CHANGED
|
@@ -27,4 +27,19 @@ Constants.SEP_NODE = ":";
|
|
|
27
27
|
Constants.SEP_TEXT_NODE = ">>";
|
|
28
28
|
/** Namespace of a node that declares none and inherits none. */
|
|
29
29
|
Constants.EMPTY_NAMESPACE = "";
|
|
30
|
+
/**
|
|
31
|
+
* Default maximum open nesting levels (STXT-SPEC §11.2); level 0 is the first. Configure it
|
|
32
|
+
* per parser with {@link ParserOptions.maxNesting}; -1 disables the limit.
|
|
33
|
+
*/
|
|
34
|
+
Constants.DEFAULT_MAX_NESTING = 100;
|
|
35
|
+
/**
|
|
36
|
+
* Default maximum length of one input line, indentation included (STXT-SPEC §11.2).
|
|
37
|
+
* Configure it per parser with {@link ParserOptions.maxLineLength}; -1 disables the limit.
|
|
38
|
+
*/
|
|
39
|
+
Constants.DEFAULT_MAX_LINE_LENGTH = 10000;
|
|
40
|
+
/**
|
|
41
|
+
* Default maximum total input consumed (STXT-SPEC §11.2). Configure it per parser with
|
|
42
|
+
* {@link ParserOptions.maxInputSize}; -1 disables the limit.
|
|
43
|
+
*/
|
|
44
|
+
Constants.DEFAULT_MAX_INPUT_SIZE = 10000000;
|
|
30
45
|
//# sourceMappingURL=Constants.js.map
|
package/out/core/Parser.d.ts
CHANGED
|
@@ -1,21 +1,64 @@
|
|
|
1
1
|
import { Node } from "./Node";
|
|
2
2
|
import { Observer } from "../processors/Observer";
|
|
3
|
+
import { StreamObserver } from "../processors/StreamObserver";
|
|
3
4
|
import { Validator } from "../processors/Validator";
|
|
4
5
|
import { ParseResult } from "./ParseResult";
|
|
6
|
+
/**
|
|
7
|
+
* Parser limits (STXT-SPEC §11.2), configurable per {@link Parser}. Every limit defaults to
|
|
8
|
+
* the `DEFAULT_MAX_*` value of {@link Constants}, and -1 disables it. Lengths are measured in
|
|
9
|
+
* UTF-16 units (`string.length`); for ASCII content they equal characters.
|
|
10
|
+
*/
|
|
11
|
+
export interface ParserOptions {
|
|
12
|
+
/** Maximum open nesting levels; level 0 is the first. Default {@link Constants.DEFAULT_MAX_NESTING}; -1 disables. */
|
|
13
|
+
maxNesting?: number;
|
|
14
|
+
/** Maximum length of one input line, indentation included. Default {@link Constants.DEFAULT_MAX_LINE_LENGTH}; -1 disables. */
|
|
15
|
+
maxLineLength?: number;
|
|
16
|
+
/** Maximum total input consumed. Default {@link Constants.DEFAULT_MAX_INPUT_SIZE}; -1 disables. */
|
|
17
|
+
maxInputSize?: number;
|
|
18
|
+
}
|
|
5
19
|
/**
|
|
6
20
|
* Line-by-line STXT parsing engine. It knows nothing about schemas: semantic validation is
|
|
7
|
-
* plugged in through {@link Parser.registerValidator}
|
|
8
|
-
*
|
|
21
|
+
* plugged in through {@link Parser.registerValidator}, process observation through
|
|
22
|
+
* {@link Parser.registerObserver} and result observation through
|
|
23
|
+
* {@link Parser.registerStreamObserver}. See {@link UnifiedSchemaProvider} for the usual way
|
|
24
|
+
* of building the validators to register.
|
|
25
|
+
*
|
|
26
|
+
* Three entry points share one traversal: {@link Parser.parse} (fail-fast),
|
|
27
|
+
* {@link Parser.parseResult} (multi-error) and {@link Parser.parseStream} (line iterator in,
|
|
28
|
+
* nothing retained). Which callbacks fire never depends on the entry point, only on what is
|
|
29
|
+
* registered.
|
|
30
|
+
*
|
|
31
|
+
* The parser aborts on inputs that exceed its limits (STXT-SPEC §11.2), set to the
|
|
32
|
+
* `DEFAULT_MAX_*` values of {@link Constants} unless configured through {@link ParserOptions}.
|
|
33
|
+
* A limit error is a {@link LimitException} and is in every case the last one emitted: the
|
|
34
|
+
* nodes still open are not closed nor notified.
|
|
9
35
|
*/
|
|
10
36
|
export declare class Parser {
|
|
11
37
|
private observers;
|
|
38
|
+
private streamObservers;
|
|
12
39
|
private validators;
|
|
40
|
+
private readonly maxNesting;
|
|
41
|
+
private readonly maxLineLength;
|
|
42
|
+
private readonly maxInputSize;
|
|
43
|
+
/**
|
|
44
|
+
* Creates a parser, optionally with its own limits.
|
|
45
|
+
*
|
|
46
|
+
* @param options the {@link ParserOptions} limits; every omitted one takes its default.
|
|
47
|
+
*/
|
|
48
|
+
constructor(options?: ParserOptions);
|
|
13
49
|
/**
|
|
14
50
|
* Registers an observer, notified when each node is opened and closed.
|
|
15
51
|
*
|
|
16
52
|
* @param observer the {@link Observer} to register, notified while parsing.
|
|
17
53
|
*/
|
|
18
54
|
registerObserver(observer: Observer): void;
|
|
55
|
+
/**
|
|
56
|
+
* Registers a stream observer, notified with each completed root node and each error, in
|
|
57
|
+
* every mode.
|
|
58
|
+
*
|
|
59
|
+
* @param streamObserver the {@link StreamObserver} to register.
|
|
60
|
+
*/
|
|
61
|
+
registerStreamObserver(streamObserver: StreamObserver): void;
|
|
19
62
|
/**
|
|
20
63
|
* Registers a validator, invoked when each node is closed.
|
|
21
64
|
*
|
|
@@ -34,12 +77,29 @@ export declare class Parser {
|
|
|
34
77
|
parse(content: string): Node[];
|
|
35
78
|
/**
|
|
36
79
|
* Multi-error mode: parses the whole content collecting every error found (both syntax and
|
|
37
|
-
* validation) without bailing out on the first one
|
|
80
|
+
* validation) without bailing out on the first one — except a {@link LimitException}, which
|
|
81
|
+
* aborts and is in every case the last error collected. See {@link ParseResult}.
|
|
38
82
|
*
|
|
39
83
|
* @param content the whole STXT document to parse.
|
|
40
84
|
* @returns the collected result, with the root nodes obtained and every error found.
|
|
41
85
|
*/
|
|
42
86
|
parseResult(content: string): ParseResult;
|
|
87
|
+
/**
|
|
88
|
+
* Streaming mode: input from a line iterable (each item one line, without its line break —
|
|
89
|
+
* e.g. a generator over a file read lazily), and nothing retained: no nodes, no errors.
|
|
90
|
+
* Results reach the program only through the registered {@link StreamObserver}s (each
|
|
91
|
+
* completed root by `onRootNode()`, each error by `onError()`), so memory holds one root
|
|
92
|
+
* tree at a time. This is the entry point for files that do not fit in memory.
|
|
93
|
+
*
|
|
94
|
+
* @param lines the input, line by line.
|
|
95
|
+
*/
|
|
96
|
+
parseStream(lines: Iterable<string>): void;
|
|
97
|
+
/**
|
|
98
|
+
* Shared traversal. With a result, roots and errors are collected into it
|
|
99
|
+
* (parse/parseResult); with null, nothing is retained (parseStream). Either way every
|
|
100
|
+
* registered callback fires the same.
|
|
101
|
+
*/
|
|
102
|
+
private parseLines;
|
|
43
103
|
private processLine;
|
|
44
104
|
/**
|
|
45
105
|
* Records an error raised while parsing or validating a line. Typed exceptions travel as they
|
|
@@ -47,6 +107,11 @@ export declare class Parser {
|
|
|
47
107
|
* when it was raised by a validator, so that the subtype still tells the phase apart).
|
|
48
108
|
*/
|
|
49
109
|
private handleError;
|
|
110
|
+
/**
|
|
111
|
+
* Every error goes through here: collected into the result when there is one, and notified
|
|
112
|
+
* to the stream observers always, in order of appearance.
|
|
113
|
+
*/
|
|
114
|
+
private emitError;
|
|
50
115
|
private closeToLevel;
|
|
51
116
|
private removeUTF8BOM;
|
|
52
117
|
}
|
package/out/core/Parser.js
CHANGED
|
@@ -4,18 +4,41 @@ exports.Parser = void 0;
|
|
|
4
4
|
const TextNode_1 = require("./TextNode");
|
|
5
5
|
const LineParser_1 = require("./LineParser");
|
|
6
6
|
const NodeCreator_1 = require("./NodeCreator");
|
|
7
|
+
const Constants_1 = require("./Constants");
|
|
7
8
|
const ParseResult_1 = require("./ParseResult");
|
|
8
9
|
const ParseException_1 = require("../exceptions/ParseException");
|
|
9
10
|
const ValidationException_1 = require("../exceptions/ValidationException");
|
|
11
|
+
const LimitException_1 = require("../exceptions/LimitException");
|
|
10
12
|
/**
|
|
11
13
|
* Line-by-line STXT parsing engine. It knows nothing about schemas: semantic validation is
|
|
12
|
-
* plugged in through {@link Parser.registerValidator}
|
|
13
|
-
*
|
|
14
|
+
* plugged in through {@link Parser.registerValidator}, process observation through
|
|
15
|
+
* {@link Parser.registerObserver} and result observation through
|
|
16
|
+
* {@link Parser.registerStreamObserver}. See {@link UnifiedSchemaProvider} for the usual way
|
|
17
|
+
* of building the validators to register.
|
|
18
|
+
*
|
|
19
|
+
* Three entry points share one traversal: {@link Parser.parse} (fail-fast),
|
|
20
|
+
* {@link Parser.parseResult} (multi-error) and {@link Parser.parseStream} (line iterator in,
|
|
21
|
+
* nothing retained). Which callbacks fire never depends on the entry point, only on what is
|
|
22
|
+
* registered.
|
|
23
|
+
*
|
|
24
|
+
* The parser aborts on inputs that exceed its limits (STXT-SPEC §11.2), set to the
|
|
25
|
+
* `DEFAULT_MAX_*` values of {@link Constants} unless configured through {@link ParserOptions}.
|
|
26
|
+
* A limit error is a {@link LimitException} and is in every case the last one emitted: the
|
|
27
|
+
* nodes still open are not closed nor notified.
|
|
14
28
|
*/
|
|
15
29
|
class Parser {
|
|
16
|
-
|
|
30
|
+
/**
|
|
31
|
+
* Creates a parser, optionally with its own limits.
|
|
32
|
+
*
|
|
33
|
+
* @param options the {@link ParserOptions} limits; every omitted one takes its default.
|
|
34
|
+
*/
|
|
35
|
+
constructor(options) {
|
|
17
36
|
this.observers = [];
|
|
37
|
+
this.streamObservers = [];
|
|
18
38
|
this.validators = [];
|
|
39
|
+
this.maxNesting = options?.maxNesting ?? Constants_1.Constants.DEFAULT_MAX_NESTING;
|
|
40
|
+
this.maxLineLength = options?.maxLineLength ?? Constants_1.Constants.DEFAULT_MAX_LINE_LENGTH;
|
|
41
|
+
this.maxInputSize = options?.maxInputSize ?? Constants_1.Constants.DEFAULT_MAX_INPUT_SIZE;
|
|
19
42
|
}
|
|
20
43
|
/**
|
|
21
44
|
* Registers an observer, notified when each node is opened and closed.
|
|
@@ -25,6 +48,15 @@ class Parser {
|
|
|
25
48
|
registerObserver(observer) {
|
|
26
49
|
this.observers.push(observer);
|
|
27
50
|
}
|
|
51
|
+
/**
|
|
52
|
+
* Registers a stream observer, notified with each completed root node and each error, in
|
|
53
|
+
* every mode.
|
|
54
|
+
*
|
|
55
|
+
* @param streamObserver the {@link StreamObserver} to register.
|
|
56
|
+
*/
|
|
57
|
+
registerStreamObserver(streamObserver) {
|
|
58
|
+
this.streamObservers.push(streamObserver);
|
|
59
|
+
}
|
|
28
60
|
/**
|
|
29
61
|
* Registers a validator, invoked when each node is closed.
|
|
30
62
|
*
|
|
@@ -52,37 +84,76 @@ class Parser {
|
|
|
52
84
|
}
|
|
53
85
|
/**
|
|
54
86
|
* Multi-error mode: parses the whole content collecting every error found (both syntax and
|
|
55
|
-
* validation) without bailing out on the first one
|
|
87
|
+
* validation) without bailing out on the first one — except a {@link LimitException}, which
|
|
88
|
+
* aborts and is in every case the last error collected. See {@link ParseResult}.
|
|
56
89
|
*
|
|
57
90
|
* @param content the whole STXT document to parse.
|
|
58
91
|
* @returns the collected result, with the root nodes obtained and every error found.
|
|
59
92
|
*/
|
|
60
93
|
parseResult(content) {
|
|
61
|
-
content = this.removeUTF8BOM(content);
|
|
62
94
|
const result = new ParseResult_1.ParseResult();
|
|
63
|
-
const stack = [];
|
|
64
|
-
const documents = [];
|
|
65
|
-
let lineNumber = 0;
|
|
66
95
|
const lines = content.split(/\r?\n/);
|
|
67
96
|
// The final line break terminates the last line, it is not an extra empty line
|
|
68
97
|
// (this avoids adding a spurious line to a >> block at EOF, spec 10.3)
|
|
69
98
|
if (lines.length > 0 && lines[lines.length - 1] === "") {
|
|
70
99
|
lines.pop();
|
|
71
100
|
}
|
|
72
|
-
|
|
101
|
+
this.parseLines(lines, result);
|
|
102
|
+
return result;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Streaming mode: input from a line iterable (each item one line, without its line break —
|
|
106
|
+
* e.g. a generator over a file read lazily), and nothing retained: no nodes, no errors.
|
|
107
|
+
* Results reach the program only through the registered {@link StreamObserver}s (each
|
|
108
|
+
* completed root by `onRootNode()`, each error by `onError()`), so memory holds one root
|
|
109
|
+
* tree at a time. This is the entry point for files that do not fit in memory.
|
|
110
|
+
*
|
|
111
|
+
* @param lines the input, line by line.
|
|
112
|
+
*/
|
|
113
|
+
parseStream(lines) {
|
|
114
|
+
this.parseLines(lines, null);
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Shared traversal. With a result, roots and errors are collected into it
|
|
118
|
+
* (parse/parseResult); with null, nothing is retained (parseStream). Either way every
|
|
119
|
+
* registered callback fires the same.
|
|
120
|
+
*/
|
|
121
|
+
parseLines(lines, result) {
|
|
122
|
+
const stack = [];
|
|
123
|
+
let lineNumber = 0;
|
|
124
|
+
let consumed = 0;
|
|
125
|
+
for (let line of lines) {
|
|
73
126
|
lineNumber++;
|
|
74
|
-
|
|
127
|
+
// A UTF-8 BOM only means anything at the very start of the input (spec 3)
|
|
128
|
+
if (lineNumber === 1) {
|
|
129
|
+
line = this.removeUTF8BOM(line);
|
|
130
|
+
}
|
|
131
|
+
// Limits first (spec 11.2): a limit error aborts, leaving the open nodes
|
|
132
|
+
// unclosed and unnotified.
|
|
133
|
+
if (this.maxLineLength !== -1 && line.length > this.maxLineLength) {
|
|
134
|
+
this.emitError(new LimitException_1.LimitException(lineNumber, "LIMIT_LINE_LENGTH_EXCEEDED", `Line longer than ${this.maxLineLength} characters`), result);
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
consumed += line.length + 1; // the line separator counts as one
|
|
138
|
+
if (this.maxInputSize !== -1 && consumed > this.maxInputSize) {
|
|
139
|
+
this.emitError(new LimitException_1.LimitException(lineNumber, "LIMIT_INPUT_SIZE_EXCEEDED", `Input larger than ${this.maxInputSize} characters`), result);
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
try {
|
|
143
|
+
this.processLine(line, lineNumber, stack, result);
|
|
144
|
+
}
|
|
145
|
+
catch (e) {
|
|
146
|
+
if (e instanceof LimitException_1.LimitException) {
|
|
147
|
+
this.emitError(e, result);
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
throw e;
|
|
151
|
+
}
|
|
75
152
|
}
|
|
76
153
|
// Close every node still open at EOF
|
|
77
154
|
this.closeToLevel(stack, 0, result);
|
|
78
|
-
// Add the nodes to the result
|
|
79
|
-
for (const doc of documents) {
|
|
80
|
-
result.addNode(doc);
|
|
81
|
-
}
|
|
82
|
-
// Return the result
|
|
83
|
-
return result;
|
|
84
155
|
}
|
|
85
|
-
processLine(lineString, lineNumber, stack,
|
|
156
|
+
processLine(lineString, lineNumber, stack, result) {
|
|
86
157
|
try {
|
|
87
158
|
const lastNode = stack.length === 0 ? null : stack[stack.length - 1];
|
|
88
159
|
// The stack holds the open nodes, one per level: its size is the level of the next line's parent
|
|
@@ -123,18 +194,22 @@ class Parser {
|
|
|
123
194
|
if (line.isEmpty()) {
|
|
124
195
|
return;
|
|
125
196
|
}
|
|
197
|
+
// Nesting limit (spec 11.2): only a node line can open a new level. Comment and
|
|
198
|
+
// block text lines returned above; with the consecutive-level rule this triggers
|
|
199
|
+
// exactly when the first node at level maxNesting opens.
|
|
200
|
+
if (this.maxNesting !== -1 && currentLevel >= this.maxNesting) {
|
|
201
|
+
throw new LimitException_1.LimitException(lineNumber, "LIMIT_NESTING_EXCEEDED", `Nesting deeper than ${this.maxNesting} levels`);
|
|
202
|
+
}
|
|
126
203
|
// Close the nodes down to the current level (this "finishes" them: validators and observers run)
|
|
127
204
|
this.closeToLevel(stack, currentLevel, result);
|
|
128
|
-
// Create the new node, attach it to its parent (or
|
|
129
|
-
// and leave it "open" on the stack. Attaching links both ends: the node already
|
|
130
|
-
// its parent, and so its effective namespace and its level, when the observers
|
|
131
|
-
// The parent is always an InlineNode: a TextNode on top of the stack only
|
|
205
|
+
// Create the new node, attach it to its parent (or keep it as a root if the stack is
|
|
206
|
+
// empty) and leave it "open" on the stack. Attaching links both ends: the node already
|
|
207
|
+
// knows its parent, and so its effective namespace and its level, when the observers
|
|
208
|
+
// see it. The parent is always an InlineNode: a TextNode on top of the stack only
|
|
209
|
+
// takes text lines.
|
|
132
210
|
const parent = stack.length === 0 ? null : stack[stack.length - 1];
|
|
133
211
|
const node = (0, NodeCreator_1.createNode)(line, lineNumber);
|
|
134
|
-
if (parent
|
|
135
|
-
documents.push(node);
|
|
136
|
-
}
|
|
137
|
-
else {
|
|
212
|
+
if (parent !== null) {
|
|
138
213
|
parent.addChild(node);
|
|
139
214
|
}
|
|
140
215
|
// Hand it over to the observers
|
|
@@ -145,6 +220,9 @@ class Parser {
|
|
|
145
220
|
stack.push(node);
|
|
146
221
|
}
|
|
147
222
|
catch (e) {
|
|
223
|
+
if (e instanceof LimitException_1.LimitException) {
|
|
224
|
+
throw e;
|
|
225
|
+
}
|
|
148
226
|
this.handleError(e, lineNumber, result);
|
|
149
227
|
}
|
|
150
228
|
}
|
|
@@ -155,15 +233,27 @@ class Parser {
|
|
|
155
233
|
*/
|
|
156
234
|
handleError(e, line, result, validating = false) {
|
|
157
235
|
if (e instanceof ParseException_1.ParseException) {
|
|
158
|
-
|
|
236
|
+
this.emitError(e, result);
|
|
159
237
|
}
|
|
160
238
|
else {
|
|
161
239
|
const message = e instanceof Error ? e.message : String(e);
|
|
162
|
-
|
|
240
|
+
this.emitError(validating
|
|
163
241
|
? new ValidationException_1.ValidationException(line, "UNEXPECTED_ERROR", message)
|
|
164
|
-
: new ParseException_1.ParseException(line, "UNEXPECTED_ERROR", message));
|
|
242
|
+
: new ParseException_1.ParseException(line, "UNEXPECTED_ERROR", message), result);
|
|
165
243
|
}
|
|
166
244
|
}
|
|
245
|
+
/**
|
|
246
|
+
* Every error goes through here: collected into the result when there is one, and notified
|
|
247
|
+
* to the stream observers always, in order of appearance.
|
|
248
|
+
*/
|
|
249
|
+
emitError(error, result) {
|
|
250
|
+
if (result !== null) {
|
|
251
|
+
result.addError(error);
|
|
252
|
+
}
|
|
253
|
+
this.streamObservers.forEach(streamObserver => {
|
|
254
|
+
streamObserver.onError(error);
|
|
255
|
+
});
|
|
256
|
+
}
|
|
167
257
|
closeToLevel(stack, targetLevel, result) {
|
|
168
258
|
while (stack.length > targetLevel) {
|
|
169
259
|
const completed = stack.pop();
|
|
@@ -172,7 +262,7 @@ class Parser {
|
|
|
172
262
|
try {
|
|
173
263
|
const errors = validator.validate(completed);
|
|
174
264
|
errors.forEach(error => {
|
|
175
|
-
|
|
265
|
+
this.emitError(error, result);
|
|
176
266
|
});
|
|
177
267
|
}
|
|
178
268
|
catch (e) {
|
|
@@ -183,6 +273,15 @@ class Parser {
|
|
|
183
273
|
this.observers.forEach(observer => {
|
|
184
274
|
observer.onFinish(completed);
|
|
185
275
|
});
|
|
276
|
+
// A closed root: the stream observers receive it, the result collects it
|
|
277
|
+
if (stack.length === 0) {
|
|
278
|
+
this.streamObservers.forEach(streamObserver => {
|
|
279
|
+
streamObserver.onRootNode(completed);
|
|
280
|
+
});
|
|
281
|
+
if (result !== null) {
|
|
282
|
+
result.addNode(completed);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
186
285
|
}
|
|
187
286
|
}
|
|
188
287
|
removeUTF8BOM(content) {
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { ParseException } from "./ParseException";
|
|
2
|
+
/**
|
|
3
|
+
* A parser limit exceeded (STXT-SPEC §11.2): nesting depth, line length or input size. Unlike
|
|
4
|
+
* any other parse error it aborts the parse: it is emitted and no further input is processed,
|
|
5
|
+
* in every mode, so it is always the last error. Exceeding a limit does not make the document
|
|
6
|
+
* invalid: the same document may parse under higher limits (see {@link ParserOptions}).
|
|
7
|
+
*/
|
|
8
|
+
export declare class LimitException extends ParseException {
|
|
9
|
+
/**
|
|
10
|
+
* Creates a limit error located at a line of the document.
|
|
11
|
+
*
|
|
12
|
+
* @param line line number where the limit was exceeded.
|
|
13
|
+
* @param code error code in UPPERCASE (`LIMIT_*`).
|
|
14
|
+
* @param message descriptive message.
|
|
15
|
+
*/
|
|
16
|
+
constructor(line: number, code: string, message: string);
|
|
17
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LimitException = void 0;
|
|
4
|
+
const ParseException_1 = require("./ParseException");
|
|
5
|
+
/**
|
|
6
|
+
* A parser limit exceeded (STXT-SPEC §11.2): nesting depth, line length or input size. Unlike
|
|
7
|
+
* any other parse error it aborts the parse: it is emitted and no further input is processed,
|
|
8
|
+
* in every mode, so it is always the last error. Exceeding a limit does not make the document
|
|
9
|
+
* invalid: the same document may parse under higher limits (see {@link ParserOptions}).
|
|
10
|
+
*/
|
|
11
|
+
class LimitException extends ParseException_1.ParseException {
|
|
12
|
+
/**
|
|
13
|
+
* Creates a limit error located at a line of the document.
|
|
14
|
+
*
|
|
15
|
+
* @param line line number where the limit was exceeded.
|
|
16
|
+
* @param code error code in UPPERCASE (`LIMIT_*`).
|
|
17
|
+
* @param message descriptive message.
|
|
18
|
+
*/
|
|
19
|
+
constructor(line, code, message) {
|
|
20
|
+
super(line, code, message);
|
|
21
|
+
this.name = "LimitException";
|
|
22
|
+
Object.setPrototypeOf(this, LimitException.prototype);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
exports.LimitException = LimitException;
|
|
26
|
+
//# sourceMappingURL=LimitException.js.map
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Node } from "../core/Node";
|
|
2
|
+
import { ParseException } from "../exceptions/ParseException";
|
|
3
|
+
/**
|
|
4
|
+
* Process hook notified by the {@link Parser} with the stream of results a parse emits: each
|
|
5
|
+
* completed root node, and every error. It complements {@link Observer}, which watches the
|
|
6
|
+
* process line by line: a StreamObserver only sees finished roots and errors, so a consumer
|
|
7
|
+
* that processes a document root by root never has to ask a node for its level. Register it
|
|
8
|
+
* with {@link Parser.registerStreamObserver}; a class may implement {@link Observer},
|
|
9
|
+
* StreamObserver or both.
|
|
10
|
+
*
|
|
11
|
+
* It fires in every entry point — `parse()`, `parseResult()` and `parseStream()` — exactly the
|
|
12
|
+
* same way; what `parseStream()` adds is that the parser retains nothing, so there these
|
|
13
|
+
* callbacks are the only way to get the results.
|
|
14
|
+
*/
|
|
15
|
+
export interface StreamObserver {
|
|
16
|
+
/**
|
|
17
|
+
* Called when a root (level 0) node is closed, with its whole subtree already complete —
|
|
18
|
+
* children, values, text lines — and its validators already run. In
|
|
19
|
+
* {@link Parser.parseStream} the parser releases the node right after this call, so the
|
|
20
|
+
* memory in use is one root tree at a time.
|
|
21
|
+
*
|
|
22
|
+
* @param node the completed root node. Do not modify it.
|
|
23
|
+
*/
|
|
24
|
+
onRootNode(node: Node): void;
|
|
25
|
+
/**
|
|
26
|
+
* Called for every error found (syntax or validation), in order of appearance. Parsing
|
|
27
|
+
* continues with the next line, except for `LIMIT_*` errors ({@link LimitException},
|
|
28
|
+
* STXT-SPEC §11.2), which abort the parse right after this call. In fail-fast
|
|
29
|
+
* {@link Parser.parse} the observer still sees every error before the first one is thrown,
|
|
30
|
+
* because `parse()` reuses the `parseResult()` traversal.
|
|
31
|
+
*
|
|
32
|
+
* @param error the error found.
|
|
33
|
+
*/
|
|
34
|
+
onError(error: ParseException): void;
|
|
35
|
+
}
|
|
@@ -30,7 +30,7 @@ export declare class NodeWriter {
|
|
|
30
30
|
* STXT-TREE-SPEC 11.1.
|
|
31
31
|
*
|
|
32
32
|
* @param parentNs effective namespace of the parent, "" for a root: the namespace is
|
|
33
|
-
*
|
|
33
|
+
* written only where it changes (rule 3), regardless of where the source declared it.
|
|
34
34
|
*/
|
|
35
35
|
private static writeNode;
|
|
36
36
|
private static indent;
|
|
@@ -48,7 +48,7 @@ class NodeWriter {
|
|
|
48
48
|
* STXT-TREE-SPEC 11.1.
|
|
49
49
|
*
|
|
50
50
|
* @param parentNs effective namespace of the parent, "" for a root: the namespace is
|
|
51
|
-
*
|
|
51
|
+
* written only where it changes (rule 3), regardless of where the source declared it.
|
|
52
52
|
*/
|
|
53
53
|
static writeNode(out, n, depth, style, parentNs) {
|
|
54
54
|
NodeWriter.indent(out, depth, style);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@stxt-lang/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Parser and schema validator for STXT, an indentation-based structured-text format.",
|
|
5
5
|
"main": "out/all.js",
|
|
6
6
|
"types": "out/all.d.ts",
|
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
"!out/**/*.js.map"
|
|
21
21
|
],
|
|
22
22
|
"scripts": {
|
|
23
|
-
"
|
|
23
|
+
"clean": "node -e \"require('node:fs').rmSync('out', { recursive: true, force: true })\"",
|
|
24
|
+
"build": "npm run clean && tsc",
|
|
24
25
|
"watch": "tsc --watch",
|
|
25
26
|
"lint": "eslint src --ext .ts",
|
|
26
27
|
"prepare": "npm run build",
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { Node } from "../core/Node";
|
|
2
|
-
import { Validator } from "../processors/Validator";
|
|
3
|
-
import { SchemaValidator } from "../schema/SchemaValidator";
|
|
4
|
-
import { ValidationException } from "../exceptions/ValidationException";
|
|
5
|
-
/**
|
|
6
|
-
* Wrapper around a {@link SchemaValidator} that only validates namespaced nodes, so that a
|
|
7
|
-
* document mixing schema-bound and free nodes does not report the free ones as unknown.
|
|
8
|
-
*
|
|
9
|
-
* @deprecated since 0.8.0: {@link SchemaValidator} applies this rule itself (STXT-SCHEMA-SPEC 5,
|
|
10
|
-
* the empty namespace is never validated), so the wrapper adds nothing. Register the
|
|
11
|
-
* `SchemaValidator` directly. Kept for compatibility; to be removed in 1.0.
|
|
12
|
-
*/
|
|
13
|
-
export declare class ConditionalValidator implements Validator {
|
|
14
|
-
private readonly schemaValidator;
|
|
15
|
-
/**
|
|
16
|
-
* Creates a validator that delegates to a schema validator.
|
|
17
|
-
*
|
|
18
|
-
* @param schemaValidator validator the namespaced nodes are handed over to.
|
|
19
|
-
*/
|
|
20
|
-
constructor(schemaValidator: SchemaValidator);
|
|
21
|
-
/**
|
|
22
|
-
* Validates a node when it has a namespace, and lets it through otherwise.
|
|
23
|
-
*
|
|
24
|
-
* @param node already closed node to validate.
|
|
25
|
-
* @returns the validation errors found, or an empty array if the node is valid or has no namespace.
|
|
26
|
-
*/
|
|
27
|
-
validate(node: Node): ValidationException[];
|
|
28
|
-
}
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ConditionalValidator = void 0;
|
|
4
|
-
/**
|
|
5
|
-
* Wrapper around a {@link SchemaValidator} that only validates namespaced nodes, so that a
|
|
6
|
-
* document mixing schema-bound and free nodes does not report the free ones as unknown.
|
|
7
|
-
*
|
|
8
|
-
* @deprecated since 0.8.0: {@link SchemaValidator} applies this rule itself (STXT-SCHEMA-SPEC 5,
|
|
9
|
-
* the empty namespace is never validated), so the wrapper adds nothing. Register the
|
|
10
|
-
* `SchemaValidator` directly. Kept for compatibility; to be removed in 1.0.
|
|
11
|
-
*/
|
|
12
|
-
class ConditionalValidator {
|
|
13
|
-
/**
|
|
14
|
-
* Creates a validator that delegates to a schema validator.
|
|
15
|
-
*
|
|
16
|
-
* @param schemaValidator validator the namespaced nodes are handed over to.
|
|
17
|
-
*/
|
|
18
|
-
constructor(schemaValidator) {
|
|
19
|
-
this.schemaValidator = schemaValidator;
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* Validates a node when it has a namespace, and lets it through otherwise.
|
|
23
|
-
*
|
|
24
|
-
* @param node already closed node to validate.
|
|
25
|
-
* @returns the validation errors found, or an empty array if the node is valid or has no namespace.
|
|
26
|
-
*/
|
|
27
|
-
validate(node) {
|
|
28
|
-
// Only validate the node when it has a namespace
|
|
29
|
-
if (node.getNamespace() !== "") {
|
|
30
|
-
return this.schemaValidator.validate(node);
|
|
31
|
-
}
|
|
32
|
-
return [];
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
exports.ConditionalValidator = ConditionalValidator;
|
|
36
|
-
//# sourceMappingURL=ConditionalValidator.js.map
|