grammar-well 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/generator/artifacts/basic.d.ts +1 -1
- package/build/generator/artifacts/basic.js.map +1 -1
- package/build/generator/artifacts/lexer.d.ts +2 -2
- package/build/generator/artifacts/lexer.js +1 -1
- package/build/generator/artifacts/lexer.js.map +1 -1
- package/build/generator/artifacts/lr.d.ts +5 -5
- package/build/generator/artifacts/lr.js +2 -2
- package/build/generator/artifacts/lr.js.map +1 -1
- package/build/generator/generator.d.ts +3 -3
- package/build/generator/generator.js +3 -3
- package/build/generator/generator.js.map +1 -1
- package/build/generator/grammars/index.d.ts +2 -2
- package/build/generator/grammars/index.js +2 -2
- package/build/generator/import-resolvers/auto.d.ts +1 -1
- package/build/generator/import-resolvers/browser.d.ts +1 -1
- package/build/generator/import-resolvers/browser.js.map +1 -1
- package/build/generator/import-resolvers/dictionary.d.ts +1 -1
- package/build/generator/import-resolvers/dictionary.js.map +1 -1
- package/build/generator/import-resolvers/filesystem.d.ts +1 -1
- package/build/generator/index.d.ts +3 -3
- package/build/generator/index.js +3 -3
- package/build/generator/state.d.ts +1 -1
- package/build/generator/stringify/common.d.ts +2 -2
- package/build/generator/stringify/common.js.map +1 -1
- package/build/generator/stringify/exports/javascript.d.ts +1 -1
- package/build/generator/stringify/exports/json.d.ts +1 -1
- package/build/generator/stringify/exports/registry.d.ts +6 -6
- package/build/generator/stringify/exports/typescript.d.ts +1 -1
- package/build/generator/stringify/grammar/v2.d.ts +1 -1
- package/build/generator/stringify/javascript.d.ts +2 -2
- package/build/generator/stringify/javascript.js.map +1 -1
- package/build/index.d.ts +7 -7
- package/build/index.js +7 -7
- package/build/lexers/character-lexer.d.ts +1 -1
- package/build/lexers/stateful-lexer.d.ts +1 -1
- package/build/lexers/token-buffer.d.ts +3 -3
- package/build/lexers/token-buffer.js +2 -2
- package/build/lexers/token-buffer.js.map +1 -1
- package/build/parser/algorithms/cyk.d.ts +2 -2
- package/build/parser/algorithms/earley.d.ts +15 -12
- package/build/parser/algorithms/earley.js +29 -17
- package/build/parser/algorithms/earley.js.map +1 -1
- package/build/parser/algorithms/lrk/algorithm.d.ts +2 -2
- package/build/parser/algorithms/lrk/algorithm.js +31 -21
- package/build/parser/algorithms/lrk/algorithm.js.map +1 -1
- package/build/parser/algorithms/lrk/canonical-collection.d.ts +10 -10
- package/build/parser/algorithms/lrk/canonical-collection.js +73 -49
- package/build/parser/algorithms/lrk/canonical-collection.js.map +1 -1
- package/build/parser/algorithms/lrk/closure.d.ts +1 -1
- package/build/parser/algorithms/lrk/closure.js.map +1 -1
- package/build/parser/algorithms/lrk/stack.d.ts +8 -18
- package/build/parser/algorithms/lrk/stack.js +11 -24
- package/build/parser/algorithms/lrk/stack.js.map +1 -1
- package/build/parser/algorithms/lrk/state.d.ts +1 -1
- package/build/parser/algorithms/lrk/typings.d.ts +16 -0
- package/build/parser/algorithms/lrk/typings.js +2 -0
- package/build/parser/algorithms/lrk/typings.js.map +1 -0
- package/build/parser/parse.d.ts +1 -1
- package/build/typings/ast.d.ts +1 -1
- package/build/typings/generator.d.ts +4 -4
- package/build/typings/index.d.ts +8 -8
- package/build/typings/index.js +4 -4
- package/build/typings/runtime.d.ts +3 -3
- package/build/utility/format.js +3 -3
- package/build/utility/general.d.ts +2 -2
- package/build/utility/general.js +1 -1
- package/build/utility/general.js.map +1 -1
- package/build/utility/index.d.ts +4 -4
- package/build/utility/index.js +4 -4
- package/build/utility/lint.d.ts +1 -1
- package/build/utility/monarch.d.ts +1 -1
- package/build/utility/parsing.d.ts +2 -2
- package/build/utility/text-format.d.ts +2 -2
- package/build/version.json +1 -1
- package/package.json +8 -9
- package/src/generator/artifacts/basic.ts +6 -4
- package/src/generator/artifacts/lexer.ts +6 -4
- package/src/generator/artifacts/lr.ts +10 -7
- package/src/generator/generator.ts +17 -11
- package/src/generator/grammars/index.ts +2 -2
- package/src/generator/grammars/v1.ts +1 -1
- package/src/generator/grammars/v2.ts +1 -1
- package/src/generator/import-resolvers/auto.ts +3 -3
- package/src/generator/import-resolvers/browser.ts +5 -2
- package/src/generator/import-resolvers/dictionary.ts +5 -2
- package/src/generator/import-resolvers/filesystem.ts +1 -1
- package/src/generator/index.ts +3 -3
- package/src/generator/state.ts +1 -1
- package/src/generator/stringify/common.ts +6 -3
- package/src/generator/stringify/exports/javascript.ts +1 -1
- package/src/generator/stringify/exports/json.ts +1 -1
- package/src/generator/stringify/exports/registry.ts +4 -4
- package/src/generator/stringify/exports/typescript.ts +1 -1
- package/src/generator/stringify/grammar/v2.ts +1 -1
- package/src/generator/stringify/javascript.ts +12 -8
- package/src/index.ts +7 -7
- package/src/lexers/character-lexer.ts +1 -1
- package/src/lexers/stateful-lexer.ts +1 -1
- package/src/lexers/token-buffer.ts +16 -3
- package/src/parser/algorithms/cyk.ts +4 -4
- package/src/parser/algorithms/earley.ts +56 -23
- package/src/parser/algorithms/lrk/algorithm.ts +40 -25
- package/src/parser/algorithms/lrk/canonical-collection.ts +84 -55
- package/src/parser/algorithms/lrk/stack.ts +12 -37
- package/src/parser/algorithms/lrk/typings.ts +13 -0
- package/src/parser/parse.ts +8 -8
- package/src/typings/ast.ts +1 -1
- package/src/typings/generator.ts +4 -4
- package/src/typings/index.ts +8 -8
- package/src/typings/runtime.ts +3 -3
- package/src/utility/format.ts +4 -4
- package/src/utility/general.ts +4 -3
- package/src/utility/index.ts +4 -4
- package/src/utility/lint.ts +1 -1
- package/src/utility/monarch.ts +1 -1
- package/src/utility/parsing.ts +1 -1
- package/src/utility/text-format.ts +2 -2
- package/src/version.json +1 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/src/parser/algorithms/lrk/closure.ts +0 -37
- package/src/parser/algorithms/lrk/state.ts +0 -10
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RuntimeLexer, RuntimeLexerToken, TQRestorePoint } from '../typings/index.
|
|
1
|
+
import type { RuntimeLexer, RuntimeLexerToken, TQRestorePoint } from '../typings/index.ts';
|
|
2
2
|
|
|
3
3
|
export class TokenBuffer {
|
|
4
4
|
private history: RuntimeLexerToken[] = [];
|
|
@@ -6,6 +6,9 @@ export class TokenBuffer {
|
|
|
6
6
|
|
|
7
7
|
private $historyIndex = -1;
|
|
8
8
|
|
|
9
|
+
public lexer: RuntimeLexer;
|
|
10
|
+
private tokenProcessor?: (token: RuntimeLexerToken) => RuntimeLexerToken;
|
|
11
|
+
|
|
9
12
|
get offset() { return this.active?.offset || 0 }
|
|
10
13
|
get line() { return this.active?.line || 0 }
|
|
11
14
|
get column() { return this.active?.column || 0; }
|
|
@@ -15,7 +18,13 @@ export class TokenBuffer {
|
|
|
15
18
|
return { historyIndex: this.$historyIndex, offset: this.offset };
|
|
16
19
|
}
|
|
17
20
|
|
|
18
|
-
constructor(
|
|
21
|
+
constructor(
|
|
22
|
+
lexer: RuntimeLexer,
|
|
23
|
+
tokenProcessor?: (token: RuntimeLexerToken) => RuntimeLexerToken
|
|
24
|
+
) {
|
|
25
|
+
this.lexer = lexer;
|
|
26
|
+
this.tokenProcessor = tokenProcessor;
|
|
27
|
+
}
|
|
19
28
|
|
|
20
29
|
reset(buffer: string) {
|
|
21
30
|
this.lexer.feed(buffer);
|
|
@@ -92,7 +101,11 @@ export class TokenBuffer {
|
|
|
92
101
|
}
|
|
93
102
|
|
|
94
103
|
class TokenIterator {
|
|
95
|
-
|
|
104
|
+
private buffer: TokenBuffer;
|
|
105
|
+
|
|
106
|
+
constructor(buffer: TokenBuffer) {
|
|
107
|
+
this.buffer = buffer;
|
|
108
|
+
}
|
|
96
109
|
|
|
97
110
|
next() {
|
|
98
111
|
const token = this.buffer.next()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol, RuntimeParserClass, RuntimeLexerToken } from "../../typings/index.
|
|
2
|
-
import { TokenBuffer } from "../../lexers/token-buffer.
|
|
3
|
-
import { Matrix } from "../../utility/general.
|
|
4
|
-
import { ParserUtility } from "../../utility/parsing.
|
|
1
|
+
import type { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol, RuntimeParserClass, RuntimeLexerToken } from "../../typings/index.ts";
|
|
2
|
+
import { TokenBuffer } from "../../lexers/token-buffer.ts";
|
|
3
|
+
import { Matrix } from "../../utility/general.ts";
|
|
4
|
+
import { ParserUtility } from "../../utility/parsing.ts";
|
|
5
5
|
|
|
6
6
|
export function CYK(language: RuntimeParserClass & { tokens: TokenBuffer }, _options = {}) {
|
|
7
7
|
const { grammar } = language.artifacts;
|
|
@@ -1,16 +1,18 @@
|
|
|
1
|
-
import { Dictionary, RuntimeGrammarProductionRule, RuntimeLexerToken, RuntimeParserClass } from "../../typings/index.
|
|
2
|
-
import { TokenBuffer } from "../../lexers/token-buffer.
|
|
3
|
-
import { TextFormatter } from "../../utility/text-format.
|
|
4
|
-
import { ParserUtility } from "../../utility/parsing.
|
|
1
|
+
import type { Dictionary, RuntimeGrammarProductionRule, RuntimeLexerToken, RuntimeParserClass } from "../../typings/index.ts";
|
|
2
|
+
import { TokenBuffer } from "../../lexers/token-buffer.ts";
|
|
3
|
+
import { TextFormatter } from "../../utility/text-format.ts";
|
|
4
|
+
import { ParserUtility } from "../../utility/parsing.ts";
|
|
5
5
|
|
|
6
6
|
export interface EarleyParserOptions {
|
|
7
7
|
keepHistory?: boolean;
|
|
8
|
+
postProcessing?: 'eager' | 'lazy';
|
|
8
9
|
}
|
|
9
10
|
|
|
10
11
|
export function Earley(language: RuntimeParserClass & { tokens: TokenBuffer }, options: EarleyParserOptions = {}) {
|
|
11
12
|
const { tokens } = language;
|
|
12
13
|
const { rules, start } = language.artifacts.grammar;
|
|
13
|
-
const
|
|
14
|
+
const StateClass = options.postProcessing === 'eager' ? EagerState : LazyState;
|
|
15
|
+
const column = new Column(rules, 0, StateClass);
|
|
14
16
|
const table: Column[] = [column];
|
|
15
17
|
column.wants[start] = [];
|
|
16
18
|
column.predict(start);
|
|
@@ -27,7 +29,7 @@ export function Earley(language: RuntimeParserClass & { tokens: TokenBuffer }, o
|
|
|
27
29
|
|
|
28
30
|
current++;
|
|
29
31
|
|
|
30
|
-
const nextColumn = new Column(rules, current);
|
|
32
|
+
const nextColumn = new Column(rules, current, StateClass);
|
|
31
33
|
table.push(nextColumn);
|
|
32
34
|
|
|
33
35
|
const literal = token.value;
|
|
@@ -58,9 +60,12 @@ export function Earley(language: RuntimeParserClass & { tokens: TokenBuffer }, o
|
|
|
58
60
|
results.push(data);
|
|
59
61
|
}
|
|
60
62
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
63
|
+
|
|
64
|
+
if (StateClass == LazyState) {
|
|
65
|
+
const clone = results.length > 1;
|
|
66
|
+
for (let i = 0; i < results.length; i++) {
|
|
67
|
+
results[i] = PostProcess(results[i], clone);
|
|
68
|
+
}
|
|
64
69
|
}
|
|
65
70
|
return { results, info: { table } };
|
|
66
71
|
}
|
|
@@ -81,11 +86,19 @@ class Column {
|
|
|
81
86
|
wants: Dictionary<State[]> = Object.create(null);// states indexed by the non-terminal they expect
|
|
82
87
|
scannable: State[] = [];// list of states that expect a token
|
|
83
88
|
completed: Dictionary<State[]> = Object.create(null); // states that are nullable
|
|
89
|
+
private rules: Dictionary<RuntimeGrammarProductionRule[]>;
|
|
90
|
+
public index: number;
|
|
91
|
+
private StateClass: Concrete<typeof State>;
|
|
84
92
|
|
|
85
93
|
constructor(
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
94
|
+
rules: Dictionary<RuntimeGrammarProductionRule[]>,
|
|
95
|
+
index: number,
|
|
96
|
+
StateClass: Concrete<typeof State>
|
|
97
|
+
) {
|
|
98
|
+
this.rules = rules;
|
|
99
|
+
this.index = index;
|
|
100
|
+
this.StateClass = StateClass;
|
|
101
|
+
}
|
|
89
102
|
|
|
90
103
|
|
|
91
104
|
process() {
|
|
@@ -136,7 +149,7 @@ class Column {
|
|
|
136
149
|
return;
|
|
137
150
|
|
|
138
151
|
for (const rule of this.rules[exp]) {
|
|
139
|
-
this.states.push(new
|
|
152
|
+
this.states.push(new this.StateClass(rule, 0, this.index, this.wants[exp]));
|
|
140
153
|
}
|
|
141
154
|
}
|
|
142
155
|
|
|
@@ -156,22 +169,30 @@ class Column {
|
|
|
156
169
|
}
|
|
157
170
|
}
|
|
158
171
|
|
|
159
|
-
class State {
|
|
172
|
+
abstract class State {
|
|
160
173
|
isComplete: boolean;
|
|
161
174
|
data: any = [];
|
|
162
175
|
left: State;
|
|
163
176
|
right: State | StateToken;
|
|
177
|
+
public rule: RuntimeGrammarProductionRule;
|
|
178
|
+
public dot: number;
|
|
179
|
+
public reference: number;
|
|
180
|
+
public wantedBy: State[];
|
|
164
181
|
constructor(
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
182
|
+
rule: RuntimeGrammarProductionRule,
|
|
183
|
+
dot: number,
|
|
184
|
+
reference: number,
|
|
185
|
+
wantedBy: State[]
|
|
169
186
|
) {
|
|
187
|
+
this.rule = rule;
|
|
188
|
+
this.dot = dot;
|
|
189
|
+
this.reference = reference;
|
|
190
|
+
this.wantedBy = wantedBy;
|
|
170
191
|
this.isComplete = this.dot === rule.symbols.length;
|
|
171
192
|
}
|
|
172
193
|
|
|
173
194
|
nextState(child: State | StateToken) {
|
|
174
|
-
const state = new
|
|
195
|
+
const state = new (this.constructor as any)(this.rule, this.dot + 1, this.reference, this.wantedBy);
|
|
175
196
|
state.left = this;
|
|
176
197
|
state.right = child;
|
|
177
198
|
if (state.isComplete) {
|
|
@@ -182,9 +203,7 @@ class State {
|
|
|
182
203
|
}
|
|
183
204
|
|
|
184
205
|
|
|
185
|
-
finish()
|
|
186
|
-
this.data = [this.rule, this.data, { reference: this.reference, dot: this.dot }];
|
|
187
|
-
}
|
|
206
|
+
abstract finish(): void;
|
|
188
207
|
|
|
189
208
|
protected build() {
|
|
190
209
|
const children = [];
|
|
@@ -198,6 +217,18 @@ class State {
|
|
|
198
217
|
}
|
|
199
218
|
}
|
|
200
219
|
|
|
220
|
+
class EagerState extends State {
|
|
221
|
+
finish() {
|
|
222
|
+
this.data = ParserUtility.PostProcess(this.rule, this.data, { reference: this.reference, dot: this.dot });
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
class LazyState extends State {
|
|
227
|
+
finish() {
|
|
228
|
+
this.data = [this.rule, this.data, { reference: this.reference, dot: this.dot }];
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
201
232
|
interface StateToken {
|
|
202
233
|
data: any,
|
|
203
234
|
token: any,
|
|
@@ -205,4 +236,6 @@ interface StateToken {
|
|
|
205
236
|
reference: number
|
|
206
237
|
}
|
|
207
238
|
|
|
208
|
-
type PreAST = [RuntimeGrammarProductionRule, (RuntimeLexerToken | PreAST)[], { reference: number, dot: number }];
|
|
239
|
+
type PreAST = [RuntimeGrammarProductionRule, (RuntimeLexerToken | PreAST)[], { reference: number, dot: number }];
|
|
240
|
+
type Concrete<T extends abstract new (...args: any) => any> =
|
|
241
|
+
new (...args: ConstructorParameters<T>) => InstanceType<T>;
|
|
@@ -1,37 +1,52 @@
|
|
|
1
|
-
import { RuntimeParserClass } from "../../../typings/index.
|
|
2
|
-
import { TokenBuffer } from "../../../lexers/token-buffer.
|
|
3
|
-
import { ParserUtility } from "../../../utility/parsing.
|
|
4
|
-
import { CanonicalCollection } from "./canonical-collection.
|
|
5
|
-
import {
|
|
1
|
+
import type { RuntimeParserClass } from "../../../typings/index.ts";
|
|
2
|
+
import { TokenBuffer } from "../../../lexers/token-buffer.ts";
|
|
3
|
+
import { ParserUtility } from "../../../utility/parsing.ts";
|
|
4
|
+
import { CanonicalCollection } from "./canonical-collection.ts";
|
|
5
|
+
import { Stack } from "./stack.ts";
|
|
6
|
+
import type { State } from "./typings.ts";
|
|
6
7
|
|
|
7
8
|
export function LRK(language: RuntimeParserClass & { tokens: TokenBuffer }, options = {}) {
|
|
8
9
|
const { grammar } = language.artifacts;
|
|
9
10
|
const { tokens } = language;
|
|
10
|
-
const {
|
|
11
|
-
const
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
const { start } = new CanonicalCollection(grammar);
|
|
12
|
+
const stateStack = new Stack<State>();
|
|
13
|
+
const inputStack = new Stack<any>();
|
|
14
|
+
stateStack.push(start);
|
|
15
|
+
|
|
15
16
|
let token;
|
|
16
17
|
|
|
17
|
-
while (token = tokens.next()) {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
18
|
+
tokenloop: while (token = tokens.next()) {
|
|
19
|
+
const match = stateStack.current.actions.find(a => ParserUtility.SymbolMatchesToken(a.symbol, token));
|
|
20
|
+
|
|
21
|
+
if (match) {
|
|
22
|
+
inputStack.push(token);
|
|
23
|
+
stateStack.push(match.state);
|
|
24
|
+
} else {
|
|
25
|
+
throw new Error("Syntax Error: Unexpected Token");
|
|
25
26
|
}
|
|
26
|
-
while (stack.current.state?.isFinal) {
|
|
27
|
-
const rule = rules.fetch(stack.current.state.reduce);
|
|
28
|
-
stack.reduce(rule);
|
|
29
|
-
stack.current.value = ParserUtility.PostProcess(rule, stack.current.children.map(v => v.value));
|
|
30
|
-
const s = stack.previous.state.goto.get(rule.name);
|
|
31
|
-
stack.shift(states.get(s));
|
|
32
27
|
|
|
28
|
+
while (stateStack.current.reduce) {
|
|
29
|
+
const rule = stateStack.current.reduce;
|
|
30
|
+
const value = inputStack.pop(rule.symbols.length);
|
|
31
|
+
stateStack.pop(rule.symbols.length);
|
|
32
|
+
inputStack.push(ParserUtility.PostProcess(rule, value));
|
|
33
|
+
const nextState = stateStack.current.goto[rule.name];
|
|
34
|
+
|
|
35
|
+
if (!nextState)
|
|
36
|
+
break tokenloop;
|
|
37
|
+
|
|
38
|
+
stateStack.push(nextState);
|
|
33
39
|
}
|
|
34
40
|
}
|
|
35
41
|
|
|
36
|
-
|
|
42
|
+
if (stateStack.size > 1) {
|
|
43
|
+
throw new Error("Syntax Error: Unexpected End of Input");
|
|
44
|
+
}
|
|
45
|
+
const peek = tokens.next();
|
|
46
|
+
if (peek) {
|
|
47
|
+
console.log(peek)
|
|
48
|
+
throw new Error("Syntax Error: Expected End of Input");
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
return { results: [inputStack.current[0]] }
|
|
37
52
|
}
|
|
@@ -1,79 +1,108 @@
|
|
|
1
|
-
import { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol, RuntimeParserClass } from "../../../typings/index.
|
|
2
|
-
import { ParserUtility } from "../../../utility/parsing.
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { State } from "./
|
|
1
|
+
import type { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol, RuntimeParserClass } from "../../../typings/index.ts";
|
|
2
|
+
import { ParserUtility } from "../../../utility/parsing.ts";
|
|
3
|
+
import { TextFormatter } from "../../../utility/text-format.ts";
|
|
4
|
+
import { BiMap } from "./bimap.ts";
|
|
5
|
+
import type { LRItem, State } from "./typings.ts";
|
|
6
6
|
|
|
7
7
|
export class CanonicalCollection {
|
|
8
|
-
|
|
9
|
-
rules: BiMap<RuntimeGrammarProductionRule> = new BiMap();
|
|
10
|
-
|
|
8
|
+
public start: State;
|
|
9
|
+
public rules: BiMap<RuntimeGrammarProductionRule> = new BiMap();
|
|
10
|
+
public grammar: RuntimeParserClass['artifacts']['grammar'];
|
|
11
|
+
private cache: { [key: string]: State } = {};
|
|
11
12
|
|
|
12
|
-
private closure: ClosureBuilder;
|
|
13
13
|
constructor(
|
|
14
|
-
|
|
14
|
+
grammar: RuntimeParserClass['artifacts']['grammar']
|
|
15
15
|
) {
|
|
16
|
+
this.grammar = grammar;
|
|
16
17
|
const augmented = {
|
|
17
18
|
name: Symbol() as unknown as string,
|
|
18
19
|
symbols: [this.grammar.start]
|
|
19
20
|
}
|
|
20
|
-
this.grammar
|
|
21
|
-
this.closure = new ClosureBuilder(this.grammar);
|
|
21
|
+
this.grammar.rules[augmented.name] = [augmented];
|
|
22
22
|
this.rules.id(augmented);
|
|
23
|
-
this.
|
|
24
|
-
this.linkStates('0.0');
|
|
23
|
+
this.start = this.generateState([{ rule: augmented, dot: 0 }]);
|
|
25
24
|
}
|
|
26
25
|
|
|
27
|
-
private
|
|
28
|
-
const id = this.
|
|
29
|
-
if (this.
|
|
30
|
-
return;
|
|
26
|
+
private generateState(kernel: LRItem[]): State {
|
|
27
|
+
const id = this.canonicalStateId(kernel);
|
|
28
|
+
if (this.cache[id])
|
|
29
|
+
return this.cache[id];
|
|
31
30
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
goto: new Map(),
|
|
37
|
-
reduce: null,
|
|
38
|
-
rule: rule
|
|
31
|
+
this.cache[id] = { id };
|
|
32
|
+
if (kernel.length == 1 && kernel[0].rule.symbols.length == kernel[0].dot) {
|
|
33
|
+
this.cache[id].reduce = kernel[0].rule;
|
|
34
|
+
return this.cache[id];
|
|
39
35
|
}
|
|
40
36
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
37
|
+
const items = [...kernel];
|
|
38
|
+
const visited = new Set<string>();
|
|
39
|
+
const refs: { [key: string]: RuntimeGrammarRuleSymbol } = {}
|
|
40
|
+
const actions: { [key: string]: LRItem[] } = {};
|
|
41
|
+
const goto: { [key: string]: LRItem[] } = {};
|
|
42
|
+
for (let i = 0; i < items.length; i++) {
|
|
43
|
+
const { rule, dot } = items[i];
|
|
44
|
+
const id = this.canonicalLRItemId(items[i]);
|
|
45
|
+
if (dot == rule.symbols.length)
|
|
46
|
+
throw new Error('Reduce Conflict on state: ' + id + `\n${items.map(v => TextFormatter.GrammarRule(v.rule, v.dot)).join('\n')}`);
|
|
44
47
|
|
|
45
|
-
|
|
48
|
+
if (visited.has(id))
|
|
49
|
+
continue;
|
|
50
|
+
visited.add(id);
|
|
51
|
+
const symbol = rule.symbols[dot];
|
|
52
|
+
const name = this.canonicalSymbolId(symbol);
|
|
53
|
+
refs[name] = symbol;
|
|
46
54
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
55
|
+
if (symbol && !ParserUtility.SymbolIsTerminal(symbol)) {
|
|
56
|
+
const prods = this.grammar.rules[symbol] || [];
|
|
57
|
+
for (const rule of prods) {
|
|
58
|
+
items.push({ rule, dot: 0 });
|
|
59
|
+
}
|
|
60
|
+
goto[name] = goto[name] || [];
|
|
61
|
+
goto[name].push({ rule, dot: dot + 1 });
|
|
62
|
+
} else {
|
|
63
|
+
actions[name] = actions[name] || [];
|
|
64
|
+
actions[name].push({ rule, dot: dot + 1 });
|
|
52
65
|
}
|
|
53
|
-
|
|
66
|
+
}
|
|
67
|
+
this.cache[id].actions = [];
|
|
68
|
+
this.cache[id].goto = {};
|
|
54
69
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
if (!state.isFinal) {
|
|
59
|
-
for (const { rule, dot } of state.items) {
|
|
60
|
-
const symbol = rule.symbols[dot];
|
|
61
|
-
const itemStateId = this.getStateId(rule, dot + 1);
|
|
62
|
-
if (ParserUtility.SymbolIsTerminal(symbol) && typeof symbol != 'symbol') {
|
|
63
|
-
state.actions.set(symbol, itemStateId);
|
|
64
|
-
} else {
|
|
65
|
-
state.goto.set(symbol, itemStateId);
|
|
66
|
-
}
|
|
70
|
+
for (const key in actions) {
|
|
71
|
+
this.cache[id].actions.push({ symbol: refs[key], state: this.generateState(actions[key]) })
|
|
72
|
+
}
|
|
67
73
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
}
|
|
71
|
-
} else {
|
|
72
|
-
state.reduce = this.rules.id(state.rule);
|
|
74
|
+
for (const key in goto) {
|
|
75
|
+
this.cache[id].goto[refs[key] as string] = this.generateState(goto[key]);
|
|
73
76
|
}
|
|
77
|
+
|
|
78
|
+
return this.cache[id];
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
private canonicalStateId(items: LRItem[]): string {
|
|
82
|
+
return items
|
|
83
|
+
.map(item => this.canonicalLRItemId(item))
|
|
84
|
+
.sort()
|
|
85
|
+
.join('|');
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
private canonicalLRItemId(item: LRItem) {
|
|
89
|
+
return `${this.rules.id(item.rule)}:${item.dot}`;
|
|
74
90
|
}
|
|
75
91
|
|
|
76
|
-
private
|
|
77
|
-
|
|
92
|
+
private canonicalSymbolId(symbol: RuntimeGrammarRuleSymbol) {
|
|
93
|
+
if (typeof symbol === 'symbol')
|
|
94
|
+
return `SY:START`;
|
|
95
|
+
if (typeof symbol === 'string')
|
|
96
|
+
return `NT:${symbol}`;
|
|
97
|
+
if (typeof symbol == 'function')
|
|
98
|
+
return `FN:${symbol.toString()}`;
|
|
99
|
+
if (!symbol)
|
|
100
|
+
return
|
|
101
|
+
if (symbol instanceof RegExp)
|
|
102
|
+
return `RG:${symbol.source}`;
|
|
103
|
+
if ("token" in symbol)
|
|
104
|
+
return `TK:${symbol.token}`;
|
|
105
|
+
if ("literal" in symbol)
|
|
106
|
+
return `LT:${symbol.literal}`;
|
|
78
107
|
}
|
|
79
|
-
}
|
|
108
|
+
}
|
|
@@ -1,53 +1,28 @@
|
|
|
1
|
-
|
|
2
|
-
import { State } from "./state.js";
|
|
1
|
+
export class Stack<T> {
|
|
3
2
|
|
|
4
|
-
|
|
3
|
+
private stack: T[] = [];
|
|
5
4
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
get current() {
|
|
9
|
-
return this.stack[this.stack.length - 1];
|
|
5
|
+
get size() {
|
|
6
|
+
return this.stack.length;
|
|
10
7
|
}
|
|
11
8
|
|
|
12
9
|
get previous() {
|
|
13
10
|
return this.stack[this.stack.length - 2];
|
|
14
11
|
}
|
|
15
12
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
this.current.state = state;
|
|
13
|
+
get current() {
|
|
14
|
+
return this.stack[this.stack.length - 1];
|
|
19
15
|
}
|
|
20
16
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const l = rule.symbols.length;
|
|
24
|
-
n.children = this.stack.splice(l * -1, l);
|
|
25
|
-
n.children.forEach(v => delete v.state);
|
|
26
|
-
n.rule = rule;
|
|
27
|
-
n.symbol = rule.name;
|
|
28
|
-
this.stack.push(n);
|
|
17
|
+
set current(item: T) {
|
|
18
|
+
this.stack[this.stack.length - 1] = item;
|
|
29
19
|
}
|
|
30
20
|
|
|
31
|
-
|
|
32
|
-
this.stack.push(
|
|
33
|
-
this.current.symbol = symbol;
|
|
21
|
+
push(...items: T[]) {
|
|
22
|
+
return this.stack.push(...items);
|
|
34
23
|
}
|
|
35
24
|
|
|
36
|
-
|
|
37
|
-
return
|
|
38
|
-
children: [],
|
|
39
|
-
state: null,
|
|
40
|
-
symbol: null,
|
|
41
|
-
rule: null,
|
|
42
|
-
value: null
|
|
43
|
-
}
|
|
25
|
+
pop(n: number = 1): T[] {
|
|
26
|
+
return this.stack.splice(n * -1, n);
|
|
44
27
|
}
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
interface LRStackItem {
|
|
48
|
-
children: LRStackItem[];
|
|
49
|
-
state: State;
|
|
50
|
-
symbol: RuntimeGrammarRuleSymbol;
|
|
51
|
-
rule: RuntimeGrammarProductionRule;
|
|
52
|
-
value: any;
|
|
53
28
|
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol } from "../../../typings/index.ts";
|
|
2
|
+
|
|
3
|
+
export interface LRItem {
|
|
4
|
+
rule: RuntimeGrammarProductionRule;
|
|
5
|
+
dot: number;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export interface State {
|
|
9
|
+
id: string;
|
|
10
|
+
actions?: { symbol: RuntimeGrammarRuleSymbol, state: State }[];
|
|
11
|
+
goto?: { [key: string]: State };
|
|
12
|
+
reduce?: RuntimeGrammarProductionRule;
|
|
13
|
+
}
|
package/src/parser/parse.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { ParserAlgorithm, RuntimeParserClass } from "../typings/index.
|
|
2
|
-
import { CharacterLexer } from "../lexers/character-lexer.
|
|
3
|
-
import { StatefulLexer } from "../lexers/stateful-lexer.
|
|
4
|
-
import { TokenBuffer } from "../lexers/token-buffer.
|
|
5
|
-
import { CYK } from "./algorithms/cyk.
|
|
6
|
-
import { Earley } from "./algorithms/earley.
|
|
7
|
-
import { LRK } from "./algorithms/lrk/algorithm.
|
|
8
|
-
import { ParserUtility } from "../utility/parsing.
|
|
1
|
+
import type { ParserAlgorithm, RuntimeParserClass } from "../typings/index.ts";
|
|
2
|
+
import { CharacterLexer } from "../lexers/character-lexer.ts";
|
|
3
|
+
import { StatefulLexer } from "../lexers/stateful-lexer.ts";
|
|
4
|
+
import { TokenBuffer } from "../lexers/token-buffer.ts";
|
|
5
|
+
import { CYK } from "./algorithms/cyk.ts";
|
|
6
|
+
import { Earley } from "./algorithms/earley.ts";
|
|
7
|
+
import { LRK } from "./algorithms/lrk/algorithm.ts";
|
|
8
|
+
import { ParserUtility } from "../utility/parsing.ts";
|
|
9
9
|
|
|
10
10
|
const ParserRegistry: { [key: string]: ParserAlgorithm } = {
|
|
11
11
|
earley: Earley,
|
package/src/typings/ast.ts
CHANGED
package/src/typings/generator.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { GeneratorState } from "../generator/state.
|
|
2
|
-
import { ASTGrammarSymbolLiteral, ASTGrammarSymbolNonTerminal, ASTGrammarSymbolRegex, ASTGrammarSymbolToken, ASTJavaScriptBuiltin, ASTJavaScriptLiteral, ASTJavaScriptTemplate, ASTLexerStateImportRule, ASTLexerStateMatchRule, ASTLexerStateNonMatchRule } from "./ast.
|
|
3
|
-
import { Dictionary } from "./common.
|
|
4
|
-
import { ImportResolver, ImportResolverConstructor } from "./index.
|
|
1
|
+
import { GeneratorState } from "../generator/state.ts";
|
|
2
|
+
import type { ASTGrammarSymbolLiteral, ASTGrammarSymbolNonTerminal, ASTGrammarSymbolRegex, ASTGrammarSymbolToken, ASTJavaScriptBuiltin, ASTJavaScriptLiteral, ASTJavaScriptTemplate, ASTLexerStateImportRule, ASTLexerStateMatchRule, ASTLexerStateNonMatchRule } from "./ast.ts";
|
|
3
|
+
import type { Dictionary } from "./common.ts";
|
|
4
|
+
import type { ImportResolver, ImportResolverConstructor } from "./index.ts";
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
export type GenerateOptions = GeneratorOptions & { output?: GeneratorOutputOptions };
|
package/src/typings/index.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { TokenBuffer } from "../lexers/token-buffer.
|
|
2
|
-
import { ParserUtility } from "../utility/parsing.
|
|
3
|
-
import { ASTLexerStateNonMatchRule } from "./ast.
|
|
4
|
-
import { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol, RuntimeParserClass, RuntimeLexerStateMatchRule } from "./runtime.
|
|
5
|
-
export * from './ast.
|
|
6
|
-
export * from './common.
|
|
7
|
-
export * from './generator.
|
|
8
|
-
export * from './runtime.
|
|
1
|
+
import type { TokenBuffer } from "../lexers/token-buffer.ts";
|
|
2
|
+
import type { ParserUtility } from "../utility/parsing.ts";
|
|
3
|
+
import type { ASTLexerStateNonMatchRule } from "./ast.ts";
|
|
4
|
+
import type { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol, RuntimeParserClass, RuntimeLexerStateMatchRule } from "./runtime.ts";
|
|
5
|
+
export * from './ast.ts';
|
|
6
|
+
export * from './common.ts';
|
|
7
|
+
export * from './generator.ts';
|
|
8
|
+
export * from './runtime.ts';
|
|
9
9
|
|
|
10
10
|
export interface ImportResolver {
|
|
11
11
|
path(path: string): string;
|
package/src/typings/runtime.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { LRState } from "./index.
|
|
2
|
-
import { ASTGrammarSymbolLiteral, ASTGrammarSymbolToken, ASTLexerStateNonMatchRule } from "./ast.
|
|
3
|
-
import { Dictionary } from "./common.
|
|
1
|
+
import type { LRState } from "./index.ts";
|
|
2
|
+
import type { ASTGrammarSymbolLiteral, ASTGrammarSymbolToken, ASTLexerStateNonMatchRule } from "./ast.ts";
|
|
3
|
+
import type { Dictionary } from "./common.ts";
|
|
4
4
|
|
|
5
5
|
export interface RuntimeParserClass {
|
|
6
6
|
artifacts: {
|
package/src/utility/format.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import V1Grammar from '../generator/grammars/v1.
|
|
2
|
-
import V2Grammar from '../generator/grammars/v2.
|
|
3
|
-
import { V2GrammarString } from "../generator/stringify/grammar/v2.
|
|
4
|
-
import { Parse } from '../parser/parse.
|
|
1
|
+
import V1Grammar from '../generator/grammars/v1.ts';
|
|
2
|
+
import V2Grammar from '../generator/grammars/v2.ts';
|
|
3
|
+
import { V2GrammarString } from "../generator/stringify/grammar/v2.ts";
|
|
4
|
+
import { Parse } from '../parser/parse.ts';
|
|
5
5
|
|
|
6
6
|
export function Format(source: string, sourceVersion: '1' | '2' = '2') {
|
|
7
7
|
const grammar = sourceVersion.toString() == '1' ? V1Grammar : V2Grammar;
|
package/src/utility/general.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Dictionary, GeneratorGrammarSymbol } from "../typings/index.
|
|
1
|
+
import type { Dictionary, GeneratorGrammarSymbol } from "../typings/index.ts";
|
|
2
2
|
|
|
3
3
|
export class Collection<T> {
|
|
4
4
|
categorized: Dictionary<Dictionary<number>> = {};
|
|
@@ -85,10 +85,11 @@ export class Matrix<T> {
|
|
|
85
85
|
set x(x: number) { x != this.$x && this.resize(x, this.y); }
|
|
86
86
|
get y() { return this.$y }
|
|
87
87
|
set y(y: number) { y != this.$y && this.resize(this.x, y); }
|
|
88
|
-
|
|
88
|
+
private initial?: T | ((...args: any) => T);
|
|
89
89
|
matrix: GetCallbackOrValue<T>[][] = [];
|
|
90
90
|
|
|
91
|
-
constructor(x: number, y: number,
|
|
91
|
+
constructor(x: number, y: number, initial?: T | ((...args: any) => T)) {
|
|
92
|
+
this.initial = initial;
|
|
92
93
|
this.resize(x, y);
|
|
93
94
|
}
|
|
94
95
|
|
package/src/utility/index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export * from './monarch.
|
|
2
|
-
export * from './general.
|
|
3
|
-
export * from './format.
|
|
4
|
-
export * from './parsing.
|
|
1
|
+
export * from './monarch.ts';
|
|
2
|
+
export * from './general.ts';
|
|
3
|
+
export * from './format.ts';
|
|
4
|
+
export * from './parsing.ts';
|