grammar-well 2.0.6 → 2.1.0

package/src/generator/state.ts

@@ -4,8 +4,9 @@ export class GeneratorState {
     grammar?: GeneratorStateGrammar;
     lexer?: GeneratorLexerConfig;
     lifecycle: {
-        import?: string;
-        new?: string;
+        import?: string[];
+        new?: string[];
+        token?: string[];
     } = {}
 
     config = {};
@@ -72,9 +73,14 @@ export class GeneratorState {
         }
     }
 
-    addLifecycle(lifecycle: string, literal: string) {
-        this.lifecycle[lifecycle] = this.lifecycle[lifecycle] || '';
-        this.lifecycle[lifecycle] += literal;
+    addLifecycle(lifecycle: string, literal: string | string[]) {
+        this.lifecycle[lifecycle] = this.lifecycle[lifecycle] || [];
+
+        if (typeof literal == 'string')
+            this.lifecycle[lifecycle].push(literal);
+
+        if (Array.isArray(literal))
+            this.lifecycle[lifecycle].push(...literal);
     }
 
     export() {

package/src/generator/stringify/exports/javascript.ts

@@ -20,10 +20,11 @@ function Generate(generator: JavaScriptGenerator) {
     return `// Generated automatically by Grammar-Well, version ${generator.state.version} 
 // https://github.com/0x6563/grammar-well
 
-${generator.lifecycle('import')}
+${generator.lifecycle('import').join('')}
 
 class ${exportName} {
+    state = {};
     artifacts = ${generator.artifacts(1)}
-    constructor(){${generator.lifecycle('new')}}
+    constructor(){${generator.lifecycle('new').join('')}}
 }`;
 }

package/src/generator/stringify/exports/typescript.ts

@@ -6,11 +6,12 @@ export function TypescriptFormat(generator: JavaScriptGenerator) {
 // https://github.com/0x6563/grammar-well
 // @ts-nocheck
 
-${generator.lifecycle('import')}
+${generator.lifecycle('import').join('')}
 
 class ${exportName} {
+    state = {};
     artifacts = ${generator.artifacts(1)}
-    constructor(){${generator.lifecycle('new')}}
+    constructor(){${generator.lifecycle('new').join('')}}
 }
 
 export default ${exportName};`;

package/src/generator/stringify/grammar/v2.ts

@@ -6,7 +6,11 @@ export class V2GrammarString {
         directives = Array.isArray(directives) ? directives : [directives];
         for (const directive of directives) {
             if ("lifecycle" in directive) {
-                this.appendSection("on:" + directive.lifecycle, directive.js.js.trim());
+                if ('js' in directive.js)
+                    this.appendSection("on:" + directive.lifecycle, directive.js.js.trim());
+                else if ('template' in directive.js)
+                    this.appendSource("on:" + directive.lifecycle + this.formatTemplate(directive.js));
+
             } else if ("import" in directive) {
                 this.appendImportDirective(directive);
             } else if ("config" in directive) {
@@ -93,12 +97,16 @@ export class V2GrammarString {
             return `=> \${ ${postProcess.js} }`;
         }
         if ('template' in postProcess) {
-            const prefix = postProcess.template.slice(0, 1);
-            const suffix = postProcess.template.slice(-1);
-            return `=> ${prefix} ${postProcess.template.slice(1, -1).trim()} ${suffix}`;
+            return this.formatTemplate(postProcess)
         }
     }
 
+    formatTemplate(postProcess: { template: string }) {
+        const prefix = postProcess.template.slice(0, 1);
+        const suffix = postProcess.template.slice(-1);
+        return `=> ${prefix} ${postProcess.template.slice(1, -1).trim()} ${suffix}`;
+    }
+
     appendLexerDirective(directive: ASTLexer) {
         let body = '';
         if (directive.lexer.start) {
@@ -211,10 +219,14 @@ export class V2GrammarString {
     }
 
     appendSection(label: string, body: string) {
+        this.appendSource(`${label} {\n${body}\n}\n`)
+    }
+
+    appendSource(source) {
         if (this.source) {
             this.source += '\n';
         }
-        this.source += `${label} {\n${body}\n}\n`;
+        this.source += source;
     }
 
     indent(depth: number = 0, content: string) {

package/src/generator/stringify/javascript.ts

@@ -22,8 +22,8 @@ export class JavaScriptGenerator {
 
     lifecycle(lifecycle: string) {
         if (this.options.noscript)
-            return '';
-        return this.state.lifecycle[lifecycle] || '';
+            return [];
+        return this.state.lifecycle[lifecycle] || [];
     }
 
     artifacts(depth: number = 0) {
@@ -48,9 +48,31 @@ export class JavaScriptGenerator {
             output.grammar = basic.stringify(depth + 1);
         }
 
+        const onToken = this.lifecycle('token');
+        if (onToken.length) {
+            output.tokenProcessor = `() => {`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 1)}return (token) => {`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 2)}const processors = [`;
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 3)}${onToken.map(v => `({ token, state }) => ${v}`).join(',' + CommonGenerator.SmartIndent(depth + 3))}`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 2)}];`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 2)}for (const processor of processors) {`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 3)}token = processor({ token, state: this.state })`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 2)}}`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 2)}return token;`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 1)}}`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth)}}`;
+        }
         return CommonGenerator.JSON(output, depth);
     }
 
+    f(token) {
+        const processors = [({ token, state }) => token]
+        for (const processor of processors) {
+            token = processor({ token, state: this.state })
+        }
+        return token;
+    }
+
     postProcess(postprocess: GeneratorGrammarProductionRule['postprocess'], alias: Dictionary<number>) {
         postprocess = this.state.grammar.config.postprocessorOverride || postprocess || this.state.grammar.config.postprocessorDefault;
         if (!postprocess)

package/src/lexers/character-lexer.ts

@@ -28,7 +28,8 @@ export class CharacterLexer implements RuntimeLexer {
                 value,
                 offset: this.index,
                 line: this.line,
-                column: this.column
+                column: this.column,
+                custom: {}
             };
         }
     }

package/src/lexers/stateful-lexer.ts

@@ -64,7 +64,8 @@ export class StatefulLexer implements RuntimeLexer {
             line: this.line,
             lines: 0,
             column: this.column,
-            state: this.current
+            state: this.current,
+            custom: {},
         }
         this.adjustPosition(text);
         token.lines = this.line - token.line;

package/src/lexers/token-buffer.ts

@@ -15,7 +15,7 @@ export class TokenBuffer {
         return { historyIndex: this.$historyIndex, offset: this.offset };
     }
 
-    constructor(public lexer: RuntimeLexer) { }
+    constructor(public lexer: RuntimeLexer, private tokenProcessor?: (token: RuntimeLexerToken) => RuntimeLexerToken) { }
 
     reset(buffer: string) {
         this.lexer.feed(buffer);
@@ -77,8 +77,11 @@ export class TokenBuffer {
             this.queued = '';
             token = this.lexer.next();
         }
-        if (token)
+        if (token) {
+            if (this.tokenProcessor)
+                token = this.tokenProcessor(token);
             this.history.push(token);
+        }
         return token;
     }
 

package/src/parser/algorithms/earley.ts

@@ -1,4 +1,4 @@
-import { Dictionary, RuntimeGrammarProductionRule, RuntimeParserClass } from "../../typings/index.js";
+import { Dictionary, RuntimeGrammarProductionRule, RuntimeLexerToken, RuntimeParserClass } from "../../typings/index.js";
 import { TokenBuffer } from "../../lexers/token-buffer.js";
 import { TextFormatter } from "../../utility/text-format.js";
 import { ParserUtility } from "../../utility/parsing.js";
@@ -58,9 +58,22 @@ export function Earley(language: RuntimeParserClass & { tokens: TokenBuffer }, o
             results.push(data);
         }
     }
+    const clone = results.length > 1;
+    for (let i = 0; i < results.length; i++) {
+        results[i] = PostProcess(results[i], clone);
+    }
     return { results, info: { table } };
 }
 
+function PostProcess(ast: PreAST | RuntimeLexerToken, clone?: boolean) {
+    if (!Array.isArray(ast))
+        return clone ? { ...ast } : ast;
+    const data = [];
+    for (let i = 0; i < ast[1].length; i++) {
+        data[i] = PostProcess(ast[1][i], clone);
+    }
+    return ParserUtility.PostProcess(ast[0], data, ast[2]);
+}
 
 class Column {
     data: any;
@@ -170,7 +183,7 @@ class State {
 
 
     finish() {
-        this.data = ParserUtility.PostProcess(this.rule, this.data, { reference: this.reference, dot: this.dot });
+        this.data = [this.rule, this.data, { reference: this.reference, dot: this.dot }];
     }
 
     protected build() {
@@ -190,4 +203,6 @@ interface StateToken {
     token: any,
     isToken: boolean,
     reference: number
-}
+}
+
+type PreAST = [RuntimeGrammarProductionRule, (RuntimeLexerToken | PreAST)[], { reference: number, dot: number }];

package/src/parser/parse.ts

@@ -29,13 +29,14 @@ export function Parse(
     return results == 'full' ? result : result.results[0];
 }
 
-function GetTokenizer({ lexer }: RuntimeParserClass['artifacts']) {
-    if (!lexer) {
-        return new TokenBuffer(new CharacterLexer());
-    } else if ("feed" in lexer && typeof lexer.feed == 'function') {
-        return new TokenBuffer(lexer);
-    } else if ('states' in lexer) {
-        return new TokenBuffer(new StatefulLexer(lexer));
+function GetTokenizer(artifacts: RuntimeParserClass['artifacts']) {
+    const tokenProcessor = artifacts?.tokenProcessor ? artifacts.tokenProcessor() : null;
+    if (!artifacts.lexer) {
+        return new TokenBuffer(new CharacterLexer(), tokenProcessor);
+    } else if ("feed" in artifacts.lexer && typeof artifacts.lexer.feed == 'function') {
+        return new TokenBuffer(artifacts.lexer, tokenProcessor);
+    } else if ('states' in artifacts.lexer) {
+        return new TokenBuffer(new StatefulLexer(artifacts.lexer), tokenProcessor);
     }
 }
 

package/src/typings/ast.ts

@@ -5,7 +5,7 @@ export type AST = ASTDirectives[];
 export type ASTJavascriptLifecycleLiteral = {
     lifecycle: string;
     path?: boolean;
-    js: ASTJavaScriptLiteral;
+    js: ASTJavaScriptLiteral | ASTJavaScriptTemplate;
 }
 export type ASTJavaScriptLiteral = { js: string }
 export type ASTJavaScriptTemplate = { template: string }

package/src/typings/generator.ts

@@ -41,7 +41,7 @@ export interface GeneratorStateGrammar {
 };
 
 export interface GeneratorLexerConfig {
-    start?: string
+    start?: string;
     states: Dictionary<GeneratorLexerState>;
 }
 

package/src/typings/runtime.ts

@@ -1,5 +1,5 @@
 import { LRState } from "./index.js";
-import { ASTGrammarSymbolLiteral, ASTGrammarSymbolToken, ASTLexerStateMatchRule, ASTLexerStateNonMatchRule } from "./ast.js";
+import { ASTGrammarSymbolLiteral, ASTGrammarSymbolToken, ASTLexerStateNonMatchRule } from "./ast.js";
 import { Dictionary } from "./common.js";
 
 export interface RuntimeParserClass {
@@ -13,6 +13,7 @@ export interface RuntimeParserClass {
             k: number;
             table: Dictionary<LRState>;
         }
+        tokenProcessor: () => ((token: RuntimeLexerToken) => RuntimeLexerToken) | undefined
     }
     new();
 }
@@ -60,6 +61,7 @@ export interface RuntimeLexerToken {
     offset: number;
     line: number;
     column: number;
+    custom?: { [key: string]: any }
 }
 
 

package/src/version.json

@@ -1 +1 @@
-{"version":"2.0.6"}
+{"version":"2.1.0"}