grammar-well 2.0.5 → 2.0.7

package/src/generator/stringify/grammar/v2.ts

@@ -6,7 +6,11 @@ export class V2GrammarString {
         directives = Array.isArray(directives) ? directives : [directives];
         for (const directive of directives) {
             if ("lifecycle" in directive) {
-                this.appendSection("on:" + directive.lifecycle, directive.js.js.trim());
+                if ('js' in directive.js)
+                    this.appendSection("on:" + directive.lifecycle, directive.js.js.trim());
+                else if ('template' in directive.js)
+                    this.appendSource("on:" + directive.lifecycle + this.formatTemplate(directive.js));
+
             } else if ("import" in directive) {
                 this.appendImportDirective(directive);
             } else if ("config" in directive) {
@@ -93,12 +97,16 @@ export class V2GrammarString {
             return `=> \${ ${postProcess.js} }`;
         }
         if ('template' in postProcess) {
-            const prefix = postProcess.template.slice(0, 1);
-            const suffix = postProcess.template.slice(-1);
-            return `=> ${prefix} ${postProcess.template.slice(1, -1).trim()} ${suffix}`;
+            return this.formatTemplate(postProcess)
         }
     }
 
+    formatTemplate(postProcess: { template: string }) {
+        const prefix = postProcess.template.slice(0, 1);
+        const suffix = postProcess.template.slice(-1);
+        return `=> ${prefix} ${postProcess.template.slice(1, -1).trim()} ${suffix}`;
+    }
+
     appendLexerDirective(directive: ASTLexer) {
         let body = '';
         if (directive.lexer.start) {
@@ -211,10 +219,14 @@ export class V2GrammarString {
     }
 
     appendSection(label: string, body: string) {
+        this.appendSource(`${label} {\n${body}\n}\n`)
+    }
+
+    appendSource(source) {
         if (this.source) {
             this.source += '\n';
         }
-        this.source += `${label} {\n${body}\n}\n`;
+        this.source += source;
     }
 
     indent(depth: number = 0, content: string) {

package/src/generator/stringify/javascript.ts

@@ -22,8 +22,8 @@ export class JavaScriptGenerator {
 
     lifecycle(lifecycle: string) {
         if (this.options.noscript)
-            return '';
-        return this.state.lifecycle[lifecycle] || '';
+            return [];
+        return this.state.lifecycle[lifecycle] || [];
     }
 
     artifacts(depth: number = 0) {
@@ -48,9 +48,31 @@ export class JavaScriptGenerator {
             output.grammar = basic.stringify(depth + 1);
         }
 
+        const onToken = this.lifecycle('token');
+        if (onToken.length) {
+            output.tokenProcessor = `() => {`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 1)}return (token) => {`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 2)}const processors = [`;
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 3)}${onToken.map(v => `({ token, state }) => ${v}`).join(',' + CommonGenerator.SmartIndent(depth + 3))}`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 2)}];`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 2)}for (const processor of processors) {`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 3)}token = processor({ token, state: this.state })`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 2)}}`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 2)}return token;`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth + 1)}}`
+            output.tokenProcessor += `${CommonGenerator.SmartIndent(depth)}}`;
+        }
         return CommonGenerator.JSON(output, depth);
     }
 
+    f(token) {
+        const processors = [({ token, state }) => token]
+        for (const processor of processors) {
+            token = processor({ token, state: this.state })
+        }
+        return token;
+    }
+
     postProcess(postprocess: GeneratorGrammarProductionRule['postprocess'], alias: Dictionary<number>) {
         postprocess = this.state.grammar.config.postprocessorOverride || postprocess || this.state.grammar.config.postprocessorDefault;
         if (!postprocess)

package/src/lexers/character-lexer.ts

@@ -28,7 +28,8 @@ export class CharacterLexer implements RuntimeLexer {
                 value,
                 offset: this.index,
                 line: this.line,
-                column: this.column
+                column: this.column,
+                custom: {}
             };
         }
     }

package/src/lexers/stateful-lexer.ts

@@ -64,7 +64,8 @@ export class StatefulLexer implements RuntimeLexer {
             line: this.line,
             lines: 0,
             column: this.column,
-            state: this.current
+            state: this.current,
+            custom: {},
         }
         this.adjustPosition(text);
         token.lines = this.line - token.line;

package/src/lexers/token-buffer.ts

@@ -15,7 +15,7 @@ export class TokenBuffer {
         return { historyIndex: this.$historyIndex, offset: this.offset };
     }
 
-    constructor(public lexer: RuntimeLexer) { }
+    constructor(public lexer: RuntimeLexer, private tokenProcessor?: (token: RuntimeLexerToken) => RuntimeLexerToken) { }
 
     reset(buffer: string) {
         this.lexer.feed(buffer);
@@ -77,8 +77,11 @@ export class TokenBuffer {
             this.queued = '';
             token = this.lexer.next();
         }
-        if (token)
+        if (token) {
+            if (this.tokenProcessor)
+                token = this.tokenProcessor(token);
             this.history.push(token);
+        }
         return token;
     }
 

package/src/parser/parse.ts

@@ -29,13 +29,14 @@ export function Parse(
     return results == 'full' ? result : result.results[0];
 }
 
-function GetTokenizer({ lexer }: RuntimeParserClass['artifacts']) {
-    if (!lexer) {
-        return new TokenBuffer(new CharacterLexer());
-    } else if ("feed" in lexer && typeof lexer.feed == 'function') {
-        return new TokenBuffer(lexer);
-    } else if ('states' in lexer) {
-        return new TokenBuffer(new StatefulLexer(lexer));
+function GetTokenizer(artifacts: RuntimeParserClass['artifacts']) {
+    const tokenProcessor = artifacts?.tokenProcessor ? artifacts.tokenProcessor() : null;
+    if (!artifacts.lexer) {
+        return new TokenBuffer(new CharacterLexer(), tokenProcessor);
+    } else if ("feed" in artifacts.lexer && typeof artifacts.lexer.feed == 'function') {
+        return new TokenBuffer(artifacts.lexer, tokenProcessor);
+    } else if ('states' in artifacts.lexer) {
+        return new TokenBuffer(new StatefulLexer(artifacts.lexer), tokenProcessor);
     }
 }
 

package/src/typings/ast.ts

@@ -5,7 +5,7 @@ export type AST = ASTDirectives[];
 export type ASTJavascriptLifecycleLiteral = {
     lifecycle: string;
     path?: boolean;
-    js: ASTJavaScriptLiteral;
+    js: ASTJavaScriptLiteral | ASTJavaScriptTemplate;
 }
 export type ASTJavaScriptLiteral = { js: string }
 export type ASTJavaScriptTemplate = { template: string }

package/src/typings/generator.ts

@@ -41,7 +41,7 @@ export interface GeneratorStateGrammar {
 };
 
 export interface GeneratorLexerConfig {
-    start?: string
+    start?: string;
     states: Dictionary<GeneratorLexerState>;
 }
 

package/src/typings/runtime.ts

@@ -1,5 +1,5 @@
 import { LRState } from "./index.js";
-import { ASTGrammarSymbolLiteral, ASTGrammarSymbolToken, ASTLexerStateMatchRule, ASTLexerStateNonMatchRule } from "./ast.js";
+import { ASTGrammarSymbolLiteral, ASTGrammarSymbolToken, ASTLexerStateNonMatchRule } from "./ast.js";
 import { Dictionary } from "./common.js";
 
 export interface RuntimeParserClass {
@@ -13,6 +13,7 @@ export interface RuntimeParserClass {
             k: number;
             table: Dictionary<LRState>;
         }
+        tokenProcessor: () => ((token: RuntimeLexerToken) => RuntimeLexerToken) | undefined
     }
     new();
 }
@@ -60,6 +61,7 @@ export interface RuntimeLexerToken {
     offset: number;
     line: number;
     column: number;
+    custom?: { [key: string]: any }
 }
 
 

package/src/version.json

@@ -1 +1 @@
-{"version":"2.0.5"}
+{"version":"2.0.7"}