grammar-well 1.2.6 → 1.3.0

package/build/compiler/index.d.ts

@@ -0,0 +1,2 @@
+export * from './compiler';
+export * from './import-resolver';

package/build/compiler/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./compiler"), exports);
+__exportStar(require("./import-resolver"), exports);
+//# sourceMappingURL=index.js.map

package/build/compiler/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/compiler/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6CAA2B;AAC3B,oDAAkC"}

package/build/compiler/outputs/typescript.js

@@ -4,98 +4,10 @@ exports.TypescriptFormat = void 0;
 function TypescriptFormat(generator, exportName) {
     return `// Generated automatically by Grammar-Well, version ${generator.state.version} 
 // https://github.com/0x6563/grammar-well
+import type { LanguageDefinition } from 'grammar-well';
 
 ${generator.serializeHead()}
 
-interface LanguageDefinition {
-    lexer?: Lexer | LexerConfig;
-    grammar: {
-        start: string;
-        rules: GrammarRule[];
-    }
-}
-
-interface Lexer {
-    next(): LexerToken | undefined;
-    feed(chunk?: string, state?: ReturnType<Lexer['state']>): void;
-    state(): any;
-    flush?(): void;
-}
-
-interface LexerConfig {
-    start?: string
-    states: LexerStateDefinition[];
-}
-
-interface LexerToken {
-    type?: string;
-    value: string;
-    offset: number;
-    line: number;
-    column: number;
-}
-
-interface LexerStatus {
-    index: number;
-    line: number;
-    column: number;
-    state: string;
-}
-
-interface LexerStateDefinition {
-    name: string;
-    unmatched?: string;
-    default?: string;
-    rules: (LexerStateImportRule | LexerStateMatchRule)[];
-}
-
-interface LexerStateImportRule {
-    import: string[]
-}
-
-interface LexerStateMatchRule {
-    when: string | RegExp
-    type?: string;
-    pop?: number | 'all';
-    highlight?: string;
-    inset?: number;
-    goto?: string; 
-    set?: string;
-}
-
-interface GrammarRule {
-    name: string;
-    symbols: RuleSymbol[];
-    postprocess?: PostProcessor;
-}
-
-type RuleSymbol = string | RegExp | RuleSymbolToken | RuleSymbolLexerToken | LexerTokenMatch | RuleSymbolTestable;
-
-interface RuleSymbolToken {
-    literal: any;
-}
-
-interface RuleSymbolTestable {
-    test: (data: any) => boolean;
-}
-
-interface RuleSymbolLexerToken {
-    type: string;
-}
-
-interface LexerTokenMatch {
-    token: string;
-}
-
-type PostProcessor = (payload: PostProcessorPayload) => any;
-
-interface PostProcessorPayload {
-    data: any[];
-    reference: number;
-    dot: number;
-    name: string;
-}
-
 function ${exportName}(): LanguageDefinition {
     ${generator.serializeBody()}
     return ${generator.serializeLanguage(1)}

package/build/compiler/outputs/typescript.js.map

@@ -1 +1 @@
-{"version":3,"file":"typescript.js","sourceRoot":"","sources":["../../../src/compiler/outputs/typescript.ts"],"names":[],"mappings":";;;AAEA,SAAgB,gBAAgB,CAAC,SAAoB,EAAE,UAAkB;IACrE,OAAO,uDAAuD,SAAS,CAAC,KAAK,CAAC,OAAO;;;EAGvF,SAAS,CAAC,aAAa,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;WA2FhB,UAAU;MACf,SAAS,CAAC,aAAa,EAAE;aAClB,SAAS,CAAC,iBAAiB,CAAC,CAAC,CAAC;;;iBAG1B,UAAU;;CAE1B,CAAC;AACF,CAAC;AAvGD,4CAuGC"}
+{"version":3,"file":"typescript.js","sourceRoot":"","sources":["../../../src/compiler/outputs/typescript.ts"],"names":[],"mappings":";;;AAEA,SAAgB,gBAAgB,CAAC,SAAoB,EAAE,UAAkB;IACrE,OAAO,uDAAuD,SAAS,CAAC,KAAK,CAAC,OAAO;;;;EAIvF,SAAS,CAAC,aAAa,EAAE;;WAEhB,UAAU;MACf,SAAS,CAAC,aAAa,EAAE;aAClB,SAAS,CAAC,iBAAiB,CAAC,CAAC,CAAC;;;iBAG1B,UAAU;;CAE1B,CAAC;AACF,CAAC;AAfD,4CAeC"}

package/build/index.d.ts

@@ -2,4 +2,5 @@ export * from './compiler/compiler';
 export * from './compiler/import-resolver';
 export * from './parser/parser';
 export * from './typings';
+export { StatefulLexer } from './lexers/stateful-lexer';
 export * from './utility/monarch';

package/build/index.js

@@ -14,9 +14,12 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.StatefulLexer = void 0;
 __exportStar(require("./compiler/compiler"), exports);
 __exportStar(require("./compiler/import-resolver"), exports);
 __exportStar(require("./parser/parser"), exports);
 __exportStar(require("./typings"), exports);
+var stateful_lexer_1 = require("./lexers/stateful-lexer");
+Object.defineProperty(exports, "StatefulLexer", { enumerable: true, get: function () { return stateful_lexer_1.StatefulLexer; } });
 __exportStar(require("./utility/monarch"), exports);
 //# sourceMappingURL=index.js.map

package/build/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,sDAAoC;AACpC,6DAA2C;AAC3C,kDAAgC;AAChC,4CAAyB;AACzB,oDAAkC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,sDAAoC;AACpC,6DAA2C;AAC3C,kDAAgC;AAChC,4CAAyB;AACzB,0DAAwD;AAA/C,+GAAA,aAAa,OAAA;AACtB,oDAAkC"}

package/build/lexers/stateful-lexer.d.ts

@@ -1,5 +1,5 @@
-import { LexerStateDefinition, LexerConfig } from "../typings";
-export declare class StatefulLexer {
+import { LexerStateDefinition, LexerConfig, Lexer } from "../typings";
+export declare class StatefulLexer implements Lexer {
     private start;
     private states;
     private buffer;
@@ -24,11 +24,15 @@ export declare class StatefulLexer {
     };
     next(): {
         type: string;
+        highlight: string;
+        open: string;
+        close: string;
         tag: Set<string>;
         value: string;
         text: string;
         offset: number;
         line: number;
+        lines: number;
         column: number;
         state: string;
     };

package/build/lexers/stateful-lexer.js

@@ -114,22 +114,27 @@ class StatefulLexer {
     createToken(rule, text, offset) {
         const token = {
             type: rule.type,
+            highlight: rule.highlight,
+            open: rule.open,
+            close: rule.close,
             tag: this.getTags(rule.tag),
             value: text,
             text: text,
             offset: offset,
             line: this.line,
+            lines: 0,
             column: this.column,
             state: this.current
         };
         for (let i = 0; i < text.length; i++) {
-            this.index++;
             this.column++;
             if (text[i] == '\n') {
-                this.line++;
+                token.lines++;
                 this.column = 1;
             }
         }
+        this.index += text.length;
+        this.line += token.lines;
         return token;
     }
     getTags(tags) {

package/build/lexers/stateful-lexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"stateful-lexer.js","sourceRoot":"","sources":["../../src/lexers/stateful-lexer.ts"],"names":[],"mappings":";;;AAEA,MAAa,aAAa;IACd,KAAK,CAAS;IACd,MAAM,GAA+C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzE,MAAM,CAAS;IACf,KAAK,CAAW;IAChB,KAAK,CAAS;IACd,IAAI,CAAS;IACb,MAAM,CAAS;IACf,UAAU,CAAmB;IAC7B,OAAO,CAAS;IAChB,SAAS,CAAsB;IAC/B,KAAK,CAAwB;IAC7B,MAAM,CAAS;IACf,IAAI,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEhD,YAAY,EAAE,MAAM,EAAE,KAAK,EAAe;QACtC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG;gBACf,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,CAA4B,CAAC;gBAC7D,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAA8B;gBACjD,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAyB,CAAC,CAAC,CAAC,IAAI;aACnG,CAAC;SACL;QACD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,CAAC,IAAa,EAAE,KAA0C;QAC1D,IAAI,CAAC,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,UAAU,GAAG,KAAK,EAAE,UAAU,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACjE,CAAC;IAED,KAAK;QACD,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,OAAO;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAA;IACL,CAAC;IAED,IAAI;QACA,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC9B,IAAI,CAAC,IAAI,EAAE;YACP,OAAM;SACT;QACD,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;QACnC,IAAI,CAAC,IAAI,EAAE;YACP,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;SACnD;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAA;QACjD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACvB,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,GAAG,CAAC,OAAe;QACvB,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO;YACpC,OAAM;QACV,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC9B,CAAC;IAEO,GAAG;QACP,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IAC/B,CAAC;IAEO,IAAI,CAAC,KAAa;QACtB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACnB,CAAC;IAEO,SAAS;QACb,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YACnC,OAAO;SACV;QAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAC/B,IAAI,IAAI,CAAC;QACT,IAAI,IAAyB,CAAC;QAC9B,IAAI,KAAK,CAAC;QAEV,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,KAAK,CAAC;QAC9B,IAAI,IAAI,CAAC,UAAU,EAAE;YACjB,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC;YACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;SAC1B;aAAM;YACH,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;SACnC;QACD,IAAI,KAAK,IAAI,IAAI,EAAE;YACf,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;YACtB,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;SAC7C;aAAM,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,EAAE;YAC9B,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;YACtB,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;SAC3B;aAAM;YACH,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAChB,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACvB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;aAC3B;SACJ;QAED,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IAChC,CAAC;IAEO,WAAW,CAAC,IAAyB,EAAE,IAAY,EAAE,MAAc;QACvE,MAAM,KAAK,GAAG;YACV,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;YAC3B,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,OAAO;SACtB,CAAA;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE;gBACjB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;aACnB;SACJ;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,OAAO,CAAC,IAAe;QAC3B,IAAI,CAAC,IAAI;YACL,OAAO,SAAS,CAAC;QACrB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,WAAW,CAAC,IAAyB;QACzC,IAAI,IAAI,CAAC,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAC1D,OAAO,CAAC,EAAE,GAAG,CAAC,EAAE;gBACZ,IAAI,CAAC,GAAG,EAAE,CAAC;aACd;SACJ;QAED,IAAI,IAAI,CAAC,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACtB;QACD,IAAI,IAAI,CAAC,IAAI,EAAE;YACX,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACxB;QAED,IAAI,IAAI,CAAC,KAAK,EAAE;YACZ,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;YACnB,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE;gBACb,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;aAC3B;SACJ;IACL,CAAC;IAEO,QAAQ,CAAC,KAAK;QAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACxC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,SAAS,EAAE;gBAC5B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aACxB;SACJ;QACD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;IAC9D,CAAC;CACJ;AArLD,sCAqLC;AAED,MAAM,QAAQ;IAEV,MAAM,CAAC,OAAO,CAAC,CAAM;QACjB,OAAO,CAAC,YAAY,MAAM,CAAA;IAC9B,CAAC;IACD,MAAM,CAAC,MAAM,CAAC,CAAS;QACnB,OAAO,CAAC,CAAC,OAAO,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAA;IACrD,CAAC;IAED,MAAM,CAAC,SAAS,CAAC,CAAS;QACtB,OAAO,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,CAAC,OAAO,CAAC,MAAc;QACzB,OAAO,GAAG,GAAG,MAAM,GAAG,GAAG,CAAA;IAC7B,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAiB;QACzB,IAAI,CAAC,OAAO,CAAC,MAAM;YACf,OAAO,MAAM,CAAC;QAClB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxD,OAAO,MAAM,MAAM,GAAG,CAAC;IAC3B,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,MAAuB;QACjC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;YAC5B,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;SAC3C;QACD,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC1B,OAAO,MAAM,CAAC,MAAM,CAAC;SACxB;QACD,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,MAAM,CAAC,CAAA;IAC/C,CAAC;CAEJ;AAED,SAAS,aAAa,CAAC,KAA8B;IACjD,MAAM,KAAK,GAAG,EAAE,CAAC;IACjB,MAAM,cAAc,GAAG,EAAE,CAAC;IAE1B,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE;QAC/B,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAChC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAc,CAAC;YACpC,IAAI,SAAS,KAAK,IAAI,EAAE;gBACpB,SAAS,GAAG,IAAI,CAAC,OAAO,CAAA;aAC3B;iBAAM,IAAI,SAAS,KAAK,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;gBACvD,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACzE;YACD,IAAI,IAAI,KAAK,IAAI,EAAE;gBACf,IAAI,GAAG,IAAI,CAAC,UAAU,CAAA;aACzB;iBAAM,IAAI,IAAI,KAAK,IAAI,CAAC,UAAU,EAAE;gBACjC,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACzE;SACJ;aAAM;YACH,IAAI,IAAI,IAAI,IAAI,EAAE;gBACd,IAAI,GAAG,KAAK,CAAC;aAChB;iBAAM,IAAI,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACzE;SACJ;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,CAAA;QAC9B,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,MAAM,CAAC,CAAA;SAC5D;QAED,IAAI,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE;YACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,MAAM,GAAG,uBAAuB,CAAC,CAAA;SACpF;QAED,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;KAC7C;IAED,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3C,IAAI,SAAS,KAAK,IAAI;QAClB,KAAK,IAAI,GAAG,CAAA;IAChB,IAAI,IAAI,KAAK,IAAI;QACb,KAAK,IAAI,GAAG,CAAA;IAChB,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,CAAC;AAC5D,CAAC;AAED,SAAgB,aAAa,CAAC,MAA+C,EAAE,KAAa;IAExF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAGhC,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9D,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;QACtB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACpB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;SACtB;KACJ;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAdD,sCAcC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,MAA+C,EAAE,QAAqB,EAAE,SAAsB,EAAE,KAAkB;IACxJ,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,EAAE,CAAC,CAAC;IAClE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QACzC,OAAO;IACX,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC;IAChC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAChB,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACzC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE;gBAC3B,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC5D,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAA8B,CAAC,CAAC;aAC7D;SACJ;aAAM;YACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBAC3C,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;aACxE;YACD,IAAI,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC7C,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;aACzE;SACJ;KACJ;IACD,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IAC1B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACvB,CAAC;AAED,MAAM,WAAW;IACL,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5B,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,GAA0B,EAAE,CAAC;IAElC,IAAI,CAAC,GAAG,KAA4B;QAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC7B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAE,IAAI,CAAC,IAAe,CAAC,MAAM,CAAC,EAAE;oBACjD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACzB;aACJ;iBAAM;gBACH,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAc,CAAC,EAAE;oBACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACzB;aACJ;SACJ;IACL,CAAC;CAEJ"}
+{"version":3,"file":"stateful-lexer.js","sourceRoot":"","sources":["../../src/lexers/stateful-lexer.ts"],"names":[],"mappings":";;;AAEA,MAAa,aAAa;IACd,KAAK,CAAS;IACd,MAAM,GAA+C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzE,MAAM,CAAS;IACf,KAAK,CAAW;IAChB,KAAK,CAAS;IACd,IAAI,CAAS;IACb,MAAM,CAAS;IACf,UAAU,CAAmB;IAC7B,OAAO,CAAS;IAChB,SAAS,CAAsB;IAC/B,KAAK,CAAwB;IAC7B,MAAM,CAAS;IACf,IAAI,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEhD,YAAY,EAAE,MAAM,EAAE,KAAK,EAAe;QACtC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG;gBACf,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,CAA4B,CAAC;gBAC7D,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAA8B;gBACjD,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAyB,CAAC,CAAC,CAAC,IAAI;aACnG,CAAC;SACL;QACD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,CAAC,IAAa,EAAE,KAA0C;QAC1D,IAAI,CAAC,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,UAAU,GAAG,KAAK,EAAE,UAAU,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACjE,CAAC;IAED,KAAK;QACD,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,OAAO;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAA;IACL,CAAC;IAED,IAAI;QACA,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC9B,IAAI,CAAC,IAAI,EAAE;YACP,OAAM;SACT;QACD,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;QACnC,IAAI,CAAC,IAAI,EAAE;YACP,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;SACnD;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAA;QACjD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACvB,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,GAAG,CAAC,OAAe;QACvB,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO;YACpC,OAAM;QACV,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC9B,CAAC;IAEO,GAAG;QACP,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IAC/B,CAAC;IAEO,IAAI,CAAC,KAAa;QACtB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACnB,CAAC;IAEO,SAAS;QACb,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YACnC,OAAO;SACV;QAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAC/B,IAAI,IAAI,CAAC;QACT,IAAI,IAAyB,CAAC;QAC9B,IAAI,KAAK,CAAC;QAEV,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,KAAK,CAAC;QAC9B,IAAI,IAAI,CAAC,UAAU,EAAE;YACjB,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC;YACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;SAC1B;aAAM;YACH,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;SACnC;QACD,IAAI,KAAK,IAAI,IAAI,EAAE;YACf,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;YACtB,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;SAC7C;aAAM,IAAI,KAAK,CAAC,KAAK,KAAK,KAAK,EAAE;YAC9B,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;YACtB,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;SAC3B;aAAM;YACH,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAChB,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACvB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;aAC3B;SACJ;QAED,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IAChC,CAAC;IAEO,WAAW,CAAC,IAAyB,EAAE,IAAY,EAAE,MAAc;QACvE,MAAM,KAAK,GAAG;YACV,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;YAC3B,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,OAAO;SACtB,CAAA;QACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE;gBACjB,KAAK,CAAC,KAAK,EAAE,CAAC;gBACd,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;aACnB;SACJ;QACD,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC;QACzB,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,OAAO,CAAC,IAAe;QAC3B,IAAI,CAAC,IAAI;YACL,OAAO,SAAS,CAAC;QACrB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,WAAW,CAAC,IAAyB;QACzC,IAAI,IAAI,CAAC,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAC1D,OAAO,CAAC,EAAE,GAAG,CAAC,EAAE;gBACZ,IAAI,CAAC,GAAG,EAAE,CAAC;aACd;SACJ;QAED,IAAI,IAAI,CAAC,GAAG,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SACtB;QACD,IAAI,IAAI,CAAC,IAAI,EAAE;YACX,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACxB;QAED,IAAI,IAAI,CAAC,KAAK,EAAE;YACZ,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;YACnB,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE;gBACb,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;aAC3B;SACJ;IACL,CAAC;IAEO,QAAQ,CAAC,KAAK;QAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACxC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,SAAS,EAAE;gBAC5B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aACxB;SACJ;QACD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;IAC9D,CAAC;CACJ;AAzLD,sCAyLC;AAED,MAAM,QAAQ;IAEV,MAAM,CAAC,OAAO,CAAC,CAAM;QACjB,OAAO,CAAC,YAAY,MAAM,CAAC;IAC/B,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,CAAS;QACnB,OAAO,CAAC,CAAC,OAAO,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,CAAC,SAAS,CAAC,CAAS;QACtB,OAAO,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,CAAC,OAAO,CAAC,MAAc;QACzB,OAAO,GAAG,GAAG,MAAM,GAAG,GAAG,CAAC;IAC9B,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAiB;QACzB,IAAI,CAAC,OAAO,CAAC,MAAM;YACf,OAAO,MAAM,CAAC;QAClB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxD,OAAO,MAAM,MAAM,GAAG,CAAC;IAC3B,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,MAAuB;QACjC,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;YAC5B,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC;SAC3C;QACD,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YAC1B,OAAO,MAAM,CAAC,MAAM,CAAC;SACxB;QACD,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,MAAM,CAAC,CAAA;IAC/C,CAAC;CAEJ;AAED,SAAS,aAAa,CAAC,KAA8B;IACjD,MAAM,KAAK,GAAG,EAAE,CAAC;IACjB,MAAM,cAAc,GAAG,EAAE,CAAC;IAE1B,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE;QAC/B,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAChC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAc,CAAC;YACpC,IAAI,SAAS,KAAK,IAAI,EAAE;gBACpB,SAAS,GAAG,IAAI,CAAC,OAAO,CAAA;aAC3B;iBAAM,IAAI,SAAS,KAAK,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;gBACvD,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACzE;YACD,IAAI,IAAI,KAAK,IAAI,EAAE;gBACf,IAAI,GAAG,IAAI,CAAC,UAAU,CAAA;aACzB;iBAAM,IAAI,IAAI,KAAK,IAAI,CAAC,UAAU,EAAE;gBACjC,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACzE;SACJ;aAAM;YACH,IAAI,IAAI,IAAI,IAAI,EAAE;gBACd,IAAI,GAAG,KAAK,CAAC;aAChB;iBAAM,IAAI,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACzE;SACJ;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,CAAA;QAC9B,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YACjB,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,MAAM,CAAC,CAAA;SAC5D;QAED,IAAI,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE;YACzB,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,MAAM,GAAG,uBAAuB,CAAC,CAAA;SACpF;QAED,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;KAC7C;IAED,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3C,IAAI,SAAS,KAAK,IAAI;QAClB,KAAK,IAAI,GAAG,CAAA;IAChB,IAAI,IAAI,KAAK,IAAI;QACb,KAAK,IAAI,GAAG,CAAA;IAChB,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,CAAC;AAC5D,CAAC;AAED,SAAgB,aAAa,CAAC,MAA+C,EAAE,KAAa;IACxF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9D,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;QACtB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACpB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;SACtB;KACJ;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAZD,sCAYC;AAED,SAAS,kBAAkB,CAAC,IAAY,EAAE,MAA+C,EAAE,QAAqB,EAAE,SAAsB,EAAE,KAAkB;IACxJ,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,EAAE,CAAC,CAAC;IAClE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QACzC,OAAO;IACX,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC;IAChC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAChB,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACzC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE;gBAC3B,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC5D,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAA8B,CAAC,CAAC;aAC7D;SACJ;aAAM;YACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBAC3C,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;aACxE;YACD,IAAI,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC7C,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;aACzE;SACJ;KACJ;IACD,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IAC1B,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnB,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACvB,CAAC;AAED,MAAM,WAAW;IACL,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5B,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,GAA0B,EAAE,CAAC;IAElC,IAAI,CAAC,GAAG,KAA4B;QAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC7B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAE,IAAI,CAAC,IAAe,CAAC,MAAM,CAAC,EAAE;oBACjD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACzB;aACJ;iBAAM;gBACH,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAc,CAAC,EAAE;oBACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACzB;aACJ;SACJ;IACL,CAAC;CACJ"}

package/build/typings.d.ts

@@ -181,8 +181,10 @@ export interface LexerStateMatchRule {
     type?: string;
     tag?: string[];
     pop?: number | 'all';
-    before?: true;
+    before?: boolean;
     highlight?: string;
+    open?: string;
+    close?: string;
     embed?: string;
     unembed?: boolean;
     inset?: number;

package/build/utility/index.d.ts

@@ -0,0 +1 @@
+export * from './monarch';

package/build/utility/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./monarch"), exports);
+//# sourceMappingURL=index.js.map

package/build/utility/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/utility/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,4CAA0B"}

package/build/utility/monarch.js

@@ -33,6 +33,9 @@ function CreateMonarchTokenizer(lexer) {
                 else if ('unembed' in rule) {
                     action.nextEmbedded = '@pop';
                 }
+                if (rule.before) {
+                    action.token = '@rematch';
+                }
                 tokenizer[name].push([TransformWhen(rule.when), action]);
             }
         }

package/build/utility/monarch.js.map

@@ -1 +1 @@
-{"version":3,"file":"monarch.js","sourceRoot":"","sources":["../../src/utility/monarch.ts"],"names":[],"mappings":";;;AAEA,SAAgB,sBAAsB,CAAC,KAAkB;IACrD,MAAM,SAAS,GAAQ,EAAE,CAAC;IAC1B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IAChC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;QACtB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACpC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,IAAI,QAAQ,IAAI,IAAI,EAAE;gBAClB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE;oBACzB,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAA;iBACvC;aACJ;iBAAM;gBACH,MAAM,MAAM,GAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,IAAI,QAAQ,EAAE,CAAC;gBAC1D,IAAI,KAAK,IAAI,IAAI,EAAE;oBACf,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC;iBACxB;qBAAM,IAAI,MAAM,IAAI,IAAI,EAAE;oBACvB,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;iBACjC;qBAAM,IAAI,KAAK,IAAI,IAAI,EAAE;oBACtB,MAAM,CAAC,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;iBACpC;qBAAM,IAAI,OAAO,IAAI,IAAI,EAAE;oBACxB,MAAM,CAAC,IAAI,GAAG,OAAO,CAAC;iBACzB;gBAED,IAAI,OAAO,IAAI,IAAI,EAAE;oBACjB,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC;iBACpC;qBAAM,IAAI,SAAS,IAAI,IAAI,EAAE;oBAC1B,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC;iBAChC;gBAED,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAA;aAC3D;SACJ;KACJ;IACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAlCD,wDAkCC;AAED,SAAS,aAAa,CAAC,GAAG;IACtB,OAAO,OAAO,GAAG,IAAI,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;AACpG,CAAC;AAED,SAAS,WAAW,CAAC,MAAM;IACvB,OAAO,MAAM,CAAC,OAAO,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAA;AAC1D,CAAC"}
+{"version":3,"file":"monarch.js","sourceRoot":"","sources":["../../src/utility/monarch.ts"],"names":[],"mappings":";;;AAEA,SAAgB,sBAAsB,CAAC,KAAkB;IACrD,MAAM,SAAS,GAAQ,EAAE,CAAC;IAC1B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IAChC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;QACtB,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACpC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,IAAI,QAAQ,IAAI,IAAI,EAAE;gBAClB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE;oBACzB,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAA;iBACvC;aACJ;iBAAM;gBACH,MAAM,MAAM,GAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,IAAI,QAAQ,EAAE,CAAC;gBAC1D,IAAI,KAAK,IAAI,IAAI,EAAE;oBACf,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC;iBACxB;qBAAM,IAAI,MAAM,IAAI,IAAI,EAAE;oBACvB,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;iBACjC;qBAAM,IAAI,KAAK,IAAI,IAAI,EAAE;oBACtB,MAAM,CAAC,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;iBACpC;qBAAM,IAAI,OAAO,IAAI,IAAI,EAAE;oBACxB,MAAM,CAAC,IAAI,GAAG,OAAO,CAAC;iBACzB;gBAED,IAAI,OAAO,IAAI,IAAI,EAAE;oBACjB,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC;iBACpC;qBAAM,IAAI,SAAS,IAAI,IAAI,EAAE;oBAC1B,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC;iBAChC;gBAED,IAAI,IAAI,CAAC,MAAM,EAAE;oBACb,MAAM,CAAC,KAAK,GAAG,UAAU,CAAC;iBAC7B;gBAED,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAA;aAC3D;SACJ;KACJ;IACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAtCD,wDAsCC;AAED,SAAS,aAAa,CAAC,GAAG;IACtB,OAAO,OAAO,GAAG,IAAI,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;AACpG,CAAC;AAED,SAAS,WAAW,CAAC,MAAM;IACvB,OAAO,MAAM,CAAC,OAAO,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAA;AAC1D,CAAC"}

package/package.json

@@ -1,8 +1,7 @@
 {
   "name": "grammar-well",
-  "version": "1.2.6",
+  "version": "1.3.0",
   "description": "Cross-platform Parser written in TypeScript",
-  "main": "build/index.js",
   "scripts": {
     "prebuild": "tsc --build --clean",
     "build": "tsc --build",
@@ -12,6 +11,13 @@
     "test": "nyc mocha tests/**/*.spec.ts",
     "profile": "node tests/performance/profile.js"
   },
+  "exports": {
+    ".": "./build/index.js",
+    "./parser": "./build/parser/parser.js",
+    "./compiler": "./build/compiler/index.js",
+    "./lexer": "./build/lexers/stateful-lexer.js",
+    "./utility": "./build/utility/index.js"
+  },
   "author": "0x6563",
   "license": "LGPL-3.0",
   "repository": {
@@ -48,6 +54,7 @@
     "compiler",
     "compile",
     "grammar",
-    "language"
+    "language",
+    "lexer"
   ]
 }

package/src/compiler/gwell.gwell

@@ -161,6 +161,8 @@ grammar: {{
         K_TAG _ L_COLON _ string_list : {{ { tag: $4 } }}
         | K_WHEN _ L_COLON _ T_STRING : {{ { when: $4 } }}
         | K_WHEN _ L_COLON _ T_REGEX : {{ { when: $4 } }}
+        | K_OPEN _ L_COLON _ T_STRING : {{ { open: $4 } }}
+        | K_CLOSE _ L_COLON _ T_STRING : {{ { close: $4 } }}
         | K_BEFORE _ L_COLON _ T_STRING : {{ { when: $4, before:true } }}
         | K_BEFORE _ L_COLON _ T_REGEX : {{ { when: $4, before:true } }}
         | K_POP : {{ { pop: 1 } }}
@@ -266,6 +268,8 @@ grammar: {{
     K_WHEN -> "when"
     K_POP -> "pop"
     K_BEFORE -> "before"
+    K_OPEN -> "open"
+    K_CLOSE -> "close"
     K_HIGHLIGHT -> "highlight"
     K_EMBED -> "embed"
     K_UNEMBED -> "unembed"

package/src/compiler/index.ts

@@ -0,0 +1,2 @@
+export * from './compiler';
+export * from './import-resolver';

package/src/compiler/outputs/typescript.ts

@@ -3,98 +3,10 @@ import { Generator } from "../generator/generator";
 export function TypescriptFormat(generator: Generator, exportName: string) {
     return `// Generated automatically by Grammar-Well, version ${generator.state.version} 
 // https://github.com/0x6563/grammar-well
+import type { LanguageDefinition } from 'grammar-well';
 
 ${generator.serializeHead()}
 
-interface LanguageDefinition {
-    lexer?: Lexer | LexerConfig;
-    grammar: {
-        start: string;
-        rules: GrammarRule[];
-    }
-}
-
-interface Lexer {
-    next(): LexerToken | undefined;
-    feed(chunk?: string, state?: ReturnType<Lexer['state']>): void;
-    state(): any;
-    flush?(): void;
-}
-
-interface LexerConfig {
-    start?: string
-    states: LexerStateDefinition[];
-}
-
-interface LexerToken {
-    type?: string;
-    value: string;
-    offset: number;
-    line: number;
-    column: number;
-}
-
-interface LexerStatus {
-    index: number;
-    line: number;
-    column: number;
-    state: string;
-}
-
-interface LexerStateDefinition {
-    name: string;
-    unmatched?: string;
-    default?: string;
-    rules: (LexerStateImportRule | LexerStateMatchRule)[];
-}
-
-interface LexerStateImportRule {
-    import: string[]
-}
-
-interface LexerStateMatchRule {
-    when: string | RegExp
-    type?: string;
-    pop?: number | 'all';
-    highlight?: string;
-    inset?: number;
-    goto?: string; 
-    set?: string;
-}
-
-interface GrammarRule {
-    name: string;
-    symbols: RuleSymbol[];
-    postprocess?: PostProcessor;
-}
-
-type RuleSymbol = string | RegExp | RuleSymbolToken | RuleSymbolLexerToken | LexerTokenMatch | RuleSymbolTestable;
-
-interface RuleSymbolToken {
-    literal: any;
-}
-
-interface RuleSymbolTestable {
-    test: (data: any) => boolean;
-}
-
-interface RuleSymbolLexerToken {
-    type: string;
-}
-
-interface LexerTokenMatch {
-    token: string;
-}
-
-type PostProcessor = (payload: PostProcessorPayload) => any;
-
-interface PostProcessorPayload {
-    data: any[];
-    reference: number;
-    dot: number;
-    name: string;
-}
-
 function ${exportName}(): LanguageDefinition {
     ${generator.serializeBody()}
     return ${generator.serializeLanguage(1)}

package/src/index.ts

@@ -2,4 +2,5 @@ export * from './compiler/compiler';
 export * from './compiler/import-resolver';
 export * from './parser/parser';
 export * from './typings'
+export { StatefulLexer } from './lexers/stateful-lexer';
 export * from './utility/monarch';

package/src/lexers/stateful-lexer.ts

@@ -1,6 +1,6 @@
-import { CompiledStateDefinition, LexerStateMatchRule, ResolvedStateDefinition, LexerStateDefinition, LexerConfig } from "../typings";
+import { CompiledStateDefinition, LexerStateMatchRule, ResolvedStateDefinition, LexerStateDefinition, LexerConfig, Lexer } from "../typings";
 
-export class StatefulLexer {
+export class StatefulLexer implements Lexer {
     private start: string;
     private states: { [key: string]: CompiledStateDefinition } = Object.create(null);
     private buffer: string;
@@ -122,23 +122,27 @@ export class StatefulLexer {
     private createToken(rule: LexerStateMatchRule, text: string, offset: number) {
         const token = {
             type: rule.type,
+            highlight: rule.highlight,
+            open: rule.open,
+            close: rule.close,
             tag: this.getTags(rule.tag),
             value: text,
             text: text,
             offset: offset,
             line: this.line,
+            lines: 0,
             column: this.column,
             state: this.current
         }
-
         for (let i = 0; i < text.length; i++) {
-            this.index++;
             this.column++;
             if (text[i] == '\n') {
-                this.line++;
+                token.lines++;
                 this.column = 1;
             }
         }
+        this.index += text.length;
+        this.line += token.lines;
         return token;
     }
 
@@ -186,18 +190,19 @@ export class StatefulLexer {
 class RegexLib {
 
     static IsRegex(o: any) {
-        return o instanceof RegExp
+        return o instanceof RegExp;
     }
+
     static Escape(s: string) {
-        return s.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&')
+        return s.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
     }
 
     static HasGroups(s: string) {
-        return (new RegExp('|' + s)).exec('').length > 1
+        return (new RegExp('|' + s)).exec('').length > 1;
     }
 
     static Capture(source: string) {
-        return '(' + source + ')'
+        return '(' + source + ')';
     }
 
     static Join(regexps: string[]) {
@@ -269,12 +274,10 @@ function CompileRegExp(state: ResolvedStateDefinition): RegExp {
 }
 
 export function ResolveStates(states: { [key: string]: LexerStateDefinition }, start: string) {
-
     const resolved = new Set<string>();
     const resolving = new Set<string>();
     const chain = new Set<string>();
 
-
     ResolveRuleImports(start, states, resolved, resolving, chain);
     for (const key in states) {
         if (!resolved.has(key)) {
@@ -335,5 +338,4 @@ class UniqueRules {
             }
         }
     }
-
 }

package/src/typings.ts

@@ -202,8 +202,10 @@ export interface LexerStateMatchRule {
     type?: string;
     tag?: string[];
     pop?: number | 'all';
-    before?: true;
+    before?: boolean;
     highlight?: string;
+    open?: string;
+    close?: string;
     embed?: string;
     unembed?: boolean;
     inset?: number;

package/src/utility/index.ts

@@ -0,0 +1 @@
+export * from './monarch';

package/src/utility/monarch.ts

@@ -28,6 +28,10 @@ export function CreateMonarchTokenizer(lexer: LexerConfig) {
                 } else if ('unembed' in rule) {
                     action.nextEmbedded = '@pop';
                 }
+                
+                if (rule.before) {
+                    action.token = '@rematch';
+                }
 
                 tokenizer[name].push([TransformWhen(rule.when), action])
             }