grammar-well 1.3.3 → 2.0.0

package/src/lexers/token-buffer.ts

@@ -1,7 +1,7 @@
-import { Lexer, TQRestorePoint, LexerToken } from '../typings';
+import { RuntimeLexer, RuntimeLexerToken, TQRestorePoint } from '../typings/index.js';
 
 export class TokenBuffer {
-    private history: LexerToken[] = [];
+    private history: RuntimeLexerToken[] = [];
     private queued: string = '';
 
     private $historyIndex = -1;
@@ -15,7 +15,7 @@ export class TokenBuffer {
         return { historyIndex: this.$historyIndex, offset: this.offset };
     }
 
-    constructor(private lexer: Lexer) { }
+    constructor(public lexer: RuntimeLexer) { }
 
     reset(buffer: string) {
         this.lexer.feed(buffer);

package/src/parser/algorithms/cyk.ts

@@ -1,13 +1,14 @@
-import { TokenBuffer } from "../../lexers/token-buffer";
-import { GrammarRule, GrammarRuleSymbol, LanguageDefinition, LexerToken } from "../../typings";
-import { Matrix } from "../../utility/general";
-import { ParserUtility } from "../parser";
+import { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol, RuntimeParserClass, RuntimeLexerToken } from "../../typings/index.js";
+import { TokenBuffer } from "../../lexers/token-buffer.js";
+import { Matrix } from "../../utility/general.js";
+import { ParserUtility } from "../../utility/parsing.js";
 
-export function CYK(language: LanguageDefinition & { tokens: TokenBuffer }, _options = {}) {
-    const { grammar, tokens } = language;
+export function CYK(language: RuntimeParserClass & { tokens: TokenBuffer }, _options = {}) {
+    const { grammar } = language.artifacts;
+    const { tokens } = language;
 
-    const terminals: GrammarRule[] = [];
-    const nonTerminals: GrammarRule[] = [];
+    const terminals: RuntimeGrammarProductionRule[] = [];
+    const nonTerminals: RuntimeGrammarProductionRule[] = [];
 
     for (const name in grammar.rules) {
         for (const rule of grammar.rules[name]) {
@@ -21,7 +22,7 @@ export function CYK(language: LanguageDefinition & { tokens: TokenBuffer }, _opt
     }
 
     let currentTokenIndex = -1;
-    const chart = new Matrix(0, 0, () => new Map<GrammarRuleSymbol, Terminal | NonTerminal>());
+    const chart = new Matrix(0, 0, () => new Map<RuntimeGrammarRuleSymbol, Terminal | NonTerminal>());
     for (const token of tokens) {
         currentTokenIndex++;
         chart.resize(currentTokenIndex + 2, currentTokenIndex + 2);
@@ -63,12 +64,12 @@ function GetValue(ref: Terminal | NonTerminal) {
 }
 
 export interface NonTerminal {
-    rule: GrammarRule;
+    rule: RuntimeGrammarProductionRule;
     left: NonTerminal | Terminal;
     right: NonTerminal | Terminal;
 }
 
 export interface Terminal {
-    rule: GrammarRule;
-    token: LexerToken;
+    rule: RuntimeGrammarProductionRule;
+    token: RuntimeLexerToken;
 }

package/src/parser/algorithms/earley.ts

@@ -1,15 +1,15 @@
-import { Dictionary, GrammarRule, LanguageDefinition } from "../../typings";
-import { TokenBuffer } from "../../lexers/token-buffer";
-import { TextFormatter } from "../../utility/text-format";
-import { ParserUtility } from "../parser";
+import { Dictionary, RuntimeGrammarProductionRule, RuntimeParserClass } from "../../typings/index.js";
+import { TokenBuffer } from "../../lexers/token-buffer.js";
+import { TextFormatter } from "../../utility/text-format.js";
+import { ParserUtility } from "../../utility/parsing.js";
 
 export interface EarleyParserOptions {
     keepHistory?: boolean;
 }
 
-export function Earley(language: LanguageDefinition & { tokens: TokenBuffer }, options: EarleyParserOptions = {}) {
+export function Earley(language: RuntimeParserClass & { tokens: TokenBuffer }, options: EarleyParserOptions = {}) {
     const { tokens } = language;
-    const { rules, start } = language.grammar;
+    const { rules, start } = language.artifacts.grammar;
     const column = new Column(rules, 0);
     const table: Column[] = [column];
     column.wants[start] = [];
@@ -70,7 +70,7 @@ class Column {
     completed: Dictionary<State[]> = Object.create(null);  // states that are nullable
 
     constructor(
-        private rules: Dictionary<GrammarRule[]>,
+        private rules: Dictionary<RuntimeGrammarProductionRule[]>,
         public index: number
     ) { }
 
@@ -127,8 +127,8 @@ class Column {
         }
     }
 
-    expects(): GrammarRule[] {
-        const result: GrammarRule[] = [];
+    expects(): RuntimeGrammarProductionRule[] {
+        const result: RuntimeGrammarProductionRule[] = [];
         for (const state of this.states) {
             if (state.rule.symbols[state.dot] && typeof state.rule.symbols[state.dot] !== 'string') {
                 result.push({ ...state.rule, index: state.dot } as any)
@@ -149,7 +149,7 @@ class State {
     left: State;
     right: State | StateToken;
     constructor(
-        public rule: GrammarRule,
+        public rule: RuntimeGrammarProductionRule,
         public dot: number,
         public reference: number,
         public wantedBy: State[]

package/src/parser/algorithms/lrk/algorithm.ts

@@ -1,11 +1,12 @@
-import { TokenBuffer } from "../../../lexers/token-buffer";
-import { LanguageDefinition } from "../../../typings";
-import { ParserUtility } from "../../parser";
-import { CanonicalCollection } from "./canonical-collection";
-import { LRStack } from "./stack";
+import { RuntimeParserClass } from "../../../typings/index.js";
+import { TokenBuffer } from "../../../lexers/token-buffer.js";
+import { ParserUtility } from "../../../utility/parsing.js";
+import { CanonicalCollection } from "./canonical-collection.js";
+import { LRStack } from "./stack.js";
 
-export function LRK(language: LanguageDefinition & { tokens: TokenBuffer }, options = {}) {
-    const { grammar, tokens } = language;
+export function LRK(language: RuntimeParserClass & { tokens: TokenBuffer }, options = {}) {
+    const { grammar } = language.artifacts;
+    const { tokens } = language;
     const { states, rules: rules } = new CanonicalCollection(grammar);
     const stack = new LRStack();
     const s = states.get('0.0');

package/src/parser/algorithms/lrk/canonical-collection.ts

@@ -1,30 +1,30 @@
-import { GrammarRule, GrammarRuleSymbol, LanguageDefinition } from "../../../typings";
-import { ParserUtility } from "../../parser";
-import { BiMap } from "./bimap";
-import { ClosureBuilder } from "./closure";
-import { State } from "./state";
+import { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol, RuntimeParserClass } from "../../../typings/index.js";
+import { ParserUtility } from "../../../utility/parsing.js";
+import { BiMap } from "./bimap.js";
+import { ClosureBuilder } from "./closure.js";
+import { State } from "./state.js";
 
 export class CanonicalCollection {
     states: Map<string, State> = new Map();
-    rules: BiMap<GrammarRule> = new BiMap();
-    terminals: BiMap<GrammarRuleSymbol> = new BiMap();
+    rules: BiMap<RuntimeGrammarProductionRule> = new BiMap();
+    terminals: BiMap<RuntimeGrammarRuleSymbol> = new BiMap();
 
     private closure: ClosureBuilder;
     constructor(
-        public grammar: LanguageDefinition['grammar']
+        public grammar:  RuntimeParserClass['artifacts']['grammar']
     ) {
         const augmented = {
             name: Symbol() as unknown as string,
-            symbols: [grammar.start]
+            symbols: [this.grammar.start]
         }
-        grammar['rules'][augmented.name] = [augmented];
-        this.closure = new ClosureBuilder(grammar);
+        this.grammar['rules'][augmented.name] = [augmented];
+        this.closure = new ClosureBuilder(this.grammar);
         this.rules.id(augmented);
-        this.addState(grammar['rules'][augmented.name][0], 0);
+        this.addState(this.grammar['rules'][augmented.name][0], 0);
         this.linkStates('0.0');
     }
 
-    private addState(rule: GrammarRule, dot: number) {
+    private addState(rule: RuntimeGrammarProductionRule, dot: number) {
         const id = this.getStateId(rule, dot);
         if (this.states.has(id))
             return;
@@ -73,7 +73,7 @@ export class CanonicalCollection {
         }
     }
 
-    private getStateId(rule: GrammarRule, dot: number) {
+    private getStateId(rule: RuntimeGrammarProductionRule, dot: number) {
         return this.rules.id(rule) + '.' + dot;
     }
 }

package/src/parser/algorithms/lrk/closure.ts

@@ -1,9 +1,9 @@
-import { GrammarRule, GrammarRuleSymbol, LanguageDefinition } from "../../../typings"
-import { ParserUtility } from "../../parser"
+import { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol, RuntimeParserClass } from "../../../typings/index.js";
+import { ParserUtility } from "../../../utility/parsing.js";
 
 export class ClosureBuilder {
     constructor(
-        private grammar: LanguageDefinition['grammar'],
+        private grammar: RuntimeParserClass['artifacts']['grammar'],
     ) { }
 
     get(rule: string) {
@@ -12,7 +12,7 @@ export class ClosureBuilder {
         return closure.items;
     }
 
-    private addClosure(closure: RuleClosure, symbol: GrammarRuleSymbol) {
+    private addClosure(closure: RuleClosure, symbol: RuntimeGrammarRuleSymbol) {
         if (!ParserUtility.SymbolIsTerminal(symbol)) {
             const key = symbol as string;
             if (!(closure.visited.has(key))) {
@@ -30,8 +30,8 @@ export class ClosureBuilder {
 
 interface RuleClosure {
     items: {
-        rule: GrammarRule,
+        rule: RuntimeGrammarProductionRule,
         dot: number,
     }[]
-    visited: Set<GrammarRuleSymbol>;
+    visited: Set<RuntimeGrammarRuleSymbol>;
 }

package/src/parser/algorithms/lrk/stack.ts

@@ -1,5 +1,5 @@
-import { GrammarRule, GrammarRuleSymbol } from "../../../typings";
-import { State } from "./state";
+import { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol } from "../../../typings/index.js";
+import { State } from "./state.js";
 
 export class LRStack {
 
@@ -18,7 +18,7 @@ export class LRStack {
         this.current.state = state;
     }
 
-    reduce(rule: GrammarRule) {
+    reduce(rule: RuntimeGrammarProductionRule) {
         const n = LRStack.NewItem();
         const l = rule.symbols.length;
         n.children = this.stack.splice(l * -1, l);
@@ -28,7 +28,7 @@ export class LRStack {
         this.stack.push(n);
     }
 
-    append(symbol: GrammarRuleSymbol) {
+    append(symbol: RuntimeGrammarRuleSymbol) {
         this.stack.push(LRStack.NewItem())
         this.current.symbol = symbol;
     }
@@ -47,7 +47,7 @@ export class LRStack {
 interface LRStackItem {
     children: LRStackItem[];
     state: State;
-    symbol: GrammarRuleSymbol;
-    rule: GrammarRule;
+    symbol: RuntimeGrammarRuleSymbol;
+    rule: RuntimeGrammarProductionRule;
     value: any;
 }

package/src/parser/algorithms/lrk/state.ts

@@ -1,10 +1,10 @@
-import { GrammarRule, GrammarRuleSymbol } from "../../../typings";
+import { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol } from "../../../typings/index.js";
 
 export interface State {
-    items: { rule: GrammarRule, dot: number }[];
+    items: { rule: RuntimeGrammarProductionRule, dot: number }[];
     isFinal: boolean;
-    actions: Map<GrammarRuleSymbol, string>;
-    goto: Map<GrammarRuleSymbol, string>;
+    actions: Map<RuntimeGrammarRuleSymbol, string>;
+    goto: Map<RuntimeGrammarRuleSymbol, string>;
     reduce: number;
-    rule: GrammarRule
+    rule: RuntimeGrammarProductionRule
 }

package/src/parser/parse.ts

@@ -0,0 +1,45 @@
+import { ParserAlgorithm, RuntimeParserClass } from "../typings/index.js";
+import { CharacterLexer } from "../lexers/character-lexer.js";
+import { StatefulLexer } from "../lexers/stateful-lexer.js";
+import { TokenBuffer } from "../lexers/token-buffer.js";
+import { CYK } from "./algorithms/cyk.js";
+import { Earley } from "./algorithms/earley.js";
+import { LRK } from "./algorithms/lrk/algorithm.js";
+import { ParserUtility } from "../utility/parsing.js";
+
+const ParserRegistry: { [key: string]: ParserAlgorithm } = {
+    earley: Earley,
+    cyk: CYK,
+    lr0: LRK
+}
+
+export function Parse(
+    language: InstanceType<RuntimeParserClass>,
+    input: string,
+    options: ParserOptions = {
+        algorithm: 'earley',
+        parserOptions: {}
+    },
+    results: 'full' | 'first' = 'first'
+) {
+    const tokenizer = GetTokenizer(language.artifacts);
+    tokenizer.feed(input);
+    const algorithm = typeof options.algorithm == 'function' ? options.algorithm : ParserRegistry[options.algorithm];
+    const result = algorithm({ ...language, tokens: tokenizer, utility: ParserUtility }, options.parserOptions);
+    return results == 'full' ? result : result.results[0];
+}
+
+function GetTokenizer({ lexer }: RuntimeParserClass['artifacts']) {
+    if (!lexer) {
+        return new TokenBuffer(new CharacterLexer());
+    } else if ("feed" in lexer && typeof lexer.feed == 'function') {
+        return new TokenBuffer(lexer);
+    } else if ('states' in lexer) {
+        return new TokenBuffer(new StatefulLexer(lexer));
+    }
+}
+
+interface ParserOptions {
+    algorithm: (keyof typeof ParserRegistry) | ParserAlgorithm,
+    parserOptions?: any;
+}

package/src/typings/ast.ts

@@ -0,0 +1,148 @@
+import { Dictionary } from "./common.js";
+
+export type AST = ASTDirectives[];
+
+export type ASTJavascriptLifecycleLiteral = {
+    lifecycle: string;
+    path?: boolean;
+    js: ASTJavaScriptLiteral;
+}
+export type ASTJavaScriptLiteral = { js: string }
+export type ASTJavaScriptTemplate = { template: string }
+
+/**
+ * Only available during the generator phase
+ *
+ * @export
+ * @interface ASTJavaScriptBuiltin
+ */
+export interface ASTJavaScriptBuiltin { builtin: string }
+
+export interface ASTImport {
+    import: string;
+    path?: boolean;
+    alias?: string;
+}
+
+export interface ASTConfig {
+    config: Dictionary<any>;
+}
+
+export interface ASTGrammar {
+    grammar: {
+        config?: Dictionary<any>;
+        rules: ASTGrammarProduction[];
+    }
+}
+
+export interface ASTLexer {
+    lexer: ASTLexerConfig;
+}
+export interface ASTLexerConfig {
+    start?: string;
+    states: { name: string, state: (ASTLexerState | ASTLexerStateSpan) }[];
+};
+export interface ASTGrammarProduction {
+    name: string;
+    expressions: ASTGrammarProductionRule[];
+    postprocess?: ASTJavaScriptLiteral | ASTJavaScriptTemplate | ASTJavaScriptBuiltin;
+}
+
+export interface ASTGrammarProductionRule {
+    symbols: ASTGrammarSymbol[];
+    postprocess?: ASTJavaScriptLiteral | ASTJavaScriptTemplate | ASTJavaScriptBuiltin;
+}
+
+export type ASTGrammarSymbol = ASTGrammarSymbolAlias & (ASTGrammarSymbolNonTerminal | ASTGrammarSymbolRegex | ASTGrammarSymbolToken | ASTGrammarSymbolLiteral | ASTGrammarSymbolRepeat | ASTGrammarSymbolGroup);
+export type ASTGrammarSymbolAlias = { alias?: string };
+
+export interface ASTGrammarSymbolGroup {
+    subexpression: ASTGrammarProductionRule[];
+}
+
+export interface ASTGrammarSymbolRepeat {
+    expression: ASTGrammarSymbol;
+    repeat: "+" | "*" | "?";
+}
+
+export interface ASTGrammarSymbolNonTerminal {
+    rule: string;
+}
+
+export interface ASTGrammarSymbolRegex {
+    regex: string;
+    flags?: string;
+    quote?: string;
+}
+
+export interface ASTGrammarSymbolToken {
+    token: string;
+}
+
+export interface ASTGrammarSymbolLiteral {
+    literal: string;
+    insensitive?: boolean;
+}
+
+export type ASTDirectives = (ASTJavascriptLifecycleLiteral | ASTImport | ASTConfig | ASTGrammar | ASTLexer);
+
+export interface ASTLexerState {
+    unmatched?: ASTLexerStateNonMatchRule;
+    default?: ASTLexerStateMatchRule;
+    rules: (ASTLexerStateImportRule | ASTLexerStateMatchRule | ASTLexerStateSpan)[];
+}
+
+export interface ASTLexerStateSpan {
+    config?: {
+        transition?: "goto" | "set"
+    }
+    span: { name: string, state: (ASTLexerState) }[];
+}
+
+export interface ASTLexerStateImportRule {
+    import: string[];
+
+    pop?: number | 'all';
+    inset?: number;
+    goto?: string;
+    set?: string;
+    stay?: boolean;
+}
+
+export interface ASTLexerStateMatchRule {
+    when: string | ASTGrammarSymbolRegex;
+    type?: string;
+    tag?: string[];
+    before?: boolean;
+    skip?: boolean;
+    highlight?: string;
+    open?: string;
+    close?: string;
+    embed?: string;
+    unembed?: boolean;
+
+    pop?: number | 'all';
+    stay?: true;
+    inset?: number;
+    goto?: string;
+    set?: string;
+}
+
+export interface ASTLexerStateNonMatchRule {
+    type?: string;
+    tag?: string[];
+    before?: undefined;
+    skip?: undefined;
+    highlight?: string;
+    open?: string;
+    close?: string;
+    embed?: string;
+    unembed?: boolean;
+
+    pop?: number | 'all';
+    inset?: number;
+    goto?: string;
+    set?: string;
+}
+
+export type StateList = { name: string, state: (ASTLexerState | ASTLexerStateSpan) }[];

package/src/typings/common.ts

@@ -0,0 +1,2 @@
+export type Dictionary<T> = { [key: string]: T }
+export type Override<T, R> = Omit<T, keyof R> & R;

package/src/typings/generator.ts

@@ -0,0 +1,62 @@
+import { GeneratorState } from "../generator/state.js";
+import { ASTGrammarSymbolLiteral, ASTGrammarSymbolNonTerminal, ASTGrammarSymbolRegex, ASTGrammarSymbolToken, ASTJavaScriptBuiltin, ASTJavaScriptLiteral, ASTJavaScriptTemplate, ASTLexerStateImportRule, ASTLexerStateMatchRule, ASTLexerStateNonMatchRule } from "./ast.js";
+import { Dictionary } from "./common.js";
+import { ImportResolver, ImportResolverConstructor } from "./index.js";
+
+
+export type GenerateOptions = GeneratorOptions & { output?: GeneratorOutputOptions };
+
+export interface GeneratorOptions {
+    version?: string;
+    basedir?: string;
+    resolver?: ImportResolverConstructor | ImportResolver;
+    overrides?: Dictionary<string>;
+}
+
+export interface GeneratorOutputOptions {
+    artifacts?: {
+        lexer?: boolean;
+        grammar?: boolean;
+        [key: string]: boolean;
+    }
+    parser?: boolean;
+    format?: GeneratorExportFormat;
+    noscript?: boolean;
+    name?: string;
+}
+
+export type GeneratorExportFormat = 'object' | 'json' | 'cjs' | 'commonjs' | 'js' | 'javascript' | 'module' | 'esmodule' | 'esm' | 'ts' | 'typescript';
+
+export interface GeneratorContext {
+    imported: Set<string>;
+    resolver: ImportResolver;
+    state: GeneratorState;
+}
+export interface GeneratorStateGrammar {
+    start: string;
+    config: {
+        algorithm?: 'earley' | 'lr0' | 'cyk';
+        postprocessorDefault?: ASTJavaScriptLiteral | ASTJavaScriptBuiltin;
+        postprocessorOverride?: ASTJavaScriptLiteral | ASTJavaScriptBuiltin;
+    }
+    rules: Dictionary<GeneratorGrammarProductionRule[]>,
+    uuids: { [key: string]: number }
+};
+
+export interface GeneratorLexerConfig {
+    start?: string
+    states: Dictionary<GeneratorLexerState>;
+}
+
+export interface GeneratorLexerState {
+    unmatched?: ASTLexerStateNonMatchRule;
+    rules: (ASTLexerStateImportRule | ASTLexerStateMatchRule)[];
+}
+
+export interface GeneratorGrammarProductionRule {
+    name: string;
+    symbols: GeneratorGrammarSymbol[];
+    postprocess?: ASTJavaScriptTemplate | ASTJavaScriptLiteral | ASTJavaScriptBuiltin;
+}
+
+export type GeneratorGrammarSymbol = { alias?: string } & (ASTGrammarSymbolNonTerminal | ASTGrammarSymbolRegex | ASTGrammarSymbolLiteral | ASTGrammarSymbolToken);

package/src/typings/index.ts

@@ -0,0 +1,38 @@
+import { TokenBuffer } from "../lexers/token-buffer.js";
+import { ParserUtility } from "../utility/parsing.js";
+import { ASTLexerStateNonMatchRule } from "./ast.js";
+import { RuntimeGrammarProductionRule, RuntimeGrammarRuleSymbol, RuntimeParserClass, RuntimeLexerStateMatchRule } from "./runtime.js";
+export * from './ast.js';
+export * from './common.js';
+export * from './generator.js';
+export * from './runtime.js';
+
+export interface ImportResolver {
+    path(path: string): string;
+    body(path: string): Promise<string>;
+}
+
+export interface ImportResolverConstructor {
+    new(basePath: string): ImportResolver;
+}
+
+export type ParserAlgorithm = ((language: RuntimeParserClass & { tokens: TokenBuffer; utility: ParserUtility }, options?: any) => { results: any[], info?: any });
+
+export interface LRState {
+    actions: Next[];
+    goto: { [key: string]: string };
+    reduce?: RuntimeGrammarProductionRule;
+    isFinal: boolean;
+}
+
+type Next = { symbol: RuntimeGrammarRuleSymbol, next: string };
+
+export interface TQRestorePoint {
+    historyIndex: number;
+    offset: number;
+}
+export interface StatefulLexerStateDefinition {
+    regex: RegExp;
+    unmatched?: ASTLexerStateNonMatchRule;
+    rules: RuntimeLexerStateMatchRule[];
+}

package/src/typings/runtime.ts

@@ -0,0 +1,82 @@
+import { LRState } from "./index.js";
+import { ASTGrammarSymbolLiteral, ASTGrammarSymbolToken, ASTLexerStateMatchRule, ASTLexerStateNonMatchRule } from "./ast.js";
+import { Dictionary } from "./common.js";
+
+export interface RuntimeParserClass {
+    artifacts: {
+        lexer?: RuntimeLexer | RuntimeLexerConfig;
+        grammar: {
+            start: string;
+            rules: Dictionary<RuntimeGrammarProductionRule[]>;
+        }
+        lr?: {
+            k: number;
+            table: Dictionary<LRState>;
+        }
+    }
+    new();
+}
+
+export interface RuntimeGrammarProductionRule {
+    name: string;
+    symbols: RuntimeGrammarRuleSymbol[];
+    postprocess?: RuntimePostProcessor;
+}
+
+export type RuntimeGrammarRuleSymbol = string | RegExp | ASTGrammarSymbolLiteral | ASTGrammarSymbolToken | RuntimeGrammarRuleSymbolFunction;
+
+type RuntimeGrammarRuleSymbolFunction = (data: RuntimeLexerToken) => boolean;
+
+export type RuntimePostProcessor = (payload: RuntimePostProcessorPayload) => any;
+
+interface RuntimePostProcessorPayload {
+    data: any[];
+    rule: RuntimeGrammarProductionRule;
+    meta: any;
+}
+
+export interface RuntimeLexerConfig {
+    start?: string
+    states: Dictionary<RuntimeLexerState>;
+}
+
+export interface RuntimeLexerState {
+    unmatched?: ASTLexerStateNonMatchRule;
+    rules: RuntimeLexerStateMatchRule[];
+    regex: RegExp;
+}
+
+export interface RuntimeLexer {
+    next(): RuntimeLexerToken | undefined;
+    feed(chunk?: string, state?: ReturnType<RuntimeLexer['state']>): void;
+    state(): any;
+    flush?(): void;
+}
+
+export interface RuntimeLexerToken {
+    type?: string | undefined;
+    tag?: Set<string>;
+    value: string;
+    offset: number;
+    line: number;
+    column: number;
+}
+
+
+export interface RuntimeLexerStateMatchRule {
+    when: string | RegExp;
+    type?: string;
+    tag?: string[];
+    before?: boolean;
+    skip?: boolean;
+    highlight?: string;
+    open?: string;
+    close?: string;
+    embed?: string;
+    unembed?: boolean;
+
+    pop?: number | 'all';
+    inset?: number;
+    goto?: string;
+    set?: string;
+}

package/src/utility/format.ts

@@ -0,0 +1,12 @@
+import V1Grammar from '../generator/grammars/v1.js';
+import V2Grammar from '../generator/grammars/v2.js';
+import { V2GrammarString } from "../generator/stringify/grammar/v2.js";
+import { Parse } from '../parser/parse.js';
+
+export function Format(source: string, sourceVersion: '1' | '2' = '2') {
+    const grammar = sourceVersion.toString() == '1' ? V1Grammar : V2Grammar;
+    const result = Parse(new grammar() as any, source);
+    const stringer = new V2GrammarString();
+    stringer.append(result);
+    return stringer.source;
+}