grammar-well 1.1.0 → 1.1.1

package/README.md

@@ -1,6 +1,109 @@
 # Grammar Well
 A cross-platform grammar compiler and interpreter. That aims to facilitate a simple way to create and evaluate custom grammars on the front-end and back-end. Formerly a TypeScript port of [Nearley](https://github.com/kach/nearley).
 
+# Quick Start
+### Install
+`npm i grammar-well`
+
+### Example
+
+```Javascript
+import { Compile, Parse } from 'grammar-well';
+
+async function GrammarWellRunner(source, input) {
+    function Evalr(source) {
+        const module = { exports: null };
+        eval(source);
+        return module.exports;
+    }
+    const compiled = await Compile(source, { exportName: 'grammar' });
+    return Parse(Evalr(compiled)(), input, { algorithm: 'earley' });
+}
+
+
+const source = `lexer: {{
+    start: "json"
+
+    json ->
+        - when: /\s+/ tag: "space"
+        - when: /-?(?:[0-9]|[1-9][0-9]+)(?:\.[0-9]+)?(?:[eE][-+]?[0-9]+)?\b/ tag: "number"
+        - when: /"(?:\\["bfnrt\/\\]|\\u[a-fA-F0-9]{4}|[^"\\])*"/ tag: "string"
+        - when: "{" tag: "{"
+        - when: "}" tag: "}"
+        - when: "[" tag: "["
+        - when: "]" tag: "]"
+        - when: "," tag: ","
+        - when: ":" tag: ":"
+        - when: "true" tag: "true"
+        - when: "false" tag: "false"
+        - when: "null" tag: "null"
+}}
+
+grammar: {{
+    json -> _ (object | array) _ : {{ $1[0] }}
+
+    object -> "{" _ "}" : {{ {} }}
+        | "{" _ pair (_ "," _ pair)* _ "}" : \${ extractObject }
+
+    array -> "[" _ "]" : {{ [] }}
+        | "[" _ value (_ "," _ value)* _ "]" : \${ extractArray }
+
+    value : {{ $0 }} ->
+        object
+        | array
+        | number
+        | string
+        | "true" : {{ true }}
+        | "false" : {{ false }}
+        | "null" : {{ null }}
+
+    number -> $number : {{ parseFloat($0.value) }}
+
+    string -> $string : {{ JSON.parse($0.value) }}
+
+    pair -> key:k _ ":" _ value:v : {{ [$k, $v] }}
+
+    key -> string : {{ $0 }}
+
+    _ -> $space? : {{ null }}
+}}
+
+head: \${
+
+function extractPair(kv, output) {
+    if(kv[0]) { output[kv[0]] = kv[1]; }
+}
+
+function extractObject({data}) {
+    let output = {};
+
+    extractPair(data[2], output);
+
+    for (let i in data[3]) {
+        extractPair(data[3][i][3], output);
+    }
+
+    return output;
+}
+
+function extractArray({data}) {
+    let output = [data[2]];
+
+    for (let i in data[3]) {
+        output.push(data[3][i][3]);
+    }
+
+    return output;
+}
+}
+`
+
+const input = `{"a":"string","b":true,"c":2}`
+
+
+console.log(await GrammarWellRunner(source, input))
+```
+
 # Warning
 A lot has changed and documentation needs to be written. For now here's the Grammar Well grammar file that parses Garmmar Well's syntax.
 

package/bootstrap.ts

@@ -9,8 +9,8 @@ const BaseDir = './src/grammars';
         try {
             console.log(fullpath(file))
             if (/\.gwell$/.test(file)) {
-                const json = await Compile(read(file), { format: 'json' });
-                const js = await Compile(read(file), { exportName: 'grammar', format: 'esmodule' });
+                const json = await Compile(read(file), { template: 'json' });
+                const js = await Compile(read(file), { exportName: 'grammar', template: 'esmodule' });
                 write(file.replace(/.gwell$/, '.json'), json);
                 write(file.replace(/.gwell$/, '.js'), js);
             }

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "grammar-well",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "Cross-platform Parser written in TypeScript",
   "main": "build/index.js",
   "scripts": {
     "prebuild": "tsc --build --clean",
     "build": "tsc --build",
-    "bootstrap": "ts-node bootstrap.ts",
-    "testing": "ts-node testing.ts",
+    "bootstrap": "tsx bootstrap.ts",
+    "testing": "tsx testing.ts",
     "benchmark": "node tests/performance/benchmark.js",
     "test": "nyc mocha tests/**/*.spec.ts",
     "profile": "node tests/performance/profile.js"
@@ -25,19 +25,20 @@
   "devDependencies": {
     "@types/chai": "^4.3.4",
     "@types/mocha": "^10.0.1",
-    "@types/node": "^18.11.18",
-    "@typescript-eslint/eslint-plugin": "^5.49.0",
-    "@typescript-eslint/parser": "^5.49.0",
+    "@types/node": "^18.15.0",
+    "@typescript-eslint/eslint-plugin": "^5.54.1",
+    "@typescript-eslint/parser": "^5.54.1",
     "babel-cli": "^6.26.0",
     "babel-preset-env": "^1.7.0",
     "benny": "^3.7.1",
     "chai": "^4.3.7",
-    "eslint": "^8.33.0",
-    "expect": "^29.4.1",
+    "eslint": "^8.36.0",
+    "expect": "^29.5.0",
     "mocha": "^10.2.0",
     "nyc": "^15.1.0",
     "ts-node": "^10.9.1",
-    "typescript": "^4.9.4",
+    "tsx": "^3.12.3",
+    "typescript": "^4.9.5",
     "yaml": "^2.2.1"
   },
   "keywords": [
@@ -49,4 +50,4 @@
     "grammar",
     "language"
   ]
-}
+}

package/src/compiler/compiler.ts

@@ -1,4 +1,4 @@
-import { CompileOptions, GrammarBuilderContext, OutputFormat, LanguageDirective, ConfigDirective, GrammarBuilderSymbolRepeat, GrammarBuilderExpression, GeneratorGrammarRule, GrammarDirective, ImportDirective, LexerDirective, GrammarBuilderSymbolSubexpression, GrammarTypeLiteral, GeneratorGrammarSymbol, GrammarBuilderSymbol, GrammarBuilderRule } from "../typings";
+import { CompileOptions, GrammarBuilderContext, TemplateFormat, LanguageDirective, ConfigDirective, GrammarBuilderSymbolRepeat, GrammarBuilderExpression, GeneratorGrammarRule, GrammarDirective, ImportDirective, LexerDirective, GrammarBuilderSymbolSubexpression, GrammarTypeLiteral, GeneratorGrammarSymbol, GrammarBuilderSymbol, GrammarBuilderRule } from "../typings";
 
 import { Parser } from "../parser/parser";
 import { FileSystemResolver } from "./import-resolver";
@@ -18,7 +18,7 @@ const BuiltInRegistry = {
     string,
     whitespace,
 }
-const OutputFormats = {
+const TemplateFormats = {
     _default: JavascriptOutput,
     object: (grammar, exportName) => ({ grammar, exportName }),
     json: JSONFormatter,
@@ -26,6 +26,7 @@ const OutputFormats = {
     javascript: JavascriptOutput,
     module: ESMOutput,
     esmodule: ESMOutput,
+    esm: ESMOutput,
     ts: TypescriptFormat,
     typescript: TypescriptFormat
 }
@@ -33,7 +34,7 @@ const OutputFormats = {
 export async function Compile(rules: string | LanguageDirective | (LanguageDirective[]), config: CompileOptions = {}) {
     const builder = new GrammarBuilder(config);
     await builder.import(rules as any);
-    return builder.export(config.format);
+    return builder.export(config.template);
 }
 
 export class GrammarBuilder {
@@ -51,11 +52,11 @@ export class GrammarBuilder {
         this.generator.state.grammar.uuids = this.context.uuids;
     }
 
-    export<T extends OutputFormat = '_default'>(format: T, name: string = 'GWLanguage'): ReturnType<typeof OutputFormats[T]> {
+    export<T extends TemplateFormat = '_default'>(format: T, name: string = 'GWLanguage'): ReturnType<typeof TemplateFormats[T]> {
         const grammar = this.generator.state;
         const output = format || grammar.config.preprocessor || '_default';
-        if (OutputFormats[output]) {
-            return OutputFormats[output](this.generator, name);
+        if (TemplateFormats[output]) {
+            return TemplateFormats[output](this.generator, name);
         }
         throw new Error("No such preprocessor: " + output)
     }

package/src/typings.ts

@@ -12,10 +12,10 @@ export interface CompileOptions {
     resolver?: ImportResolverConstructor;
     resolverInstance?: ImportResolver;
     exportName?: string;
-    format?: OutputFormat;
+    template?: TemplateFormat;
 }
 
-export type OutputFormat = '_default' | 'object' | 'json' | 'js' | 'javascript' | 'module' | 'esmodule' | 'ts' | 'typescript'
+export type TemplateFormat = '_default' | 'object' | 'json' | 'js' | 'javascript' | 'module' | 'esmodule' | 'esm' | 'ts' | 'typescript'
 
 export interface GrammarBuilderContext {
     alreadyCompiled: Set<string>;

package/src/utility/general.ts

@@ -4,7 +4,7 @@ import { Dictionary, GrammarRuleSymbol } from "../typings";
 export class Collection<T> {
     categorized: Dictionary<Dictionary<number>> = {};
     private uncategorized = new Map<T, number>();
-    private items: T[] = [];
+    items: T[] = [];
 
     constructor(ref: T[] = []) {
         for (const s of ref) {
@@ -23,6 +23,17 @@ export class Collection<T> {
         return this.items[typeof id == 'string' ? parseInt(id) : id];
     }
 
+    has(ref: T) {
+        const c = this.resolve(ref);
+        if (c)
+            return (c.key in this.categorized[c.category])
+        return this.uncategorized.has(ref);
+    }
+
+    redirect(source: T, target: T) {
+        this.items[this.encode(source)] = target;
+    }
+
     // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-function
     resolve(_: T): { category: keyof Collection<T>['categorized'], key: string } | void { }
 
@@ -63,7 +74,7 @@ export class SymbolCollection extends Collection<GrammarRuleSymbol>{
             return { category: 'literalS', key: symbol.literal }
         } else if ('token' in symbol) {
             return { category: 'token', key: symbol.token }
-        } else if ('test' in symbol) {
+        } else if (symbol instanceof RegExp) {
             return { category: 'regex', key: symbol.toString() }
         } else if (typeof symbol == 'function') {
             return { category: 'function', key: symbol.toString() }
@@ -117,4 +128,48 @@ export class Matrix<T> {
     }
 }
 
-type GetCallbackOrValue<T> = T extends (...args: any) => any ? ReturnType<T> : T;
+export function Flatten(obj: any[] | { [key: string]: any }): FlatObject {
+    const collection = new Collection();
+    function Traverse(ref: any) {
+        if (collection.has(ref)) {
+            return collection.encode(ref)
+        }
+        if (Array.isArray(ref)) {
+            collection.redirect(ref, ref.map(v => Traverse(v)));
+        } else if (typeof ref === 'object') {
+            const o = {};
+            for (const k in ref) {
+                o[k] = Traverse(ref[k])
+            }
+            collection.redirect(ref, o);
+        } else if (typeof ref === 'function') {
+            return collection.encode(ref.toString());
+        }
+        return collection.encode(ref);
+    }
+    Traverse(obj);
+    return collection.items as any;
+}
+
+export function Unflatten(items: FlatObject) {
+    const visited = new Set();
+    function Traverse(id: number) {
+        if (visited.has(id)) {
+            return items[id];
+        }
+        visited.add(id);
+        if (Array.isArray(items[id])) {
+            return (items[id] as any[]).map(v => Traverse(id));
+        } else if (typeof items[id] === 'object') {
+            for (const k in items[id] as { [key: string]: any }) {
+                items[id][k] = Traverse(id[k])
+            }
+        }
+        return items[id];
+    }
+    return Traverse(0);
+}
+
+type FlatObject = (boolean | number | string | (number[]) | { [key: string]: number })[];
+
+type GetCallbackOrValue<T> = T extends (...args: any) => any ? ReturnType<T> : T;

package/src/utility/lr.ts

@@ -5,10 +5,9 @@ import { Collection, SymbolCollection } from "./general";
 export class CanonicalCollection {
     rules: Collection<GrammarRule> = new Collection();
     states: { [key: string]: State } = Object.create(null)
-    symbols: SymbolCollection;
-    
-    constructor(public grammar: LanguageGrammar & { symbols?: SymbolCollection }) {
-        this.symbols = grammar.symbols || new SymbolCollection();
+    symbols: SymbolCollection = new SymbolCollection();
+
+    constructor(public grammar: LanguageGrammar) {
         const augmented = { name: Symbol() as unknown as string, symbols: [grammar.start] }
         grammar.rules[augmented.name] = [augmented];
         this.addState([{ rule: augmented, dot: 0 }]);