@scinorandex/sparse 0.0.7 → 0.0.8

package/example/selfhosted/codegen-meta.ts

@@ -380,11 +380,11 @@ export const states = [
   {
     "[R_ANGLE]": {
       "type": "reduce",
-      "value": 16
+      "value": 18
     },
     "[COLON]": {
       "type": "reduce",
-      "value": 16
+      "value": 18
     },
     "[PIPE]": {
       "type": "shift",
@@ -404,11 +404,11 @@ export const states = [
   {
     "[R_BRACKET]": {
       "type": "reduce",
-      "value": 16
+      "value": 18
     },
     "[COLON]": {
       "type": "reduce",
-      "value": 16
+      "value": 18
     },
     "[PIPE]": {
       "type": "shift",
@@ -459,6 +459,14 @@ export const states = [
     "[QUESTION_MARK]": {
       "type": "shift",
       "value": 41
+    },
+    "[PLUS]": {
+      "type": "shift",
+      "value": 42
+    },
+    "[STAR]": {
+      "type": "shift",
+      "value": 43
     }
   },
   {
@@ -470,83 +478,83 @@ export const states = [
   {
     "[SEMICOLON]": {
       "type": "reduce",
-      "value": 12
+      "value": 14
     },
     "[R_PAREN]": {
       "type": "reduce",
-      "value": 12
+      "value": 14
     },
     "[L_PAREN]": {
       "type": "reduce",
-      "value": 12
+      "value": 14
     },
     "[L_ANGLE]": {
       "type": "reduce",
-      "value": 12
+      "value": 14
     },
     "[L_BRACKET]": {
       "type": "reduce",
-      "value": 12
+      "value": 14
     }
   },
   {
     "[IDENTIFIER]": {
       "type": "shift",
-      "value": 42
+      "value": 44
     }
   },
   {
     "[IDENTIFIER]": {
       "type": "shift",
-      "value": 44
+      "value": 46
     },
     "<INSIDE>": {
       "type": "goto",
-      "value": 43
+      "value": 45
     }
   },
   {
     "[SEMICOLON]": {
       "type": "reduce",
-      "value": 14
+      "value": 16
     },
     "[R_PAREN]": {
       "type": "reduce",
-      "value": 14
+      "value": 16
     },
     "[L_PAREN]": {
       "type": "reduce",
-      "value": 14
+      "value": 16
     },
     "[L_ANGLE]": {
       "type": "reduce",
-      "value": 14
+      "value": 16
     },
     "[L_BRACKET]": {
       "type": "reduce",
-      "value": 14
+      "value": 16
     }
   },
   {
     "[IDENTIFIER]": {
       "type": "shift",
-      "value": 45
+      "value": 47
     }
   },
   {
     "[IDENTIFIER]": {
       "type": "shift",
-      "value": 44
+      "value": 46
     },
     "<INSIDE>": {
       "type": "goto",
-      "value": 46
+      "value": 48
     }
   },
   {
     "[SEMICOLON]": {
       "type": "shift",
-      "value": 47
+      "value": 49
     }
   },
   {
@@ -571,54 +579,98 @@ export const states = [
       "value": 11
     }
   },
+  {
+    "[SEMICOLON]": {
+      "type": "reduce",
+      "value": 12
+    },
+    "[R_PAREN]": {
+      "type": "reduce",
+      "value": 12
+    },
+    "[L_PAREN]": {
+      "type": "reduce",
+      "value": 12
+    },
+    "[L_ANGLE]": {
+      "type": "reduce",
+      "value": 12
+    },
+    "[L_BRACKET]": {
+      "type": "reduce",
+      "value": 12
+    }
+  },
+  {
+    "[SEMICOLON]": {
+      "type": "reduce",
+      "value": 13
+    },
+    "[R_PAREN]": {
+      "type": "reduce",
+      "value": 13
+    },
+    "[L_PAREN]": {
+      "type": "reduce",
+      "value": 13
+    },
+    "[L_ANGLE]": {
+      "type": "reduce",
+      "value": 13
+    },
+    "[L_BRACKET]": {
+      "type": "reduce",
+      "value": 13
+    }
+  },
   {
     "[R_ANGLE]": {
       "type": "shift",
-      "value": 48
+      "value": 50
     }
   },
   {
     "[R_ANGLE]": {
       "type": "reduce",
-      "value": 17
+      "value": 19
     },
     "[COLON]": {
       "type": "reduce",
-      "value": 17
+      "value": 19
     }
   },
   {
     "[R_ANGLE]": {
       "type": "reduce",
-      "value": 16
+      "value": 18
     },
     "[COLON]": {
       "type": "reduce",
-      "value": 16
+      "value": 18
     },
     "[R_BRACKET]": {
       "type": "reduce",
-      "value": 16
+      "value": 18
     },
     "[PIPE]": {
       "type": "shift",
-      "value": 49
+      "value": 51
     }
   },
   {
     "[R_BRACKET]": {
       "type": "shift",
-      "value": 50
+      "value": 52
     }
   },
   {
     "[R_BRACKET]": {
       "type": "reduce",
-      "value": 17
+      "value": 19
     },
     "[COLON]": {
       "type": "reduce",
-      "value": 17
+      "value": 19
     }
   },
   {
@@ -634,69 +686,69 @@ export const states = [
   {
     "[SEMICOLON]": {
       "type": "reduce",
-      "value": 13
+      "value": 15
     },
     "[R_PAREN]": {
       "type": "reduce",
-      "value": 13
+      "value": 15
     },
     "[L_PAREN]": {
       "type": "reduce",
-      "value": 13
+      "value": 15
     },
     "[L_ANGLE]": {
       "type": "reduce",
-      "value": 13
+      "value": 15
     },
     "[L_BRACKET]": {
       "type": "reduce",
-      "value": 13
+      "value": 15
     }
   },
   {
     "[IDENTIFIER]": {
       "type": "shift",
-      "value": 44
+      "value": 46
     },
     "<INSIDE>": {
       "type": "goto",
-      "value": 51
+      "value": 53
     }
   },
   {
     "[SEMICOLON]": {
       "type": "reduce",
-      "value": 15
+      "value": 17
     },
     "[R_PAREN]": {
       "type": "reduce",
-      "value": 15
+      "value": 17
     },
     "[L_PAREN]": {
       "type": "reduce",
-      "value": 15
+      "value": 17
     },
     "[L_ANGLE]": {
       "type": "reduce",
-      "value": 15
+      "value": 17
     },
     "[L_BRACKET]": {
       "type": "reduce",
-      "value": 15
+      "value": 17
     }
   },
   {
     "[R_ANGLE]": {
       "type": "reduce",
-      "value": 17
+      "value": 19
     },
     "[COLON]": {
       "type": "reduce",
-      "value": 17
+      "value": 19
     },
     "[R_BRACKET]": {
       "type": "reduce",
-      "value": 17
+      "value": 19
     }
   }
 ];
@@ -1183,7 +1235,121 @@ export const states = [
           "type": 0
         },
         "identifier": "[QUESTION_MARK]",
-        "name": "question_mark"
+        "name": "modifier"
+      }
+    ]
+  },
+  {
+    "lhs": {
+      "column": 1,
+      "lexeme": "TOKEN",
+      "line": 7,
+      "type": 0
+    },
+    "identifier": "<TOKEN>",
+    "name": "grouped_token",
+    "originalProductionIndex": 6,
+    "rhs": [
+      {
+        "type": "terminal",
+        "token": {
+          "column": 25,
+          "lexeme": "L_PAREN",
+          "line": 7,
+          "type": 0
+        },
+        "identifier": "[L_PAREN]",
+        "name": null
+      },
+      {
+        "type": "variable",
+        "token": {
+          "column": 35,
+          "lexeme": "TOKENS",
+          "line": 7,
+          "type": 0
+        },
+        "identifier": "<TOKENS>",
+        "name": "tokens"
+      },
+      {
+        "type": "terminal",
+        "token": {
+          "column": 52,
+          "lexeme": "R_PAREN",
+          "line": 7,
+          "type": 0
+        },
+        "identifier": "[R_PAREN]",
+        "name": null
+      },
+      {
+        "type": "terminal",
+        "token": {
+          "column": 78,
+          "lexeme": "PLUS",
+          "line": 7,
+          "type": 0
+        },
+        "identifier": "[PLUS]",
+        "name": "modifier"
+      }
+    ]
+  },
+  {
+    "lhs": {
+      "column": 1,
+      "lexeme": "TOKEN",
+      "line": 7,
+      "type": 0
+    },
+    "identifier": "<TOKEN>",
+    "name": "grouped_token",
+    "originalProductionIndex": 6,
+    "rhs": [
+      {
+        "type": "terminal",
+        "token": {
+          "column": 25,
+          "lexeme": "L_PAREN",
+          "line": 7,
+          "type": 0
+        },
+        "identifier": "[L_PAREN]",
+        "name": null
+      },
+      {
+        "type": "variable",
+        "token": {
+          "column": 35,
+          "lexeme": "TOKENS",
+          "line": 7,
+          "type": 0
+        },
+        "identifier": "<TOKENS>",
+        "name": "tokens"
+      },
+      {
+        "type": "terminal",
+        "token": {
+          "column": 52,
+          "lexeme": "R_PAREN",
+          "line": 7,
+          "type": 0
+        },
+        "identifier": "[R_PAREN]",
+        "name": null
+      },
+      {
+        "type": "terminal",
+        "token": {
+          "column": 85,
+          "lexeme": "STAR",
+          "line": 7,
+          "type": 0
+        },
+        "identifier": "[STAR]",
+        "name": "modifier"
       }
     ]
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scinorandex/sparse",
-  "version": "0.0.7",
+  "version": "0.0.8",
   "description": "Yet another parser generator",
   "main": "dist/index.js",
   "scripts": {

package/src/meta/common.ts

@@ -60,6 +60,8 @@ export enum GrammarTokenType {
   R_PAREN,
   PIPE,
   QUESTION_MARK,
+  STAR,
+  PLUS,
   COLON,
   SEMICOLON,
   NUMBER,

package/src/meta/selfhosted.ts

@@ -38,6 +38,8 @@ grammarLexerGenerator.addRule("pipe", "$|", GrammarTokenType.PIPE);
 grammarLexerGenerator.addRule("equals", "$=", GrammarTokenType.EQUALS);
 grammarLexerGenerator.addRule("comma", "$,", GrammarTokenType.COMMA);
 grammarLexerGenerator.addRule("question_mark", "$?", GrammarTokenType.QUESTION_MARK);
+grammarLexerGenerator.addRule("star", "$*", GrammarTokenType.STAR);
+grammarLexerGenerator.addRule("plus", "$+", GrammarTokenType.PLUS);
 grammarLexerGenerator.addRule("number", "(${digit})+", GrammarTokenType.NUMBER);
 
 grammarLexerGenerator.addRule("single_line_comment", "$/$/(($\n)!)*", GrammarTokenType.SINGLE_LINE_COMMENT);
@@ -74,11 +76,16 @@ class ProductionNode extends BaseNode {
     public readonly lhs: GrammarToken,
     public readonly rhs: ListNode<TokenNode | GroupedTokenNode>,
     public originalProductionIndex: number,
+    public readonly identifier: string,
     public readonly name?: string | undefined,
   ) {
     super();
   }
 
+  fork(rhs: ListNode<TokenNode | GroupedTokenNode>) {
+    return new ProductionNode(this.lhs, rhs, this.originalProductionIndex, this.identifier, this.name);
+  }
+
   toStruct(): Production {
     if (this.originalProductionIndex === -1) throw new Error("ProductionNode has no originalProductionIndex");
 
@@ -87,7 +94,7 @@ class ProductionNode extends BaseNode {
 
     return {
       lhs: this.lhs,
-      identifier: `<${this.lhs.lexeme}>`,
+      identifier: `<${this.identifier}>`,
       name: this.name ?? null,
       originalProductionIndex: this.originalProductionIndex,
       rhs: this.rhs.getItemsReversed().map((node) => (node as TokenNode).toStruct()),
@@ -99,7 +106,21 @@ class ProductionNode extends BaseNode {
   }
 }
 
-function expandProduction(production: ProductionNode): ProductionNode[] {
+class ExpanderContext {
+  current = 0;
+
+  constructor(public currentProductionIndex: number) {}
+
+  getNextProductionIndex() {
+    return this.currentProductionIndex++;
+  }
+
+  getAutogeneratedId() {
+    return `autogen-${this.current++}`;
+  }
+}
+
+function expandProduction(production: ProductionNode, ctx: ExpanderContext): ProductionNode[] {
   const rhs = production.rhs.getItemsReversed();
 
   for (let i = 0; i < rhs.length; i++) {
@@ -109,35 +130,53 @@ function expandProduction(production: ProductionNode): ProductionNode[] {
     const afterItems = rhs.slice(i + 1);
 
     if (item instanceof GroupedTokenNode) {
-      // need to create two new productions, one optional and the other not
-      // then apply unrolling on both
-
-      const withoutCurrentItem = new ProductionNode(
-        production.lhs,
-        new ListNode<TokenNode | GroupedTokenNode>([...beforeItems, ...afterItems].toReversed()),
-        production.originalProductionIndex,
-        production.name,
-      );
-
-      const withCurrentItem = new ProductionNode(
-        production.lhs,
-        new ListNode<TokenNode | GroupedTokenNode>(
-          [...beforeItems, ...item.inside.getItemsReversed(), ...afterItems].toReversed(),
-        ),
-        production.originalProductionIndex,
-        production.name,
-      );
-
-      const newProductions: ProductionNode[] = [withoutCurrentItem, withCurrentItem];
-      return newProductions.flatMap((prod) => expandProduction(prod));
+      // The unrolling rules here depend on the modifier in the GroupedTokenNode
+
+      if (item.modifier === GrammarTokenType.QUESTION_MARK) {
+        const withoutCurrentItem = production.fork(
+          new ListNode<TokenNode | GroupedTokenNode>([...beforeItems, ...afterItems].toReversed()),
+        );
+
+        const withCurrentItem = production.fork(
+          new ListNode<TokenNode | GroupedTokenNode>(
+            [...beforeItems, ...item.inside.getItemsReversed(), ...afterItems].toReversed(),
+          ),
+        );
+
+        const newProductions: ProductionNode[] = [withoutCurrentItem, withCurrentItem];
+        return newProductions.flatMap((prod) => expandProduction(prod, ctx));
+      } else if (item.modifier === GrammarTokenType.STAR || item.modifier === GrammarTokenType.PLUS) {
+        // need to handle kleene-star and kleene-plus, which create entirely unique new productions
+        const inside = item.inside.getItemsReversed();
+        const newProductionIdentifier = ctx.getAutogeneratedId();
+
+        // create a dummy production that contains the contents of inside
+        const raw = new StubTokenNode("variable", new ListNode([production.lhs]), "rest", newProductionIdentifier);
+        const pump = new GroupedTokenNode(new ListNode([raw]), GrammarTokenType.QUESTION_MARK);
+
+        const stub = new ProductionNode(
+          production.lhs,
+          new ListNode<TokenNode | GroupedTokenNode>([...inside, pump].toReversed()),
+          ctx.getNextProductionIndex(),
+          newProductionIdentifier,
+          "autogenerated-kleene", // the user needs to implement a reducer named "autogenerated-kleene"
+        );
+
+        const currentReplacement = production.fork(
+          new ListNode(
+            [...beforeItems, item.modifier === GrammarTokenType.STAR ? pump : raw, ...afterItems].toReversed(),
+          ),
+        );
+
+        return [currentReplacement, stub].flatMap((prod) => expandProduction(prod, ctx));
+      } else throw new Error(`Unknown modifier ${GrammarTokenType[item.modifier]}`);
     } else if (item instanceof TokenNode) {
       if (item.variables.getItems().length > 1) {
         // this token node has many variants and we should unroll it
         const variables = item.variables.getItemsReversed();
 
         const newProductions = variables.map((variable) => {
-          return new ProductionNode(
-            production.lhs,
+          return production.fork(
             new ListNode<TokenNode | GroupedTokenNode>(
               [
                 ...beforeItems,
@@ -145,12 +184,10 @@ function expandProduction(production: ProductionNode): ProductionNode[] {
                 ...afterItems,
               ].toReversed(),
             ),
-            production.originalProductionIndex,
-            production.name,
           );
         });
 
-        return newProductions.flatMap((prod) => expandProduction(prod));
+        return newProductions.flatMap((prod) => expandProduction(prod, ctx));
       }
     } else {
       throw new Error("shoudn't reach here, ProductionNode RHS is neither GroupedTokenNode nor TokenNode");
@@ -170,11 +207,17 @@ class TokenNode extends BaseNode {
     super();
   }
 
+  public getIdentifier() {
+    const items = this.variables.getItems();
+    if (items.length > 1) throw new Error("Not yet unrolled");
+    return items[0].lexeme;
+  }
+
   toStruct() {
     const items = this.variables.getItems();
     if (items.length > 1) throw new Error("Not yet unrolled");
 
-    const lexeme = items[0].lexeme;
+    const lexeme = this.getIdentifier();
     const identifier = this.type === "variable" ? `<${lexeme}>` : `[${lexeme}]`;
     const name = this.name ?? null;
 
@@ -182,8 +225,26 @@ class TokenNode extends BaseNode {
   }
 }
 
+class StubTokenNode extends TokenNode {
+  constructor(
+    public readonly type: "terminal" | "variable",
+    public readonly variables: ListNode<GrammarToken>,
+    public readonly name: string | undefined,
+    public readonly identifier: string,
+  ) {
+    super(type, variables, name);
+  }
+
+  public getIdentifier() {
+    return this.identifier;
+  }
+}
+
 class GroupedTokenNode extends BaseNode {
-  constructor(public readonly inside: ListNode<TokenNode>) {
+  constructor(
+    public readonly inside: ListNode<TokenNode>,
+    public readonly modifier: GrammarTokenType,
+  ) {
     super();
   }
 }
@@ -195,8 +256,10 @@ class ProgramNode extends BaseNode {
 
   unrollProductions() {
     const productions = this.productions.getItemsReversed();
+    const expanderContext = new ExpanderContext(this.productions.getItems().length);
+
     for (let i = 0; i < productions.length; i++) productions[i].setOriginalProductionIndex(i);
-    return productions.flatMap((node) => expandProduction(node));
+    return productions.flatMap((node) => expandProduction(node, expanderContext));
   }
 
   getProductions() {
@@ -226,7 +289,7 @@ const reducers: { [key: string]: Reducer } = {
     production_name: GrammarToken;
     tokens: ListNode<TokenNode | GroupedTokenNode>;
     uuid?: GrammarToken;
-  }) => new ProductionNode(bag.production_name, bag.tokens, -1, bag.uuid?.lexeme),
+  }) => new ProductionNode(bag.production_name, bag.tokens, -1, bag.production_name.lexeme, bag.uuid?.lexeme),
 
   tokens: (bag: { token: TokenNode | GroupedTokenNode; rest?: ListNode<TokenNode | GroupedTokenNode> }) => {
     if (bag.rest == null) return new ListNode<TokenNode | GroupedTokenNode>([bag.token]);
@@ -234,7 +297,8 @@ const reducers: { [key: string]: Reducer } = {
   },
 
   token: (bag: { token: TokenNode }) => bag.token,
-  grouped_token: (bag: { tokens: ListNode<TokenNode> }) => new GroupedTokenNode(bag.tokens),
+  grouped_token: (bag: { tokens: ListNode<TokenNode>; modifier: GrammarToken }) =>
+    new GroupedTokenNode(bag.tokens, bag.modifier.type),
 
   variable: (bag: { inside: ListNode<GrammarToken>; token_name?: GrammarToken }) =>
     new TokenNode("variable", bag.inside, bag.token_name?.lexeme),