@scinorandex/sparse 0.0.2 → 0.0.4

package/src/meta/common.ts

@@ -0,0 +1,74 @@
+import { ColumnAndRow, Token } from "@scinorandex/slex";
+
+export type Production = {
+  lhs: GrammarToken;
+  identifier: string;
+  originalProductionIndex: number;
+  name: string | null;
+  rhs: { type: "terminal" | "variable"; token: GrammarToken; identifier: string; name: string | null }[];
+};
+
+const dehydateGrammarToken = ({ column, lexeme, line, type }: GrammarToken) => {
+  return { column, lexeme, line, type };
+};
+
+const hydrateGrammarToken = (
+  token: ReturnType<typeof dehydateGrammarToken>,
+): Token<GrammarTokenType, GrammarTokenMetadata> => {
+  return new Token(token.type, token.lexeme, new ColumnAndRow(token.line, token.column), {});
+};
+
+export const dehydateProduction = (opts: Production) => {
+  const { lhs, identifier, rhs, originalProductionIndex, name } = opts;
+  return {
+    lhs: dehydateGrammarToken(lhs),
+    identifier,
+    name,
+    originalProductionIndex,
+    rhs: rhs.map(({ type, token, identifier, name }) => ({
+      type,
+      token: dehydateGrammarToken(token),
+      identifier,
+      name,
+    })),
+  };
+};
+
+export const hydrateProduction = (production: ReturnType<typeof dehydateProduction>): Production => {
+  return {
+    lhs: hydrateGrammarToken(production.lhs),
+    identifier: production.identifier,
+    originalProductionIndex: production.originalProductionIndex,
+    name: production.name,
+    rhs: production.rhs.map(({ type, token, name }) => ({
+      type,
+      token: hydrateGrammarToken(token),
+      identifier: token.lexeme,
+      name,
+    })),
+  };
+};
+
+export type GrammarTokenMetadata = {};
+export enum GrammarTokenType {
+  IDENTIFIER,
+  L_ANGLE,
+  R_ANGLE,
+  L_BRACKET,
+  R_BRACKET,
+  L_PAREN,
+  R_PAREN,
+  PIPE,
+  QUESTION_MARK,
+  COLON,
+  SEMICOLON,
+  NUMBER,
+  EQUALS,
+  COMMA,
+  EOF,
+
+  PRODUCTION_NAME,
+  TOKEN_NAME,
+}
+
+export type GrammarToken = Token<GrammarTokenType, GrammarTokenMetadata>;

package/src/{grammarParser.ts → meta/handwritten.ts}

@@ -1,18 +1,6 @@
-import { Slex, Token } from "@scinorandex/slex";
-import { Result } from "./utils/Result";
-
-export type Production = { lhs: GrammarToken; rhs: { type: "terminal" | "variable"; token: GrammarToken }[] };
-
-export type GrammarTokenMetadata = {};
-export enum GrammarTokenType {
-  PRODUCTION_NAME,
-  TOKEN_NAME,
-  COLON,
-  SEMICOLON,
-  EOF,
-}
-
-export type GrammarToken = Token<GrammarTokenType, GrammarTokenMetadata>;
+import { Slex } from "@scinorandex/slex";
+import { Result } from "../utils/Result";
+import { GrammarToken, GrammarTokenMetadata, GrammarTokenType, Production } from "./common";
 
 export const grammarLexerGenerator = new Slex<GrammarTokenType, GrammarTokenMetadata>({
   EOF_TYPE: GrammarTokenType.EOF,
@@ -21,11 +9,11 @@ export const grammarLexerGenerator = new Slex<GrammarTokenType, GrammarTokenMeta
 
 grammarLexerGenerator.addRule(
   "lowercase",
-  "a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z"
+  "a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z",
 );
 grammarLexerGenerator.addRule(
   "uppercase",
-  "A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V | W | X | Y | Z"
+  "A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V | W | X | Y | Z",
 );
 grammarLexerGenerator.addRule("letter", "${lowercase} | ${uppercase}");
 grammarLexerGenerator.addRule("digit", "0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9");
@@ -53,19 +41,27 @@ export const tryBuildProductions = (lexer: LexerInterface | string): Result<Prod
     lexer.getNextToken();
   };
 
+  let productionIndex = 0;
+
   const buildProduction = (): Result<Production> => {
     const productionLHS = lexer.getNextToken();
     expect(GrammarTokenType.COLON);
 
-    const production: Production = { lhs: productionLHS, rhs: [] };
+    const production: Production = {
+      lhs: productionLHS,
+      identifier: productionLHS.lexeme,
+      rhs: [],
+      name: null,
+      originalProductionIndex: productionIndex++,
+    };
 
     while (lexer.peekNextToken().type != GrammarTokenType.SEMICOLON) {
       const productionRHS = lexer.getNextToken();
 
       if (productionRHS.type == GrammarTokenType.PRODUCTION_NAME)
-        production.rhs.push({ type: "variable", token: productionRHS });
+        production.rhs.push({ type: "variable", token: productionRHS, identifier: productionRHS.lexeme, name: null });
       else if (productionRHS.type == GrammarTokenType.TOKEN_NAME)
-        production.rhs.push({ type: "terminal", token: productionRHS });
+        production.rhs.push({ type: "terminal", token: productionRHS, identifier: productionRHS.lexeme, name: null });
       else
         return {
           success: false,
@@ -91,7 +87,7 @@ export const buildProductions = (lexer: LexerInterface | string): Production[] =
   const result = tryBuildProductions(lexer);
   if (result.success === false)
     throw new Error(
-      `Encountered error "${result.reason}" while parsing grammar at ${result.token.line}:${result.token.column}`
+      `Encountered error "${result.reason}" while parsing grammar at ${result.token.line}:${result.token.column}`,
     );
   return result.value;
 };

package/src/meta/selfhosted.ts

@@ -0,0 +1,272 @@
+import { RegexEngine, Slex } from "@scinorandex/slex";
+import { Result, Sparse } from "../index";
+import { selfhosted } from "./states";
+import { TableState } from "../parser";
+import { GrammarToken, GrammarTokenMetadata, GrammarTokenType, hydrateProduction, Production } from "./common";
+
+export const grammarLexerGenerator = new Slex<GrammarTokenType, GrammarTokenMetadata>({
+  EOF_TYPE: GrammarTokenType.EOF,
+  isHigherPrecedence: () => false,
+});
+
+grammarLexerGenerator.addRule(
+  "lowercase",
+  "a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z",
+);
+grammarLexerGenerator.addRule(
+  "uppercase",
+  "A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V | W | X | Y | Z",
+);
+grammarLexerGenerator.addRule("letter", "${lowercase} | ${uppercase}");
+grammarLexerGenerator.addRule("digit", "0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9");
+grammarLexerGenerator.addRule("alphanumeric", "${letter} | ${digit}");
+grammarLexerGenerator.addRule(
+  "identifier",
+  "(${letter} | $_) (${letter} | ${digit} | $_)*",
+  GrammarTokenType.IDENTIFIER,
+);
+grammarLexerGenerator.addRule("colon", "$:", GrammarTokenType.COLON);
+grammarLexerGenerator.addRule("semicolon", "$;", GrammarTokenType.SEMICOLON);
+grammarLexerGenerator.addRule("langle", "$<", GrammarTokenType.L_ANGLE);
+grammarLexerGenerator.addRule("rangle", "$>", GrammarTokenType.R_ANGLE);
+grammarLexerGenerator.addRule("lbracket", "$[", GrammarTokenType.L_BRACKET);
+grammarLexerGenerator.addRule("rbracket", "$]", GrammarTokenType.R_BRACKET);
+grammarLexerGenerator.addRule("lparen", "$(", GrammarTokenType.L_PAREN);
+grammarLexerGenerator.addRule("rparen", "$)", GrammarTokenType.R_PAREN);
+grammarLexerGenerator.addRule("pipe", "$|", GrammarTokenType.PIPE);
+grammarLexerGenerator.addRule("equals", "$=", GrammarTokenType.EQUALS);
+grammarLexerGenerator.addRule("comma", "$,", GrammarTokenType.COMMA);
+grammarLexerGenerator.addRule("question_mark", "$?", GrammarTokenType.QUESTION_MARK);
+grammarLexerGenerator.addRule("number", "(${digit})+", GrammarTokenType.NUMBER);
+
+export abstract class BaseNode {}
+
+export class ListNode<T> extends BaseNode {
+  private items: T[];
+
+  constructor(items: T | T[]) {
+    super();
+
+    if (Array.isArray(items)) this.items = items;
+    else this.items = [items];
+  }
+
+  add(item: T) {
+    this.items.push(item);
+    return this;
+  }
+
+  getItems() {
+    return this.items;
+  }
+
+  getItemsReversed() {
+    return this.items.toReversed();
+  }
+}
+
+class ProductionNode extends BaseNode {
+  constructor(
+    public readonly lhs: GrammarToken,
+    public readonly rhs: ListNode<TokenNode | GroupedTokenNode>,
+    public originalProductionIndex: number,
+    public readonly name?: string | undefined,
+  ) {
+    super();
+  }
+
+  toStruct(): Production {
+    if (this.originalProductionIndex === -1) throw new Error("ProductionNode has no originalProductionIndex");
+
+    const hasGrouped = this.rhs.getItemsReversed().some((node) => node instanceof GroupedTokenNode);
+    if (hasGrouped) throw new Error("Need to unroll before serializing");
+
+    return {
+      lhs: this.lhs,
+      identifier: `<${this.lhs.lexeme}>`,
+      name: this.name ?? null,
+      originalProductionIndex: this.originalProductionIndex,
+      rhs: this.rhs.getItemsReversed().map((node) => (node as TokenNode).toStruct()),
+    };
+  }
+
+  setOriginalProductionIndex(index: number) {
+    this.originalProductionIndex = index;
+  }
+}
+
+function expandProduction(production: ProductionNode): ProductionNode[] {
+  const rhs = production.rhs.getItemsReversed();
+
+  for (let i = 0; i < rhs.length; i++) {
+    const item = rhs[i];
+
+    const beforeItems = rhs.slice(0, i);
+    const afterItems = rhs.slice(i + 1);
+
+    if (item instanceof GroupedTokenNode) {
+      // need to create two new productions, one optional and the other not
+      // then apply unrolling on both
+
+      const withoutCurrentItem = new ProductionNode(
+        production.lhs,
+        new ListNode<TokenNode | GroupedTokenNode>([...beforeItems, ...afterItems].toReversed()),
+        production.originalProductionIndex,
+        production.name,
+      );
+
+      const withCurrentItem = new ProductionNode(
+        production.lhs,
+        new ListNode<TokenNode | GroupedTokenNode>(
+          [...beforeItems, ...item.inside.getItemsReversed(), ...afterItems].toReversed(),
+        ),
+        production.originalProductionIndex,
+        production.name,
+      );
+
+      const newProductions: ProductionNode[] = [withoutCurrentItem, withCurrentItem];
+      return newProductions.flatMap((prod) => expandProduction(prod));
+    } else if (item instanceof TokenNode) {
+      if (item.variables.getItems().length > 1) {
+        // this token node has many variants and we should unroll it
+        const variables = item.variables.getItemsReversed();
+
+        const newProductions = variables.map((variable) => {
+          return new ProductionNode(
+            production.lhs,
+            new ListNode<TokenNode | GroupedTokenNode>([
+              ...beforeItems,
+              new TokenNode(item.type, new ListNode<GrammarToken>([variable]), item.name),
+            ]),
+            production.originalProductionIndex,
+            production.name,
+          );
+        });
+
+        return newProductions.flatMap((prod) => expandProduction(prod));
+      }
+    } else {
+      throw new Error("shoudn't reach here, ProductionNode RHS is neither GroupedTokenNode nor TokenNode");
+    }
+  }
+
+  // if it reached here then its normal
+  return [production];
+}
+
+class TokenNode extends BaseNode {
+  constructor(
+    public readonly type: "terminal" | "variable",
+    public readonly variables: ListNode<GrammarToken>,
+    public readonly name: string | undefined,
+  ) {
+    super();
+  }
+
+  toStruct() {
+    const items = this.variables.getItems();
+    if (items.length > 1) throw new Error("Not yet unrolled");
+
+    const lexeme = items[0].lexeme;
+    const identifier = this.type === "variable" ? `<${lexeme}>` : `[${lexeme}]`;
+    const name = this.name ?? null;
+
+    return { type: this.type, token: items[0], identifier, name };
+  }
+}
+
+class GroupedTokenNode extends BaseNode {
+  constructor(public readonly inside: ListNode<TokenNode>) {
+    super();
+  }
+}
+
+class ProgramNode extends BaseNode {
+  constructor(public readonly productions: ListNode<ProductionNode>) {
+    super();
+  }
+
+  unrollProductions() {
+    const productions = this.productions.getItemsReversed();
+    for (let i = 0; i < productions.length; i++) productions[i].setOriginalProductionIndex(i);
+    return productions.flatMap((node) => expandProduction(node));
+  }
+
+  getProductions() {
+    return this.unrollProductions().map((node) => node.toStruct());
+  }
+}
+
+export function getSelfHostedParserGenerator(e: typeof selfhosted = selfhosted) {
+  return new Sparse<GrammarTokenType, GrammarTokenMetadata, BaseNode>({
+    productions: e.productions.map((production) => hydrateProduction(production as any)),
+    states: e.states.map((state) => TableState.fromJSObject(state as any)),
+    toStringifiedTokenType: (type: GrammarTokenType) => GrammarTokenType[type],
+  });
+}
+
+type Reducer = (bag: any) => BaseNode;
+const reducers: { [key: string]: Reducer } = {
+  program: (bag: { productions?: ListNode<ProductionNode> }) =>
+    new ProgramNode(bag.productions ?? new ListNode<ProductionNode>([])),
+
+  productions: (bag: { production: ProductionNode; rest?: ListNode<ProductionNode> }) => {
+    if (bag.rest == null) return new ListNode<ProductionNode>([bag.production]);
+    else return bag.rest.add(bag.production);
+  },
+
+  production: (bag: {
+    production_name: GrammarToken;
+    tokens: ListNode<TokenNode | GroupedTokenNode>;
+    uuid?: GrammarToken;
+  }) => new ProductionNode(bag.production_name, bag.tokens, -1, bag.uuid?.lexeme),
+
+  tokens: (bag: { token: TokenNode | GroupedTokenNode; rest?: ListNode<TokenNode | GroupedTokenNode> }) => {
+    if (bag.rest == null) return new ListNode<TokenNode | GroupedTokenNode>([bag.token]);
+    else return bag.rest.add(bag.token);
+  },
+
+  token: (bag: { token: TokenNode }) => bag.token,
+  grouped_token: (bag: { tokens: ListNode<TokenNode> }) => new GroupedTokenNode(bag.tokens),
+
+  variable: (bag: { inside: ListNode<GrammarToken>; token_name?: GrammarToken }) =>
+    new TokenNode("variable", bag.inside, bag.token_name?.lexeme),
+
+  terminal: (bag: { inside: ListNode<GrammarToken>; token_name?: GrammarToken }) =>
+    new TokenNode("terminal", bag.inside, bag.token_name?.lexeme),
+
+  inside: (bag: { identifier: GrammarToken; rest?: ListNode<GrammarToken> }) => {
+    if (bag.rest == null) return new ListNode<GrammarToken>([bag.identifier]);
+    else return bag.rest.add(bag.identifier);
+  },
+};
+
+export const tryBuildProductions = (lexer: LexerInterface | string): Result<Production[]> => {
+  if (typeof lexer === "string") lexer = grammarLexerGenerator.generate(lexer, () => ({}));
+
+  const parserGenerator = getSelfHostedParserGenerator();
+  const parser = parserGenerator.generate(lexer as RegexEngine<GrammarTokenType, GrammarTokenMetadata>, {
+    reducer: ({ bag, name }) => {
+      return reducers[name ?? ""](bag);
+    },
+  });
+
+  const parsingResult = parser.parse().result as ProgramNode;
+  const productions = parsingResult.getProductions();
+  return { success: true, value: productions };
+};
+
+interface LexerInterface {
+  peekNextToken(): GrammarToken;
+  getNextToken(): GrammarToken;
+  hasNextToken(): boolean;
+}
+
+export const buildProductions = (lexer: LexerInterface | string): Production[] => {
+  const result = tryBuildProductions(lexer);
+  if (result.success === false)
+    throw new Error(
+      `Encountered error "${result.reason}" while parsing grammar at ${result.token.line}:${result.token.column}`,
+    );
+  return result.value;
+};