drift-parser 0.0.11

package/src/parser/type.ts

@@ -0,0 +1,89 @@
+// import { LocationInterface, TokenType } from "../lexer/tokens";
+// // import { Type, SymbolType, ArrayType } from "../ast/exports";
+// import { Parser } from "./parser";
+
+// export enum TypeBP {
+//     Default = 0,
+    
+// }
+
+// export type TypeNudHandler = (p: Parser) => Type;
+// export type TypeLedHandler = (p: Parser, left: Type, bp: TypeBP) => Type;
+
+
+// const _type_bp_lu: Partial<Record<TokenType, TypeBP>> = {};
+// const _type_nud_lu: Partial<Record<TokenType, TypeNudHandler>> = {};
+// const _type_led_lu: Partial<Record<TokenType, TypeLedHandler>> = {};
+
+
+// export function type_led(kind: TokenType, bp: TypeBP, ledFn: TypeLedHandler): void {
+//     _type_bp_lu[kind] = bp;
+//     _type_led_lu[kind] = ledFn;
+// }
+
+// export function type_nud(kind: TokenType, nudFn: TypeNudHandler): void {
+//     _type_nud_lu[kind] = nudFn;
+// }
+
+
+// export function createTokenTypeLookups(): void {
+//     type_nud(TokenType.IDENTIFIER, parse_symbol_type);
+    
+    
+// }
+
+// createTokenTypeLookups()
+
+// function parse_symbol_type(p: Parser): Type {
+//     const token = p.expect(TokenType.IDENTIFIER);
+//     return new SymbolType(token.value, token.loc, token.range);
+// }
+
+
+// export function parse_type(p: Parser, bp: TypeBP): Type {
+//     const tokenKind = p.currentToken().type;
+//     const nudFn = _type_nud_lu[tokenKind];
+
+//     if (!nudFn) {
+//         throw new Error(`No nud handler for type token: ${TokenType[tokenKind]}`);
+//     }
+
+    
+//     let left = nudFn(p);
+
+    
+//     while (p.currentToken().type === TokenType.LEFT_BRACKET) {
+//         const leftBracket = p.expect(TokenType.LEFT_BRACKET);
+//         p.expect(TokenType.RIGHT_BRACKET);
+//         const rightBracket = p.previousToken();
+
+        
+//         const loc: LocationInterface = {
+//             start: left.loc.start,
+//             end: rightBracket.loc.end,
+//         };
+
+//         const range: [number, number] = [left.range[0], rightBracket.range[1]];
+
+//         left = new ArrayType(left, loc, range);
+//     }
+
+    
+//     while ((_type_bp_lu[p.currentToken().type] ?? TypeBP.Default) > bp) {
+//         const tokenKind = p.currentToken().type;
+//         const ledFn = _type_led_lu[tokenKind];
+//         if (!ledFn) {
+//             throw new Error(`No led handler for type token: ${TokenType[tokenKind]}`);
+//         }
+//         left = ledFn(p, left, _type_bp_lu[tokenKind]!);
+//     }
+
+//     return left;
+// }
+
+
+// export {
+//     _type_bp_lu as type_bp_lu,
+//     _type_nud_lu as type_nud_lu,
+//     _type_led_lu as type_led_lu,
+// };

package/src/utils/combineLocation.ts

@@ -0,0 +1,7 @@
+import { LocationInterface } from "../lexer/tokens";
+export function combineLocations(loc1: LocationInterface, loc2: LocationInterface): LocationInterface {
+  return {
+    start: loc1.start,
+    end: loc2.end,
+  };
+}

package/src/utils/mapAll.ts

@@ -0,0 +1,43 @@
+// import { ASTNode } from '../ast/ast';
+// import {
+//   Statement,
+//   BlockStatement,
+//   ExpressionStatement,
+//   TerminatorStatement
+// } from '../ast/stmt';
+// import { Expression } from '../ast/expr';
+
+// type Mapper = (node: ASTNode) => ASTNode;
+
+// export function mapAll(node: ASTNode, mapper: Mapper): ASTNode {
+//   // Apply mapper to current node first
+//   const newNode = mapper(node);
+
+//   // Recursively map children depending on node type
+//   if (newNode instanceof BlockStatement) {
+//     return new BlockStatement(
+//       newNode.body.map(stmt => mapAll(stmt, mapper) as Statement),
+//       newNode.loc,
+//       newNode.range
+//     );
+//   } else if (newNode instanceof ExpressionStatement) {
+//     return new ExpressionStatement(
+//       mapAll(newNode.expression, mapper) as Expression,
+//       newNode.loc,
+//       newNode.range
+//     );
+//   } else if (newNode instanceof TerminatorStatement) {
+//     // No children, return as is
+//     return newNode;
+//   }
+
+//   // Add other node types here if needed
+
+//   // Default: no children, return node as is
+//   return newNode;
+// }
+
+// export function _toJSON(n: ASTNode) {
+//   // Just call toJSON directly on root node
+//   return n.toJSON();
+// }

package/src/utils/registerParse.ts

@@ -0,0 +1,117 @@
+// import { Statement } from "../ast/stmt";
+import { Stmt } from "../ast/type";
+import { Token, TokenType, TypeName } from "../lexer/tokens";
+import { Parser } from "../parser/parser";
+
+// export function stmt_registerParse({
+//   consume,
+//   handler
+// }: {
+//   consume: string[];
+//   handler: (...tokens: Token[]) => Statement[];
+// }) {
+//   return function parse(p: Parser): Statement[] {
+//     const args: Token[] = [];
+
+//     for (const entry of consume) {
+//       if (entry === "_") {
+//         p.advance(); // discard
+//         continue;
+//       }
+
+//       const typeNames = entry.split("|");
+//       const expectedTypes = typeNames.map(name => {
+//         const upper = name.toUpperCase();
+//         if (!(upper in TokenType)) {
+//           throw new Error(`Unknown token type: '${name}'`);
+//         }
+//         return TokenType[upper as keyof typeof TokenType];
+//       });
+
+//       const token = p.currentToken();
+//       if (!token.IsA(...expectedTypes)) {
+//         throw new Error(`Expected ${typeNames.join(" or ")}, but got ${TypeName(token.type)}`);
+//       }
+
+//       args.push(token);
+//       p.advance();
+//     }
+
+//     return handler(...args);
+//   };
+// }
+
+export function stmt_registerParse({
+  consume,
+  handler
+}: {
+  consume: (string | string[])[];  // allow optional groups as nested arrays
+  handler: (...tokens: Token[]) => Stmt[];
+}) {
+  return function parse(p: Parser): Stmt[] {
+    const args: Token[] = [];
+
+    for (const entry of consume) {
+      if (entry === "_") {
+        p.advance(); // discard
+        continue;
+      }
+
+      if (Array.isArray(entry)) {
+        // Optional group: try to parse it, else reset
+        const checkpoint = p.mark();
+
+        try {
+          for (const subEntry of entry) {
+            if (subEntry === "_") {
+              p.advance(); // discard
+              continue;
+            }
+
+            const types = subEntry.split("|").map(name => {
+              const upper = name.toUpperCase();
+              if (!(upper in TokenType)) {
+                throw new Error(`Unknown token type: '${name}'`);
+              }
+              return TokenType[upper as keyof typeof TokenType];
+            });
+
+            const token = p.currentToken();
+            if (!token.IsA(...types)) {
+              throw new Error();
+            }
+
+            args.push(token);
+            p.advance();
+          }
+        } catch {
+          // rollback if any token didn't match
+          p.reset(checkpoint);
+          continue; // skip optional group
+        }
+
+        continue;
+      }
+
+      // single token
+      const typeNames = entry.split("|");
+      const expectedTypes = typeNames.map(name => {
+        const upper = name.toUpperCase();
+        if (!(upper in TokenType)) {
+          throw new Error(`Unknown token type: '${name}'`);
+        }
+        return TokenType[upper as keyof typeof TokenType];
+      });
+
+      const token = p.currentToken();
+      if (!token.IsA(...expectedTypes)) {
+        throw new Error(`Expected ${typeNames.join(" or ")}, but got ${TokenType[token.type]}`);
+      }
+
+      args.push(token);
+      p.advance();
+    }
+
+    return handler(...args);
+  };
+}

package/tests/astTest.js

@@ -0,0 +1,44 @@
+const drift = require('../dist/index.js');
+
+const program = `5;`;
+
+console.log(JSON.stringify(program))
+// Tokenize the input string using drift.Lexer.Tokenize
+const tokens = drift.Lexer.Tokenize(program);
+
+// Parse tokens into AST
+const ast = drift.Parser.parse(tokens);
+
+// Pretty print the AST with full depth and colors
+console.dir(ast, { depth: null, colors: true });
+
+// function mapAll(node, mapper) {
+//   // Apply mapper to current node first
+//   const newNode = mapper(node);
+
+//   // Recursively map children depending on node type
+//   if (newNode.constructor.name === 'BlockStatement') {
+//     return new BlockStatement(
+//       newNode.body.map(stmt => mapAll(stmt, mapper)),
+//       newNode.loc,
+//       newNode.range
+//     );
+//   } else if (newNode.constructor.name === 'ExpressionStatement') {
+//     return new ExpressionStatement(
+//       mapAll(newNode.expression, mapper),
+//       newNode.loc,
+//       newNode.range
+//     );
+//   } else if (newNode.constructor.name === 'TerminatorStatement') {
+//     // No children, return as is
+//     return newNode;
+//   }
+
+//   // Add other node types here if needed
+
+//   // Default: no children, return node as is
+//   return newNode;
+// }
+
+// const op = mapAll(ast, m => m.toJSON())
+// require('fs').writeFileSync('./ast.json', JSON.stringify(op, null, 2))

package/tests/printTest.mjs

@@ -0,0 +1,7 @@
+import { Lexer } from "../dist/index.js";
+
+const output = Lexer.Print([], {
+    shortenWhitespace: false,
+})
+
+console.log(output)

package/tests/tokenize.js

@@ -0,0 +1,92 @@
+const drift = require('../dist/index.js');
+
+const code = 'let                b = 5;'
+const tokens = drift.Lexer.Tokenize(code)
+
+
+function updateTokenAtIndex(tokenArray, index, newValue) {
+  if (index < 0 || index >= tokenArray.length) {
+    throw new Error("Index out of bounds");
+  }
+
+  const token = tokenArray[index];
+
+  // Preserve start info
+  const startIndex = token.range[0];
+  const startLine = token.loc.start.line;
+  const startColumn = token.loc.start.column;
+
+  const oldLength = token.range[1] - token.range[0];
+  const newLength = newValue.length;
+  const lengthDiff = newLength - oldLength;
+
+  // Update the token value
+  token.value = newValue;
+
+  // Update the token range for the modified token
+  token.range = [startIndex, startIndex + newLength];
+
+  // Update token.loc.end based on newValue
+  const lines = newValue.split('\n');
+
+  if (lines.length === 1) {
+    token.loc.end = {
+      line: startLine,
+      column: startColumn + newLength,
+    };
+  } else {
+    token.loc.end = {
+      line: startLine + lines.length - 1,
+      column: lines[lines.length - 1].length,
+    };
+  }
+
+  // Now shift all tokens after the modified one by lengthDiff
+  for (let i = index + 1; i < tokenArray.length; i++) {
+    const t = tokenArray[i];
+
+    // Shift range by lengthDiff
+    t.range = [t.range[0] + lengthDiff, t.range[1] + lengthDiff];
+
+    // Adjust loc.start and loc.end accordingly
+    // If the token is on the same line as the modified token's end line,
+    // update the column; otherwise, just shift lines if needed
+
+    // Simple approximation: if the token is on the same line as the changed token's end line,
+    // shift the column by lengthDiff, else leave columns alone.
+    if (t.loc.start.line === token.loc.end.line) {
+      t.loc.start = {
+        line: t.loc.start.line,
+        column: t.loc.start.column + lengthDiff,
+      };
+    } else if (t.loc.start.line > token.loc.end.line) {
+      // If the token is on a line after the modified token, shift the line number by the number of new lines added or removed
+      t.loc.start = {
+        line: t.loc.start.line + (lines.length - 1),
+        column: t.loc.start.column,
+      };
+    }
+    // Similarly for loc.end:
+    if (t.loc.end.line === token.loc.end.line) {
+      t.loc.end = {
+        line: t.loc.end.line,
+        column: t.loc.end.column + lengthDiff,
+      };
+    } else if (t.loc.end.line > token.loc.end.line) {
+      t.loc.end = {
+        line: t.loc.end.line + (lines.length - 1),
+        column: t.loc.end.column,
+      };
+    }
+  }
+}
+
+
+// updateTokenAtIndex(tokens, 0, 'latest!value')
+
+// const cj = tokens.slice(0,-1).map(t => t.value).join('')
+// console.log(tokens, JSON.stringify(cj).slice(1, -1));
+
+console.log(drift.Lexer.Print(tokens, {
+
+}))

package/tests/typenames.js

@@ -0,0 +1,15 @@
+/*
+type names for later reference
+uint8_e
+uint16_e
+uint32_e
+uint64_e
+
+int8_e
+int16_e
+int32_e
+int64_e
+
+float32_e
+float64_e
+*/

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "ESNext",
+    "declaration": true,                
+    "emitDeclarationOnly": true,       
+    "outDir": "dist",                   
+    "strict": true,
+    "moduleResolution": "node",
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["index.ts", "src/**/*"]
+}