dal-ast-js 0.0.1-dev

package/package.json

@@ -0,0 +1,28 @@
+{
+    "name": "dal-ast-js",
+    "version": "0.0.1-dev",
+    "main": "dist/index.cjs",
+    "module": "dist/index.esm.js",
+    "type": "module",
+    "scripts": {
+        "build": "rollup -c",
+        "test": "vitest",
+        "test:watch": "vitest --watch",
+        "test:run": "vitest run"
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/vishalpalaniappan/program-synthesizer-python.git"
+    },
+    "author": "Vishal Palaniappan",
+    "license": "MIT",
+    "bugs": {
+        "url": "https://github.com/vishalpalaniappan/program-synthesizer-python/issues"
+    },
+    "homepage": "https://github.com/vishalpalaniappan/program-synthesizer-python",
+    "devDependencies": {
+        "@rollup/plugin-node-resolve": "^16.0.3",
+        "vitest": "^4.0.18",
+        "rollup": "^4.57.1"
+    }
+}

package/rollup.config.js

@@ -0,0 +1,18 @@
+import resolve from "@rollup/plugin-node-resolve";
+
+export default {
+    input: "src/DalAstGenerator.js",
+    output: [
+        {
+            file: "dist/index.cjs",
+            format: "cjs",
+            inlineDynamicImports: true,
+        },
+        {
+            file: "dist/index.esm.js",
+            format: "esm",
+            inlineDynamicImports: true,
+        },
+    ],
+    plugins: [resolve()],
+};

package/src/ArgsParser.js

@@ -0,0 +1,152 @@
+import { Untokenizer } from "./Untokenizer";
+
+/**
+ * Structure
+ * command(args...)
+ * 
+ * Example:
+ * set(book, ["name"], "Harry Potter")
+ * 
+ * Returns:
+ * [
+ *      {
+ *          "type": "name",
+ *          "value": "book"
+ *      },
+ *      {
+ *          "type": "list",
+ *          "value": ["name"]
+ *      },
+ *      {
+ *          "type": "string",
+ *          "value": "Harry Potter"
+ *      }    
+ * ]
+ * 
+ * This class parses the arguments from parser
+ * for a keyword or command instruction.
+ */
+export class ArgsParser {
+    constructor (tokens) {
+        this.tokens = tokens;
+        this.groupedTokens = [];
+        this.parsedArgs = [];
+
+        this.pairs = [
+            {
+                start:"QUOTE", 
+                end:"QUOTE", 
+                type:"string"
+            },
+            {
+                start:"LBRACE", 
+                end:"RBRACE", 
+                type:"object"
+            },
+            {
+                start:"LBRACKET", 
+                end:"RBRACKET", 
+                type: "list"
+            }
+        ]
+    }
+
+    run () {
+        if (this.tokens.length === 0) {
+            return this.parsedArgs;
+        }
+        this.splitByComma(this.tokens);
+        for (const group of this.groupedTokens) {
+            this.parsedArgs.push(this.processGroup(group));
+        }
+        return this.parsedArgs;
+    }
+
+    /**
+     * Splits the tokens by commas so they can be processed independently.
+     * For objects, arrays and strings, it flags that pair is being tracked
+     * so that any commmas inside the object or array is ignored.
+     * @param {Array} tokens 
+     */
+    splitByComma (tokens) {
+        let pos = 0;
+        let group = [];
+        this.trackingPair = null;
+        do {
+            const token = this.tokens[pos];
+            if (token.type === "COMMA" && !this.trackingPair) {
+                this.groupedTokens.push(group);
+                group = [];
+                continue;
+            } 
+            
+            if (this.trackingPair && token.type === this.trackingPair) {
+                this.trackingPair = null;
+            } else if (!this.trackingPair) {
+                this.trackingPair = this.isPair(token);
+            }
+            
+            group.push(token);
+
+        } while (++pos < this.tokens.length);
+
+        this.groupedTokens.push(group);
+    }
+
+    /**
+     * Checks if the token is a pair.
+     * @param {Object} token Token being checked.
+     * @returns {String|null}
+     */
+    isPair (token) {
+        for(const pair of this.pairs) {
+            if (token.type === pair.start) {
+                return pair.end;
+            }
+        }
+    }
+
+    /**
+     * Process the grouped tokens and returns
+     * the type and value of the arguments.
+     * 
+     * Types:
+     * ------
+     * Name (this is a participant name)
+     * String
+     * List or Array
+     * Object
+     * Number
+     * 
+     * @param {Array} tokens 
+     */
+    processGroup (tokens) {
+        let type;
+        let value = new Untokenizer(tokens).buildString();
+        if (tokens[0].type === "QUOTE") {
+            type = "string";
+            value = JSON.parse(value)
+        } else if (tokens[0].type === "LBRACKET") {
+            type = "list"
+            value = JSON.parse(value)
+        } else if (tokens[0].type === "LBRACE") {
+            type = "object"
+            value = JSON.parse(value)
+        } else if (!Number.isNaN(Number(value))) {
+            type = "number"
+            value = parseFloat(value)
+        } else {
+            type = "name";
+            if (value === "true") {
+                value = true;
+            } else if (value === "false") {
+                value = false;
+            }
+        }
+
+        return {
+            type: type,
+            value: value
+        }
+    }
+}

package/src/DalAstGenerator.js

@@ -0,0 +1,35 @@
+import { DalLexer } from "./Lexer";
+import { DalParser } from "./Parser";
+/**
+ * This class produces an AST output given a DAL source file.
+ * 
+ * Internally it invokes:
+ * - Lexer
+ * - Parser
+ * - AST output
+ * 
+ * The AST output is sent to the synthesizer and synthesized
+ * into a python program.
+ * 
+ * In the future, this will also:
+ * - Identify syntax errors
+ * - Lint
+ * - Validate the internal consistency of the design
+ */
+export class DalAstGenerator {
+
+    constructor() {
+        this.ast = null;
+    }
+
+    run (source) {
+        try {
+            const lexer = new DalLexer(source);
+            const parser = new DalParser(lexer.scannedTokens);
+            this.ast = parser.ast;
+        } catch(e) {
+            console.error("Error generating the AST:", e);
+        }
+        return this.ast;
+    }
+}

package/src/DesignValidator.js

@@ -0,0 +1,37 @@
+// This will evaluate the semantic world for inconsistencies.
+// Example: Type mistmatch, invalid participants etc. 
+
+// This requires knowledge about the actual transformations semantics, so I will establish that before moving forward with the validation. 
+// The synthesis development can happen in parallel because it doesn't impact this process.
+
+export class DesignValidator {
+
+    constructor (ast) {
+        this.ast = ast;
+        this.currentBehavior;
+    }
+
+    run() {
+        this.processTree(this.ast);
+    }
+
+    processTree(node) {
+        if ("body" in node) {
+            for (const child of node["body"]) {
+                if (child["type"] === "behavior") {
+                    this.currentBehavior = child;
+                }
+                this.processTree(child)
+            }
+        } else {
+            this.processNode(node);
+        }
+
+    }
+
+    processNode(node) {
+        if (node.type === "cmd" && node.command === "create") {
+            console.log(`Participant ${node.args[0].value} created in behavior ${this.currentBehavior["behaviorName"]}`, )
+        }
+    }
+}

package/src/KEYWORDS.js

@@ -0,0 +1,11 @@
+let KEYWORDS = [
+    "design",
+    "behavior",
+    "if",
+    "else",
+    "for",
+    "while"
+]
+KEYWORDS = Object.freeze(KEYWORDS);
+
+export default KEYWORDS;

package/src/Lexer.js

@@ -0,0 +1,174 @@
+import TOKENS from "./TOKENS";
+
+/**
+ * This lexer works as follows:
+ * - Reads character at current position
+ * - If it is not a token:
+ *      - adds to identifier accumulator
+ * - If it is a token:
+ *      - saves what is in the accumulator to scannedTokens
+ *      - saves the token to scannedTokens
+ * - If a token is a quote:
+ *      - saves quote to scanned token
+ *      - scans forward to end of quote while accumulating identifier
+ *      - saves accumulator to scanned tokens
+ *      - saves quote to scanned tokens
+ * 
+ * I am not scanning for keywords here, I just save them as
+ * identifiers and then in the parse stage I will classify them.
+ * This makes the algorithm very simple.
+ * 
+ * Example:
+ * design ("name")
+ * IDENTIFIER LPAREN QUOTE IDENTIFIER QUOTE RPAREN
+ * 
+ * I also save the line/col number (starting and ending). This will be
+ * useful for visualization in the workbench.
+ */
+export class DalLexer {
+    constructor (source) {
+        this.currPos = 0;
+        this.source = [...source];
+        this.scannedTokens = [];
+
+        // Current line and colno
+        this.lineno = 1;
+        this.colno = 0;
+
+        // Accumulates identifiers until tokens are visited.
+        this.accumulatedIdentifier = [];
+        this.startColIdentifier;
+        this.startLineIdentifier;
+
+        this.run();
+    }
+
+    /**
+     * Runs the lexer from the current position.
+     */
+    run() {
+        do  {
+            const character = this.source[this.currPos];
+            this.processCurrentPosition(character);
+        } while (++this.currPos < this.source.length);
+    }
+
+    /**
+     * Processes the character at the current position.
+     */
+    processCurrentPosition(character) {
+        this.colno++;
+
+        if (character == " ") {
+            this.addAccumulatedIdentifierToken();
+            return;
+        } else if (character == "\n") {
+            this.lineno++;
+            this.colno = 0;
+            return;
+        }
+
+        const token = this.getToken(character);
+        if (token) {
+            this.addAccumulatedIdentifierToken();
+            this.addToken(token);
+            if (token === "QUOTE") {
+                this.extractStringFromQuotes();
+            }
+            return;
+        }
+        this.addToAccumulator(character);
+    }
+
+    /**
+     * Finds the identifier given the character.
+     * @param {String} character Character from scan.
+     * @returns {String|null} Identifier if valid token, else null.
+     */
+    getToken (character) {
+        for (const [identifier, value] of Object.entries(TOKENS)) {
+            if (value[0] !== character) {
+                continue;
+            }
+            return identifier;
+        }
+    }
+
+    /**
+     * If a quote is encountered, scan forward until end of quote is found.
+     * Everything inside the quote is part of the string, this ignores any
+     * of the tokens like comma.
+     * 
+     * TODO: Add support for escaped quotes inside quotes.
+     */
+    extractStringFromQuotes () {
+        this.colno++;
+        this.currPos++;
+        do {
+            const character = this.source[this.currPos];
+            const identifier = this.getToken(character);
+            if (character === "\n") {
+                this.lineno++;
+                this.colno = 0;
+                continue;
+            }
+            if (identifier !== "QUOTE") {
+                this.addToAccumulator(character);
+                this.colno++;
+                continue;
+            }
+            this.addAccumulatedIdentifierToken();
+            this.addToken(identifier);
+            break;
+        } while (this.currPos++ < this.source.length)
+    }
+
+    /**
+     * Adds to accumulator. Saves starting position of
+     * identifier being accumulated.
+     * 
+     * @param {String} character Character to add to accumulate.
+     */
+    addToAccumulator (character) {
+        if (this.accumulatedIdentifier.length === 0) {
+            this.startColIdentifier = this.colno;
+            this.startLineIdentifier = this.lineno;
+        }
+        this.accumulatedIdentifier.push(character);
+    }
+
+    /**
+     * Add the accumulated identifier to the scannedTokens list.
+     */
+    addAccumulatedIdentifierToken () {
+        // Subtract 1 from endColno beause we have to reach token
+        // to identify that the accumulator is done.
+        if (this.accumulatedIdentifier.length > 0) {
+            this.scannedTokens.push({
+                type: "IDENTIFIER",
+                value: this.accumulatedIdentifier.join(""),
+                startLineno: this.startLineIdentifier,
+                startColno: this.startColIdentifier,
+                endLineno: this.lineno,
+                endColno: this.colno - 1
+            })
+            this.accumulatedIdentifier = [];
+        }
+    }
+
+    /**
+     * Adds the token to the scannedTokens list.
+     * @param {String} type Type for the token.
+     * @param {Number} value Value of token (example identifier)
+     */
+    addToken (type, value) {
+        this.scannedTokens.push({
+            type: type,
+            value: value,
+            startLineno: this.lineno,
+            startColno: this.colno,
+            endLineno: this.lineno,
+            endColno: this.colno
+        })
+    }
+}