dal-ast-js 0.0.1-dev

package/dist/index.esm.js

@@ -0,0 +1,694 @@
+let TOKENS$1 = {
+    "LPAREN": "(",
+    "RPAREN": ")",
+    "QUOTE": "\"",
+    "COMMA": ",",
+    "LBRACE": "{",
+    "RBRACE": "}",
+    "LBRACKET": "[",
+    "RBRACKET": "]"
+};
+TOKENS$1 = Object.freeze(TOKENS$1);
+
+var TOKENS = TOKENS$1;
+
+/**
+ * This lexer works as follows:
+ * - Reads character at current position
+ * - If it is not a token:
+ *      - adds to identifier accumulator
+ * - If it is a token:
+ *      - saves what is in the accumulator to scannedTokens
+ *      - saves the token to scannedTokens
+ * - If a token is a quote:
+ *      - saves quote to scanned token
+ *      - scans forward to end of quote while accumulating identifier
+ *      - saves accumulator to scanned tokens
+ *      - saves quote to scanned tokens
+ * 
+ * I am not scanning for keywords here, I just save them as
+ * identifiers and then in the parse stage I will classify them.
+ * This makes the algorithm very simple.
+ * 
+ * Example:
+ * design ("name")
+ * IDENTIFIER LPAREN QUOTE IDENTIFIER QUOTE RPAREN
+ * 
+ * I also save the line/col number (starting and ending). This will be
+ * useful for visualization in the workbench.
+ */
+class DalLexer {
+    constructor (source) {
+        this.currPos = 0;
+        this.source = [...source];
+        this.scannedTokens = [];
+
+        // Current line and colno
+        this.lineno = 1;
+        this.colno = 0;
+
+        // Accumulates identifiers until tokens are visited.
+        this.accumulatedIdentifier = [];
+        this.startColIdentifier;
+        this.startLineIdentifier;
+
+        this.run();
+    }
+
+    /**
+     * Runs the lexer from the current position.
+     */
+    run() {
+        do  {
+            const character = this.source[this.currPos];
+            this.processCurrentPosition(character);
+        } while (++this.currPos < this.source.length);
+    }
+
+    /**
+     * Processes the character at the current position.
+     */
+    processCurrentPosition(character) {
+        this.colno++;
+
+        if (character == " ") {
+            this.addAccumulatedIdentifierToken();
+            return;
+        } else if (character == "\n") {
+            this.lineno++;
+            this.colno = 0;
+            return;
+        }
+
+        const token = this.getToken(character);
+        if (token) {
+            this.addAccumulatedIdentifierToken();
+            this.addToken(token);
+            if (token === "QUOTE") {
+                this.extractStringFromQuotes();
+            }
+            return;
+        }
+        this.addToAccumulator(character);
+    }
+
+    /**
+     * Finds the identifier given the character.
+     * @param {String} character Character from scan.
+     * @returns {String|null} Identifier if valid token, else null.
+     */
+    getToken (character) {
+        for (const [identifier, value] of Object.entries(TOKENS)) {
+            if (value[0] !== character) {
+                continue;
+            }
+            return identifier;
+        }
+    }
+
+    /**
+     * If a quote is encountered, scan forward until end of quote is found.
+     * Everything inside the quote is part of the string, this ignores any
+     * of the tokens like comma.
+     * 
+     * TODO: Add support for escaped quotes inside quotes.
+     */
+    extractStringFromQuotes () {
+        this.colno++;
+        this.currPos++;
+        do {
+            const character = this.source[this.currPos];
+            const identifier = this.getToken(character);
+            if (character === "\n") {
+                this.lineno++;
+                this.colno = 0;
+                continue;
+            }
+            if (identifier !== "QUOTE") {
+                this.addToAccumulator(character);
+                this.colno++;
+                continue;
+            }
+            this.addAccumulatedIdentifierToken();
+            this.addToken(identifier);
+            break;
+        } while (this.currPos++ < this.source.length)
+    }
+
+    /**
+     * Adds to accumulator. Saves starting position of
+     * identifier being accumulated.
+     * 
+     * @param {String} character Character to add to accumulate.
+     */
+    addToAccumulator (character) {
+        if (this.accumulatedIdentifier.length === 0) {
+            this.startColIdentifier = this.colno;
+            this.startLineIdentifier = this.lineno;
+        }
+        this.accumulatedIdentifier.push(character);
+    }
+
+    /**
+     * Add the accumulated identifier to the scannedTokens list.
+     */
+    addAccumulatedIdentifierToken () {
+        // Subtract 1 from endColno beause we have to reach token
+        // to identify that the accumulator is done.
+        if (this.accumulatedIdentifier.length > 0) {
+            this.scannedTokens.push({
+                type: "IDENTIFIER",
+                value: this.accumulatedIdentifier.join(""),
+                startLineno: this.startLineIdentifier,
+                startColno: this.startColIdentifier,
+                endLineno: this.lineno,
+                endColno: this.colno - 1
+            });
+            this.accumulatedIdentifier = [];
+        }
+    }
+
+    /**
+     * Adds the token to the scannedTokens list.
+     * @param {String} type Type for the token.
+     * @param {Number} value Value of token (example identifier)
+     */
+    addToken (type, value) {
+        this.scannedTokens.push({
+            type: type,
+            value: value,
+            startLineno: this.lineno,
+            startColno: this.colno,
+            endLineno: this.lineno,
+            endColno: this.colno
+        });
+    }
+}
+
+let KEYWORDS = [
+    "design",
+    "behavior",
+    "if",
+    "else",
+    "for",
+    "while"
+];
+KEYWORDS = Object.freeze(KEYWORDS);
+
+/**
+ * Builds string back from the tokens.
+ * 
+ * Ex:
+ * LBRACKET QUOTE IDENTIFIER("test") QUOTE RBRACKET
+ * ("test")
+ */
+class Untokenizer {
+
+    constructor (tokens) {
+        this.tokens = tokens;
+    }
+
+    /**
+     * Builds string from tokens.
+     * @param {Array} tokens 
+     * @returns 
+     */
+    buildString(tokens) {
+        let str = "";
+
+        for (const token of this.tokens) {
+            if (token.type === "IDENTIFIER") {
+                str = str + token.value;
+            } else {
+                str = str + TOKENS[token.type];
+            }
+        }
+
+        return str;
+    }
+}
+
+/**
+ * Structure
+ * command(args...)
+ * 
+ * Example:
+ * set(book, ["name"], "Harry Potter")
+ * 
+ * Returns:
+ * [
+ *      {
+ *          "type": "name",
+ *          "value": "book"
+ *      },
+ *      {
+ *          "type": "list",
+ *          "value": ["name"]
+ *      },
+ *      {
+ *          "type": "string",
+ *          "value": "Harry Potter"
+ *      }    
+ * ]
+ * 
+ * This class parses the arguments from parser
+ * for a keyword or command instruction.
+ */
+class ArgsParser {
+    constructor (tokens) {
+        this.tokens = tokens;
+        this.groupedTokens = [];
+        this.parsedArgs = [];
+
+        this.pairs = [
+            {
+                start:"QUOTE", 
+                end:"QUOTE", 
+                type:"string"
+            },
+            {
+                start:"LBRACE", 
+                end:"RBRACE", 
+                type:"object"
+            },
+            {
+                start:"LBRACKET", 
+                end:"RBRACKET", 
+                type: "list"
+            }
+        ];
+    }
+
+    run () {
+        if (this.tokens.length === 0) {
+            return this.parsedArgs;
+        }
+        this.splitByComma(this.tokens);
+        for (const group of this.groupedTokens) {
+            this.parsedArgs.push(this.processGroup(group));
+        }
+        return this.parsedArgs;
+    }
+
+    /**
+     * Splits the tokens by commas so they can be processed independently.
+     * For objects, arrays and strings, it flags that pair is being tracked
+     * so that any commmas inside the object or array is ignored.
+     * @param {Array} tokens 
+     */
+    splitByComma (tokens) {
+        let pos = 0;
+        let group = [];
+        this.trackingPair = null;
+        do {
+            const token = this.tokens[pos];
+            if (token.type === "COMMA" && !this.trackingPair) {
+                this.groupedTokens.push(group);
+                group = [];
+                continue;
+            } 
+            
+            if (this.trackingPair && token.type === this.trackingPair) {
+                this.trackingPair = null;
+            } else if (!this.trackingPair) {
+                this.trackingPair = this.isPair(token);
+            }
+            
+            group.push(token);
+
+        } while (++pos < this.tokens.length);
+
+        this.groupedTokens.push(group);
+    }
+
+    /**
+     * Checks if the token is a pair.
+     * @param {Object} token Token being checked.
+     * @returns {String|null}
+     */
+    isPair (token) {
+        for(const pair of this.pairs) {
+            if (token.type === pair.start) {
+                return pair.end;
+            }
+        }
+    }
+
+    /**
+     * Process the grouped tokens and returns
+     * the type and value of the arguments.
+     * 
+     * Types:
+     * ------
+     * Name (this is a participant name)
+     * String
+     * List or Array
+     * Object
+     * Number
+     * 
+     * @param {Array} tokens 
+     */
+    processGroup (tokens) {
+        let type;
+        let value = new Untokenizer(tokens).buildString();
+        if (tokens[0].type === "QUOTE") {
+            type = "string";
+            value = JSON.parse(value);
+        } else if (tokens[0].type === "LBRACKET") {
+            type = "list";
+            value = JSON.parse(value);
+        } else if (tokens[0].type === "LBRACE") {
+            type = "object";
+            value = JSON.parse(value);
+        } else if (!Number.isNaN(Number(value))) {
+            type = "number";
+            value = parseFloat(value);
+        } else {
+            type = "name";
+            if (value === "true") {
+                value = true;
+            } else if (value === "false") {
+                value = false;
+            }
+        }
+
+        return {
+            type: type,
+            value: value
+        }
+    }
+}
+
+/**
+ * This class produces an AST given the scanned tokens.
+ * 
+ * Steps:
+ * - Process current token
+ * - Visit keyword and create node 
+ * - Parse the args and save node metadata
+ * - If node has a body (if, behavior), add to stack
+ * - If node doesn't have body, add to body of stack top
+ * - When rbrace is encountered, pop stack and add
+ *   to body of top of stack.
+ * 
+ * It is a very simple algorithm because my language is basic.
+ * I create stack to track the nested body being processed. 
+ * I use rbrace as marker to finish a nested block and build the tree.
+ * 
+ * Note: This was the first way I thought to implement this
+ * but I think there is a better way, I am going to iterate
+ * on this. This approach does not identify syntax errors 
+ * properly.
+ * 
+ * The actual commands are very simple in this language
+ * because I am always following the format shown below:
+ * 
+ * command(args)
+ * 
+ * So every identifier that isn't a keyword while scanning
+ * forward is a command.
+ */
+class DalParser {
+
+    constructor (tokens) {
+        this.tokens = tokens;
+        this.currPos = 0;
+        this.ast = {
+            type: "root",
+            body: []
+        };
+        this.stack = [this.ast];
+        this.run();
+    }
+
+    run () {
+        do {
+            const token = this.tokens[this.currPos];
+            this.processToken(token);
+        } while (++this.currPos < this.tokens.length);
+    }
+
+    /**
+     * Process the current token.
+     * 
+     * RBRACE indicates current block is over.
+     * behavior and if are blocks and have a body
+     * design is a statement.
+     * 
+     * @param {String} token 
+     */
+    processToken(token) {
+        if (token.type === "RBRACE") {
+            this.closeBlock();
+        } else if (token.value === "design") {
+            this.processDesignKeyword();
+        } else if (token.value === "behavior") {
+            this.processBehavior();
+        } else if (token.value === "if") {
+            this.processIf();
+        } else if (token.value === "else") {
+            this.processElse();
+        } else if (token.value === "for") {
+            this.processFor();
+        } else if (token.value === "while") {
+            this.processWhile();
+        }else if (token.type === "IDENTIFIER") {
+            this.processCmd(token.value);
+        }
+    }
+
+
+    /**
+     * Closes the nested block by removing it from
+     * the top of the stack and adding it to the new
+     * node at the top of the stack.
+     * 
+     * It uses the RBRACE identifier to determine if
+     * the top block is closed. 
+     * 
+     * For if statements, it appends the elif and else
+     * blocks to the if block that was already processed
+     * so that they can be synthesized in a single AST node.
+     */
+    closeBlock () {
+        const node = this.stack.pop();
+
+        if (node.type === "else") {
+            const body = this.stack[this.stack.length - 1].body;
+            const ifNode = body[body.length - 1];
+            ifNode["else"] = node;
+        } else {
+            this.stack[this.stack.length - 1].body.push(node); 
+        }
+    }
+
+    /**
+     * Processes the behavior block.
+     * 
+     * behavior <behavior_name>(args1, arg2...argN) {
+     * 
+     * }
+     */
+    processBehavior () {
+        const behaviorName = this.tokens[++this.currPos].value;
+        const node = {
+            type: "behavior",
+            behaviorName: behaviorName,
+            body: []
+        };
+        this.stack.push(node);
+    }
+
+    /**
+     * Processes if block.
+     * 
+     * if (condition) {
+     * 
+     * }
+     */
+    processIf () {
+        const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const node = {
+            type: "if",
+            args: parsedArgs,
+            body: []
+        };
+        this.stack.push(node);
+    }
+
+    /**
+     * Processes while block.
+     * 
+     * while (condition) {
+     * 
+     * }
+     */
+    processWhile () {
+        const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const node = {
+            type: "while",
+            args: parsedArgs,
+            body: []
+        };
+        this.stack.push(node);
+    }
+
+    /**
+     * Processes else block.
+     * 
+     * else (condition) {
+     * 
+     * }
+     */
+    processElse () {
+        const node = {
+            type: "else",
+            body: []
+        };
+        this.stack.push(node);
+    }
+
+    /**
+     * Processes for block.
+     * 
+     * for (<participan>, <start>,<end>) {
+     * 
+     * }
+     */
+    processFor () {
+        const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const node = {
+            type: "for",
+            participant: parsedArgs[0],
+            start: parsedArgs[1],
+            end: parsedArgs[2],
+            body: []
+        };
+        this.stack.push(node);
+    }
+
+    /**
+     * Process design keyword.
+     * 
+     * design(<design_name>)
+     */
+    processDesignKeyword () {
+        const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const node = {
+            "type": "design",
+            "design_name": parsedArgs
+        };
+        this.stack[this.stack.length - 1].body.push(node); 
+    }
+
+    /**
+     * Process design keyword.
+     * 
+     * cmd(args1, arg2...argN)
+     */
+    processCmd (cmd) {
+        const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const node = {
+            "type": "cmd",
+            "command": cmd,
+            "args": parsedArgs
+        };
+        this.stack[this.stack.length - 1].body.push(node); 
+    }
+
+    /**
+     * Parse the args.
+     * 
+     * command("test",varName, 123,["count"])
+     * 
+     * "args": [
+     *      {
+     *          "type": "string",
+     *          "value": "test"
+     *      },
+     *      {
+     *           "type": "name",
+     *          "value": "varName"
+     *      },
+     *      {
+     *          "type": "number",
+     *          "value": 123
+     *      },
+     *      {
+     *          "type": "list",
+     *          "value": [
+     *              "count"
+     *          ]
+     *      }
+     *  ]
+     * 
+     * @returns {Object} Returns the processed args.
+     */
+    getArgs () {
+        const token = this.tokens[++this.currPos];
+        if (token.type !== "LPAREN") {
+            throw new Error("Expected LPAREN, Syntax error")
+        }
+
+        const args = [];
+        let foundRParen = false;
+        do {
+            const token = this.tokens[++this.currPos];
+            if (token.type === "RPAREN") {
+                foundRParen = true;
+                break;
+            } else {
+                args.push(token);
+            }
+        } while (this.currPos < this.tokens.length)
+
+        // This is a crude attempt to see if brackets are closed
+        // It has obvious flaws, if the parenthesis isn't closed and
+        // then another identifier opens and closes parenthesis, it
+        // will keep moving forward until it reaches it. 
+ 
+        // There is a better way to do this, so this will get replaced.
+        if (!foundRParen) {
+            throw new Error("Expected RPAREN, Syntax error")
+        }
+
+        return args;
+    }
+
+}
+
+/**
+ * This class produces an AST output given a DAL source file.
+ * 
+ * Internally it invokes:
+ * - Lexer
+ * - Parser
+ * - AST output
+ * 
+ * The AST output is sent to the synthesizer and synthesized
+ * into a python program.
+ * 
+ * In the future, this will also:
+ * - Identify syntax errors
+ * - Lint
+ * - Validate the internal consistency of the design
+ */
+class DalAstGenerator {
+
+    constructor() {
+        this.ast = null;
+    }
+
+    run (source) {
+        try {
+            const lexer = new DalLexer(source);
+            const parser = new DalParser(lexer.scannedTokens);
+            this.ast = parser.ast;
+        } catch(e) {
+            console.error("Error generating the AST:", e);
+        }
+        return this.ast;
+    }
+}
+
+export { DalAstGenerator };

package/docs/notes.md

@@ -0,0 +1,36 @@
+## Planning
+
+- Lexer tokenizes the script
+- Parser uses the grammar to build the AST.
+- Grammar establishes how to build tree from tokens.
+
+## Sample Script
+```
+design(sample_design)
+
+behavior(getValue) {
+    create("basket","list",[])    
+}
+```
+
+## Tree Output
+```
+SemanticModel
+    - Design
+        - name:sample Design
+    - Behavior
+        - name:getValue
+        - block
+            - Call
+                - name:create
+                - arguments
+                    - participant: basket
+                    - type: list
+                    - value: []
+```
+
+## Synthesis Output
+```
+def getValue():
+    basket = []
+```