dal-ast-js 0.0.1-dev

package/src/Parser.js

@@ -0,0 +1,281 @@
+import KEYWORDS from "./KEYWORDS";
+import { ArgsParser } from "./ArgsParser";
+
+/**
+ * This class produces an AST given the scanned tokens.
+ * 
+ * Steps:
+ * - Process current token
+ * - Visit keyword and create node 
+ * - Parse the args and save node metadata
+ * - If node has a body (if, behavior), add to stack
+ * - If node doesn't have body, add to body of stack top
+ * - When rbrace is encountered, pop stack and add
+ *   to body of top of stack.
+ * 
+ * It is a very simple algorithm because my language is basic.
+ * I create stack to track the nested body being processed. 
+ * I use rbrace as marker to finish a nested block and build the tree.
+ * 
+ * Note: This was the first way I thought to implement this
+ * but I think there is a better way, I am going to iterate
+ * on this. This approach does not identify syntax errors 
+ * properly.
+ * 
+ * The actual commands are very simple in this language
+ * because I am always following the format shown below:
+ * 
+ * command(args)
+ * 
+ * So every identifier that isn't a keyword while scanning
+ * forward is a command.
+ */
+export class DalParser {
+
+    constructor (tokens) {
+        this.tokens = tokens;
+        this.currPos = 0;
+        this.ast = {
+            type: "root",
+            body: []
+        }
+        this.stack = [this.ast];
+        this.run();
+    }
+
+    run () {
+        do {
+            const token = this.tokens[this.currPos];
+            this.processToken(token);
+        } while (++this.currPos < this.tokens.length);
+    }
+
+    /**
+     * Process the current token.
+     * 
+     * RBRACE indicates current block is over.
+     * behavior and if are blocks and have a body
+     * design is a statement.
+     * 
+     * @param {String} token 
+     */
+    processToken(token) {
+        let output;
+        if (token.type === "RBRACE") {
+            this.closeBlock();
+        } else if (token.value === "design") {
+            this.processDesignKeyword();
+        } else if (token.value === "behavior") {
+            this.processBehavior();
+        } else if (token.value === "if") {
+            this.processIf();
+        } else if (token.value === "else") {
+            this.processElse();
+        } else if (token.value === "for") {
+            this.processFor();
+        } else if (token.value === "while") {
+            this.processWhile();
+        }else if (token.type === "IDENTIFIER") {
+            this.processCmd(token.value);
+        }
+    }
+
+
+    /**
+     * Closes the nested block by removing it from
+     * the top of the stack and adding it to the new
+     * node at the top of the stack.
+     * 
+     * It uses the RBRACE identifier to determine if
+     * the top block is closed. 
+     * 
+     * For if statements, it appends the elif and else
+     * blocks to the if block that was already processed
+     * so that they can be synthesized in a single AST node.
+     */
+    closeBlock () {
+        const node = this.stack.pop();
+
+        if (node.type === "else") {
+            const body = this.stack[this.stack.length - 1].body;
+            const ifNode = body[body.length - 1]
+            ifNode["else"] = node;
+        } else {
+            this.stack[this.stack.length - 1].body.push(node); 
+        }
+    }
+
+    /**
+     * Processes the behavior block.
+     * 
+     * behavior <behavior_name>(args1, arg2...argN) {
+     * 
+     * }
+     */
+    processBehavior () {
+        const behaviorName = this.tokens[++this.currPos].value;
+        const node = {
+            type: "behavior",
+            behaviorName: behaviorName,
+            body: []
+        }
+        this.stack.push(node)
+    }
+
+    /**
+     * Processes if block.
+     * 
+     * if (condition) {
+     * 
+     * }
+     */
+    processIf () {
+        const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const node = {
+            type: "if",
+            args: parsedArgs,
+            body: []
+        }
+        this.stack.push(node)
+    }
+
+    /**
+     * Processes while block.
+     * 
+     * while (condition) {
+     * 
+     * }
+     */
+    processWhile () {
+        const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const node = {
+            type: "while",
+            args: parsedArgs,
+            body: []
+        }
+        this.stack.push(node)
+    }
+
+    /**
+     * Processes else block.
+     * 
+     * else (condition) {
+     * 
+     * }
+     */
+    processElse () {
+        const node = {
+            type: "else",
+            body: []
+        }
+        this.stack.push(node)
+    }
+
+    /**
+     * Processes for block.
+     * 
+     * for (<participan>, <start>,<end>) {
+     * 
+     * }
+     */
+    processFor () {
+        const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const node = {
+            type: "for",
+            participant: parsedArgs[0],
+            start: parsedArgs[1],
+            end: parsedArgs[2],
+            body: []
+        }
+        this.stack.push(node)
+    }
+
+    /**
+     * Process design keyword.
+     * 
+     * design(<design_name>)
+     */
+    processDesignKeyword () {
+        const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const node = {
+            "type": "design",
+            "design_name": parsedArgs
+        }
+        this.stack[this.stack.length - 1].body.push(node); 
+    }
+
+    /**
+     * Process design keyword.
+     * 
+     * cmd(args1, arg2...argN)
+     */
+    processCmd (cmd) {
+        const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const node = {
+            "type": "cmd",
+            "command": cmd,
+            "args": parsedArgs
+        }
+        this.stack[this.stack.length - 1].body.push(node); 
+    }
+
+    /**
+     * Parse the args.
+     * 
+     * command("test",varName, 123,["count"])
+     * 
+     * "args": [
+     *      {
+     *          "type": "string",
+     *          "value": "test"
+     *      },
+     *      {
+     *           "type": "name",
+     *          "value": "varName"
+     *      },
+     *      {
+     *          "type": "number",
+     *          "value": 123
+     *      },
+     *      {
+     *          "type": "list",
+     *          "value": [
+     *              "count"
+     *          ]
+     *      }
+     *  ]
+     * 
+     * @returns {Object} Returns the processed args.
+     */
+    getArgs () {
+        const token = this.tokens[++this.currPos];
+        if (token.type !== "LPAREN") {
+            throw new Error("Expected LPAREN, Syntax error")
+        }
+
+        const args = [];
+        let foundRParen = false;
+        do {
+            const token = this.tokens[++this.currPos];
+            if (token.type === "RPAREN") {
+                foundRParen = true;
+                break;
+            } else {
+                args.push(token);
+            }
+        } while (this.currPos < this.tokens.length)
+
+        // This is a crude attempt to see if brackets are closed
+        // It has obvious flaws, if the parenthesis isn't closed and
+        // then another identifier opens and closes parenthesis, it
+        // will keep moving forward until it reaches it. 
+ 
+        // There is a better way to do this, so this will get replaced.
+        if (!foundRParen) {
+            throw new Error("Expected RPAREN, Syntax error")
+        }
+
+        return args;
+    }
+
+}

package/src/TOKENS.js

@@ -0,0 +1,13 @@
+let TOKENS = {
+    "LPAREN": "(",
+    "RPAREN": ")",
+    "QUOTE": "\"",
+    "COMMA": ",",
+    "LBRACE": "{",
+    "RBRACE": "}",
+    "LBRACKET": "[",
+    "RBRACKET": "]"
+}
+TOKENS = Object.freeze(TOKENS);
+
+export default TOKENS;

package/src/Untokenizer.js

@@ -0,0 +1,34 @@
+import TOKENS from "./TOKENS";
+
+/**
+ * Builds string back from the tokens.
+ * 
+ * Ex:
+ * LBRACKET QUOTE IDENTIFIER("test") QUOTE RBRACKET
+ * ("test")
+ */
+export class Untokenizer {
+
+    constructor (tokens) {
+        this.tokens = tokens;
+    }
+
+    /**
+     * Builds string from tokens.
+     * @param {Array} tokens 
+     * @returns 
+     */
+    buildString(tokens) {
+        let str = "";
+
+        for (const token of this.tokens) {
+            if (token.type === "IDENTIFIER") {
+                str = str + token.value
+            } else {
+                str = str + TOKENS[token.type];
+            }
+        }
+
+        return str;
+    }
+}

package/src/docs/DesignValidatorNotes.md

@@ -0,0 +1,6 @@
+Writing down some notes here as I am working through the design validator:
+
+- As I was processing this, I decided that I would create a participant declaration block and in it, I would establish the participants, their types, allowed transformations domain structure etc.
+- When I call the create command, it will create a participant of a particular type and assign it a UID.
+- Then when I save that participant to the world state or access it anywhere else, I have the UID that traces it back to the creation. 
+- When a participant interacts with a transformation, for example, insert book into basket, their types will be checked to see if book is allowed to be inserted into basket.

package/src/grammar.json

@@ -0,0 +1,57 @@
+[
+    {
+        "token": "IDENTIFIER",
+        "value": "design",
+        "description": "design(\"lexer\")",
+        "order": [
+            {
+                "token": "LPAREN",
+                "endToken": "RPAREN",
+                "contentType": "args"
+            }
+        ]
+    },
+    {
+        "token": "IDENTIFIER",
+        "value": "behavior",
+        "order": [
+            {
+                "token": "IDENTIFIER",
+                "type": "string",
+                "identity": "behaviorName"
+            },
+            {
+                "token": "LBRACE",
+                "endToken": "RBRACE",
+                "contentType": "body"
+            }
+        ]
+    },
+    {
+        "token": "IDENTIFIER",
+        "value": "if",
+        "order": [
+            {
+                "token": "LPAREN",
+                "endToken": "RPAREN",
+                "contentType": "args"
+            },
+            {
+                "token": "LBRACE",
+                "endToken": "RBRACE",
+                "contentType": "body"
+            }
+        ]
+    },
+    {
+        "token": "IDENTIFIER",
+        "value": "else",
+        "order": [
+            {
+                "token": "LBRACE",
+                "endToken": "RBRACE",
+                "contentType": "body"
+            }
+        ]
+    }
+]

package/synthesizer/Synthesizer.py

@@ -0,0 +1,88 @@
+import ast
+from pathlib import Path
+from getSynthesizedNode import getSynthesizedNode
+import shutil
+
+class Synthesizer:
+    
+    def __init__(self, dalAst):
+        self.dalAst = dalAst
+        self.pythonAst = ast.Module(
+            body=[],
+            type_ignores=[]
+        )
+
+    def run(self):
+        '''
+            Run the synthesizer
+        '''
+        # Process each node in the DAL ast.
+        for node in self.dalAst["body"]:
+            self.processTree(node, self.pythonAst, 0)
+
+        importNode = ast.parse("from LoggingHelper import semanticLogger").body[0]
+        self.pythonAst.body.insert(0, importNode)
+
+        self.writeToOutputFolder()
+
+
+    def writeToOutputFolder(self):
+        '''
+            Writes the synthesized output to output folder and
+            also adds the logging helper.
+
+            If the output folder has files, clear it.
+        '''
+        outputFolder = Path(__file__).parent / "output"
+
+        # Make folder if it doesn't exist
+        outputFolder.mkdir(parents=True, exist_ok=True)
+
+        # Clear the output folder (assuming it existed)
+        for item in outputFolder.iterdir():
+            if item.is_dir():
+                shutil.rmtree(item)
+            else:
+                item.unlink()
+
+        # Write the synthesized output
+        outputFile = Path(__file__).parent / "output" / "synthesized.py"
+        with open(outputFile,"w+") as f:
+            f.write(ast.unparse(self.pythonAst))
+
+        # Copyt logging helper
+        src = Path(__file__).parent / "output_helpers" / "LoggingHelper.py"
+        dst = Path(__file__).parent / "output" / "LoggingHelper.py"
+        shutil.copy(src, dst)
+
+
+    def processTree(self, dalAstNode, pythonAstNode, indent):
+        '''
+            Process the tree node. If there is a body, process
+            each node in the body.
+
+            Writes the synthesized ast node to the ast tree.
+        '''
+        # self.printTree(indent, dalAstNode["type"])
+        astNodeBody = getSynthesizedNode(dalAstNode)
+
+        if astNodeBody is None:
+            if dalAstNode['type'] == "cmd":
+                type = dalAstNode['type'] + "," + dalAstNode['command']
+            else:
+                type = dalAstNode['type']
+            print(f"Unable to synthesize node of type {type}")
+        else:
+            ast.fix_missing_locations(astNodeBody)
+            pythonAstNode.body.append(astNodeBody)
+        
+            if "body" in dalAstNode:
+                for node in dalAstNode["body"]:
+                    self.processTree(node, astNodeBody, indent + 1)
+
+    def printTree(self, indent, value):
+        '''
+            Prints Tree with indentation for inspection.
+        '''
+        spaces = (indent * 4) * " "
+        print(f"{spaces}{value}")