dal-ast-js 0.0.2-dev → 0.0.4-dev

package/README.md

@@ -1,8 +1,25 @@
-# program-synthesizer-python
-Currently, this tool builds an Abstract Syntax Tree representation of a script written in the Design Abstraction Language (DAL).
+# DAL Abstract Syntax Tree Library
+This tool currently takes a design defined in a Design Abstraction Language (DAL) and produces an Abstract Syntax Tree (AST). 
 
-# Usage
-Currently, I am using test cases to establish functionality.
+To achieve this it implements the following tools:
+- Lexer
+- Parser
+
+This tool will be extended to also do the following:
+- Syntactic Validation: 
+    - Provide metadata about the result of the lexing and parsing to identify syntax errors. This will be used by the workbench to visually provide feedback in realtime.
+- Semantic Validation: 
+    - Validate the design by ensuring that it is internally consistent in realtime.
+
+This initial commit contains a working workflow but it is clearly not complete. I wanted to commit a working example to the repo to establish the workflow with the engine/workbench before I move forward. 
+
+Some relevant files:
+- [Library Manager Design](./tests/designs/library_manager.dal)
+- [Lexer Output](./tests/output/test_tokens.json)
+- [Generated AST](./tests/output/ast_direct_gen.json)
+
+# Test
+While it does have test cases to verify functionality, currently this is more functional than complete. 
 
 Install libraries:
 ```sh
@@ -19,8 +36,15 @@ To run specific test case:
 npm run test tests/Lexer.test.js
 ```
 
-This library will likely be integrated into the workbench directly and will be maintained there.
+Run the lexer.test.js while providing it with a design file with a .dal extension. This will produce an AST output. Currently, I am reading the designs from the designs folder and writing the ast's to the output folder.
+
+This will be streamlined by integrating everything into the workbench. It will automatically produce the AST and synthesize the program by invoking the python [synthesizer][synth].
+
+# Providing feedback
+
+You can use GitHub issues to [report a bug][bug-report] or [request a feature][feature-req].
 
-# Synthesizer
+[bug-report]: https://github.com/vishalpalaniappan/dal-ast-js/issues
+[feature-req]: https://github.com/vishalpalaniappan/dal-ast-js/issues
+[synth]: https://github.com/vishalpalaniappan/dal-synthesizer-python
 
-This repo also contains a synthesizer that can take the generated AST and synthesize it into an implementation.

package/dist/index.cjs

@@ -8,7 +8,8 @@ let TOKENS = {
     "LBRACE": "{",
     "RBRACE": "}",
     "LBRACKET": "[",
-    "RBRACKET": "]"
+    "RBRACKET": "]",
+    "HASH": "#"
 };
 TOKENS = Object.freeze(TOKENS);
 
@@ -84,6 +85,10 @@ class DalLexer {
 
         const token = this.getToken(character);
         if (token) {
+            if (token === "HASH") {
+                this.scanToNewLine();
+                return;
+            }
             this.addAccumulatedIdentifierToken();
             this.addToken(token);
             if (token === "QUOTE") {
@@ -94,6 +99,20 @@ class DalLexer {
         this.addToAccumulator(character);
     }
 
+    /**
+     * Scan to new line to ignore comment.
+     * @returns {null}
+     */
+    scanToNewLine () {
+        do  {
+            const character = this.source[this.currPos];
+            if (character == "\n") {
+                console.log("SCANNED NEW LINE");
+                return;
+            }
+        } while (++this.currPos < this.source.length);
+    }
+
     /**
      * Finds the identifier given the character.
      * @param {String} character Character from scan.
@@ -371,6 +390,9 @@ class ArgsParser {
                 value = true;
             } else if (value === "false") {
                 value = false;
+            } else if (value === "null") {
+                type = "null";
+                value = null;
             }
         }
 
@@ -587,11 +609,16 @@ class DalParser {
      * Process design keyword.
      * 
      * cmd(args1, arg2...argN)
+     * 
+     * Type:
+     * _cmd -> "registeredCmd"
+     * cmd -> "cmd"
      */
     processCmd (cmd) {
         const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const type = (cmd[0] === "_")?"registeredCmd":"cmd";
         const node = {
-            "type": "cmd",
+            "type": type,
             "command": cmd,
             "args": parsedArgs
         };
@@ -682,6 +709,7 @@ class DalAstGenerator {
     }
 
     run (source) {
+        this.ast = null;
         try {
             const lexer = new DalLexer(source);
             const parser = new DalParser(lexer.scannedTokens);

package/dist/index.esm.js

@@ -6,7 +6,8 @@ let TOKENS$1 = {
     "LBRACE": "{",
     "RBRACE": "}",
     "LBRACKET": "[",
-    "RBRACKET": "]"
+    "RBRACKET": "]",
+    "HASH": "#"
 };
 TOKENS$1 = Object.freeze(TOKENS$1);
 
@@ -82,6 +83,10 @@ class DalLexer {
 
         const token = this.getToken(character);
         if (token) {
+            if (token === "HASH") {
+                this.scanToNewLine();
+                return;
+            }
             this.addAccumulatedIdentifierToken();
             this.addToken(token);
             if (token === "QUOTE") {
@@ -92,6 +97,20 @@ class DalLexer {
         this.addToAccumulator(character);
     }
 
+    /**
+     * Scan to new line to ignore comment.
+     * @returns {null}
+     */
+    scanToNewLine () {
+        do  {
+            const character = this.source[this.currPos];
+            if (character == "\n") {
+                console.log("SCANNED NEW LINE");
+                return;
+            }
+        } while (++this.currPos < this.source.length);
+    }
+
     /**
      * Finds the identifier given the character.
      * @param {String} character Character from scan.
@@ -369,6 +388,9 @@ class ArgsParser {
                 value = true;
             } else if (value === "false") {
                 value = false;
+            } else if (value === "null") {
+                type = "null";
+                value = null;
             }
         }
 
@@ -585,11 +607,16 @@ class DalParser {
      * Process design keyword.
      * 
      * cmd(args1, arg2...argN)
+     * 
+     * Type:
+     * _cmd -> "registeredCmd"
+     * cmd -> "cmd"
      */
     processCmd (cmd) {
         const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const type = (cmd[0] === "_")?"registeredCmd":"cmd";
         const node = {
-            "type": "cmd",
+            "type": type,
             "command": cmd,
             "args": parsedArgs
         };
@@ -680,6 +707,7 @@ class DalAstGenerator {
     }
 
     run (source) {
+        this.ast = null;
         try {
             const lexer = new DalLexer(source);
             const parser = new DalParser(lexer.scannedTokens);

package/docs/validation_planning

@@ -0,0 +1,12 @@
+## Syntax Validation Planning
+- For a given keyword, it is easy to identify if the tokens match the pattern that is established in the grammar.
+- How should I detect when blocks are not closed?
+- I think there is actually a simple answer to this, since I am creating a stack for each nested block I encounter. I can tell at the end which stacks were not closed.
+- I also cannot encounter an opening brace when I am inside a block without a new entry in the stack being created.
+- Since I've already gathered the line numbers, its easy to visually identify exactly where the syntax error is.
+
+It would be great to have a formatting class that will pretty print the script as well.
+
+## Semantic Validation Planning
+- I think this is very simple because it just requires setting up the rules and then seeing if they are enforced.
+- If the rule isn't specified, that is also relevant information.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "dal-ast-js",
-    "version": "0.0.2-dev",
+    "version": "0.0.4-dev",
     "main": "dist/index.cjs",
     "module": "dist/index.esm.js",
     "type": "module",

package/src/ArgsParser.js

@@ -141,6 +141,9 @@ export class ArgsParser {
                 value = true;
             } else if (value === "false") {
                 value = false;
+            } else if (value === "null") {
+                type = "null";
+                value = null;
             }
         }
 

package/src/DalAstGenerator.js

@@ -23,6 +23,7 @@ export class DalAstGenerator {
     }
 
     run (source) {
+        this.ast = null;
         try {
             const lexer = new DalLexer(source);
             const parser = new DalParser(lexer.scannedTokens);

package/src/DesignValidator.js

@@ -4,6 +4,10 @@
 // This requires knowledge about the actual transformations semantics, so I will establish that before moving forward with the validation. 
 // The synthesis development can happen in parallel because it doesn't impact this process.
 
+/**
+ * Note: This is not a working implementation, it is just the skeleton to
+ * traverse the tree. This will developed further in followup PRs.
+ */
 export class DesignValidator {
 
     constructor (ast) {
@@ -23,6 +27,10 @@ export class DesignValidator {
                 }
                 this.processTree(child)
             }
+            // Process else blocks in if statements.
+            if ("else" in node) {
+                this.processTree(node["else"]);
+            }
         } else {
             this.processNode(node);
         }

package/src/Lexer.js

@@ -70,6 +70,10 @@ export class DalLexer {
 
         const token = this.getToken(character);
         if (token) {
+            if (token === "HASH") {
+                this.scanToNewLine();
+                return;
+            }
             this.addAccumulatedIdentifierToken();
             this.addToken(token);
             if (token === "QUOTE") {
@@ -80,6 +84,20 @@ export class DalLexer {
         this.addToAccumulator(character);
     }
 
+    /**
+     * Scan to new line to ignore comment.
+     * @returns {null}
+     */
+    scanToNewLine () {
+        do  {
+            const character = this.source[this.currPos];
+            if (character == "\n") {
+                console.log("SCANNED NEW LINE");
+                return;
+            }
+        } while (++this.currPos < this.source.length);
+    }
+
     /**
      * Finds the identifier given the character.
      * @param {String} character Character from scan.

package/src/Parser.js

@@ -208,11 +208,16 @@ export class DalParser {
      * Process design keyword.
      * 
      * cmd(args1, arg2...argN)
+     * 
+     * Type:
+     * _cmd -> "registeredCmd"
+     * cmd -> "cmd"
      */
     processCmd (cmd) {
         const parsedArgs = new ArgsParser(this.getArgs()).run();
+        const type = (cmd[0] === "_")?"registeredCmd":"cmd";
         const node = {
-            "type": "cmd",
+            "type": type,
             "command": cmd,
             "args": parsedArgs
         }

package/src/TOKENS.js

@@ -6,7 +6,8 @@ let TOKENS = {
     "LBRACE": "{",
     "RBRACE": "}",
     "LBRACKET": "[",
-    "RBRACKET": "]"
+    "RBRACKET": "]",
+    "HASH": "#"
 }
 TOKENS = Object.freeze(TOKENS);
 

package/src/docs/commands.json

@@ -0,0 +1,56 @@
+{
+    "set": {
+        "description": "Set the value of the target with the given keys",
+        "syntax": "set(<target>,<keys>,<value>)",
+        "synthesized": "target[<keys>] = <value>"
+    },
+    "create": {
+        "description": "Design metadata, declares participants.",
+        "syntax": "create(<name>,<type>,<role>)"
+    },
+    "log": {
+        "description": "Adds a log statement for the given participant.",
+        "syntax": "log(<behavior>, <name>, <type>, <value>)",
+        "synthesized": "semanticLogger.logParticipant(<behavior>, <name>, <type>, <value>)"
+    },
+    "insert": {
+        "description": "Inserts the value at the given index in the target with the provided keys",
+        "syntax": "insert(<target>, <keys>,  <value>, <index>)",
+        "synthesized": " <target>[<keys>].insert(<index>, <value>)"
+    },
+    "get": {
+        "description": "Get the value from the source provided the keys and save in target.",
+        "syntax": "get(<target>, <source>, <keys>)",
+        "synthesized": "<target> = <source>[<keys>]"
+    },
+    "select": {
+        "description": "Selects the next behavior to exhibit.",
+        "syntax": "select(<nextBehavior>)",
+        "synthesized": "return \"<nextBehavior>\""
+    },
+    "display": {
+        "description": "Prints the promopt to screen (prompt is an f string)",
+        "example": "display(\"f'Got book named {name} and it has first letter {firstLetter}'\")",
+        "syntax": "display(<prompt>)",
+        "synthesized": "print(\"<prompt>\")"
+    },
+    "isEqual": {
+        "description": "Checks the equality of cmp1 and cmp2 and stores result.",
+        "syntax": "isEqual(<cmp1>, <cmp2>, <result>)",
+        "synthesized": "<result> = <cmp1> == <cmp2>"
+    },
+    "getFromPos": {
+        "description": "Get the value at pos from source and saves it in target.",
+        "syntax": "getFromPos(<target>, <source>, <pos>)",
+        "synthesized": "<target> = <source>[<pos>]"
+    },
+    "removeFromPos": {
+        "description": "Removes the value at pos from source and saves it in target.",
+        "syntax": "getFromPos(<target>, <source>, <pos>)",
+        "synthesized": "<target> = <source>.pop(<pos>)"
+    },
+    "run": {
+        "description": "Synthesizes the block which runs the desig.",
+        "syntax": "See geCmdRunAst in getSynthesizedNode.py"
+    }
+}

package/tests/Lexer.test.js

@@ -9,7 +9,7 @@ import { DalAstGenerator } from "../src/DalAstGenerator";
 
 describe("Lexer", () => {
     it("basic source", async () => {
-        const filePath = resolve(__dirname, "./designs/library_manager.dal")
+        const filePath = resolve(__dirname, "./designs/reverse_name_persist.dal")
         const source = await readFile(filePath)
         const lexer = new DalLexer(source.toString());
 
@@ -32,7 +32,7 @@ describe("Lexer", () => {
     });
 
     it("tests direct ast generation", async () => {
-        const filePath = resolve(__dirname, "./designs/library_manager.dal")
+        const filePath = resolve(__dirname, "./designs/reverse_name_persist.dal")
         const source = await readFile(filePath)
         const ast = new DalAstGenerator().run(source.toString());
         const ast_output_path = resolve(__dirname, "./output/ast_direct_gen.json")

package/tests/designs/reverse_name_persist.dal

@@ -0,0 +1,52 @@
+design ("reverse_name_persist")
+
+# Example comment that is ignored.
+behavior b_createDatabaseConnection {
+    create(name=connection, type="object", role="connection")
+    _connectToDatabase(connection) 
+    set(worldState, ["connection"], connection)
+    select(b_createCursor)    
+}
+
+behavior b_createCursor {
+    create(name=cursor, type="object", role="cursor")
+    get(connection, worldState, ["connection"])
+    _createCursor(cursor, connection)
+    set(worldState, ["cursor"], cursor)
+    select(b_createTable)
+}
+
+behavior b_createTable {
+    get(cursor, worldState, ["cursor"])
+    _createTable(null, cursor)
+    select(b_commitConnection)
+}
+
+behavior b_commitConnection {
+    get(connection, worldState, ["connection"])
+    _commitConnection(null, connection)
+    select(b_receiveName)
+}
+
+behavior b_receiveName {
+    create(name=name, type="string", role="name")
+    _receiveName(name)
+    set(worldState, ["name"], name)
+    select(b_reverse)
+}
+
+behavior b_reverse {
+    get(name, worldState, ["name"])
+    _reverse(reversedName, name)
+    set(worldState, ["reversedName"], reversedName)
+    select(b_writeToDatabase)
+}
+
+behavior b_writeToDatabase {
+    get(cursor, worldState, ["cursor"])
+    get(reversedName, worldState, ["reversedName"])
+    _writeToDatabase(null, cursor, reversedName)
+    select(b_commitConnection)
+}
+
+run(b_createDatabaseConnection)