@markw65/monkeyc-optimizer 1.0.17 → 1.0.20

package/README.md

@@ -193,3 +193,45 @@ More fixes found via open source projects.
 
 - Bug Fixes
   - Fix a bug affecting lookup of types, which could cause definitions, references and links to the api docs to be missed in the vscode extension
+
+### 1.0.18
+
+- Bug Fixes
+  - The new inliner was too agressive at constant propagating literal parameters to their point of use.
+
+### 1.0.19
+
+- Upgrade to @markw65/prettier-plugin-monkeyc@1.0.22
+
+  - fixes some minor typing issues for mctree
+  - special handling for certain parenthesized expressions.
+
+- Optimizer
+
+  - Handle more unused expressions, add tests, and prettify the OptimizerTests project
+  - Allow statement-style inlining in assignent and return contexts
+  - Add diagnostics for failure to inline
+
+- Tests
+
+  - More tweaks to pragma-checker
+  - Add launch and task configs for building/running tests
+
+- Code cleanup
+  - Properly type the results of JSON.parse
+  - Switch over to using ParenthesizedExpression for formatAst (depends on @markw65/prettier-plugin-monkeyc@1.0.22)
+
+### 1.0.20
+
+- Bug fixes
+
+  - Fix a bug marking unknown callees
+  - Fix a bug in test.js that didn't notice when tests failed, and fix a failing test
+
+- Optimizer enhancements
+
+  - Re-run the main optimization step, to properly account for functions that are unused after optimization
+  - Call the optimizer on 'unused' nodes before returning them
+
+- Code cleanup
+  - Called function cleanup

package/build/api.cjs

@@ -181,7 +181,7 @@ function getArgSafety(state, func, args, requireAll) {
     })) {
         return false;
     }
-    if (allSafe)
+    if (allSafe && requireAll)
         return true;
     let callSeen = false;
     let ok = true;
@@ -204,10 +204,16 @@ function getArgSafety(state, func, args, requireAll) {
             .map(([key]) => key);
     }, (node) => {
         switch (node.type) {
+            case "AssignmentExpression":
+            case "UpdateExpression": {
+                const v = node.type == "UpdateExpression" ? node.argument : node.left;
+                if (v.type === "Identifier" && hasProperty(params, v.name)) {
+                    safeArgs[params[v.name]] = null;
+                }
+            }
+            // fall through
             case "CallExpression":
             case "NewExpression":
-            case "AssignmentExpression":
-            case "UpdateExpression":
                 callSeen = true;
                 break;
             case "Identifier":
@@ -250,15 +256,30 @@ function inliningLooksUseful(func, node) {
     }
     return false;
 }
-var inliner_InlineStatus;
+var InlineStatus;
 (function (InlineStatus) {
     InlineStatus[InlineStatus["Never"] = 0] = "Never";
     InlineStatus[InlineStatus["AsExpression"] = 1] = "AsExpression";
     InlineStatus[InlineStatus["AsStatement"] = 2] = "AsStatement";
-})(inliner_InlineStatus || (inliner_InlineStatus = {}));
-function inliner_shouldInline(state, func, args) {
+})(InlineStatus || (InlineStatus = {}));
+function inlineRequested(state, func) {
+    const excludeAnnotations = (func.node.loc?.source &&
+        state.fnMap[func.node.loc?.source]?.excludeAnnotations) ||
+        {};
+    if (func.node.attrs &&
+        func.node.attrs.attrs &&
+        func.node.attrs.attrs.some((attr) => attr.type === "UnaryExpression" &&
+            (attr.argument.name === "inline" ||
+                (attr.argument.name.startsWith("inline_") &&
+                    hasProperty(excludeAnnotations, attr.argument.name.substring(7)))))) {
+        return true;
+    }
+    return false;
+}
+function inliner_shouldInline(state, func, call, context) {
     let autoInline = false;
     let inlineAsExpression = false;
+    const args = call.arguments;
     if (func.node.body &&
         func.node.body.body.length === 1 &&
         func.node.body.body[0].type === "ReturnStatement" &&
@@ -268,39 +289,30 @@ function inliner_shouldInline(state, func, args) {
         autoInline = inliningLooksUseful(func.node, func.node.body.body[0].argument);
     }
     if (autoInline === 1) {
-        return inliner_InlineStatus.AsExpression;
+        return true;
     }
-    const excludeAnnotations = (func.node.loc?.source &&
-        state.fnMap[func.node.loc?.source]?.excludeAnnotations) ||
-        {};
-    const inlineRequested = func.node.attrs &&
-        func.node.attrs.attrs &&
-        func.node.attrs.attrs.some((attr) => attr.type === "UnaryExpression" &&
-            (attr.argument.name === "inline" ||
-                (attr.argument.name.startsWith("inline_") &&
-                    hasProperty(excludeAnnotations, attr.argument.name.substring(7)))));
-    if (autoInline || inlineRequested) {
-        return inlineAsExpression && canInline(state, func, args)
-            ? inliner_InlineStatus.AsExpression
-            : inliner_InlineStatus.AsStatement;
+    const requested = inlineRequested(state, func);
+    if (autoInline || requested) {
+        if (inlineAsExpression) {
+            if (canInline(state, func, args)) {
+                return true;
+            }
+        }
+        if (!context && requested) {
+            inlineDiagnostic(state, func, call, "This function can only be inlined in statement, assignment, or return contexts");
+        }
+        return context != null;
     }
-    return inliner_InlineStatus.Never;
+    return false;
 }
 function processInlineBody(state, func, call, root, insertedVariableDecls, params) {
-    if (!params) {
-        const safeArgs = getArgSafety(state, func, call.arguments, false);
-        params = Object.fromEntries(func.node.params.map((param, i) => {
-            const argnum = safeArgs === true || (safeArgs !== false && safeArgs[i] !== null)
-                ? i
-                : -1;
-            const name = variableDeclarationName(param);
-            return [name, argnum];
-        }));
-    }
+    let failed = false;
     const pre = state.pre;
     const post = state.post;
     try {
         state.pre = (node) => {
+            if (failed)
+                return [];
             node.start = call.start;
             node.end = call.end;
             node.loc = call.loc;
@@ -312,6 +324,11 @@ function processInlineBody(state, func, call, root, insertedVariableDecls, param
                 const { map } = locals;
                 if (!map)
                     throw new Error("No local variable map!");
+                // We still need to keep track of every local name that was
+                // already in use, but we don't want to use any of its renames.
+                // We also want to know whether a local is from the function being
+                // inlined, or the calling function, so set every element to false.
+                Object.keys(map).forEach((key) => (map[key] = false));
                 const declarations = func.node.params
                     .map((param, i) => {
                     const paramName = variableDeclarationName(param);
@@ -336,6 +353,8 @@ function processInlineBody(state, func, call, root, insertedVariableDecls, param
             return result;
         };
         state.post = (node) => {
+            if (failed)
+                return null;
             let replacement = null;
             switch (node.type) {
                 case "Identifier": {
@@ -350,22 +369,28 @@ function processInlineBody(state, func, call, root, insertedVariableDecls, param
                     }
                     replacement = fixNodeScope(state, node, func.stack);
                     if (!replacement) {
-                        throw new Error(`Inliner: Couldn't fix the scope of '${node.name}`);
+                        failed = true;
+                        inlineDiagnostic(state, func, call, `Failed to resolve '${node.name}'`);
+                        return null;
                     }
                     break;
                 }
             }
             return post(replacement || node, state) || replacement;
         };
-        return state.traverse(root) || null;
-    }
-    catch (ex) {
-        if (ex instanceof Error) {
-            if (ex.message.startsWith("Inliner: ")) {
-                return null;
-            }
+        let ret = state.traverse(root);
+        if (failed) {
+            return null;
+        }
+        if (ret === null) {
+            ret = root;
+        }
+        if (!ret) {
+            inlineDiagnostic(state, func, call, `Internal error`);
+            return null;
         }
-        throw ex;
+        inlineDiagnostic(state, func, call, null);
+        return ret;
     }
     finally {
         state.pre = pre;
@@ -388,7 +413,16 @@ function inliner_unused(expression, top) {
         case "UnaryExpression":
             return inliner_unused(expression.argument);
         case "MemberExpression":
-            return inliner_unused(expression.object).concat(inliner_unused(expression.property));
+            if (expression.computed) {
+                return inliner_unused(expression.object).concat(inliner_unused(expression.property));
+            }
+            return inliner_unused(expression.object);
+        case "ArrayExpression":
+            return expression.elements.map((e) => inliner_unused(e)).flat(1);
+        case "ObjectExpression":
+            return expression.properties
+                .map((p) => inliner_unused(p.key).concat(inliner_unused(p.value)))
+                .flat(1);
     }
     return top
         ? null
@@ -396,30 +430,98 @@ function inliner_unused(expression, top) {
             {
                 type: "ExpressionStatement",
                 expression,
+                start: expression.start,
+                end: expression.end,
+                loc: expression.loc,
             },
         ];
 }
-function inlineWithArgs(state, func, call) {
+function diagnostic(state, loc, message) {
+    if (!loc || !loc.source)
+        return;
+    const source = loc.source;
+    if (!state.diagnostics)
+        state.diagnostics = {};
+    if (!hasProperty(state.diagnostics, source)) {
+        if (!message)
+            return;
+        state.diagnostics[source] = [];
+    }
+    const diags = state.diagnostics[source];
+    let index = diags.findIndex((item) => item.loc === loc);
+    if (message) {
+        if (index < 0)
+            index = diags.length;
+        diags[index] = { type: "INFO", loc, message };
+    }
+    else if (index >= 0) {
+        diags.splice(index, 1);
+    }
+}
+function inlineDiagnostic(state, func, call, message) {
+    if (inlineRequested(state, func)) {
+        diagnostic(state, call.loc, message && `While inlining ${func.node.id.name}: ${message}`);
+    }
+}
+function inlineWithArgs(state, func, call, context) {
     if (!func.node || !func.node.body) {
         return null;
     }
     let retStmtCount = 0;
-    traverseAst(func.node.body, (node) => {
-        node.type === "ReturnStatement" && retStmtCount++;
-    });
-    if (retStmtCount > 1 ||
-        (retStmtCount === 1 &&
-            func.node.body.body.slice(-1)[0].type !== "ReturnStatement")) {
-        return null;
+    if (context.type === "ReturnStatement") {
+        const last = func.node.body.body.slice(-1)[0];
+        if (!last || last.type !== "ReturnStatement") {
+            inlineDiagnostic(state, func, call, "Function didn't end with a return statement");
+            return null;
+        }
+    }
+    else {
+        traverseAst(func.node.body, (node) => {
+            node.type === "ReturnStatement" && retStmtCount++;
+        });
+        if (retStmtCount > 1) {
+            inlineDiagnostic(state, func, call, "Function had more than one return statement");
+        }
+        else if (context.type === "AssignmentExpression" && retStmtCount !== 1) {
+            inlineDiagnostic(state, func, call, "Function did not have a return statement");
+            return null;
+        }
+        if (retStmtCount === 1) {
+            const last = func.node.body.body.slice(-1)[0];
+            if (!last ||
+                last.type !== "ReturnStatement" ||
+                (context.type === "AssignmentExpression" && !last.argument)) {
+                inlineDiagnostic(state, func, call, "There was a return statement, but not at the end of the function");
+                return null;
+            }
+        }
     }
     const body = JSON.parse(JSON.stringify(func.node.body));
-    processInlineBody(state, func, call, body, func.node.params.length ? false : true);
-    if (retStmtCount) {
+    const safeArgs = getArgSafety(state, func, call.arguments, false);
+    const params = Object.fromEntries(func.node.params.map((param, i) => {
+        const argnum = safeArgs === true || (safeArgs !== false && safeArgs[i] !== null)
+            ? i
+            : -1;
+        const name = variableDeclarationName(param);
+        return [name, argnum];
+    }));
+    if (!processInlineBody(state, func, call, body, func.node.params.length ? false : true, params)) {
+        return null;
+    }
+    diagnostic(state, call.loc, null);
+    if (context.type !== "ReturnStatement" && retStmtCount) {
         const last = body.body[body.body.length - 1];
         if (last.type != "ReturnStatement") {
             throw new Error("ReturnStatement got lost!");
         }
-        if (last.argument) {
+        if (context.type === "AssignmentExpression") {
+            context.right = last.argument;
+            body.body[body.body.length - 1] = {
+                type: "ExpressionStatement",
+                expression: context,
+            };
+        }
+        else if (last.argument) {
             const side_exprs = inliner_unused(last.argument);
             body.body.splice(body.body.length - 1, 1, ...side_exprs);
         }
@@ -429,13 +531,13 @@ function inlineWithArgs(state, func, call) {
     }
     return body;
 }
-function inliner_inlineFunction(state, func, call, inlineStatus) {
-    if (inlineStatus == inliner_InlineStatus.AsStatement) {
-        return inlineWithArgs(state, func, call);
+function inliner_inlineFunction(state, func, call, context) {
+    if (context) {
+        return inlineWithArgs(state, func, call, context);
     }
     const retArg = JSON.parse(JSON.stringify(func.node.body.body[0].argument));
     const params = Object.fromEntries(func.node.params.map((param, i) => [variableDeclarationName(param), i]));
-    return processInlineBody(state, func, call, retArg, true, params) || retArg;
+    return processInlineBody(state, func, call, retArg, true, params);
 }
 function applyTypeIfNeeded(node) {
     if ("enumType" in node && node.enumType) {
@@ -450,13 +552,14 @@ function applyTypeIfNeeded(node) {
 }
 function fixNodeScope(state, lookupNode, nodeStack) {
     if (lookupNode.type === "Identifier") {
-        for (let i = state.stack.length; --i > nodeStack.length;) {
-            const si = state.stack[i];
-            if (hasProperty(si.decls, lookupNode.name)) {
-                // its a local from the inlined function.
-                // Nothing to do.
-                return lookupNode;
-            }
+        const locals = state.localsStack[state.localsStack.length - 1];
+        const { map } = locals;
+        if (!map)
+            throw new Error("No local variable map!");
+        if (hasProperty(map, lookupNode.name) && map[lookupNode.name] !== false) {
+            // map[name] !== false means its an entry that was created during inlining
+            // so its definitely one of our locals.
+            return lookupNode;
         }
     }
     const [, original] = state.lookup(lookupNode, null, nodeStack);
@@ -932,13 +1035,15 @@ function optimizeNode(node) {
     return null;
 }
 function evaluateFunction(func, args) {
-    if (args && args.length != func.params.length) {
+    if (!func.body || (args && args.length != func.params.length)) {
         return false;
     }
     const paramValues = args &&
         Object.fromEntries(func.params.map((p, i) => [variableDeclarationName(p), args[i]]));
     let ret = null;
-    const body = args ? JSON.parse(JSON.stringify(func.body)) : func.body;
+    const body = args
+        ? JSON.parse(JSON.stringify(func.body))
+        : func.body;
     try {
         traverseAst(body, (node) => {
             switch (node.type) {
@@ -981,6 +1086,13 @@ function evaluateFunction(func, args) {
         return false;
     }
 }
+function markFunctionCalled(state, func) {
+    if (!hasProperty(state.calledFunctions, func.id.name)) {
+        state.calledFunctions[func.id.name] = [func];
+        return;
+    }
+    pushUnique(state.calledFunctions[func.id.name], func);
+}
 async function optimizeMonkeyC(fnMap) {
     const state = {
         ...(await analyze(fnMap)),
@@ -1139,7 +1251,7 @@ async function optimizeMonkeyC(fnMap) {
                 if (map) {
                     if (hasProperty(map, node.name)) {
                         const name = map[node.name];
-                        if (name !== true) {
+                        if (typeof name === "string") {
                             node.name = name;
                         }
                     }
@@ -1190,10 +1302,7 @@ async function optimizeMonkeyC(fnMap) {
                         used = checkInherited(parent, node.id.name);
                     }
                     if (used) {
-                        if (!hasProperty(state.calledFunctions, node.id.name)) {
-                            state.calledFunctions[node.id.name] = [];
-                        }
-                        state.calledFunctions[node.id.name].push(node);
+                        markFunctionCalled(state, node);
                     }
                 }
             }
@@ -1230,27 +1339,61 @@ async function optimizeMonkeyC(fnMap) {
                     return node.body;
                 }
                 break;
+            case "ReturnStatement":
+                if (node.argument && node.argument.type === "CallExpression") {
+                    return optimizeCall(state, node.argument, node);
+                }
+                break;
             case "CallExpression": {
-                const ret = optimizeCall(state, node, false);
+                const ret = optimizeCall(state, node, null);
                 if (ret) {
                     replace(node, ret);
                 }
                 break;
             }
+            case "AssignmentExpression":
+                if (node.operator === "=" &&
+                    node.left.type === "Identifier" &&
+                    node.right.type === "Identifier" &&
+                    node.left.name === node.right.name) {
+                    return { type: "Literal", value: null, raw: "null" };
+                }
+                break;
             case "ExpressionStatement":
                 if (node.expression.type === "CallExpression") {
-                    const ret = optimizeCall(state, node.expression, true);
-                    if (ret) {
-                        if (ret.type === "BlockStatement") {
-                            return ret;
+                    return optimizeCall(state, node.expression, node);
+                }
+                else if (node.expression.type === "AssignmentExpression") {
+                    if (node.expression.right.type === "CallExpression") {
+                        let ok = false;
+                        if (node.expression.left.type === "Identifier") {
+                            if (hasProperty(topLocals().map, node.expression.left.type)) {
+                                ok = true;
+                            }
+                        }
+                        if (!ok && node.expression.operator == "=") {
+                            const [, result] = state.lookup(node.expression.left);
+                            ok = result != null;
+                        }
+                        if (ok) {
+                            const ret = optimizeCall(state, node.expression.right, node.expression);
+                            if (ret && ret.type === "BlockStatement") {
+                                const r2 = state.traverse(ret);
+                                return r2 === false || r2 ? r2 : ret;
+                            }
                         }
-                        node.expression = ret;
                     }
                 }
                 else {
                     const ret = unused(node.expression, true);
                     if (ret) {
-                        return ret;
+                        return ret
+                            .map((s) => {
+                            const r2 = state.traverse(s);
+                            return r2 === false || r2 ? r2 : s;
+                        })
+                            .flat(1)
+                            .filter((s) => s !== false);
                     }
                 }
                 break;
@@ -1260,6 +1403,11 @@ async function optimizeMonkeyC(fnMap) {
     Object.values(fnMap).forEach((f) => {
         collectNamespaces(f.ast, state);
     });
+    state.calledFunctions = {};
+    state.exposed = {};
+    Object.values(fnMap).forEach((f) => {
+        collectNamespaces(f.ast, state);
+    });
     delete state.pre;
     delete state.post;
     const cleanup = (node) => {
@@ -1335,9 +1483,12 @@ async function optimizeMonkeyC(fnMap) {
             return ret;
         });
     });
+    return state.diagnostics;
 }
-function optimizeCall(state, node, asStatement) {
-    const [name, callees] = state.lookup(node.callee);
+function optimizeCall(state, node, context) {
+    const [name, results] = state.lookup(node.callee);
+    const callees = results &&
+        results.filter((c) => c.type === "FunctionDeclaration");
     if (!callees || !callees.length) {
         const n = name ||
             ("name" in node.callee && node.callee.name) ||
@@ -1356,7 +1507,8 @@ function optimizeCall(state, node, asStatement) {
     }
     if (callees.length == 1 && callees[0].type === "FunctionDeclaration") {
         const callee = callees[0].node;
-        if (callee.optimizable &&
+        if (!context &&
+            callee.optimizable &&
             !callee.hasOverride &&
             node.arguments.every((n) => getNodeValue(n)[0] !== null)) {
             const ret = evaluateFunction(callee, node.arguments);
@@ -1364,19 +1516,14 @@ function optimizeCall(state, node, asStatement) {
                 return ret;
             }
         }
-        const inlineStatus = shouldInline(state, callees[0], node.arguments);
-        if (inlineStatus === InlineStatus.AsExpression ||
-            (asStatement && inlineStatus === InlineStatus.AsStatement)) {
-            const ret = inlineFunction(state, callees[0], node, inlineStatus);
+        if (shouldInline(state, callees[0], node, context)) {
+            const ret = inlineFunction(state, callees[0], node, context);
             if (ret) {
                 return ret;
             }
         }
     }
-    if (!hasProperty(state.calledFunctions, name)) {
-        state.calledFunctions[name] = [];
-    }
-    callees.forEach((c) => isStateNode(c) && state.calledFunctions[name].push(c.node));
+    callees.forEach((c) => markFunctionCalled(state, c.node));
     return null;
 }
 
@@ -1921,10 +2068,10 @@ function formatAst(node, monkeyCSource = null) {
      * The estree printer sometimes looks at the parent node without
      * checking that there *is* a parent node (eg it assumes all
      * BinaryExpressions have a parent node, and crashes if they don't).
-     * To avoid issues, wrap nodes in an ExpressionStatement, and
-     * then remove the added ";" from the end.
+     * To avoid issues, wrap nodes in an ParenthesizedExpression.
+     * The printer knows that parentless ParenthesizedExpressions
+     * should be ignored.
      */
-    let unwrap = false;
     switch (node.type) {
         case "Program":
         case "BlockStatement":
@@ -1932,11 +2079,10 @@ function formatAst(node, monkeyCSource = null) {
             break;
         default: {
             const e = {
-                type: "ExpressionStatement",
+                type: "ParenthesizedExpression",
                 expression: node,
             };
             node = e;
-            unwrap = true;
         }
     }
     // If we *do* have the original source, pass that in ahead of the
@@ -1944,15 +2090,11 @@ function formatAst(node, monkeyCSource = null) {
     // as the ast itself, and the printers will find what they're
     // looking for in the source.
     const source = (monkeyCSource || "") + "\n" + JSON.stringify(node);
-    const result = external_prettier_namespaceObject.format(source, {
+    return external_prettier_namespaceObject.format(source, {
         parser: "monkeyc-json",
         plugins: [(prettier_plugin_monkeyc_default())],
         endOfLine: "lf",
     });
-    if (unwrap) {
-        return result.replace(/;$/, "");
-    }
-    return result;
 }
 function handleException(state, node, exception) {
     try {