@qooxdoo/framework 7.0.0-beta.6 → 7.0.0-beta.7

package/source/class/qx/tool/compiler/Es6ify.js

@@ -0,0 +1,368 @@
+/* ************************************************************************
+
+   qooxdoo - the new era of web development
+
+   http://qooxdoo.org
+
+   Copyright:
+     2021 Zenesis Ltd
+
+   License:
+     MIT: https://opensource.org/licenses/MIT
+     See the LICENSE file in the project's top-level directory for details.
+
+   Authors:
+     * John Spackman (john.spackman@zenesis.com, @johnspackman)
+
+************************************************************************ */
+
+const fs = require("fs");
+const path = require("path");
+const babelCore = require("@babel/core");
+
+const types = require("@babel/types");
+const babylon = require("@babel/parser");
+const prettier = require("prettier");
+
+/**
+ * Helper method that collapses the MemberExpression into a string
+ * @param node
+ * @returns {string}
+ */
+ function collapseMemberExpression(node) {
+  var done = false;
+  function doCollapse(node) {
+    if (node.type == "ThisExpression") {
+      return "this";
+    }
+    if (node.type == "Identifier") {
+      return node.name;
+    }
+    if (node.type == "ArrayExpression") {
+      var result = [];
+      node.elements.forEach(element => result.push(doCollapse(element)));
+      return result;
+    }
+    if (node.type != "MemberExpression") {
+      return "(" + node.type + ")";
+    }
+    if (types.isIdentifier(node.object)) {
+      let str = node.object.name;
+      if (node.property.name) {
+        str += "." + node.property.name;
+      } else {
+        done = true;
+      }
+      return str;
+    }
+    var str;
+    if (node.object.type == "ArrayExpression") {
+      str = "[]";
+    } else {
+      str = doCollapse(node.object);
+    }
+    if (done) {
+      return str;
+    }
+    // `computed` is set if the expression is a subscript, eg `abc[def]`
+    if (node.computed) {
+      done = true;
+    } else if (node.property.name) {
+      str += "." + node.property.name;
+    } else {
+      done = true;
+    }
+    return str;
+  }
+
+  return doCollapse(node);
+}
+
+/**
+ * Processes a .js source file and tries to upgrade to ES6 syntax
+ * 
+ * This is a reliable but fairly unintrusive upgrade, provided that `arrowFunctions` property is
+ * `careful`.  The issue is that this code: `setTimeout(function() { something(); })` can be 
+ * changed to `setTimeout(() => something())` and that is often desirable, but it also means that
+ * the `this` will be different because an arrow function always has the `this` from where the 
+ * code is written.
+ * 
+ * However, if you use an API which changes `this` then the switch to arrow functions will break 
+ * your code.  Mostly, in Qooxdoo, changes to `this` are done via an explicit API (eg 
+ * `obj.addListener("changeXyx", function() {}, this)`) and so those known APIs can be translated,
+ * but there are places which do not work this way (eg the unit tests `qx.dev.unit.TestCase.resume()`).
+ * Third party integrations are of course completely unknown.
+ * 
+ * If `arrowFunctions` is set to aggressive, then all functions are switched to arrow functions except
+ * where there is a known API that does not support it (eg any call to `.resume` in a test class); this
+ * could break your code.
+ * 
+ * If `arrowFunctions is set to `careful` (the default), then functions are only switched to arrow 
+ * functions where the API is known  (eg `.addListener`).
+ * 
+ * The final step is that the ES6ify will use https://prettier.io/ to reformat the code, and will use
+ * the nearest `prettierrc.json` for configuration
+ */
+qx.Class.define("qx.tool.compiler.Es6ify", {
+  extend: qx.core.Object,
+
+  construct(filename) {
+    this.base(arguments);
+    this.__filename = filename;
+  },
+
+  properties: {
+    /** Whether to convert functions to arrow functions; careful means only on things like addListener callbacks */
+    arrowFunctions: {
+      init: "careful",
+      check: [ "never", "always", "careful", "aggressive" ],
+      nullable: true
+    },
+
+    /** Whether to overwrite the original file */
+    overwrite: {
+      init: false,
+      check: "Boolean"
+    }
+  },
+
+  members: {
+    __filename: null,
+
+    async transform() {
+      let src = await fs.promises.readFile(this.__filename, "utf8");
+
+      let babelConfig = {};
+      let options = qx.lang.Object.clone(babelConfig.options || {}, true);
+      options.modules = false;
+      let plugins = [
+        require("@babel/plugin-syntax-jsx"),
+        this.__pluginFunctionExpressions()
+      ];
+      if (this.getArrowFunctions() != "never") {
+        plugins.push(this.__pluginArrowFunctions());
+      }
+      plugins.push(this.__pluginRemoveUnnecessaryThis());
+      plugins.push(this.__pluginSwitchToSuper());
+      var config = {
+        ast: true,
+        babelrc: false,
+        sourceFileName: this.__filename,
+        filename: this.__filename,
+        sourceMaps: false,
+        presets: [
+          [
+            {
+              plugins: plugins
+            }
+          ]
+        ],
+        parserOpts: { 
+          allowSuperOutsideMethod: true,
+          sourceType: "script" 
+        },
+        generatorOpts: {
+          retainLines: true
+        },
+        passPerPreset: true
+      };
+      let result = babelCore.transform(src, config);
+
+      let prettierConfig = await prettier.resolveConfig(this.__filename, { editorConfig: true })||{};
+      prettierConfig.parser = "babel";
+      let prettyCode = prettier.format(result.code, prettierConfig);
+      
+      let outname = this.__filename + (this.isOverwrite() ? "" : ".es6ify");
+      await fs.promises.writeFile(outname, prettyCode, "utf8");
+    },
+
+    /**
+     * Plugin that converts object properties which are functions into object methods, eg
+     * ```
+     * {
+     *   myMethod: function() {}
+     * }
+     * ```
+     * becomes
+     * ```
+     * {
+     *   myMethod() {}
+     * }
+     * ```
+     * @returns 
+     */
+    __pluginFunctionExpressions() {
+      return {
+        visitor: {
+          ObjectExpression(path) {
+            for (let i = 0; i < path.node.properties.length; i++) {
+              let propNode = path.node.properties[i];
+              if (
+                propNode.type == "ObjectProperty" &&
+                propNode.value.type == "FunctionExpression"
+              ) {
+                let replacement = types.objectMethod(
+                  "method",
+                  propNode.key,
+                  propNode.value.params,
+                  propNode.value.body,
+                  propNode.value.computed,
+                  propNode.value.generator,
+                  propNode.value.async
+                );
+                replacement.loc = propNode.loc;
+                replacement.start = propNode.start;
+                replacement.end = propNode.end;
+                replacement.leadingComments = propNode.leadingComments;
+                path.node.properties[i] = replacement;
+              }
+            }
+          }
+        }
+      };
+    },
+
+    __toArrowExpression(argNode) {
+      let body = argNode.body;
+      if (body.body.length == 1 && body.body[0].type == "ReturnStatement") {
+        body = body.body[0].argument;
+      }
+      let replacement = types.arrowFunctionExpression(
+        argNode.params,
+        body,
+        argNode.async
+      );
+      replacement.loc = argNode.loc;
+      replacement.start = argNode.start;
+      replacement.end = argNode.end;
+      replacement.leadingComments = argNode.leadingComments;
+      return replacement;
+    },
+
+    /**
+     * Tries to convert functions into arrow functions
+     * @returns 
+     */
+    __pluginArrowFunctions() {
+      let t = this;
+      const isTest = this.__filename.indexOf("/test/") > -1;
+      let arrowFunctions = this.getArrowFunctions();
+
+      return {
+        visitor: {
+          CallExpression(path) {
+            if (path.node.callee.type == "MemberExpression") {
+              let callee = collapseMemberExpression(path.node.callee);
+              if (arrowFunctions == "careful") {
+                if (!callee.endsWith(".addListener")) {
+                  return;
+                }
+                if (
+                  path.node.arguments.length != 3 ||
+                  path.node.arguments[0].type != "StringLiteral" ||
+                  path.node.arguments[1].type != "ArrowFunctionExpression" ||
+                  path.node.arguments[2].type != "ThisExpression"
+                ) {
+                  return;
+                }
+    
+              } else if (arrowFunctions == "aggressive") {
+                if (callee == "qx.event.GlobalError.observeMethod" || 
+                    callee == "this.assertException" || 
+                    callee == "this.assertEventFired" || 
+                    callee == "qx.core.Assert.assertEventFired" ||
+                    (isTest && callee.endsWith(".resume"))) {
+                  return;
+                }
+              }
+            } else if (arrowFunctions == "careful") {
+              return;
+            }
+            for (let i = 0; i < path.node.arguments.length; i++){
+              let argNode = path.node.arguments[i];
+              if (argNode.type == "FunctionExpression") {
+                path.node.arguments[i] = t.__toArrowExpression(argNode);
+              }
+            }
+          }
+        }
+      }
+    },
+
+    /**
+     * Where a function has been translated into an arrow function, the this binding is not needed
+     * and can be removed
+     * @returns 
+     */
+    __pluginRemoveUnnecessaryThis() {
+      return {
+        visitor: {
+          CallExpression(path) {
+            if (
+              path.node.callee.type == "MemberExpression" &&
+              path.node.callee.object.type == "ThisExpression" &&
+              path.node.callee.property.type == "Identifier" &&
+              path.node.callee.property.name == "addListener" &&
+              path.node.arguments.length == 3 &&
+              path.node.arguments[0].type == "StringLiteral" &&
+              path.node.arguments[1].type == "ArrowFunctionExpression" &&
+              path.node.arguments[2].type == "ThisExpression"
+            ) {
+              qx.lang.Array.removeAt(path.node.arguments, 2);
+            }
+          }
+        }
+      }
+    },
+
+    /**
+     * Translates `this.base(arguments...)` into `super`
+     * @returns 
+     */
+    __pluginSwitchToSuper() {
+      let methodNameStack = [];
+      function peekMethodName() {
+        for (let i = methodNameStack.length - 1; i >= 0; i--) {
+          let methodName = methodNameStack[i];
+          if (methodName) {
+            return methodName;
+          }
+        }
+        return null;
+      }
+      return {
+        visitor: {
+          ObjectMethod: {
+            enter(path) {
+              methodNameStack.push(path.node.key.name||null);
+            },
+            exit(path) {
+              methodNameStack.pop();
+            }
+          },
+          CallExpression(path) {
+            if (
+              path.node.callee.type == "MemberExpression" &&
+              path.node.callee.object.type == "ThisExpression" &&
+              path.node.callee.property.type == "Identifier" &&
+              path.node.callee.property.name == "base" &&
+              path.node.arguments.length >= 1
+            ) {
+              let args = qx.lang.Array.clone(path.node.arguments);
+              args.shift();
+              let methodName = peekMethodName();
+              if (methodName == "construct") {
+                path.node.callee = types.super();
+                path.node.arguments = args;
+              } else if (methodName) {
+                let replacement = types.memberExpression(types.super(), types.identifier(methodName), false, false);
+                path.node.callee = replacement;
+                path.node.arguments = args;
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+});

package/source/class/qx/tool/compiler/targets/Target.js

@@ -81,7 +81,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
       inheritable: true,
       nullable: true
     },
-    
+
     /**
      * List of environment keys to preserve in code, ie reserve for runtime detection
      * and exclude from code elimination
@@ -98,7 +98,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
     analyser: {
       nullable: false
     },
-    
+
     /**
      * Whether to inline external scripts
      */
@@ -121,7 +121,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
       init: [ "en" ],
       transform: "_transformLocales"
     },
-    
+
     /** Whether to break locale & translation data out into separate parts */
     i18nAsParts: {
       init: false,
@@ -142,7 +142,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
       nullable: false,
       check: "Boolean"
     },
-    
+
     /** What to do with library transation strings */
     libraryPoPolicy: {
       init: "ignore",
@@ -166,6 +166,15 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
       init: true,
       nullable: false,
       check: "Boolean"
+    },
+
+    /**
+     * Whether to use relative paths in source maps
+     */
+     sourceMapRelativePaths: {
+      init: false,
+      nullable: false,
+      check: "Boolean"
     }
 
   },
@@ -178,15 +187,15 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
      *  enviroment: {Object} enviroment data
      */
     "checkEnvironment": "qx.event.type.Data",
-    
-    /** 
+
+    /**
      * Fired when an application is about to be serialized to disk; the appMeta is fully
      * populated, and this is an opportunity to amend the meta data before it is serialized
-     * into files on disk 
+     * into files on disk
      */
     "writingApplication": "qx.event.type.Event",
-    
-    /** 
+
+    /**
      * Fired when an application has been serialized to disk
      */
     "writtenApplication": "qx.event.type.Event"
@@ -196,10 +205,10 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
   members: {
     /** @type {Map} maps filenames to uris */
     __pathMappings: null,
-    
+
     /** @type {qx.tool.compiler.targets.meta.ApplicationMeta} for the current application */
     __appMeta: null,
-    
+
     /**
      * Initialises the target, creating directories etc
      */
@@ -221,7 +230,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
       }
       return value;
     },
-    
+
     /**
      * Returns the root for applications
      */
@@ -237,7 +246,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
     getProjectDir: function (application) {
       return application.getOutputPath() || application.getName();
     },
-    
+
     /**
      * Returns the URI for the root of the output, relative to the application
      */
@@ -331,22 +340,22 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
 
       let appMeta = this.__appMeta = new qx.tool.compiler.targets.meta.ApplicationMeta(this, application);
       appMeta.setAddTimestampsToUrls(this.getAddTimestampsToUrls());
-      /*      
+      /*
       if (!appMeta.getAppLibrary()) {
         qx.tool.compiler.Console.print("qx.tool.compiler.target.missingAppLibrary", application.getClassName());
         return;
       }
-*/      
+*/
       let targetUri = t._getOutputRootUri(application);
       var appRootDir = this.getApplicationRoot(application);
-      
+
       let mapTo = this.getPathMapping(path.join(appRootDir, this.getOutputDir(), "transpiled/"));
       appMeta.setSourceUri(mapTo ? mapTo : targetUri + "transpiled/");
       mapTo = this.getPathMapping(path.join(appRootDir, this.getOutputDir(), "resource"));
       appMeta.setResourceUri(mapTo ? mapTo : targetUri + "resource");
-      
+
       const requiredLibs = application.getRequiredLibraries();
-      
+
       await qx.tool.utils.Utils.makeDirs(appRootDir);
 
       appMeta.setEnvironment({
@@ -357,7 +366,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
         "qx.compiler.targetType": this.getType(),
         "qx.compiler.outputDir": this.getOutputDir()
       });
-      
+
       let externals = {};
       const addExternal = (arr, type) => {
         if (arr) {
@@ -380,7 +389,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
           });
         }
       };
-      
+
       requiredLibs.forEach(libnamespace => {
         var library = analyser.findLibrary(libnamespace);
         appMeta.addLibrary(library);
@@ -400,7 +409,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
       }
       await t.fireDataEventAsync("checkEnvironment", { application: application, environment: appMeta.getEnvironment() });
 
-      
+
       /*
        * Boot files
        */
@@ -408,7 +417,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
       let bootPackage = appMeta.createPackage();
       appMeta.setBootMetaJs(bootJs);
       bootPackage.addJavascriptMeta(new qx.tool.compiler.targets.meta.PolyfillJs(appMeta));
-      
+
 
       /*
        * Assemble the Parts
@@ -425,7 +434,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
         if (index == 0) {
           partMeta.addPackage(bootPackage);
         }
-        
+
         partData.classes.forEach(classname => {
           let classFilename = classname.replace(/\./g, "/") + ".js";
 
@@ -435,10 +444,10 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
           let library = analyser.findLibrary(dbClassInfo.libraryName);
           let sourcePath = library.getFilename(classFilename);
           let jsMeta = new qx.tool.compiler.targets.meta.Javascript(appMeta, transpiledClassFilename, sourcePath);
-          
+
           let packageName = matchBundle(classname) ? "__bundle" : partData.name;
           let pkg = packages[packageName];
-          
+
           if (!pkg || pkg !== lastPackage) {
             pkg = packages[packageName] = appMeta.createPackage();
             if (packageName == "__bundle") {
@@ -454,7 +463,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
           lastPackage = pkg;
         });
       });
-      
+
       var assetUris = application.getAssetUris(t, rm, appMeta.getEnvironment()); // Save any changes that getAssets collected
       await rm.saveDatabase();
 
@@ -462,8 +471,8 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
         analyser.getCldr("en")
           .then(cldr => bootPackage.addLocale("C", cldr)),
         t._writeTranslations()
-      ];      
-      
+      ];
+
       var fontCntr = 0;
       var assets = {};
       rm.getAssetsForPaths(assetUris).forEach(asset => {
@@ -475,7 +484,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
         // Get a list of all fonts to load; use the font name as a unique identifier, and
         //  prioritise the application's library's definitions - this allows the application
         //  the opportunity to override the font definitions.  This is important when the
-        //  library uses the open source/free versions of a font but the application 
+        //  library uses the open source/free versions of a font but the application
         //  developer has purchased the commercial/full version of the font (eg FontAwesome)
         let appLibrary = appMeta.getAppLibrary();
         let fontsToLoad = { };
@@ -507,7 +516,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
               return;
             }
             font.setResources(res);
-        
+
             await font.generateForTarget(t);
             let resources = await font.generateForApplication(t, application);
             for (var key in resources) {
@@ -617,10 +626,10 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
         return accumulateCldr(localeId);
       }
 
-      var promises = t.getLocales().map(async localeId => { 
+      var promises = t.getLocales().map(async localeId => {
         let cldr = await loadLocaleData(localeId);
         let pkg = this.isI18nAsParts() ? appMeta.getLocalePackage(localeId) : bootPackage;
-        pkg.addLocale(localeId, cldr); 
+        pkg.addLocale(localeId, cldr);
       });
 
       await qx.Promise.all(promises);
@@ -656,8 +665,8 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
           promises.push(
             addTrans(library, localeId)
           );
-        });  
-        // translation from main app should overwrite package translations 
+        });
+        // translation from main app should overwrite package translations
         promises.push(
           addTrans(appMeta.getAppLibrary(), localeId)
         );
@@ -726,24 +735,24 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
         });
       });
     },
-    
+
     /**
      * Writes the application
      */
     async _writeApplication() {
       var t = this;
-      
+
       await this.fireEventAsync("writingApplication");
-      
+
       let appMeta = this.getAppMeta();
       var application = appMeta.getApplication();
       var appRootDir = appMeta.getApplicationRoot();
-      
+
       if (!appMeta.getAppLibrary()) {
         qx.tool.compiler.Console.print("qx.tool.compiler.target.missingAppLibrary", application.getName());
         return;
       }
-        
+
       let bootMeta = appMeta.getBootMetaJs();
       for (let arr = appMeta.getPackages(), i = 0; i < arr.length; i++) {
         let pkg = arr[i];
@@ -753,16 +762,16 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
         }
         await pkg.getJavascript().unwrap().writeToDisk();
       }
-      
+
       await appMeta.getBootMetaJs().unwrap().writeToDisk();
-      
+
       await this._writeIndexHtml();
 
       if (!t.isWriteCompileInfo()) {
         await this.fireEventAsync("writtenApplication");
         return;
       }
-      
+
       let bootPackage = appMeta.getPackages()[0];
       let appSummary = {
         appClass: application.getClassName(),
@@ -783,11 +792,11 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
           name: partData.name
         });
       });
-        
+
       await fs.writeFileAsync(appRootDir + "/compile-info.json",
         JSON.stringify(appSummary, null, 2) + "\n",
         { encoding: "utf8" });
-        
+
       await this.fireEventAsync("writtenApplication");
     },
 
@@ -808,7 +817,7 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
       }
 
       var resDir = this.getApplicationRoot(application);
-      
+
       let timeStamp = (new Date()).getTime();
       let pathToTarget = path.relative(path.join(t.getOutputDir(), t.getProjectDir(application)), t.getOutputDir()) + "/";
       let indexJsTimestamp = "";
@@ -895,12 +904,12 @@ qx.Class.define("qx.tool.compiler.targets.Target", {
           "preBootJs": "",
           "appTitle": (application.getTitle()||"Qooxdoo Application"),
           "timeStamp": timeStamp,
-          "indexJsTimestamp": indexJsTimestamp        
+          "indexJsTimestamp": indexJsTimestamp
         };
         await fs.writeFileAsync(t.getOutputDir() + "index.html", replaceVars(indexHtml), { encoding: "utf8" });
       }
     },
-    
+
     getAppMeta() {
       return this.__appMeta;
     }