datagrok-tools 4.5.11 → 4.6.0

package/bin/commands/check.js

@@ -0,0 +1,367 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+var _typeof = require("@babel/runtime/helpers/typeof");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.check = check;
+
+var _fs = _interopRequireDefault(require("fs"));
+
+var _path = _interopRequireDefault(require("path"));
+
+var _ignoreWalk = _interopRequireDefault(require("ignore-walk"));
+
+var utils = _interopRequireWildcard(require("../utils/utils"));
+
+var color = _interopRequireWildcard(require("../utils/color-utils"));
+
+function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+
+function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { "default": obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj["default"] = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+
+function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it["return"] != null) it["return"](); } finally { if (didErr) throw err; } } }; }
+
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
+
+function check(args) {
+  var nOptions = Object.keys(args).length - 1;
+  if (args['_'].length !== 1 || nOptions > 0) return false;
+  var curDir = process.cwd();
+
+  if (!utils.isPackageDir(curDir)) {
+    color.error('File `package.json` not found. Run the command from the package directory');
+    return false;
+  }
+
+  var files = _ignoreWalk["default"].sync({
+    ignoreFiles: ['.npmignore', '.gitignore']
+  });
+
+  var jsTsFiles = files.filter(function (f) {
+    return f.endsWith('.js') || f.endsWith('.ts');
+  });
+  var packageFiles = ['src/package.ts', 'src/detectors.ts', 'src/package.js', 'src/detectors.js', 'src/package-test.ts', 'src/package-test.js', 'package.js', 'detectors.js'];
+  var funcFiles = jsTsFiles.filter(function (f) {
+    return packageFiles.includes(f);
+  });
+
+  var webpackConfigPath = _path["default"].join(curDir, 'webpack.config.js');
+
+  var isWebpack = _fs["default"].existsSync(webpackConfigPath);
+
+  if (isWebpack) {
+    var content = _fs["default"].readFileSync(webpackConfigPath, {
+      encoding: 'utf-8'
+    });
+
+    var externals = extractExternals(content);
+    if (externals) checkImportStatements(jsTsFiles, externals).forEach(function (warning) {
+      return color.warn(warning);
+    });
+  }
+
+  checkFuncSignatures(funcFiles).forEach(function (warning) {
+    return color.warn(warning);
+  });
+  return true;
+}
+
+function extractExternals(config) {
+  var externalsRegex = /(?<=externals\s*:\s*)(\{[\S\s]*?\})/;
+  var match = config.match(externalsRegex);
+
+  if (match) {
+    // Replace single quotes and a trailing comma to make a string JSON-like
+    var externalStr = match[1].replace(/'/g, '"').replace(/(?<=[\S\s]),(?=\s*\})/, '');
+
+    try {
+      var externals = JSON.parse(externalStr);
+      return externals;
+    } catch (e) {
+      return null;
+    }
+  }
+
+  return null;
+}
+
+function checkImportStatements(files, externals) {
+  var modules = [];
+
+  for (var key in externals) {
+    modules.push(key);
+    if (key.includes('/')) modules.push(key.split('/', 1)[0]);
+  }
+
+  var importRegex = new RegExp("import\\s+.*(".concat(modules.join('|'), ").*(?=\\s+?)"), 'g');
+  var validImportRegex = new RegExp("import\\s+.*(".concat(Object.keys(externals).join('|'), ")['\"]{1}"));
+  var warnings = [];
+
+  function validateImport(file, s) {
+    var value = validImportRegex.test(s);
+    var message = value ? '' : 'Pay attention to file ' + file + ': import statement `' + s + '` differs from the path given in the webpack config as an external module. ' + 'It can increase the bundle size.';
+    return {
+      value: value,
+      message: message
+    };
+  }
+
+  var _iterator = _createForOfIteratorHelper(files),
+      _step;
+
+  try {
+    for (_iterator.s(); !(_step = _iterator.n()).done;) {
+      var file = _step.value;
+
+      var content = _fs["default"].readFileSync(_path["default"].join(process.cwd(), file), {
+        encoding: 'utf-8'
+      });
+
+      var matchedImports = content.match(importRegex);
+
+      if (matchedImports) {
+        var _iterator2 = _createForOfIteratorHelper(matchedImports),
+            _step2;
+
+        try {
+          for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
+            var match = _step2.value;
+            var vr = validateImport(file, match);
+            if (!vr.value) warnings.push(vr.message);
+          }
+        } catch (err) {
+          _iterator2.e(err);
+        } finally {
+          _iterator2.f();
+        }
+      }
+    }
+  } catch (err) {
+    _iterator.e(err);
+  } finally {
+    _iterator.f();
+  }
+
+  return warnings;
+}
+
+function checkFuncSignatures(files) {
+  var warnings = [];
+  var checkFunctions = {
+    semTypeDetector: function semTypeDetector(_ref) {
+      var inputs = _ref.inputs,
+          outputs = _ref.outputs;
+      var value = true;
+      var message = '';
+
+      if (inputs.length !== 1 || inputs[0].type !== 'column') {
+        value = false;
+        message += 'Semantic type detectors must have one input of type "column"';
+      }
+
+      if (outputs.length !== 1 || outputs[0].type !== 'string') {
+        value = false;
+        message += 'Semantic type detectors must have one output of type "string"';
+      }
+
+      return {
+        value: value,
+        message: message
+      };
+    },
+    cellRenderer: function cellRenderer(_ref2) {
+      var inputs = _ref2.inputs,
+          outputs = _ref2.outputs;
+      var value = true;
+      var message = '';
+
+      if (inputs.length !== 0) {
+        value = false;
+        message += 'Cell renderer functions should take no arguments';
+      }
+
+      if (outputs.length !== 1 || outputs[0].type !== 'grid_cell_renderer') {
+        value = false;
+        message += 'Cell renderer functions must have one output of type "grid_cell_renderer"';
+      }
+
+      return {
+        value: value,
+        message: message
+      };
+    },
+    fileViewer: function fileViewer(_ref3) {
+      var inputs = _ref3.inputs,
+          outputs = _ref3.outputs,
+          tags = _ref3.tags;
+      var value = true;
+      var message = '';
+
+      if (tags == null || tags.filter(function (t) {
+        return t.startsWith('fileViewer');
+      }).length < 2) {
+        value = false;
+        message += 'File viewers must have at least two special tags: "fileViewer" and "fileViewer-<extension>"';
+      }
+
+      if (inputs.length !== 1 || inputs[0].type !== 'file') {
+        value = false;
+        message += 'File viewers must have one input of type "file"';
+      }
+
+      if (outputs.length !== 1 || outputs[0].type !== 'view') {
+        value = false;
+        message += 'File viewers must have one output of type "view"';
+      }
+
+      return {
+        value: value,
+        message: message
+      };
+    },
+    fileExporter: function fileExporter(_ref4) {
+      var description = _ref4.description;
+      var value = true;
+      var message = '';
+
+      if (description == null || description === '') {
+        value = false;
+        message += 'File exporters should have a description parameter';
+      }
+
+      return {
+        value: value,
+        message: message
+      };
+    },
+    packageSettingsEditor: function packageSettingsEditor(_ref5) {
+      var outputs = _ref5.outputs;
+      var value = true;
+      var message = '';
+
+      if (outputs.length === 1 && outputs[0].type === 'widget') {
+        value = false;
+        message += 'Package settings editors must have one output of type "widget"';
+      }
+
+      return {
+        value: value,
+        message: message
+      };
+    }
+  };
+  var functionRoles = Object.keys(checkFunctions);
+
+  var _iterator3 = _createForOfIteratorHelper(files),
+      _step3;
+
+  try {
+    for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
+      var file = _step3.value;
+
+      var content = _fs["default"].readFileSync(_path["default"].join(process.cwd(), file), {
+        encoding: 'utf-8'
+      });
+
+      var functions = getFuncMetadata(content);
+
+      var _iterator4 = _createForOfIteratorHelper(functions),
+          _step4;
+
+      try {
+        var _loop = function _loop() {
+          var f = _step4.value;
+          var roles = functionRoles.filter(function (role) {
+            var _f$tags;
+
+            return (_f$tags = f.tags) === null || _f$tags === void 0 ? void 0 : _f$tags.includes(role);
+          });
+
+          if (roles.length > 1) {
+            warnings.push("File ".concat(file, ", function ").concat(f.name, ": several function roles are used (").concat(roles.join(', '), ")"));
+          } else if (roles.length === 1) {
+            var vr = checkFunctions[roles[0]](f);
+            if (!vr.value) warnings.push("File ".concat(file, ", function ").concat(f.name, ":\n").concat(vr.message));
+          }
+        };
+
+        for (_iterator4.s(); !(_step4 = _iterator4.n()).done;) {
+          _loop();
+        }
+      } catch (err) {
+        _iterator4.e(err);
+      } finally {
+        _iterator4.f();
+      }
+    }
+  } catch (err) {
+    _iterator3.e(err);
+  } finally {
+    _iterator3.f();
+  }
+
+  return warnings;
+}
+
+function getFuncMetadata(script) {
+  var funcData = [];
+  var headerTags = ['name', 'description', 'help-url', 'input', 'output', 'tags', 'sample', 'language', 'returns', 'test', 'sidebar', 'condition', 'top-menu', 'environment', 'require', 'editor-for', 'schedule', 'reference', 'editor'];
+  var paramRegex = new RegExp("//\\s*(".concat(headerTags.join('|'), "|meta\\.[^:]*): *(\\S+) ?(\\S+)?"));
+  var nameRegex = /(?:|static|export\s+function|export\s+async\s+function)\s+([a-zA-Z_][a-zA-Z0-9_$]*)\s*\((.*?)\).*/;
+  var isHeader = false;
+  var data = {
+    name: '',
+    inputs: [],
+    outputs: []
+  };
+
+  var _iterator5 = _createForOfIteratorHelper(script.split('\n')),
+      _step5;
+
+  try {
+    for (_iterator5.s(); !(_step5 = _iterator5.n()).done;) {
+      var line = _step5.value;
+      if (!line) continue;
+      var match = line.match(paramRegex);
+
+      if (match) {
+        if (!isHeader) isHeader = true;
+        var param = match[1];
+        if (param === 'name') data.name = match[2];else if (param === 'description') data.description = match[2];else if (param === 'input') data.inputs.push({
+          type: match[2]
+        });else if (param === 'output') data.outputs.push({
+          type: match[2]
+        });else if (param === 'tags') data.tags = match.input && match[3] ? match.input.split(':')[1].split(',').map(function (t) {
+          return t.trim();
+        }) : [match[2]];
+      }
+
+      if (isHeader) {
+        var nm = line.match(nameRegex);
+
+        if (nm && !line.match(paramRegex)) {
+          data.name = data.name || nm[1];
+          funcData.push(data);
+          data = {
+            name: '',
+            inputs: [],
+            outputs: []
+          };
+          isHeader = false;
+        }
+      }
+    }
+  } catch (err) {
+    _iterator5.e(err);
+  } finally {
+    _iterator5.f();
+  }
+
+  return funcData;
+}

package/bin/commands/help.js

@@ -4,16 +4,18 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.help = void 0;
-var HELP = "\nUsage: grok <command>\n\nDatagrok's package management tool\n\nCommands:\n    add       Add an object template\n    api       Create wrapper functions\n    config    Create and manage config files\n    create    Create a package\n    publish   Upload a package\n\nTo get help on a particular command, use:\n    grok <command> --help\n\nRead more about the package development workflow:\nhttps://datagrok.ai/help/develop/develop\n";
+var HELP = "\nUsage: grok <command>\n\nDatagrok's package management tool\n\nCommands:\n    add       Add an object template\n    api       Create wrapper functions\n    check     Check package content (function signatures, etc.)\n    config    Create and manage config files\n    create    Create a package\n    publish   Upload a package\n\nTo get help on a particular command, use:\n    grok <command> --help\n\nRead more about the package development workflow:\nhttps://datagrok.ai/help/develop/develop\n";
 var HELP_ADD = "\nUsage: grok add <entity> <name>\n\nAdd an object template to your package:\n\ngrok add app <name>\ngrok add connection <name>\ngrok add detector <semantic-type-name>\ngrok add function [tag] <name>\ngrok add query <name>\ngrok add script [tag] <language> <name>\ngrok add view <name>\ngrok add viewer <name>\ngrok add tests\n\nPlease note that entity names may only include letters and numbers\n\nSupported languages for scripts:\njavascript, julia, node, octave, python, r\n\nAvailable tags:\npanel, init\n";
 var HELP_API = "\nUsage: grok api\n\nCreate wrapper functions for package scripts and queries\n";
 var HELP_CONFIG = "\nUsage: grok config\n\nCreate or update a configuration file\n\nOptions:\n[--reset] [--server] [--alias] [--key]\n\n--reset     Restore the default config file template\n--server    Use to add a server to the config (`grok config add --alias alias --server url --key key`)\n--alias     Use in conjunction with the `server` option to set the server name\n--key       Use in conjunction with the `server` option to set the developer key\n--default   Use in conjunction with the `server` option to set the added server as default\n";
 var HELP_CREATE = "\nUsage: grok create [name]\n\nCreate a package:\n\ngrok create         Create a package in the current working directory\ngrok create <name>  Create a package in a folder with the specified name\n\nPlease note that the package name may only include letters, numbers, underscores, or hyphens\n\nOptions:\n[--eslint] [--ide] [--js|--ts] [--jest]\n\n--eslint    Add a configuration for eslint\n--ide       Add an IDE-specific configuration for debugging (vscode)\n--js        Create a JavaScript package\n--ts        Create a TypeScript package (default)\n--jest      Add a configuration for jest\n";
 var HELP_PUBLISH = "\nUsage: grok publish [host]\n\nUpload a package\n\nOptions:\n[--build|--rebuild] [--debug|--release] [--key] [--suffix]\n\nRunning `grok publish` is the same as running `grok publish defaultHost --build --debug`\n";
+var HELP_CHECK = "\nUsage: grok check\n\nCheck package content (function signatures, import statements of external modules, etc.)\n";
 var HELP_MIGRATE = "\nUsage: grok migrate\n\nSwitch to `grok` tools by copying your keys to the config\nfile and converting your scripts in the `package.json` file\n";
 var help = {
   add: HELP_ADD,
   api: HELP_API,
+  check: HELP_CHECK,
   config: HELP_CONFIG,
   create: HELP_CREATE,
   publish: HELP_PUBLISH,

package/bin/commands/publish.js

@@ -294,7 +294,7 @@ function publish(args) {
   })) return false;
 
   if (args.build && args.rebuild) {
-    color.error('Incompatible options: --build and --rebuild');
+    color.error('Incompatible options: --build and');
     return false;
   }
 

package/bin/grok.js

@@ -5,6 +5,7 @@ const help = require('./commands/help').help;
 const commands = {
   add: require('./commands/add').add,
   api: require('./commands/api').api,
+  check: require('./commands/check').check,
   config: require('./commands/config').config,
   create: require('./commands/create').create,
   publish: require('./commands/publish').publish,

package/package-template/src/__jest__/test-node.ts

@@ -75,7 +75,7 @@ export async function getBrowserPage(puppeteer: any): Promise<{browser: any, pag
   }, token);
   await page.goto(url);
   try {
-    await page.waitForSelector('.grok-preloader', { timeout: 1800000 });
+//    await page.waitForSelector('.grok-preloader', { timeout: 1800000 });
     await page.waitForFunction(() => document.querySelector('.grok-preloader') == null, {timeout: 3600000});
   } catch (error) {
     throw error;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "4.5.11",
+  "version": "4.6.0",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {