datagrok-tools 4.2.1 → 4.4.1

package/README.md

@@ -10,16 +10,17 @@ npm install datagrok-tools -g
 
 ## Usage
 
-1. Configure your environment with the following command:  
+1. Configure your environment with the following command:
     ```
     grok config
     ```
-    Enter developer keys and set the default server. The developer key can be retrieved from your user profile (for example, see https://public.datagrok.ai/u).
+   Enter developer keys and set the default server. The developer key can be retrieved from your user profile (for
+   example, see https://public.datagrok.ai/u).
 2. Create a new package by running this command:
     ```
     grok create <package-name>
     ```
-    A new folder `<package-name>` will be created automatically as well as its contents.
+   A new folder `<package-name>` will be created automatically as well as its contents.
 3. Run `npm install` in your package directory to get the required dependencies.
 4. Start working on the functionality of your package. Use `  grok add` to create function templates.
 5. Once you have completed the work on your package, upload it by running:
@@ -33,11 +34,10 @@ Read more about package development in [Datagrok's documentation](https://datagr
 
 ## Commands
 
-- `config` creates or updates a configuration file.
-  The command shows the location of the config file with the developer keys and offers to interactively change them.
-  It is also possible to reset the current configuration.
-- `create` adds a package template to the current working directory when used without the `name` argument.
-  The directory must be empty:
+- `config` creates or updates a configuration file. The command shows the location of the config file with the developer
+  keys and offers to interactively change them. It is also possible to reset the current configuration.
+- `create` adds a package template to the current working directory when used without the `name` argument. The directory
+  must be empty:
   ```
   grok create
   ```
@@ -45,12 +45,12 @@ Read more about package development in [Datagrok's documentation](https://datagr
   ```
   grok create <package-name>
   ```
-  Package name may only include letters, numbers, underscores, or hyphens.
-  Read more about naming conventions [here](https://datagrok.ai/help/develop/develop#naming-conventions).
-  Options:
+  Package name may only include letters, numbers, underscores, or hyphens. Read more about naming
+  conventions [here](https://datagrok.ai/help/develop/develop#naming-conventions). Options:
     - `--eslint` adds a basic configuration for `eslint`
     - `--ide` adds an IDE-specific configuration for debugging (vscode)
     - `--ts` creates a TypeScript package template
+    - `--jest` adds a basic configuration for `jest`
 - `add` puts an object template to your package:
   ```
   cd <package-name>
@@ -63,22 +63,22 @@ Read more about package development in [Datagrok's documentation](https://datagr
   grok add view <name>
   grok add viewer <name>
   ```
-  In general, entity names follow naming rules for functions. Views and viewers should have class names,
-  we recommend that you postfix them with 'View' and 'Viewer' respectively.
-  Supported languages for scripts are `javascript`, `julia`, `node`, `octave`, `python`, `r`.
-  Available function tags: `panel`, `init`.
-- `api` creates wrapper functions for package scripts and queries. The output is stored in files `/src/scripts-api.ts` and `/src/queries-api.ts` respectively.
+  In general, entity names follow naming rules for functions. Views and viewers should have class names, we recommend
+  that you postfix them with 'View' and 'Viewer' respectively. Supported languages for scripts are `javascript`, `julia`
+  , `node`, `octave`, `python`, `r`. Available function tags: `panel`, `init`.
+- `api` creates wrapper functions for package scripts and queries. The output is stored in files `/src/scripts-api.ts`
+  and `/src/queries-api.ts` respectively.
 - `publish` uploads a package to the specified server (pass either a URL or a server alias from the `config.yaml` file).
   ```
   cd <package-name>
   grok publish [host]
   ```
   Options:
-    - `--build` or `--rebuild`: boolean flags that indicate whether a local webpack bundle should be used
-    or it should be generated on the server side
-    - `--debug` or `--release`: boolean flags that determine whether to publish a debug version of the package
-    visible only to the developer or a release version accessible by all eligible users and user groups
+    - `--build` or `--rebuild`: boolean flags that indicate whether a local webpack bundle should be used or it should
+      be generated on the server side
+    - `--debug` or `--release`: boolean flags that determine whether to publish a debug version of the package visible
+      only to the developer or a release version accessible by all eligible users and user groups
     - `--key`: a string containing a developer key that is not listed in the config file, e.g., the key for a new server
     - `--suffix`: a string containing package version hash
-  
+
   Running `grok publish` is the same as running `grok publish defaultHost --build --debug`.

package/bin/commands/create.js

@@ -55,6 +55,7 @@ function createDirectoryContents(name, config, templateDir, packageDir) {
   var ide = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : '';
   var ts = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : false;
   var eslint = arguments.length > 6 && arguments[6] !== undefined ? arguments[6] : false;
+  var jest = arguments.length > 7 && arguments[7] !== undefined ? arguments[7] : false;
 
   var filesToCreate = _fs["default"].readdirSync(templateDir);
 
@@ -108,6 +109,20 @@ function createDirectoryContents(name, config, templateDir, packageDir) {
             'lint': "eslint src".concat(ts ? ' --ext .ts' : ''),
             'lint-fix': "eslint src".concat(ts ? ' --ext .ts' : '', " --fix")
           });
+        }
+
+        if (jest) {
+          Object.assign(_package.devDependencies, {
+            'jest': 'latest',
+            'jest-html-reporter': 'latest',
+            '@types/jest': 'latest',
+            'puppeteer': 'latest'
+          }, ts ? {
+            'ts-jest': 'latest'
+          } : {});
+          Object.assign(_package.scripts, {
+            'test': 'jest'
+          });
         } // Save module names for installation prompt
 
 
@@ -127,6 +142,9 @@ function createDirectoryContents(name, config, templateDir, packageDir) {
       if (file === 'package-test.ts' && !ts) return false;
       if (file === 'tsconfig.json' && !ts) return false;
       if (file === 'ts.webpack.config.js') return false;
+      if (file === 'remote.test.ts' && (!ts || !jest)) return false;
+      if (file === 'test-node.ts' && (!ts || !jest)) return false;
+      if (file === 'jest.config.js' && !jest) return false;
 
       if (file === '.eslintrc.json') {
         if (!eslint) return false;
@@ -151,7 +169,7 @@ function createDirectoryContents(name, config, templateDir, packageDir) {
       _fs["default"].mkdirSync(copyFilePath); // recursive call
 
 
-      createDirectoryContents(name, config, origFilePath, copyFilePath, ide, ts);
+      createDirectoryContents(name, config, origFilePath, copyFilePath, ide, ts, eslint, jest);
     }
   });
 }
@@ -161,7 +179,7 @@ function create(args) {
   var nArgs = args['_'].length;
   if (nArgs > 2 || nOptions > 3) return false;
   if (nOptions && !Object.keys(args).slice(1).every(function (op) {
-    return ['ide', 'ts', 'eslint'].includes(op);
+    return ['ide', 'ts', 'eslint', 'jest'].includes(op);
   })) return false; // Create `config.yaml` if it doesn't exist yet
 
   if (!_fs["default"].existsSync(grokDir)) _fs["default"].mkdirSync(grokDir);
@@ -198,7 +216,7 @@ function create(args) {
       return false;
     }
 
-    createDirectoryContents(name, config, templateDir, packageDir, args.ide, args.ts, args.eslint);
+    createDirectoryContents(name, config, templateDir, packageDir, args.ide, args.ts, args.eslint, args.jest);
     console.log(_entHelpers.help["package"](name, args.ts));
     console.log("\nThe package has the following dependencies:\n".concat(dependencies.join(' '), "\n"));
     console.log('Running `npm install` to get the required dependencies...\n');

package/bin/commands/help.js

@@ -8,7 +8,7 @@ var HELP = "\nUsage: grok <command>\n\nDatagrok's package management tool\n\nCom
 var HELP_ADD = "\nUsage: grok add <entity> <name>\n\nAdd an object template to your package:\n\ngrok add app <name>\ngrok add connection <name>\ngrok add detector <semantic-type-name>\ngrok add function [tag] <name>\ngrok add query <name>\ngrok add script [tag] <language> <name>\ngrok add view <name>\ngrok add viewer <name>\n\nPlease note that entity names may only include letters and numbers\n\nSupported languages for scripts:\njavascript, julia, node, octave, python, r\n\nAvailable tags:\npanel, init\n";
 var HELP_API = "\nUsage: grok api\n\nCreate wrapper functions for package scripts and queries\n";
 var HELP_CONFIG = "\nUsage: grok config\n\nCreate or update a configuration file\n\nOptions:\n[--reset] [--server] [--alias] [--key]\n\n--reset     Restore the default config file template\n--server    Use to add a server to the config (`grok config add --alias alias --server url --key key`)\n--alias     Use in conjunction with the `server` option to set the server name\n--key       Use in conjunction with the `server` option to set the developer key\n--default   Use in conjunction with the `server` option to set the added server as default\n";
-var HELP_CREATE = "\nUsage: grok create [name]\n\nCreate a package:\n\ngrok create         Create a package in the current working directory\ngrok create <name>  Create a package in a folder with the specified name\n\nPlease note that the package name may only include letters, numbers, underscores, or hyphens\n\nOptions:\n[--eslint] [--ide] [--ts]\n\n--eslint    Add a configuration for eslint\n--ide       Add an IDE-specific configuration for debugging (vscode)\n--ts        Create a TypeScript package\n";
+var HELP_CREATE = "\nUsage: grok create [name]\n\nCreate a package:\n\ngrok create         Create a package in the current working directory\ngrok create <name>  Create a package in a folder with the specified name\n\nPlease note that the package name may only include letters, numbers, underscores, or hyphens\n\nOptions:\n[--eslint] [--ide] [--ts] [--jest]\n\n--eslint    Add a configuration for eslint\n--ide       Add an IDE-specific configuration for debugging (vscode)\n--ts        Create a TypeScript package\n--jest      Add a configuration for jest\n";
 var HELP_PUBLISH = "\nUsage: grok publish [host]\n\nUpload a package\n\nOptions:\n[--build|--rebuild] [--debug|--release] [--key] [--suffix]\n\nRunning `grok publish` is the same as running `grok publish defaultHost --build --debug`\n";
 var HELP_MIGRATE = "\nUsage: grok migrate\n\nSwitch to `grok` tools by copying your keys to the config\nfile and converting your scripts in the `package.json` file\n";
 var help = {

package/bin/commands/publish.js

@@ -163,7 +163,7 @@ function _processPackage() {
               if (relativePath.startsWith('dist') && rebuild) return;
 
               if (relativePath.startsWith('src') && !rebuild && isWebpack) {
-                if (!relativePath.startsWith('src/package')) return;
+                if (!relativePath.startsWith('src/package') && !relativePath.startsWith('src\\package')) return;
               }
 
               if (relativePath.startsWith('upload.keys.json')) return;

package/package-template/jest.config.js

@@ -0,0 +1,33 @@
+module.exports = {
+  "roots": [
+    "<rootDir>/src"
+  ],
+  "testMatch": [
+    "**/__jest__/**/*.test.+(ts|tsx)"
+  ],
+  moduleFileExtensions: [
+    'ts',
+    'js',
+  ],
+  "transform": {
+    "^.+\\.(ts|tsx)$": "ts-jest"
+  },
+  transformIgnorePatterns: ['^.+\\.js$'],
+  globals: {
+    'ts-jest': {
+      'tsconfig': {
+          "target": "es6",
+          "module": "es2020",
+      }
+    }
+  },
+  reporters: [
+    "default",
+    [
+      "./node_modules/jest-html-reporter",
+      {
+        "includeConsoleLog": true
+      }
+    ]
+  ]
+}

package/package-template/npmignore

@@ -0,0 +1,26 @@
+# Developer keys
+upload.keys.json
+
+# Dependency directories
+node_modules/
+
+# IDEs
+# VS Code (https://github.com/github/gitignore/blob/master/Global/VisualStudioCode.gitignore)
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+.history/
+
+# Intellij IDEA/WebStorm (https://github.com/github/gitignore/blob/master/Global/JetBrains.gitignore)
+.idea/inspectionProfiles/
+.idea/**/compiler.xml
+.idea/**/encodings.xml
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+.idea/codeStyles/

package/package-template/package.json

@@ -7,7 +7,8 @@
   "dependencies": {
     "datagrok-api": "latest",
     "cash-dom": "latest",
-    "dayjs": "latest"
+    "dayjs": "latest",
+    "@datagrok-libraries/utils": "latest"
   },
   "devDependencies": {
     "webpack": "latest",

package/package-template/src/__jest__/remote.test.ts

@@ -0,0 +1,49 @@
+/**
+ * @jest-environment jsdom
+ */
+
+import * as utils from './test-node';
+import puppeteer from 'puppeteer';
+
+const P_START_TIMEOUT: number = 100000;
+let browser: puppeteer.Browser;
+let page: puppeteer.Page;
+
+beforeAll(async () => {
+  let out = await utils.getBrowserPage(puppeteer);
+  browser = out.browser;
+  page = out.page;
+}, P_START_TIMEOUT);
+
+afterAll(async () => {
+  await browser.close();
+});
+
+it('TEST', async () => {
+  const target_package:string = process.env.TARGET_PACKAGE;
+  console.log(`Testing ${target_package} package`);
+
+  //console.log(require('root-require')('package.json').version);
+  let r = await page.evaluate((target_package):Promise<object> => {
+    return new Promise<object>((resolve, reject) => {
+      (<any>window).grok.functions.eval(target_package + ':test()').then((df: any) => {
+        let cStatus = df.columns.byName('success');
+        let cMessage = df.columns.byName('result');
+        let cCat = df.columns.byName('category');
+        let cName = df.columns.byName('name');
+        let failed = false;
+        let report = '';
+        for (let i = 0; i < df.rowCount; i++)
+          if (!cStatus.get(i)) {
+            report += `${cCat.get(i)}.${cName.get(i)}: ${cMessage.get(i)}\n`;
+            failed = true;
+          }
+        resolve({report, failed});
+      }).catch((e: any) => reject(e));
+    });
+  }, target_package);
+  // @ts-ignore
+  console.log(r.report);
+  // @ts-ignore
+  expect(r.failed).toBe(false);
+}, 100000);

package/package-template/src/__jest__/test-node.ts

@@ -0,0 +1,96 @@
+import * as path from "path";
+import * as os from "os";
+import * as fs from "fs";
+// @ts-ignore
+import * as yaml from 'js-yaml';
+const fetch = require('node-fetch');
+
+export async function getToken(url: string, key: string) {
+  let response = await fetch(`${url}/users/login/dev/${key}`, {method: 'POST'});
+  let json = await response.json();
+  if (json.isSuccess == true)
+    return json.token;
+  else
+    throw 'Unable to login to server. Check your dev key';
+}
+
+export async function getWebUrl(url: string, token: string) {
+  let response = await fetch(`${url}/admin/plugins/admin/settings`, {headers: {Authorization: token}});
+  let json = await response.json();
+  return json.settings.webRoot;
+}
+
+const grokDir = path.join(os.homedir(), '.grok');
+const confPath = path.join(grokDir, 'config.yaml');
+
+function mapURL(conf: Config): Indexable {
+  let urls: Indexable = {};
+  for (let server in conf.servers) {
+    urls[conf['servers'][server]['url']] = conf['servers'][server];
+  }
+  return urls;
+}
+
+export function getDevKey(hostKey: string): {url: string, key: string} {
+  let config = yaml.load(fs.readFileSync(confPath, 'utf8')) as any;
+  let host = hostKey == '' ? config.default : hostKey;
+  host = host.trim();
+  let urls = mapURL(config);
+  let key = '';
+  let url = '';
+  try {
+    let url = new URL(host).href;
+    if (url.endsWith('/')) url = url.slice(0, -1);
+    if (url in urls) key = config['servers'][urls[url]]['key'];
+  } catch (error) {
+    if (config['servers'][host] == null)
+      throw `Unknown server alias. Please add it to ${confPath}`;
+    url = config['servers'][host]['url'];
+    key = config['servers'][host]['key'];
+  }
+  return {url, key};
+}
+
+export async function getBrowserPage(puppeteer: any): Promise<{browser: any, page: any}> {
+  let url:string = process.env.HOST ?? '';
+  let cfg = getDevKey(url);
+  url = cfg.url;
+
+  let key = cfg.key;
+  let token = await getToken(url, key);
+  url = await getWebUrl(url, token);
+  console.log(`Using web root: ${url}`);
+
+  let browser = await puppeteer.launch({
+    args: ['--disable-dev-shm-usage', '--disable-features=site-per-process'],
+    ignoreHTTPSErrors: true,
+  });
+
+  let page = await browser.newPage();
+  await page.goto(`${url}/oauth/`);
+  await page.setCookie({name: 'auth', value: token});
+  await page.evaluate((token: any) => {
+    window.localStorage.setItem('auth', token);
+  }, token);
+  await page.goto(url);
+  try {
+    await page.waitForSelector('.grok-preloader');
+    await page.waitForFunction(() => document.querySelector('.grok-preloader') == null, {timeout: 100000});
+  } catch (error) {
+    throw error;
+  }
+  return {browser, page};
+}
+
+
+interface Config {
+  servers: {
+    [alias: string]: {
+      url: string,
+      key: string
+    }
+  },
+  default: string,
+}
+
+interface Indexable { [key: string]: any }

package/package-template/src/package-test.js

@@ -1,7 +1,6 @@
 import * as DG from "datagrok-api/dg";
 import * as grok from "datagrok-api/grok";
 import {runTests} from "@datagrok-libraries/utils/src/test";
-import "./tests/chem-tests";
 
 export let _package = new DG.Package();
 

package/package-template/src/package-test.ts

@@ -1,7 +1,6 @@
 import * as DG from "datagrok-api/dg";
 import * as grok from "datagrok-api/grok";
 import {runTests, tests} from '@datagrok-libraries/utils/src/test';
-import "./tests/chem-tests";
 
 export let _package = new DG.Package();
 export {tests}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "4.2.1",
+  "version": "4.4.1",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {