datagrok-tools 4.1.20 → 4.2.0

package/README.md

@@ -1,84 +1,84 @@
-# Datagrok-tools
-
-Utility to upload and publish [packages](https://datagrok.ai/help/develop/develop#packages) to Datagrok.
-
-## Installation
-
-```
-npm install datagrok-tools -g
-```
-
-## Usage
-
-1. Configure your environment with the following command:  
-    ```
-    grok config
-    ```
-    Enter developer keys and set the default server. The developer key can be retrieved from your user profile (for example, see https://public.datagrok.ai/u).
-2. Create a new package by running this command:
-    ```
-    grok create <package-name>
-    ```
-    A new folder `<package-name>` will be created automatically as well as its contents.
-3. Run `npm install` in your package directory to get the required dependencies.
-4. Start working on the functionality of your package. Use `  grok add` to create function templates.
-5. Once you have completed the work on your package, upload it by running:
-    ```
-    grok publish
-    ```
-
-Run `grok` for instructions and `grok <command> --help` to get help on a particular command.
-
-Read more about package development in [Datagrok's documentation](https://datagrok.ai/help/develop/develop).
-
-## Commands
-
-- `config` creates or updates a configuration file.
-  The command shows the location of the config file with the developer keys and offers to interactively change them.
-  It is also possible to reset the current configuration.
-- `create` adds a package template to the current working directory when used without the `name` argument.
-  The directory must be empty:
-  ```
-  grok create
-  ```
-  When called with an argument, the command creates a package in a folder with the specified name:
-  ```
-  grok create <package-name>
-  ```
-  Package name may only include letters, numbers, underscores, or hyphens.
-  Read more about naming conventions [here](https://datagrok.ai/help/develop/develop#naming-conventions).
-  Options:
-    - `--eslint` adds a basic configuration for `eslint`
-    - `--ide` adds an IDE-specific configuration for debugging (vscode)
-    - `--ts` creates a TypeScript package template
-- `add` puts an object template to your package:
-  ```
-  cd <package-name>
-  grok add app <name>
-  grok add connection <name>
-  grok add detector <semantic-type-name>
-  grok add function [tag] <name>
-  grok add query <name>
-  grok add script [tag] <language> <name>
-  grok add view <name>
-  grok add viewer <name>
-  ```
-  In general, entity names follow naming rules for functions. Views and viewers should have class names,
-  we recommend that you postfix them with 'View' and 'Viewer' respectively.
-  Supported languages for scripts are `javascript`, `julia`, `node`, `octave`, `python`, `r`.
-  Available function tags: `panel`, `init`.
-- `api` creates wrapper functions for package scripts and queries. The output is stored in files `/src/scripts-api.ts` and `/src/queries-api.ts` respectively.
-- `publish` uploads a package to the specified server (pass either a URL or a server alias from the `config.yaml` file).
-  ```
-  cd <package-name>
-  grok publish [host]
-  ```
-  Options:
-    - `--build` or `--rebuild`: boolean flags that indicate whether a local webpack bundle should be used
-    or it should be generated on the server side
-    - `--debug` or `--release`: boolean flags that determine whether to publish a debug version of the package
-    visible only to the developer or a release version accessible by all eligible users and user groups
-    - `--key`: a string containing a developer key that is not listed in the config file, e.g., the key for a new server
-    - `--suffix`: a string containing package version hash
-  
-  Running `grok publish` is the same as running `grok publish defaultHost --build --debug`.
+# Datagrok-tools
+
+Utility to upload and publish [packages](https://datagrok.ai/help/develop/develop#packages) to Datagrok.
+
+## Installation
+
+```
+npm install datagrok-tools -g
+```
+
+## Usage
+
+1. Configure your environment with the following command:  
+    ```
+    grok config
+    ```
+    Enter developer keys and set the default server. The developer key can be retrieved from your user profile (for example, see https://public.datagrok.ai/u).
+2. Create a new package by running this command:
+    ```
+    grok create <package-name>
+    ```
+    A new folder `<package-name>` will be created automatically as well as its contents.
+3. Run `npm install` in your package directory to get the required dependencies.
+4. Start working on the functionality of your package. Use `  grok add` to create function templates.
+5. Once you have completed the work on your package, upload it by running:
+    ```
+    grok publish
+    ```
+
+Run `grok` for instructions and `grok <command> --help` to get help on a particular command.
+
+Read more about package development in [Datagrok's documentation](https://datagrok.ai/help/develop/develop).
+
+## Commands
+
+- `config` creates or updates a configuration file.
+  The command shows the location of the config file with the developer keys and offers to interactively change them.
+  It is also possible to reset the current configuration.
+- `create` adds a package template to the current working directory when used without the `name` argument.
+  The directory must be empty:
+  ```
+  grok create
+  ```
+  When called with an argument, the command creates a package in a folder with the specified name:
+  ```
+  grok create <package-name>
+  ```
+  Package name may only include letters, numbers, underscores, or hyphens.
+  Read more about naming conventions [here](https://datagrok.ai/help/develop/develop#naming-conventions).
+  Options:
+    - `--eslint` adds a basic configuration for `eslint`
+    - `--ide` adds an IDE-specific configuration for debugging (vscode)
+    - `--ts` creates a TypeScript package template
+- `add` puts an object template to your package:
+  ```
+  cd <package-name>
+  grok add app <name>
+  grok add connection <name>
+  grok add detector <semantic-type-name>
+  grok add function [tag] <name>
+  grok add query <name>
+  grok add script [tag] <language> <name>
+  grok add view <name>
+  grok add viewer <name>
+  ```
+  In general, entity names follow naming rules for functions. Views and viewers should have class names,
+  we recommend that you postfix them with 'View' and 'Viewer' respectively.
+  Supported languages for scripts are `javascript`, `julia`, `node`, `octave`, `python`, `r`.
+  Available function tags: `panel`, `init`.
+- `api` creates wrapper functions for package scripts and queries. The output is stored in files `/src/scripts-api.ts` and `/src/queries-api.ts` respectively.
+- `publish` uploads a package to the specified server (pass either a URL or a server alias from the `config.yaml` file).
+  ```
+  cd <package-name>
+  grok publish [host]
+  ```
+  Options:
+    - `--build` or `--rebuild`: boolean flags that indicate whether a local webpack bundle should be used
+    or it should be generated on the server side
+    - `--debug` or `--release`: boolean flags that determine whether to publish a debug version of the package
+    visible only to the developer or a release version accessible by all eligible users and user groups
+    - `--key`: a string containing a developer key that is not listed in the config file, e.g., the key for a new server
+    - `--suffix`: a string containing package version hash
+  
+  Running `grok publish` is the same as running `grok publish defaultHost --build --debug`.

package/bin/_deprecated/migrate.js

@@ -1,83 +1,83 @@
-const fs = require('fs');
-const path = require('path');
-const os = require('os');
-const yaml = require('js-yaml');
-const utils = require('../utils/utils.js');
-
-module.exports = {
-  migrate: migrate
-};
-
-const curDir = process.cwd();
-const keysDir = path.join(curDir, 'upload.keys.json');
-const packDir = path.join(curDir, 'package.json');
-const grokDir = path.join(os.homedir(), '.grok');
-const confPath = path.join(grokDir, 'config.yaml');
-const confTemplateDir = path.join(path.dirname(path.dirname(__dirname)), 'config-template.yaml');
-const confTemplate = yaml.load(fs.readFileSync(confTemplateDir));
-
-const grokMap = {
-  'datagrok-upload': 'grok publish',
-  'debug': '',
-  'deploy': '--release',
-  'build': '',
-  'rebuild': '--rebuild'
-};
-
-const replRegExp = new RegExp(Object.keys(grokMap).join("|"), "g");
-
-function migrate(args) {
-  const nOptions = Object.keys(args).length - 1;
-  const nArgs = args['_'].length;
-
-  if (nArgs > 1 || nOptions > 0) return false;
-
-  // Create `config.yaml` if it doesn't exist yet
-  if (!fs.existsSync(grokDir)) fs.mkdirSync(grokDir);
-  if (!fs.existsSync(confPath)) fs.writeFileSync(confPath, yaml.dump(confTemplate));
-
-  let config = yaml.load(fs.readFileSync(confPath));
-
-  // Copy keys to the `config.yaml` file
-  if (fs.existsSync(keysDir)) {
-    try {
-      const keys = JSON.parse(fs.readFileSync(keysDir));
-      let urls = utils.mapURL(config);
-      for (const url in keys) {
-        try {
-          let hostname = (new URL(url)).hostname;
-          if (url in urls) hostname = urls[url];
-          config['servers'][hostname] = {};
-          config['servers'][hostname]['url'] = url;
-          config['servers'][hostname]['key'] = keys[url];
-        } catch (error) {
-          console.log(`Skipping an invalid URL in \`upload.keys.json\`: ${url}`);
-        }
-      }
-      fs.writeFileSync(confPath, yaml.dump(config));
-      console.log(`Migrated data from local \`upload.keys.json\` to ${confPath}`);
-      fs.unlinkSync(keysDir);
-      console.log('Successfully deleted the file');
-    } catch (error) {
-      console.error(error);
-    }
-  } else {
-    console.log('Unable to locate `upload.keys.json`');
-  }
-
-  // Rewrite scripts in `package.json`
-  if (!fs.existsSync(packDir)) return console.log('`package.json` doesn\'t exist');
-  try {
-    let _package = JSON.parse(fs.readFileSync(packDir));
-    for (let script in _package.scripts) {
-      if (!_package['scripts'][script].includes('datagrok-upload')) continue;
-      _package['scripts'][script] = _package['scripts'][script].replace(replRegExp, (match) => grokMap[match]);
-    }
-    fs.writeFileSync(packDir, JSON.stringify(_package, null, '\t'));
-    console.log('Converting scripts in `package.json`... Done!');
-  } catch (error) {
-    console.error(error);
-  }
-
-  return true;
-}
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const yaml = require('js-yaml');
+const utils = require('../utils/utils.js');
+
+module.exports = {
+  migrate: migrate
+};
+
+const curDir = process.cwd();
+const keysDir = path.join(curDir, 'upload.keys.json');
+const packDir = path.join(curDir, 'package.json');
+const grokDir = path.join(os.homedir(), '.grok');
+const confPath = path.join(grokDir, 'config.yaml');
+const confTemplateDir = path.join(path.dirname(path.dirname(__dirname)), 'config-template.yaml');
+const confTemplate = yaml.load(fs.readFileSync(confTemplateDir));
+
+const grokMap = {
+  'datagrok-upload': 'grok publish',
+  'debug': '',
+  'deploy': '--release',
+  'build': '',
+  'rebuild': '--rebuild'
+};
+
+const replRegExp = new RegExp(Object.keys(grokMap).join("|"), "g");
+
+function migrate(args) {
+  const nOptions = Object.keys(args).length - 1;
+  const nArgs = args['_'].length;
+
+  if (nArgs > 1 || nOptions > 0) return false;
+
+  // Create `config.yaml` if it doesn't exist yet
+  if (!fs.existsSync(grokDir)) fs.mkdirSync(grokDir);
+  if (!fs.existsSync(confPath)) fs.writeFileSync(confPath, yaml.dump(confTemplate));
+
+  let config = yaml.load(fs.readFileSync(confPath));
+
+  // Copy keys to the `config.yaml` file
+  if (fs.existsSync(keysDir)) {
+    try {
+      const keys = JSON.parse(fs.readFileSync(keysDir));
+      let urls = utils.mapURL(config);
+      for (const url in keys) {
+        try {
+          let hostname = (new URL(url)).hostname;
+          if (url in urls) hostname = urls[url];
+          config['servers'][hostname] = {};
+          config['servers'][hostname]['url'] = url;
+          config['servers'][hostname]['key'] = keys[url];
+        } catch (error) {
+          console.log(`Skipping an invalid URL in \`upload.keys.json\`: ${url}`);
+        }
+      }
+      fs.writeFileSync(confPath, yaml.dump(config));
+      console.log(`Migrated data from local \`upload.keys.json\` to ${confPath}`);
+      fs.unlinkSync(keysDir);
+      console.log('Successfully deleted the file');
+    } catch (error) {
+      console.error(error);
+    }
+  } else {
+    console.log('Unable to locate `upload.keys.json`');
+  }
+
+  // Rewrite scripts in `package.json`
+  if (!fs.existsSync(packDir)) return console.log('`package.json` doesn\'t exist');
+  try {
+    let _package = JSON.parse(fs.readFileSync(packDir));
+    for (let script in _package.scripts) {
+      if (!_package['scripts'][script].includes('datagrok-upload')) continue;
+      _package['scripts'][script] = _package['scripts'][script].replace(replRegExp, (match) => grokMap[match]);
+    }
+    fs.writeFileSync(packDir, JSON.stringify(_package, null, '\t'));
+    console.log('Converting scripts in `package.json`... Done!');
+  } catch (error) {
+    console.error(error);
+  }
+
+  return true;
+}

package/bin/_deprecated/upload.js

@@ -1,162 +1,162 @@
-#!/usr/bin/env node
-const argv = require('minimist')(process.argv.slice(2));
-const getFiles = require('node-recursive-directory');
-const fs = require('fs');
-const fetch = require('node-fetch');
-const path = require('path');
-const archiver = require('archiver-promise');
-const walk = require('ignore-walk')
-
-// The script is no longer supported
-console.log(`\`datagrok-upload\` is not available, please use \`grok publish\` instead.
-Run \`grok migrate\` to convert your scripts in \`package.json\` and to copy your keys to \`config.yaml\``);
-console.log(`Exiting with code 1`);
-process.exit(1);
-
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
-
-let mode = argv['_'][1];
-let host = argv['_'][0];
-let rebuild = argv['_'].includes('rebuild');
-
-if (mode !== 'debug' && mode !== 'deploy')
-  return console.log('Mode must be either debug or deploy');
-
-let debug = mode === 'debug';
-
-let keys = {
-  [host]: ''
-};
-
-//get key from keys file
-if (fs.existsSync('upload.keys.json')) {
-  keys = JSON.parse(fs.readFileSync('upload.keys.json'));
-} else {
-  fs.writeFileSync('upload.keys.json', JSON.stringify(keys));
-}
-let devKey = keys[host];
-if (devKey === undefined) {
-  devKey = '';
-  keys[host] = devKey;
-  fs.writeFileSync('upload.keys.json', JSON.stringify(keys));
-}
-
-if (devKey === '')
-  return console.log('Empty developer key. See upload.keys.json file.');
-
-
-//check if package.json exists, get package name
-let packageName = '';
-if (!fs.existsSync('package.json'))
-  return console.log('package.js doesn\'t exists');
-else {
-  let packageJson = JSON.parse(fs.readFileSync('package.json'));
-  packageName = packageJson['name'];
-}
-
-process.on('beforeExit', async () => {
-  let code = 0;
-  try {
-    code = await processPackage()
-
-  } catch (err) {
-    console.log(err);
-    code = 1;
-  }
-  console.log(`Exiting with code ${code}`);
-  process.exit(code);
-});
-
-async function processPackage() {
-//get server timestamps
-  let timestamps = {};
-  if (debug) {
-    try {
-      timestamps = await (await fetch(`${host}/packages/dev/${devKey}/${packageName}/timestamps`)).json();
-      if (timestamps['#type'] === 'ApiError') {
-        console.log(timestamps.message);
-        return 1;
-      }
-    } catch (error) {
-      console.log(error);
-      return 1;
-    }
-  }
-
-  let zip = archiver('zip', {store: false});
-
-//gather files
-  let localTimestamps = {};
-  let files = await walk({
-    path: '.',
-    ignoreFiles: ['.npmignore', '.gitignore'],
-    includeEmpty: true,
-    follow: true
-  });
-
-  if (!rebuild) {
-    const distFiles = await walk({
-      path: './dist',
-      ignoreFiles: [],
-      includeEmpty: true,
-      follow: true
-    });
-    distFiles.forEach((df) => {
-      files.push(`dist/${df}`);
-    })
-  }
-  files.forEach((file) => {
-    let fullPath = file;
-    let relativePath = path.relative(process.cwd(), fullPath);
-    let canonicalRelativePath = relativePath.replace(/\\/g, '/');
-    if (canonicalRelativePath.includes('/.'))
-      return;
-    if (canonicalRelativePath.startsWith('.'))
-      return;
-    if (relativePath.startsWith('node_modules'))
-      return;
-    if (relativePath.startsWith('dist') && rebuild)
-      return;
-    if (relativePath.startsWith('upload.keys.json'))
-      return;
-    if (relativePath === 'zip')
-      return;
-    let t = fs.statSync(fullPath).mtime.toUTCString();
-    localTimestamps[canonicalRelativePath] = t;
-    if (debug && timestamps[canonicalRelativePath] === t) {
-      console.log(`Skipping ${canonicalRelativePath}`);
-      return;
-    }
-    zip.append(fs.createReadStream(fullPath), {name: relativePath});
-    console.log(`Adding ${relativePath}...`);
-  });
-  zip.append(JSON.stringify(localTimestamps), {name: 'timestamps.json'});
-
-//upload
-  let uploadPromise = new Promise((resolve, reject) => {
-      fetch(`${host}/packages/dev/${devKey}/${packageName}?debug=${debug.toString()}&rebuild=${rebuild.toString()}`, {
-        method: 'POST',
-        body: zip
-      }).then(body => body.json()).then(j => resolve(j)).catch(err => {
-        reject(err);
-      });
-    }
-  )
-  await zip.finalize();
-
-  try {
-    let log = await uploadPromise;
-
-    fs.unlinkSync('zip');
-    if (log['#type'] === 'ApiError') {
-      console.log(log['message']);
-      console.log(log['innerMessage']);
-      return 1;
-    } else
-      console.log(log);
-  } catch (error) {
-    console.log(error);
-    return 1;
-  }
-  return 0;
+#!/usr/bin/env node
+const argv = require('minimist')(process.argv.slice(2));
+const getFiles = require('node-recursive-directory');
+const fs = require('fs');
+const fetch = require('node-fetch');
+const path = require('path');
+const archiver = require('archiver-promise');
+const walk = require('ignore-walk')
+
+// The script is no longer supported
+console.log(`\`datagrok-upload\` is not available, please use \`grok publish\` instead.
+Run \`grok migrate\` to convert your scripts in \`package.json\` and to copy your keys to \`config.yaml\``);
+console.log(`Exiting with code 1`);
+process.exit(1);
+
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
+
+let mode = argv['_'][1];
+let host = argv['_'][0];
+let rebuild = argv['_'].includes('rebuild');
+
+if (mode !== 'debug' && mode !== 'deploy')
+  return console.log('Mode must be either debug or deploy');
+
+let debug = mode === 'debug';
+
+let keys = {
+  [host]: ''
+};
+
+//get key from keys file
+if (fs.existsSync('upload.keys.json')) {
+  keys = JSON.parse(fs.readFileSync('upload.keys.json'));
+} else {
+  fs.writeFileSync('upload.keys.json', JSON.stringify(keys));
+}
+let devKey = keys[host];
+if (devKey === undefined) {
+  devKey = '';
+  keys[host] = devKey;
+  fs.writeFileSync('upload.keys.json', JSON.stringify(keys));
+}
+
+if (devKey === '')
+  return console.log('Empty developer key. See upload.keys.json file.');
+
+
+//check if package.json exists, get package name
+let packageName = '';
+if (!fs.existsSync('package.json'))
+  return console.log('package.js doesn\'t exists');
+else {
+  let packageJson = JSON.parse(fs.readFileSync('package.json'));
+  packageName = packageJson['name'];
+}
+
+process.on('beforeExit', async () => {
+  let code = 0;
+  try {
+    code = await processPackage()
+
+  } catch (err) {
+    console.log(err);
+    code = 1;
+  }
+  console.log(`Exiting with code ${code}`);
+  process.exit(code);
+});
+
+async function processPackage() {
+//get server timestamps
+  let timestamps = {};
+  if (debug) {
+    try {
+      timestamps = await (await fetch(`${host}/packages/dev/${devKey}/${packageName}/timestamps`)).json();
+      if (timestamps['#type'] === 'ApiError') {
+        console.log(timestamps.message);
+        return 1;
+      }
+    } catch (error) {
+      console.log(error);
+      return 1;
+    }
+  }
+
+  let zip = archiver('zip', {store: false});
+
+//gather files
+  let localTimestamps = {};
+  let files = await walk({
+    path: '.',
+    ignoreFiles: ['.npmignore', '.gitignore'],
+    includeEmpty: true,
+    follow: true
+  });
+
+  if (!rebuild) {
+    const distFiles = await walk({
+      path: './dist',
+      ignoreFiles: [],
+      includeEmpty: true,
+      follow: true
+    });
+    distFiles.forEach((df) => {
+      files.push(`dist/${df}`);
+    })
+  }
+  files.forEach((file) => {
+    let fullPath = file;
+    let relativePath = path.relative(process.cwd(), fullPath);
+    let canonicalRelativePath = relativePath.replace(/\\/g, '/');
+    if (canonicalRelativePath.includes('/.'))
+      return;
+    if (canonicalRelativePath.startsWith('.'))
+      return;
+    if (relativePath.startsWith('node_modules'))
+      return;
+    if (relativePath.startsWith('dist') && rebuild)
+      return;
+    if (relativePath.startsWith('upload.keys.json'))
+      return;
+    if (relativePath === 'zip')
+      return;
+    let t = fs.statSync(fullPath).mtime.toUTCString();
+    localTimestamps[canonicalRelativePath] = t;
+    if (debug && timestamps[canonicalRelativePath] === t) {
+      console.log(`Skipping ${canonicalRelativePath}`);
+      return;
+    }
+    zip.append(fs.createReadStream(fullPath), {name: relativePath});
+    console.log(`Adding ${relativePath}...`);
+  });
+  zip.append(JSON.stringify(localTimestamps), {name: 'timestamps.json'});
+
+//upload
+  let uploadPromise = new Promise((resolve, reject) => {
+      fetch(`${host}/packages/dev/${devKey}/${packageName}?debug=${debug.toString()}&rebuild=${rebuild.toString()}`, {
+        method: 'POST',
+        body: zip
+      }).then(body => body.json()).then(j => resolve(j)).catch(err => {
+        reject(err);
+      });
+    }
+  )
+  await zip.finalize();
+
+  try {
+    let log = await uploadPromise;
+
+    fs.unlinkSync('zip');
+    if (log['#type'] === 'ApiError') {
+      console.log(log['message']);
+      console.log(log['innerMessage']);
+      return 1;
+    } else
+      console.log(log);
+  } catch (error) {
+    console.log(error);
+    return 1;
+  }
+  return 0;
 }

package/bin/commands/create.js

@@ -123,6 +123,8 @@ function createDirectoryContents(name, config, templateDir, packageDir) {
       }
 
       if (file === 'package.js' && ts) copyFilePath = _path["default"].join(packageDir, 'package.ts');
+      if (file === 'package-test.js' && ts) return false;
+      if (file === 'package-test.ts' && !ts) return false;
       if (file === 'tsconfig.json' && !ts) return false;
       if (file === 'ts.webpack.config.js') return false;