datagrok-tools 4.1.8 → 4.1.12

package/README.md

@@ -1,4 +1,4 @@
-# datagrok-tools
+# Datagrok-tools
 
 Utility to upload and publish [packages](https://datagrok.ai/help/develop/develop#packages) to Datagrok.
 
@@ -48,6 +48,7 @@ Read more about package development in [Datagrok's documentation](https://datagr
   Package name may only include letters, numbers, underscores, or hyphens.
   Read more about naming conventions [here](https://datagrok.ai/help/develop/develop#naming-conventions).
   Options:
+    - `--eslint` adds a basic configuration for `eslint`
     - `--ide` adds an IDE-specific configuration for debugging (vscode)
     - `--ts` creates a TypeScript package template
 - `add` puts an object template to your package:
@@ -66,6 +67,7 @@ Read more about package development in [Datagrok's documentation](https://datagr
   we recommend that you postfix them with 'View' and 'Viewer' respectively.
   Supported languages for scripts are `javascript`, `julia`, `node`, `octave`, `python`, `r`.
   Available function tags: `panel`, `init`.
+- `api` creates wrapper functions for package scripts. The output is stored in `/src/scripts-api.ts`.
 - `publish` uploads a package to the specified server (pass either a URL or a server alias from the `config.yaml` file).
   ```
   cd <package-name>

package/bin/commands/add.js

@@ -29,9 +29,6 @@ function add(args) {
   if (!fs.existsSync(packagePath)) return console.log('`package.json` not found');
   try {
     const package = JSON.parse(fs.readFileSync(packagePath));
-    if (package.friendlyName !== curFolder && package.fullName !== curFolder) {
-      return console.log('The package name differs from the one in `package.json`');
-    }
   } catch (error) {
     console.error(`Error while reading ${packagePath}:`)
     console.error(error);

package/bin/commands/api.js

@@ -0,0 +1,69 @@
+const fs = require('fs');
+const path = require('path');
+const utils = require('../utils.js');
+const walk = require('ignore-walk');
+
+
+function generateWrappers() {
+  const curDir = process.cwd();
+  const scriptsDir = path.join(curDir, 'scripts');
+  if (!fs.existsSync(scriptsDir)) {
+    console.log(`Directory ${scriptsDir} not found\n` +
+      'Place your scripts there before running this command');
+    return;
+  }
+
+  const files = walk.sync({
+    path: './scripts',
+    ignoreFiles: ['.npmignore', '.gitignore'],
+  });
+
+  const wrappers = [];
+  for (const file of files) {
+    let extension = null;
+    if (!utils.scriptExtensions.some((ext) => (extension = ext, file.endsWith(ext)))) continue;
+
+    const filepath = path.join(scriptsDir, file);
+    const script = fs.readFileSync(filepath, 'utf8');
+    if (!script) continue;
+
+    const name = utils.getScriptName(script);
+    if (!name) continue;
+
+    let tb = new utils.TemplateBuilder(utils.scriptWrapperTemplate)
+      .replace('FUNC_NAME', name)
+      .replace('FUNC_NAME_LOWERCASE', name);
+
+    const packagePath = path.join(curDir, 'package.json');
+    const package = JSON.parse(fs.readFileSync(packagePath));
+    tb.replace('PACKAGE_NAMESPACE', package.name);
+
+    const inputs = utils.getScriptInputs(script);
+    const outputType = utils.getScriptOutputType(script);
+    tb.replace('PARAMS_OBJECT', inputs)
+      .replace('TYPED_PARAMS', inputs)
+      .replace('OUTPUT_TYPE', outputType);
+
+    wrappers.push(tb.build());
+  }
+
+  const srcDir = path.join(curDir, 'src');
+  let funcFilePath = path.join(fs.existsSync(srcDir) ? srcDir : curDir, 'scripts-api.ts');
+  if (fs.existsSync(funcFilePath)) {
+    console.log(`The file ${funcFilePath} already exists\nRewriting its contents...`);
+  }
+
+  const sep = '\n';
+  fs.writeFileSync(funcFilePath, utils.dgImports + sep + wrappers.join(sep.repeat(2)) + sep, 'utf8');
+}
+
+function api(args) {
+  const nOptions = Object.keys(args).length - 1;
+  if (args['_'].length !== 1 || nOptions > 0) return false;
+  generateWrappers();
+  return true;
+}
+
+module.exports = {
+  api: api,
+};

package/bin/commands/create.js

@@ -25,7 +25,7 @@ const confTemplate = yaml.safeLoad(fs.readFileSync(confTemplateDir));
 
 let dependencies = [];
 
-function createDirectoryContents(name, config, templateDir, packageDir, ide = '', ts = false) {
+function createDirectoryContents(name, config, templateDir, packageDir, ide = '', ts = false, eslint = false) {
   const filesToCreate = fs.readdirSync(templateDir);
 
   filesToCreate.forEach(file => {
@@ -55,6 +55,19 @@ function createDirectoryContents(name, config, templateDir, packageDir, ide = ''
           package['scripts'][`release-${name.toLowerCase()}-${server}`] = `grok publish ${server} --rebuild --release`;
         }
         if (ts) Object.assign(package.dependencies, { 'ts-loader': 'latest', 'typescript': 'latest' });
+        if (eslint) {
+          Object.assign(package.devDependencies, {
+            'eslint': 'latest',
+            'eslint-config-google': 'latest',
+          }, ts ? {
+            '@typescript-eslint/eslint-plugin': 'latest',
+            '@typescript-eslint/parser': 'latest',
+          } : {});
+          Object.assign(package.scripts, {
+            'lint': `eslint src${ts ? ' --ext .ts' : ''}`,
+            'lint-fix': `eslint src${ts ? ' --ext .ts' : ''} --fix`,
+          });
+        }
         // Save module names for installation prompt
         for (let [module, tag] of Object.entries(Object.assign({}, package.dependencies, package.devDependencies)))
           dependencies.push(`${module}@${tag}`);
@@ -63,6 +76,15 @@ function createDirectoryContents(name, config, templateDir, packageDir, ide = ''
       if (file === 'package.js' && ts) copyFilePath = path.join(packageDir, 'package.ts');
       if (file === 'tsconfig.json' && !ts) return false;
       if (file === 'ts.webpack.config.js') return false;
+      if (file === '.eslintrc.json') {
+        if (!eslint) return false;
+        if (ts) {
+          let eslintConf = JSON.parse(contents);
+          eslintConf.parser = '@typescript-eslint/parser';
+          eslintConf.plugins = ['@typescript-eslint'];
+          contents = JSON.stringify(eslintConf, null, '\t');
+        }
+      }
       if (file === 'gitignore') {
         copyFilePath = path.join(packageDir, '.gitignore');
         if (ts) contents += '\n# Emitted *.js files\nsrc/**/*.js\n';
@@ -80,8 +102,8 @@ function createDirectoryContents(name, config, templateDir, packageDir, ide = ''
 function create(args) {
   const nOptions = Object.keys(args).length - 1;
   const nArgs = args['_'].length;
-  if (nArgs > 2 || nOptions > 2) return false;
-  if (nOptions && !Object.keys(args).slice(1).every(op => op == 'ide' || op == 'ts')) return false;
+  if (nArgs > 2 || nOptions > 3) return false;
+  if (nOptions && !Object.keys(args).slice(1).every(op => ['ide', 'ts', 'eslint'].includes(op))) return false;
 
   // Create `config.yaml` if it doesn't exist yet
   if (!fs.existsSync(grokDir)) fs.mkdirSync(grokDir);
@@ -105,7 +127,7 @@ function create(args) {
       console.log('The package directory should be empty');
       return false;
     }
-    createDirectoryContents(name, config, templateDir, packageDir, args.ide, args.ts);
+    createDirectoryContents(name, config, templateDir, packageDir, args.ide, args.ts, args.eslint);
     console.log(help.package(name, args.ts));
     console.log(`\nThe package has the following dependencies:\n${dependencies.join(' ')}\n`);
 

package/bin/commands/help.js

@@ -40,6 +40,12 @@ Available tags:
 panel, init
 `;
 
+const HELP_API = `
+Usage: grok api
+
+Create wrapper functions for package scripts
+`;
+
 const HELP_CONFIG = `
 Usage: grok config
 
@@ -62,8 +68,9 @@ grok create <name>  Create a package in a folder with the specified name
 Please note that the package name may only include letters, numbers, underscores, or hyphens
 
 Options:
-[--ide] [--ts]
+[--eslint] [--ide] [--ts]
 
+--eslint    Add a configuration for eslint
 --ide       Add an IDE-specific configuration for debugging (vscode)
 --ts        Create a TypeScript package
 `;
@@ -88,6 +95,7 @@ file and converting your scripts in the \`package.json\` file
 
 module.exports = {
   add: HELP_ADD,
+  api: HELP_API,
   config: HELP_CONFIG,
   create: HELP_CREATE,
   publish: HELP_PUBLISH,

package/bin/commands/publish.js

@@ -9,7 +9,8 @@ const utils = require('../utils.js');
 
 
 module.exports = {
-  publish: publish
+  publish: publish,
+  processPackage: processPackage
 };
 
 const grokDir = path.join(os.homedir(), '.grok');

package/bin/grok.js

@@ -4,6 +4,7 @@ const help = require('./commands/help');
 
 const commands = {
   add: require('./commands/add').add,
+  api: require('./commands/api').api,
   config: require('./commands/config').config,
   create: require('./commands/create').create,
   publish: require('./commands/publish').publish,

package/bin/utils.js

@@ -2,9 +2,20 @@ const fs = require('fs');
 
 exports.isEmpty = (dir) => fs.readdirSync(dir).length === 0;
 
-exports.kebabToCamelCase = (s) => {
+exports.kebabToCamelCase = (s, firstUpper = true) => {
   s = s.replace(/-./g, x => x.toUpperCase()[1]);
-  return s[0].toUpperCase() + s.slice(1);
+  return (firstUpper ? s[0].toUpperCase() : s[0].toLowerCase()) + s.slice(1);
+}
+
+exports.spaceToCamelCase = (s, firstUpper = true) => {
+  s = s.replace(/\s+./g, x => x[x.length - 1].toUpperCase());
+  return (firstUpper ? s[0].toUpperCase() : s[0].toLowerCase()) + s.slice(1);
+}
+
+exports.wordsToCamelCase = (s, firstUpper = true) => {
+  const m = s.match(/^[A-Z]+/); // only abbreviation handling
+  const lastIndex = m ? m[0].length : 1;
+  return s.slice(0, lastIndex).toLowerCase() + s.slice(lastIndex);
 }
 
 exports.camelCaseToKebab = (s) => {
@@ -25,8 +36,32 @@ exports.replacers = {
   NAME_LOWERCASE: (s, name) => s.replace(/#{NAME_LOWERCASE}/g, name.toLowerCase()),
   NAME_PREFIX: (s, name) => s.replace(/#{NAME_PREFIX}/g, name.slice(0, 3)),
   PACKAGE_DETECTORS_NAME: (s, name) => s.replace(/#{PACKAGE_DETECTORS_NAME}/g, this.kebabToCamelCase(name)),
+  PACKAGE_NAMESPACE: (s, name) => s.replace(/#{PACKAGE_NAMESPACE}/g, this.kebabToCamelCase(name)),
+  FUNC_NAME: (s, name) => s.replace(/#{FUNC_NAME}/g, name.includes('-') ? this.kebabToCamelCase(name)
+    : name.includes(' ') ? this.spaceToCamelCase(name) : name[0].toUpperCase() + name.slice(1)),
+  FUNC_NAME_LOWERCASE: (s, name) => s.replace(/#{FUNC_NAME_LOWERCASE}/g, name.includes('-') ?
+    this.kebabToCamelCase(name, false) : name.includes(' ') ?
+    this.spaceToCamelCase(name, false) : this.wordsToCamelCase(name, false)),
+  PARAMS_OBJECT: (s, params) => s.replace(/#{PARAMS_OBJECT}/g, params.length ? `{ ${params.map((p) => p.name).join(', ')} }` : `{}`),
+  OUTPUT_TYPE: (s, type) => s.replace(/#{OUTPUT_TYPE}/g, type),
+  TYPED_PARAMS: (s, params) => s.replace(/#{TYPED_PARAMS}/g, params.map((p) => `${p.name}: ${p.type}`).join(', '))
 };
 
+exports.TemplateBuilder = class TemplateBuilder {
+  constructor(template) {
+    this.template = template;
+  }
+
+  replace(pattern, value) {
+    this.template = exports.replacers[pattern](this.template, value);
+    return this;
+  }
+
+  build() {
+    return this.template;
+  }
+}
+
 exports.scriptLangExtMap = {
   javascript: 'js',
   julia: 'jl',
@@ -44,3 +79,51 @@ exports.checkScriptLocation = (filepath) => {
   }
   return true;
 };
+
+exports.getScriptName = (script, comment = '#') => {
+  const regex = new RegExp(`${comment}\\s*name:\\s*(.*)`);
+  const match = script.match(regex);
+  return match ? match[1]?.trim() : null;
+};
+
+exports.dgToTsTypeMap = {
+  int: 'number',
+  double: 'number',
+  bigint: 'bigint',
+  bool: 'boolean',
+  string: 'string',
+  dataframe: 'DG.DataFrame',
+  column: 'DG.Column',
+  column_list: 'string[]',
+  file: 'DG.FileInfo',
+  // graphics: '',
+  // datetime: '',
+  // blob: '',
+  // map: '',
+  // list: '',
+  //'list<T>'
+};
+
+exports.getScriptOutputType = (script, comment = '#') => {
+  const regex = new RegExp(`${comment}\\s*output:\\s?([a-z_]+)\\s*`);
+  const match = script.match(regex);
+  if (!match) return 'void';
+  return this.dgToTsTypeMap[match[1]] || 'any';
+};
+
+exports.getScriptInputs = (script, comment = '#') => {
+  const regex = new RegExp(`${comment}\\s*input:\\s?([a-z_]+)\\s+(\\w+)`, 'g');
+  const inputs = [];
+  for (let match of script.matchAll(regex)) {
+    const type = this.dgToTsTypeMap[match[1]] || 'any';
+    const name = match[2];
+    inputs.push({ type, name });
+  }
+  return inputs;
+};
+
+exports.dgImports = `import * as grok from 'datagrok-api/grok';\nimport * as DG from 'datagrok-api/dg';\n\n`;
+
+exports.scriptWrapperTemplate = `export async function #{FUNC_NAME_LOWERCASE}(#{TYPED_PARAMS}): Promise<#{OUTPUT_TYPE}> {
+  return await grok.functions.call('#{PACKAGE_NAMESPACE}:#{FUNC_NAME}', #{PARAMS_OBJECT});
+}`;

package/package-template/.eslintrc.json

@@ -0,0 +1,25 @@
+{
+  "env": {
+    "browser": true,
+    "es2021": true
+  },
+  "extends": [
+    "google"
+  ],
+  "parserOptions": {
+    "ecmaVersion": 12,
+    "sourceType": "module"
+  },
+  "rules": {
+    "indent": [
+      "error",
+      2
+    ],
+    "max-len": [
+      "error",
+      120
+    ],
+    "require-jsdoc": "off",
+    "spaced-comment": "off"
+  }
+}

package/package-template/src/package.js

@@ -1,11 +1,11 @@
-/* Do not change these import lines to match external modules in webpack configuration */
-import * as grok from 'datagrok-api/grok';
-import * as ui from 'datagrok-api/ui';
-import * as DG from 'datagrok-api/dg';
-
-export let _package = new DG.Package();
-
-//name: test
-export function test() {
-  grok.shell.info(_package.webRoot);
-}
+/* Do not change these import lines to match external modules in webpack configuration */
+import * as grok from 'datagrok-api/grok';
+import * as ui from 'datagrok-api/ui';
+import * as DG from 'datagrok-api/dg';
+
+export const _package = new DG.Package();
+
+//name: test
+export function test() {
+  grok.shell.info(_package.webRoot);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "4.1.8",
+  "version": "4.1.12",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {

package/src/test-utils.ts

@@ -0,0 +1,74 @@
+import path from "path";
+import os from "os";
+import fs from "fs";
+// @ts-ignore
+import * as yaml from 'js-yaml';
+// @ts-ignore
+import * as utils from '../bin/utils'
+
+export async function getToken(devKey: string) {
+  return 'GGuDtdZ1Qu3sHOwscchQdl2ozmzXwzcjJ52VByCyYqOQS6v5XzMHcLtOs8ehgqJeWlMkU48AXWkLwEkx2L516gRoRzXBQd5oT3784YyeysOP1za9BebKpJ3ZxsQAb1SJ';
+}
+
+const grokDir = path.join(os.homedir(), '.grok');
+const confPath = path.join(grokDir, 'config.yaml');
+
+function mapURL(conf: object):object {
+  let urls = {};
+  // @ts-ignore
+  for (let server in conf.servers) {
+    // @ts-ignore
+    urls[conf['servers'][server]['url']] = conf['servers'][server];
+  }
+  return urls;
+}
+
+export function getDevKey(hostKey: string): {url: string, key: string} {
+  let config = yaml.safeLoad(fs.readFileSync(confPath));
+  let host = hostKey == '' ? config.default : hostKey;
+  host = host.trim();
+  let urls = mapURL(config);
+  let key = '';
+  let url = '';
+  try {
+    let url = new URL(host).href;
+    if (url.endsWith('/')) url = url.slice(0, -1);
+    if (url in urls)
+      { // @ts-ignore
+        key = config['servers'][urls[url]]['key'];
+      }
+  } catch (error) {
+    if (config['servers'][host] == null)
+      throw `Unknown server alias. Please add it to ${confPath}`;
+    url = config['servers'][host]['url'];
+    key = config['servers'][host]['key'];
+  }
+  return {url, key};
+}
+
+export async function getBrowserPage(puppeteer: any): Promise<{browser: any, page: any}> {
+  let url:string = process.env.HOST ?? '';
+  let cfg = getDevKey(url);
+  url = cfg.url;
+  if (url.endsWith('/api'))
+    url = url.slice(0, -4);
+  let key = cfg.key;
+  let browser = await puppeteer.launch({
+    args: ['--disable-dev-shm-usage', '--disable-features=site-per-process'],
+    ignoreHTTPSErrors: true,
+  });
+  let page = await browser.newPage();
+  let token = await getToken(key);
+  await page.goto(`${url}/api/info/server`);
+  await page.setCookie({name: 'auth', value: token});
+  await page.evaluate((token: any) => {
+    window.localStorage.setItem('auth', token);
+  }, token);
+  await page.goto(url);
+  try {
+    await page.waitForSelector('.grok-view');
+  } catch (error) {
+    throw 'Can`t load the page';
+  }
+  return {browser, page};
+}