datagrok-tools 4.14.69 → 4.14.71

package/bin/commands/help.js

@@ -20,6 +20,7 @@ Commands:
     publish   Upload a package
     test      Run package tests
     testall      Run packages tests
+    migrate   Migrate legacy tags to meta.role
 
 To get help on a particular command, use:
     grok <command> --help
@@ -209,6 +210,16 @@ Options:
 --verbose           Prints detailed information about linked packages  
 --all               Links all available packages(run in packages directory)
 `;
+const HELP_MIGRATE = `
+Usage: grok migrate
+
+Migrates legacy function tags into the meta.role field.
+
+Example:
+  tags: ['viewer', 'ml']
+  ⟶
+  meta: { role: 'viewer,ml' }
+`;
 
 // const HELP_MIGRATE = `
 // Usage: grok migrate
@@ -228,5 +239,6 @@ const help = exports.help = {
   publish: HELP_PUBLISH,
   test: HELP_TEST,
   testall: HELP_TESTALL,
+  migrate: HELP_MIGRATE,
   help: HELP
 };

package/bin/commands/migrate.js

@@ -0,0 +1,74 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.migrate = migrate;
+exports.toCamelCase = void 0;
+var _tsMorph = require("ts-morph");
+var path = _interopRequireWildcard(require("path"));
+var _const = require("datagrok-api/src/const");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+const FUNC_TYPE_VALUES = new Set(Object.values(_const.FUNC_TYPES));
+const toCamelCase = str => str.replace(/[-_ ]+(\w)/g, (_, c) => c.toUpperCase()).replace(/^[A-Z]/, c => c.toLowerCase());
+exports.toCamelCase = toCamelCase;
+function getProp(obj, name) {
+  return obj.getProperties().find(p => {
+    if (![_tsMorph.SyntaxKind.PropertyAssignment, _tsMorph.SyntaxKind.ShorthandPropertyAssignment].includes(p.getKind())) return false;
+    const n = p.getName?.();
+    return !!n && n.replace(/['"`]/g, '') === name;
+  });
+}
+const parseRoleString = value => value.split(',').map(v => v.trim()).filter(Boolean);
+function migrate(argv) {
+  const FILE_PATH = path.resolve(process.cwd(), 'src/package.ts');
+  const project = new _tsMorph.Project({
+    manipulationSettings: {
+      quoteKind: _tsMorph.QuoteKind.Single,
+      indentationText: _tsMorph.IndentationText.TwoSpaces,
+      useTrailingCommas: true
+    }
+  });
+  const source = project.addSourceFileAtPath(FILE_PATH);
+  source.getDescendantsOfKind(_tsMorph.SyntaxKind.Decorator).forEach(decorator => {
+    const call = decorator.getCallExpression();
+    if (!call || !call.getExpression().getText().includes('grok.decorators.')) return;
+    const arg = call.getArguments()[0]?.asKind(_tsMorph.SyntaxKind.ObjectLiteralExpression);
+    if (!arg) return;
+    const tagsProp = getProp(arg, 'tags');
+    if (!tagsProp) return;
+    const tagsArray = tagsProp.getFirstDescendantByKind(_tsMorph.SyntaxKind.ArrayLiteralExpression);
+    if (!tagsArray) return;
+    const validTags = [];
+    const remainingTags = [];
+    tagsArray.getElements().forEach(el => {
+      const tag = el.getText().replace(/['"`]/g, '');
+      FUNC_TYPE_VALUES.has(tag) ? validTags.push(toCamelCase(tag)) : remainingTags.push(tag);
+    });
+    if (remainingTags.length === 0) tagsProp.remove();else {
+      tagsProp.asKindOrThrow(_tsMorph.SyntaxKind.PropertyAssignment).setInitializer(`[${remainingTags.map(t => `'${t}'`).join(', ')}]`);
+    }
+    if (validTags.length === 0) return;
+    const metaProp = getProp(arg, 'meta');
+    if (!metaProp) {
+      arg.addPropertyAssignment({
+        name: 'meta',
+        initializer: `{role: '${validTags.join(',')}'}`
+      });
+      return;
+    }
+    const metaObj = metaProp.getFirstChildByKind(_tsMorph.SyntaxKind.ObjectLiteralExpression);
+    if (!metaObj) return;
+    const roleProp = getProp(metaObj, 'role');
+    const existingRoles = roleProp ? parseRoleString(roleProp.asKindOrThrow(_tsMorph.SyntaxKind.PropertyAssignment).getInitializer()?.getText().replace(/['"`]/g, '') ?? '') : [];
+    const mergedRoles = Array.from(new Set([...existingRoles, ...validTags]));
+    const otherProps = metaObj.getProperties().filter(p => p.asKind(_tsMorph.SyntaxKind.PropertyAssignment)?.getName() !== 'role').map(p => p.getText()).join(', ');
+    metaProp.asKindOrThrow(_tsMorph.SyntaxKind.PropertyAssignment).setInitializer(writer => {
+      writer.write('{');
+      if (otherProps) writer.write(otherProps + ', ');
+      writer.write(`role: '${mergedRoles.join(',')}'`);
+      writer.write('}');
+    });
+  });
+  source.saveSync();
+}

package/bin/grok.js

@@ -16,7 +16,8 @@ const commands = {
   publish: require('./commands/publish').publish,
   test: require('./commands/test').test,
   testall: require('./commands/test-all').testAll,
-  stresstest: require('./commands/stress-tests').stressTests
+  stresstest: require('./commands/stress-tests').stressTests,
+  migrate: require('./commands/migrate').migrate,
 };
 
 const onPackageCommandNames = ['api', 'check', 'link', 'publish', 'test'];

package/bin/utils/func-generation.js

@@ -136,7 +136,7 @@ function getFuncAnnotation(data, comment = '//', sep = '\n') {
   }
   for (const input of data.inputs ?? []) {
     if (!input) continue;
-    let type = input?.type?.replaceAll(" ", "");
+    let type = input?.type?.replaceAll(' ', '');
     if (type?.includes(`|undefined`)) {
       type = type.replaceAll(`|undefined`, '');
       // modify original payload
@@ -216,7 +216,7 @@ function buildStringOfOptions(input) {
 const reservedDecorators = exports.reservedDecorators = {
   viewer: {
     metadata: {
-      tags: [FUNC_TYPES.VIEWER],
+      role: FUNC_TYPES.VIEWER,
       inputs: [],
       outputs: [{
         name: 'result',
@@ -227,7 +227,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   filter: {
     metadata: {
-      tags: [FUNC_TYPES.FILTER],
+      role: FUNC_TYPES.FILTER,
       inputs: [],
       outputs: [{
         name: 'result',
@@ -238,7 +238,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   cellRenderer: {
     metadata: {
-      tags: [FUNC_TYPES.CELL_RENDERER],
+      role: FUNC_TYPES.CELL_RENDERER,
       inputs: [],
       outputs: [{
         name: 'renderer',
@@ -264,7 +264,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   fileExporter: {
     metadata: {
-      tags: [FUNC_TYPES.FILE_EXPORTER],
+      role: FUNC_TYPES.FILE_EXPORTER,
       inputs: [],
       outputs: []
     },
@@ -272,7 +272,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   fileHandler: {
     metadata: {
-      tags: [FUNC_TYPES.FILE_HANDLER],
+      role: FUNC_TYPES.FILE_HANDLER,
       inputs: [{
         name: 'content',
         type: 'string'
@@ -286,7 +286,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   fileViewer: {
     metadata: {
-      tags: [FUNC_TYPES.FILE_VIEWER],
+      role: FUNC_TYPES.FILE_VIEWER,
       inputs: [{
         name: 'f',
         type: 'file'
@@ -300,7 +300,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   settingsEditor: {
     metadata: {
-      tags: [FUNC_TYPES.SETTINGS_EDITOR],
+      role: FUNC_TYPES.SETTINGS_EDITOR,
       inputs: [],
       outputs: [{
         name: 'result',
@@ -322,7 +322,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   app: {
     metadata: {
-      tags: [FUNC_TYPES.APP],
+      role: FUNC_TYPES.APP,
       inputs: [],
       outputs: [{
         name: 'result',
@@ -333,7 +333,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   autostart: {
     metadata: {
-      tags: [FUNC_TYPES.AUTOSTART],
+      role: FUNC_TYPES.AUTOSTART,
       inputs: [],
       outputs: []
     },
@@ -341,7 +341,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   init: {
     metadata: {
-      tags: [FUNC_TYPES.INIT],
+      role: FUNC_TYPES.INIT,
       inputs: [],
       outputs: []
     },
@@ -349,7 +349,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   editor: {
     metadata: {
-      tags: [FUNC_TYPES.EDITOR],
+      role: FUNC_TYPES.EDITOR,
       inputs: [{
         name: 'call',
         type: 'funccall'
@@ -360,7 +360,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   panel: {
     metadata: {
-      tags: [FUNC_TYPES.PANEL],
+      role: FUNC_TYPES.PANEL,
       inputs: [],
       outputs: [{
         name: 'result',
@@ -371,7 +371,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   folderViewer: {
     metadata: {
-      tags: [FUNC_TYPES.FOLDER_VIEWER],
+      role: FUNC_TYPES.FOLDER_VIEWER,
       inputs: [{
         name: 'folder',
         type: 'file'
@@ -388,7 +388,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   semTypeDetector: {
     metadata: {
-      tags: [FUNC_TYPES.SEM_TYPE_DETECTOR],
+      role: FUNC_TYPES.SEM_TYPE_DETECTOR,
       inputs: [{
         name: 'col',
         type: 'column'
@@ -402,7 +402,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   dashboard: {
     metadata: {
-      tags: [FUNC_TYPES.DASHBOARD],
+      role: FUNC_TYPES.DASHBOARD,
       inputs: [],
       outputs: [{
         name: 'result',
@@ -413,7 +413,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   functionAnalysis: {
     metadata: {
-      tags: [FUNC_TYPES.FUNCTION_ANALYSIS],
+      role: FUNC_TYPES.FUNCTION_ANALYSIS,
       inputs: [],
       outputs: [{
         name: 'result',
@@ -424,7 +424,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   converter: {
     metadata: {
-      tags: [FUNC_TYPES.CONVERTER],
+      role: FUNC_TYPES.CONVERTER,
       inputs: [{
         name: 'value',
         type: 'dynamic'
@@ -460,7 +460,7 @@ const reservedDecorators = exports.reservedDecorators = {
   },
   model: {
     metadata: {
-      tags: [FUNC_TYPES.MODEL],
+      role: FUNC_TYPES.MODEL,
       inputs: [],
       outputs: []
     },

package/bin/utils/utils.js

@@ -187,10 +187,10 @@ const dgToTsTypeMap = exports.dgToTsTypeMap = {
 const propertyTypes = exports.propertyTypes = ['bool', 'int', 'double', 'string', 'datetime', 'object', 'column', 'dataframe', 'bitset', 'cell', 'string_list', 'map'];
 const headerTags = exports.headerTags = ['name', 'description', 'help-url', 'input', 'output', 'tags', 'sample', 'language', 'returns', 'test', 'sidebar', 'condition', 'top-menu', 'environment', 'require', 'editor-for', 'schedule', 'reference', 'editor', 'meta', 'connection', 'friendlyName'];
 const fileParamRegex = exports.fileParamRegex = {
-  py: new RegExp(`^\#\\s*([^:]*): *([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?[^\\n]*$`),
-  ts: new RegExp(`^\/\/\\s*([^:]*): *([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?[^\\n]*$`),
-  js: new RegExp(`^\/\/\\s*([^:]*): *([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?[^\\n]*$`),
-  sql: new RegExp(`^--\\s*([^:]*): *([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?[^\\n]*$`)
+  py: new RegExp(`^#\\s*(?!\\s*#)([^:]+):\\s+([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?`),
+  ts: new RegExp(`^//\\s*(?!\\s*//)([^:]+):\\s+([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?`),
+  js: new RegExp(`^//\\s*(?!\\s*//)([^:]+):\\s+([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?`),
+  sql: new RegExp(`^--\\s*([^:]+):\\s+([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?`)
 };
 const nameAnnRegex = exports.nameAnnRegex = /\s*(name[^:]*): ([^\n\r\[\{]+)/;
 const nameRegex = exports.nameRegex = /(?:|(?:static)(?:export )(?:async ))\s+function\s+([a-zA-Z_][a-zA-Z0-9_$]*)\s*\((.*?).*/;
@@ -202,7 +202,7 @@ function getScriptOutputType(script, comment = '#') {
   let resType = 'void';
   let firstItemName = '';
   let wasSecond = false;
-  for (let match of matches) {
+  for (const match of matches) {
     if (resType === 'void') {
       resType = dgToTsTypeMap[match[1]] ?? 'any';
       firstItemName = match[2];
@@ -268,12 +268,8 @@ async function runScript(script, path, verbose = false) {
     } = await execAsync(script, {
       cwd: path
     });
-    if (stderr && verbose) {
-      console.error(`Warning/Error: ${stderr}`);
-    }
-    if (stdout && verbose) {
-      console.log(`Output: ${stdout}`);
-    }
+    if (stderr && verbose) console.error(`Warning/Error: ${stderr}`);
+    if (stdout && verbose) console.log(`Output: ${stdout}`);
   } catch (error) {
     console.error(`Execution failed: ${error.message}`);
     throw new Error(`Error executing '${script}'. Error message: ${error.message}`);
@@ -316,12 +312,8 @@ function optionsToString(options) {
   for (const [key, value] of Object.entries(options)) {
     if (key === '_' || key === 'all') continue;
     if (typeof value === 'boolean') {
-      if (value) {
-        parts.push(`--${key}`);
-      }
-    } else if (value !== undefined && value !== null) {
-      parts.push(`--${key}="${value}"`);
-    }
+      if (value) parts.push(`--${key}`);
+    } else if (value !== undefined && value !== null) parts.push(`--${key}="${value}"`);
   }
   return parts.join(' ');
 }

package/entity-template/app.js

@@ -1,6 +1,6 @@
 
 //name: #{NAME}
-//tags: app
+//meta.role: app
 export function #{NAME}() {
   grok.shell.info('Hello!');
 }

package/entity-template/init.js

@@ -1,4 +1,3 @@
-
-//tags: init
+//meta.role: init
 export async function #{NAME}() {
 }

package/entity-template/panel.ts

@@ -1,7 +1,7 @@
 
 //name: #{NAME}
 //description: Creates an info panel
-//tags: panel
+//meta.role: panel
 //input: string smiles {semType: Molecule}
 //output: widget result
 //condition: true

package/entity-template/sem-type-detector.js

@@ -1,5 +1,5 @@
 
-  //tags: semTypeDetector
+  //meta.role: semTypeDetector
   //input: column col
   //output: string semType
   detect#{PACKAGE_DETECTORS_NAME}(col) {

package/entity-template/test.ts

@@ -1,4 +1,4 @@
-import { category, expect, test } from '@datagrok-libraries/utils/src/test';
+import {category, expect, test} from '@datagrok-libraries/utils/src/test';
 
 
 category('Examples', () => {
@@ -11,6 +11,6 @@ category('Examples', () => {
   });
 
   test('Skipped', async () => {
-    expect(1 === 1 , false);
+    expect(1 === 1, false);
   }, {skipReason: 'TASK-ID'});
 });

package/entity-template/view.js

@@ -1,7 +1,7 @@
 
 //name: #{NAME}
 //description: Creates #{NAME} view
-//tags: view
+//meta.role: view
 //input: map params
 //input: string path
 //output: view result

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "4.14.69",
+  "version": "4.14.71",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {
@@ -11,6 +11,7 @@
     "@typescript-eslint/visitor-keys": "^8.31.1",
     "archiver": "^4.0.2",
     "archiver-promise": "^1.0.0",
+    "datagrok-api": "../js-api",
     "estraverse": "^5.3.0",
     "fs": "^0.0.1-security",
     "glob": "^11.0.2",
@@ -24,8 +25,7 @@
     "papaparse": "^5.4.1",
     "path": "^0.12.7",
     "puppeteer": "22.10.0",
-    "puppeteer-screen-recorder": "3.0.3",
-    "datagrok-api": "^1.26.0"
+    "puppeteer-screen-recorder": "3.0.3"
   },
   "scripts": {
     "link": "npm link",
@@ -75,6 +75,7 @@
     "@typescript-eslint/parser": "^5.62.0",
     "eslint": "^8.56.0",
     "eslint-config-google": "^0.14.0",
+    "ts-morph": "^27.0.2",
     "typescript": "^5.3.3",
     "webpack": "^5.89.0",
     "webpack-cli": "^5.1.4"

package/plugins/func-gen-plugin.js

@@ -15,6 +15,8 @@ const {
   inputOptionsNames,
 } = require('../bin/utils/func-generation');
 
+const {toCamelCase} = require('../bin/commands/migrate');
+
 const {api} = require('../bin/commands/api');
 
 const baseImport = 'import * as DG from \'datagrok-api/dg\';\n';
@@ -132,13 +134,17 @@ class FuncGeneratorPlugin {
       ...(reservedDecorators[name]['metadata']['tags'] ?? []),
       ...(decoratorOptions.get('tags') ?? []),
     ]);
-
-    if ((reservedDecorators[name]['metadata']['role']?.length > 0) ) {
-      if (!decoratorOptions.get('meta'))
-        decoratorOptions.set('meta', {role: reservedDecorators[name]['metadata']['role']});
-      else if (!decoratorOptions.get('meta')['role']) 
-        decoratorOptions.get('meta')['role'] = reservedDecorators[name]['metadata']['role'];
-      delete reservedDecorators[name]['metadata']['role'];
+    
+    const role = reservedDecorators[name]['metadata']['role'];
+    if (role?.length > 0) {
+      const camelRole = toCamelCase(role);
+
+      if (!decoratorOptions.has('meta')) 
+        decoratorOptions.set('meta', {role: camelRole});
+      else {
+        const meta = decoratorOptions.get('meta');
+        meta['role'] = meta['role'] ? `${meta['role']},${camelRole}` : camelRole;
+      }
     }
 
     const functionParams =
@@ -150,6 +156,8 @@ class FuncGeneratorPlugin {
       node?.type === 'MethodDefinition' ? className : identifierName,
       modifyImportPath(path.dirname(this.options.outputPath), file),
     );
+    const metadataCopy = {...reservedDecorators[name]['metadata']};
+    delete metadataCopy.role;
     
     imports.add(importString);
     const funcName = `${
@@ -157,7 +165,7 @@ class FuncGeneratorPlugin {
     }${identifierName}`;
     const funcAnnotaionOptions = {
       ...{name: funcName},
-      ...reservedDecorators[name]['metadata'],
+      ...metadataCopy,
       ...(annotationByReturnObj ?
         {outputs: annotationByReturnObj ?? []} :
         {}),

package/script-template/javascript.js

@@ -1,6 +1,5 @@
 //name: #{NAME}
 //description: Hello world script
 //language: javascript
-//tags: template, demo
 
 alert('Hello, World!');

package/script-template/julia.jl

@@ -1,7 +1,6 @@
 #name: #{NAME}
 #description: Following template calculates number of cells in table
 #language: julia
-#tags: template, demo
 #sample: cars.csv
 #input: dataframe table [Data table]
 #output: int count [Number of cells in table]

package/script-template/node.js

@@ -1,7 +1,6 @@
 //name: #{NAME}
 //description: Following template calculates number of cells in table
 //language: nodejs
-//tags: template, demo
 //sample: cars.csv
 //input: dataframe table [Data table]
 //output: int count [Number of cells in table]

package/script-template/octave.m

@@ -1,7 +1,6 @@
 #name: #{NAME}
 #description: Following template calculates number of cells in table
 #language: octave
-#tags: template, demo
 #sample: cars.csv
 #input: dataframe table [Data table]
 #output: int count [Number of cells in table]

package/script-template/python.py

@@ -1,7 +1,6 @@
 #name: #{NAME}
 #description: Following template calculates number of cells in table
 #language: python
-#tags: template, demo
 #sample: cars.csv
 #input: dataframe table [Data table]
 #output: int count [Number of cells in table]

package/script-template/r.R

@@ -1,7 +1,6 @@
 #name: #{NAME}
 #description: Following template calculates number of cells in table
 #language: r
-#tags: template, demo
 #sample: cars.csv
 #input: dataframe table [Data table]
 #output: int count [Number of cells in table]