datagrok-tools 4.13.79 → 4.14.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Datagrok-tools changelog
 
+## 4.14.0 (2025-05-12)
+
+### Features
+
+* Added annotation creation by decorators
+
 ## 4.13.70 (2025-04-14)
 
 ### Features

package/bin/utils/func-generation.js

@@ -20,11 +20,13 @@ let pseudoParams = exports.pseudoParams = /*#__PURE__*/function (pseudoParams) {
   pseudoParams["INPUT_TYPE"] = "inputType";
   return pseudoParams;
 }({});
+const nonMetaData = ['sidebar', 'editor'];
+const decoratorOptionToAnnotation = new Map([['initialValue', 'default']]);
 let FUNC_TYPES = exports.FUNC_TYPES = /*#__PURE__*/function (FUNC_TYPES) {
   FUNC_TYPES["APP"] = "app";
   FUNC_TYPES["CELL_RENDERER"] = "cellRenderer";
   FUNC_TYPES["FILE_EXPORTER"] = "fileExporter";
-  FUNC_TYPES["FILE_IMPORTER"] = "file-handler";
+  FUNC_TYPES["FILE_HANDLER"] = "file-handler";
   FUNC_TYPES["FILE_VIEWER"] = "fileViewer";
   FUNC_TYPES["SETTINGS_EDITOR"] = "packageSettingsEditor";
   FUNC_TYPES["VIEWER"] = "viewer";
@@ -38,6 +40,7 @@ let FUNC_TYPES = exports.FUNC_TYPES = /*#__PURE__*/function (FUNC_TYPES) {
   FUNC_TYPES["DASHBOARD"] = "dashboard";
   FUNC_TYPES["FUNCTION_ANALYSIS"] = "functionAnalysis";
   FUNC_TYPES["CONVERTER"] = "converter";
+  FUNC_TYPES["MODEL"] = "model";
   return FUNC_TYPES;
 }({});
 const typesToAnnotation = exports.typesToAnnotation = {
@@ -63,17 +66,19 @@ const typesToAnnotation = exports.typesToAnnotation = {
   'FuncCall': 'funccall',
   'DG.SemanticValue': 'semantic_value',
   'SemanticValue': 'semantic_value',
-  'any': 'dynamic'
+  'any': 'dynamic',
+  'void': 'void',
+  'string': 'string'
 };
 
 /** Generates an annotation header for a function based on provided metadata. */
 function getFuncAnnotation(data, comment = '//', sep = '\n') {
   const isFileViewer = data.tags?.includes(FUNC_TYPES.FILE_VIEWER) ?? false;
-  const isFileImporter = data.tags?.includes(FUNC_TYPES.FILE_IMPORTER) ?? false;
+  const isFileImporter = data.tags?.includes(FUNC_TYPES.FILE_HANDLER) ?? false;
   let s = '';
   if (data.name) s += `${comment}name: ${data.name}${sep}`;
   if (pseudoParams.EXTENSION in data && data.tags != null && data.tags.includes(FUNC_TYPES.FILE_EXPORTER)) s += `${comment}description: Save as ${data[pseudoParams.EXTENSION]}${sep}`;else if (data.description) s += `${comment}description: ${data.description}${sep}`;
-  if (data.tags) {
+  if (data.tags && data.tags?.length > 0) {
     s += `${comment}tags: ${isFileViewer && data[pseudoParams.EXTENSIONS] ? data.tags.concat(data[pseudoParams.EXTENSIONS].map(ext => 'fileViewer-' + ext)).join(', ') : data.tags.join(', ')}${sep}`;
   }
   for (let input of data.inputs ?? []) {
@@ -88,22 +93,30 @@ function getFuncAnnotation(data, comment = '//', sep = '\n') {
     if (input?.options?.type) type = input?.options?.type;else if (annotationType) {
       if (isArray) type = `list<${annotationType}>`;else type = annotationType;
     } else type = 'dynamic';
-    console.log(input);
     const options = input?.options?.options ? buildStringOfOptions(input.options.options ?? {}) : '';
-    const functionName = (input.options?.name ? input?.options?.name ?? ` ${input.name?.replaceAll('.', '')}` : '')?.trim();
+    const functionName = (input.options?.name ? input?.options?.name : ` ${input.name?.replaceAll('.', '')}`)?.trim();
     s += comment + 'input: ' + type + ' ' + functionName + (input.defaultValue !== undefined ? `= ${input.defaultValue}` : '') + ' ' + options.replaceAll('"', '\'') + sep;
   }
   if (data.outputs) {
-    for (const output of data.outputs) s += comment + 'output: ' + output.type + (output.name ? ` ${output.name}` : '') + sep;
+    for (const output of data.outputs) if (output.type !== 'void') s += comment + 'output: ' + output.type + (output.name ? ` ${output.name}${output.options ? ` ${buildStringOfOptions(output.options)}` : ''}` : '') + sep;
   }
   if (data.meta) {
     for (let entry of Object.entries(data.meta)) s += `${comment}meta.${entry[0]}: ${entry[1]}${sep}`;
   }
   for (const parameter in data) {
-    if (parameter === pseudoParams.EXTENSION || parameter === pseudoParams.INPUT_TYPE || parameter === 'meta') continue;else if (parameter === pseudoParams.EXTENSIONS) {
+    if (parameter === pseudoParams.EXTENSION || parameter === pseudoParams.INPUT_TYPE || parameter === 'meta' || parameter === 'isAsync' || parameter === 'test') continue;else if (parameter === pseudoParams.EXTENSIONS) {
       if (isFileViewer) continue;
       s += `${comment}meta.ext: ${data[parameter]}${sep}`;
-    } else if (!headerParams.includes(parameter)) s += `${comment}meta.${parameter}: ${data[parameter]}${sep}`;
+    } else if (!headerParams.includes(parameter)) {
+      if (nonMetaData.includes(parameter)) s += `${comment}${parameter}: ${data[parameter]}${sep}`;else s += `${comment}meta.${parameter}: ${data[parameter]}${sep}`;
+    }
+  }
+  if (data.test) {
+    for (let entry of Object.entries(data.test)) {
+      if (entry[0] === 'test' || entry[0] === 'wait') s += `${comment}`;else s += `, `;
+      s += `${entry[0]}: ${entry[1]} `;
+    }
+    s += `${sep}`;
   }
   return s;
 }
@@ -111,8 +124,10 @@ function buildStringOfOptions(options) {
   let optionsInString = [];
   for (const [key, value] of Object.entries(options ?? {})) {
     let val = value;
+    let option = key;
+    option = decoratorOptionToAnnotation.get(option) ?? option;
     if (Array.isArray(value)) val = JSON.stringify(value);
-    optionsInString.push(`${key}: ${val}`);
+    optionsInString.push(`${option}: ${val}`);
   }
   return `{ ${optionsInString.join('; ')} }`;
 }
@@ -158,9 +173,9 @@ const reservedDecorators = exports.reservedDecorators = {
     },
     genFunc: generateFunc
   },
-  fileImporter: {
+  fileHandler: {
     metadata: {
-      tags: [FUNC_TYPES.FILE_IMPORTER],
+      tags: [FUNC_TYPES.FILE_HANDLER],
       inputs: [{
         name: 'content',
         type: 'string'
@@ -323,6 +338,36 @@ const reservedDecorators = exports.reservedDecorators = {
       }]
     },
     genFunc: generateFunc
+  },
+  demo: {
+    metadata: {
+      tags: [],
+      inputs: [],
+      outputs: []
+    },
+    genFunc: generateFunc
+  },
+  treeBrowser: {
+    metadata: {
+      tags: [],
+      inputs: [{
+        name: 'treeNode',
+        type: 'dynamic'
+      }, {
+        name: 'browseView',
+        type: 'view'
+      }],
+      outputs: []
+    },
+    genFunc: generateFunc
+  },
+  model: {
+    metadata: {
+      tags: [FUNC_TYPES.MODEL],
+      inputs: [],
+      outputs: []
+    },
+    genFunc: generateFunc
   }
 };
 
@@ -332,10 +377,10 @@ function generateClassFunc(annotation, className, sep = '\n') {
 }
 
 /** Generates a DG function. */
-function generateFunc(annotation, funcName, sep = '\n', className = '', inputs = []) {
+function generateFunc(annotation, funcName, sep = '\n', className = '', inputs = [], isAsync = false) {
   let funcSigNature = inputs.map(e => `${e.name}: ${e.type}`).join(', ');
   let funcArguments = inputs.map(e => e.name).join(', ');
-  return annotation + `export function _${funcName}(${funcSigNature}) {${sep}  return ${className.length > 0 ? `${className}.` : ''}${funcName}(${funcArguments});${sep}}${sep.repeat(2)}`;
+  return annotation + `export ${isAsync ? 'async ' : ''}function ${funcName}(${funcSigNature}) {${sep}  return ${className.length > 0 ? `${className}.` : ''}${funcName}(${funcArguments});${sep}}${sep.repeat(2)}`;
 }
 function generateImport(className, path, sep = '\n') {
   return `import {${className}} from '${path}';${sep}`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "4.13.79",
+  "version": "4.14.0",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {

package/plugins/func-gen-plugin.js

@@ -12,7 +12,7 @@ const {
   typesToAnnotation
 } = require("../bin/utils/func-generation");
 
-const baseImport = '\nimport * as DG from \'datagrok-api/dg\';\n\n';
+const baseImport = '\n';
 
 class FuncGeneratorPlugin {
   constructor(options = { outputPath: "./src/package.g.ts" }) {
@@ -33,6 +33,9 @@ class FuncGeneratorPlugin {
 
       for (const file of tsFiles) {
         const content = fs.readFileSync(file, "utf-8");
+        if(!content.includes('@grok.decorators.'))
+          continue;
+        
         if (!content) continue;
         const ast = tsParser.parse(content, {
           sourceType: "module",
@@ -44,11 +47,6 @@ class FuncGeneratorPlugin {
         });
         const functions = [];
         let imports = new Set();
-
-        const fileName = path.basename(file);
-        if(fileName === 'package-functions.ts')
-          imports = new Set([... this._readImports(content)]);
-
         this._walk(ast, (node, parentClass) => {
           const decorators = node.decorators;
           if (!decorators || decorators.length === 0) return;
@@ -58,12 +56,14 @@ class FuncGeneratorPlugin {
 
           if (node?.type === "MethodDefinition")
             this._addNodeData(node, file, srcDirPath, functions, imports, genImports, genExports, parentClass);
-          });
+        });
         this._insertImports([...imports]);
-        fs.appendFileSync(this.options.outputPath, functions.join("\n"), "utf-8");
+        fs.appendFileSync(this.options.outputPath, functions.join(""), "utf-8");
       }
+      this._checkPackageFileForDecoratorsExport(packageFilePath);
+      // Uncommment to add obvious import/export 
+      // this._writeToPackageFile(packageFilePath, genImports, genExports);
 
-      this._writeToPackageFile(packageFilePath, genImports, genExports);
     });
   }
 
@@ -87,26 +87,33 @@ class FuncGeneratorPlugin {
     const decoratorOptions = this._readDecoratorOptions(exp.arguments[0].properties);
     decoratorOptions.set('tags', [...(reservedDecorators[name]['metadata']['tags'] ?? [] ), ...(decoratorOptions.get('tags') ?? [])])
     const functionParams = node?.type === 'MethodDefinition' ? this._readMethodParamas(node) : [];
-
     const annotationByReturnType = node?.type === 'MethodDefinition' ? this._readFunctionReturnTypeInfo(node) : '';
     const annotationByReturnTypeObj = { name: 'result', type: annotationByReturnType };
-    
+    const isMethodAsync = this._isMethodAsync(node);
     let importString = generateImport(node?.type === 'MethodDefinition' ? className : identifierName, modifyImportPath(path.dirname(this.options.outputPath), file));
     imports.add(importString);
-    const funcName = `_${identifierName}`;
+    const funcName = `${node?.type === 'MethodDefinition' ? '' : '_'}${identifierName}`;
     const funcAnnotaionOptions = {
       ...reservedDecorators[name]['metadata'],
       ...(annotationByReturnType ? {outputs: [annotationByReturnTypeObj ?? {}]} : {}),
       ...Object.fromEntries(decoratorOptions),
       ...{inputs: functionParams},
+      ...{isAsync: isMethodAsync}
     };    
-
-    functions.push(reservedDecorators[name]['genFunc'](getFuncAnnotation(funcAnnotaionOptions), identifierName,'\n', (className ?? ''), functionParams));
-
+    if (!funcAnnotaionOptions.name)
+      funcAnnotaionOptions.name = identifierName;
+    functions.push(reservedDecorators[name]['genFunc'](getFuncAnnotation(funcAnnotaionOptions), identifierName,'\n', (className ?? ''), functionParams, funcAnnotaionOptions.isAsync ?? false));
     genImports.push(generateImport(funcName, modifyImportPath(srcDirPath, this.options.outputPath)));
     genExports.push(generateExport(funcName));
   }
 
+  _isMethodAsync(node){
+    let result = false;
+    if (node.type === 'MethodDefinition')
+      result = node.value.async;
+    return result;
+  }
+
   _readImports (content) {
     const importRegex = /(import(?:[\s\w{},*]+from\s*)?['"]([^'"]+)['"];)/g;
     const results = [];
@@ -152,13 +159,14 @@ class FuncGeneratorPlugin {
   _readMethodParamas(node) {
     const params = node?.value?.params?.map(param => {
       let baseParam = param.type === 'TSNonNullExpression' ? param.expression : param;
-      let defaultValue = undefined;
       const options   = param.decorators?.length > 0?  Object.fromEntries(this._readDecoratorOptions(param.decorators[0]?.expression?.arguments[0].properties)) : undefined;
       
+      // Commented code finds value by default of function's variable  
+      // let defaultValue = undefined; 
       if(baseParam.type === 'AssignmentPattern')
       {
-        if (baseParam?.right?.type  === 'Literal')
-          defaultValue = baseParam?.right?.raw;
+        // if (baseParam?.right?.type  === 'Literal')
+        //   defaultValue = baseParam?.right?.raw;
         baseParam = baseParam?.left;
       } 
 
@@ -167,50 +175,52 @@ class FuncGeneratorPlugin {
           baseParam.type === 'RestElement'
             ? `...${baseParam.argument.name}`
             : baseParam.name;
-    
         if (baseParam?.argument?.typeAnnotation)
           name += ': ' + generate(baseParam.argument.typeAnnotation.typeAnnotation).code;
     
         let type = '';
-        if (baseParam?.typeAnnotation)
-          type = generate(baseParam.typeAnnotation.typeAnnotation).code;
-        else
+        if (baseParam?.typeAnnotation?.typeAnnotation?.typeName || baseParam?.typeAnnotation?.typeAnnotation?.elementType?.typeName)
           type = 'any';
-    
-        return { name: name, type: type, defaultValue: defaultValue, options: options };
-      }
-      else if (baseParam.type === 'ObjectPattern' || baseParam.type === 'ArrayPattern') {
-        let name = '';
-        if (baseParam.type === 'ObjectPattern') {
-          const properties = baseParam.properties.map(prop => {
-            if (prop.type === 'Property' && prop.key.type === 'Identifier')
-              return prop.key.name;
-            else if (prop.type === 'RestElement' && prop.argument.type === 'Identifier')
-              return `...${prop.argument.name}`;
-            else
-              return generate(prop).code;
-          });
-          name = `{ ${properties.join(', ')} }`;
-        } else {
-          const elements = baseParam.elements.map(elem => {
-            if (elem) {
-              if (elem.type === 'Identifier')
-                return elem.name;
-              else
-                return generate(elem).code;
-            } else return '';
-          });
-          name = `[${elements.join(', ')}]`;
-        }
-    
-        let type = '';
-        if (baseParam.typeAnnotation)
+        else if (baseParam?.typeAnnotation?.typeAnnotation)
           type = generate(baseParam.typeAnnotation.typeAnnotation).code;
-        else
-          type = 'any';
-
-        return { name: name, type: type, defaultValue: defaultValue, options: options };
+        let params = baseParam.typeAnnotation.typeAnnotation.typeArguments?.params;
+        if(type !== 'any' && params && params.length > 0)
+          type += `<${params.map((e)=>e.typeName.name).join(',')}>`;
+        return { name: name, type: type, options: options };
       }
+      // Commented code belove sets more strong types for ObjectPatterns and ArrayPatterns
+      // else if (baseParam.type === 'ObjectPattern' || baseParam.type === 'ArrayPattern') {
+      //   let name = '';
+      //   if (baseParam.type === 'ObjectPattern') {
+      //     const properties = baseParam.properties.map(prop => {
+      //       if (prop.type === 'Property' && prop.key.type === 'Identifier')
+      //         return prop.key.name;
+      //       else if (prop.type === 'RestElement' && prop.argument.type === 'Identifier')
+      //         return `...${prop.argument.name}`;
+      //       else
+      //         return generate(prop).code;
+      //     });
+      //     name = `{ ${properties.join(', ')} }`;
+      //   } else {
+      //     const elements = baseParam.elements.map(elem => {
+      //       if (elem) {
+      //         if (elem.type === 'Identifier')
+      //           return elem.name;
+      //         else
+      //           return generate(elem).code;
+      //       } else return '';
+      //     });
+      //     name = `[${elements.join(', ')}]`;
+      //   }
+    
+      //   let type = '';
+      //   if (baseParam.typeAnnotation)
+      //     type = generate(baseParam.typeAnnotation.typeAnnotation).code;
+      //   else
+      //     type = 'any';
+
+      //   return { name: name, type: type, options: options };
+      // }
       return { name: 'value', type: 'any', options: undefined };
     });
     return params;
@@ -219,7 +229,7 @@ class FuncGeneratorPlugin {
   _getTsFiles(root) {
     const tsFiles = [];
     const extPattern = /\.tsx?$/;
-    const excludedFiles = ["package.ts", "package-test.ts", "package.g.ts"];
+    const excludedFiles = ["package-test.ts", "package.g.ts"];
 
     function findFiles(dir) {
       const files = fs.readdirSync(dir);
@@ -257,41 +267,42 @@ class FuncGeneratorPlugin {
   _readFunctionReturnTypeInfo(node) { 
     let resultType = 'any';
     let isArray = false;
-    const annotation = node.value.returnType.typeAnnotation;
-    if (annotation?.typeName?.name === 'Promise')
-    {
-      const argumnets = annotation.typeArguments?.params;
-      if (argumnets && argumnets.length===1)
+    const annotation = node.value?.returnType?.typeAnnotation;
+    if (annotation) {
+      if (annotation?.typeName?.name === 'Promise')
       {
-        if (argumnets[0].typeName)
-          resultType = argumnets[0].typeName?.right?.name ?? argumnets[0].typeName?.name;
-        else if (argumnets[0].type !== 'TSArrayType')
-          resultType = this._getTypeNameFromNode(argumnets[0]);
-        else if (argumnets[0].elementType.type !== 'TSTypeReference'){
+        const argumnets = annotation.typeArguments?.params;
+        if (argumnets && argumnets.length===1)
+        {
+          if (argumnets[0].typeName)
+            resultType = argumnets[0].typeName?.right?.name ?? argumnets[0].typeName?.name;
+          else if (argumnets[0].type !== 'TSArrayType')
+            resultType = this._getTypeNameFromNode(argumnets[0]);
+          else if (argumnets[0].elementType.type !== 'TSTypeReference'){
+            isArray = true;
+            resultType = this._getTypeNameFromNode(argumnets[0]?.elementType);
+          }
+          else{
+            isArray = true;
+            resultType = argumnets[0].elementType?.typeName?.name || argumnets[0].elementType?.typeName?.right?.name;
+          }
+        }
+      }    
+      else{      
+        if (annotation.type === 'TSTypeReference')
+          resultType = annotation.typeName?.right?.name ?? annotation.typeName?.name;
+        else if (annotation.type !== 'TSArrayType')
+          resultType = this._getTypeNameFromNode(annotation);
+        else if  (annotation.elementType.type !== 'TSTypeReference'){
           isArray = true;
-          resultType = this._getTypeNameFromNode(argumnets[0]?.elementType);
+          resultType = this._getTypeNameFromNode(annotation?.elementType);
         }
         else{
           isArray = true;
-          resultType = argumnets[0].elementType?.typeName?.name || argumnets[0].elementType?.typeName?.right?.name;
+          resultType = (annotation?.elementType?.typeName?.name || annotation?.elementType?.typeName?.right?.name);
         }
       }
     }    
-    else{      
-      if (annotation.type === 'TSTypeReference')
-        resultType = annotation.typeName?.right?.name ?? annotation.typeName?.name;
-      else if (annotation.type !== 'TSArrayType')
-        resultType = this._getTypeNameFromNode(annotation);
-      else if  (annotation.elementType.type !== 'TSTypeReference'){
-        isArray = true;
-        resultType = this._getTypeNameFromNode(annotation?.elementType);
-      }
-      else{
-        isArray = true;
-        resultType = (annotation?.elementType?.typeName?.name || annotation?.elementType?.typeName?.right?.name);
-      }
-    }
-    
     resultType = typesToAnnotation[resultType];
     if (isArray && resultType)
       resultType = `list<${resultType}>`
@@ -318,12 +329,19 @@ class FuncGeneratorPlugin {
     fs.writeFileSync(this.options.outputPath, baseImport);
   }
 
-  _writeToPackageFile(filePath, imports, exports) {
-    if (imports.length !== exports.length) return;
+  _checkPackageFileForDecoratorsExport(packagePath){    
+    const content = fs.readFileSync(packagePath, "utf-8");
+    const decoratorsExportRegex = /export\s*\*\s*from\s*'\.\/package\.g';/;
+    if (!decoratorsExportRegex.test(content))
+      console.warn(`\nWARNING: Your package doesn't export package.g.ts file to package.ts \n please add "export * from './package.g';" to the package.ts file.\n`);
+  }
+
+  _writeToPackageFile(filePath, imports, exp) {
+    if (imports.length !== exp.length) return;
     let content = fs.readFileSync(filePath, "utf-8");
     for (let i = 0; i < imports.length; i++) {
       const importStatement = imports[i];
-      const exportStatement = exports[i];
+      const exportStatement = exp[i];
       if (!content.includes(importStatement.trim()))
         content = importStatement + content + exportStatement;
     }