datagrok-tools 4.14.6 → 4.14.8

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Datagrok-tools changelog
 
+## 4.14.6 (2025-06-06)
+
+### Features
+
+* Grok Check added collision checks 
+* Grok Publish runs check before invocation 
+* Grok Api creates api for the scripts in the python directory.
+
+
 ## 4.14.6 (2025-05-29)
 
 ### Features

package/bin/commands/api.js

@@ -55,31 +55,37 @@ function generateQueryWrappers() {
 }
 function generateScriptWrappers() {
   const scriptsDir = _path.default.join(curDir, 'scripts');
+  const pythonDir = _fs.default.existsSync(srcDir) ? _path.default.join(srcDir, 'python') : _path.default.join(curDir, 'python');
   if (!_fs.default.existsSync(scriptsDir)) {
     color.warn(`Directory ${scriptsDir} not found`);
     console.log('Skipping API generation for scripts...');
     return;
   }
-  const files = _ignoreWalk.default.sync({
-    path: scriptsDir,
-    ignoreFiles: ['.npmignore', '.gitignore']
-  });
   const wrappers = [];
-  for (const file of files) {
-    let extension;
-    if (!utils.scriptExtensions.some(ext => (extension = ext, file.endsWith(ext)))) continue;
-    const filepath = _path.default.join(scriptsDir, file);
-    const script = _fs.default.readFileSync(filepath, 'utf8');
-    if (!script) continue;
-    const name = utils.getScriptName(script, utils.commentMap[extension]);
-    if (!name) continue;
-    const description = utils.getScriptDescription(script);
-    checkNameColision(name);
-    const tb = new utils.TemplateBuilder(utils.scriptWrapperTemplate).replace('FUNC_NAME', name).replace('FUNC_NAME_LOWERCASE', name).replace('PACKAGE_NAMESPACE', _package.name);
-    const inputs = utils.getScriptInputs(script);
-    const outputType = utils.getScriptOutputType(script);
-    tb.replace('PARAMS_OBJECT', inputs).replace('TYPED_PARAMS', inputs).replace('FUNC_DESCRIPTION', description).replace('OUTPUT_TYPE', outputType);
-    wrappers.push(tb.build(1));
+  for (let dir of [scriptsDir, pythonDir]) {
+    const files = _ignoreWalk.default.sync({
+      path: dir,
+      ignoreFiles: ['.npmignore', '.gitignore']
+    });
+    console.log(files);
+    for (const file of files) {
+      let extension;
+      if (!utils.scriptExtensions.some(ext => (extension = ext, file.endsWith(ext)))) continue;
+      const filepath = _path.default.join(dir, file);
+      const script = _fs.default.readFileSync(filepath, 'utf8');
+      if (!script) continue;
+      console.log(filepath);
+      console.log(script);
+      const name = utils.getScriptName(script, utils.commentMap[extension]);
+      if (!name) continue;
+      const description = utils.getScriptDescription(script);
+      checkNameColision(name);
+      const tb = new utils.TemplateBuilder(utils.scriptWrapperTemplate).replace('FUNC_NAME', name).replace('FUNC_NAME_LOWERCASE', name).replace('PACKAGE_NAMESPACE', _package.name);
+      const inputs = utils.getScriptInputs(script);
+      const outputType = utils.getScriptOutputType(script);
+      tb.replace('PARAMS_OBJECT', inputs).replace('TYPED_PARAMS', inputs).replace('FUNC_DESCRIPTION', description).replace('OUTPUT_TYPE', outputType);
+      wrappers.push(tb.build(1));
+    }
   }
   saveWrappersToFile('scripts', wrappers);
 }

package/bin/commands/check.js

@@ -21,20 +21,21 @@ var testUtils = _interopRequireWildcard(require("../utils/test-utils"));
 function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
 function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
 const warns = ['Latest package version', 'Datagrok API version should contain'];
-const forbidenNames = ['function', 'class', 'export'];
+const forbiddenNames = ['function', 'class', 'export'];
+const namesInFiles = new Map();
 function check(args) {
   const nOptions = Object.keys(args).length - 1;
   if (args['_'].length !== 1 || nOptions > 2 || nOptions > 0 && !args.r && !args.recursive) return false;
   const curDir = process.cwd();
-  if (args.recursive) return runChecksRec(curDir);else {
+  if (args.recursive) return runChecksRec(curDir, args.soft ?? false);else {
     if (!utils.isPackageDir(curDir)) {
       color.error('File `package.json` not found. Run the command from the package directory');
       return false;
     }
-    return runChecks(curDir);
+    return runChecks(curDir, args.soft ?? false);
   }
 }
-function runChecks(packagePath) {
+function runChecks(packagePath, soft = false) {
   const files = _ignoreWalk.default.sync({
     path: packagePath,
     ignoreFiles: ['.npmignore', '.gitignore']
@@ -77,20 +78,20 @@ function runChecks(packagePath) {
   if (errors.length) {
     console.log(`Checking package ${_path.default.basename(packagePath)}...`);
     showError(errors);
-    if (json.version.startsWith('0') || errors.every(w => warns.some(ww => w.includes(ww)))) return true;
+    if (soft || json.version.startsWith('0') || errors.every(w => warns.some(ww => w.includes(ww)))) return true;
     testUtils.exitWithCode(1);
   }
   console.log(`Checking package ${_path.default.basename(packagePath)}...\t\t\t\u2713 OK`);
   return true;
 }
-function runChecksRec(dir) {
+function runChecksRec(dir, soft = false) {
   const files = _fs.default.readdirSync(dir);
   for (const file of files) {
     const filepath = _path.default.join(dir, file);
     const stats = _fs.default.statSync(filepath);
     if (stats.isDirectory()) {
-      if (utils.isPackageDir(filepath)) return runChecks(filepath);else {
-        if (file !== 'node_modules' && !file.startsWith('.')) runChecksRec(_path.default.join(dir, file));
+      if (utils.isPackageDir(filepath)) return runChecks(filepath, soft);else {
+        if (file !== 'node_modules' && !file.startsWith('.')) runChecksRec(_path.default.join(dir, file), soft);
       }
     }
   }
@@ -315,13 +316,19 @@ function checkFuncSignatures(packagePath, files) {
           warnings.push(`File ${file}, function ${f.name}:\n${vr.message}`);
         }
       }
-      let wrongInputNames = f.inputs.filter(e => forbidenNames.includes(e?.name ?? ''));
+      let wrongInputNames = f.inputs.filter(e => forbiddenNames.includes(e?.name ?? ''));
+      if (f.name) {
+        if (namesInFiles.has(f.name)) namesInFiles.get(f.name)?.push(file);else namesInFiles.set(f.name, [file]);
+      }
       if (wrongInputNames.length > 0) warnings.push(`File ${file}, function ${f.name}: Wrong input names: (${wrongInputNames.map(e => e.name).join(', ')})`);
       if (f.isInvalidateOnWithoutCache) warnings.push(`File ${file}, function ${f.name}: Can't use invalidateOn without cache, please follow this example: 'meta.cache.invalidateOn'`);
-      if (f.cache) if (!utils.cahceValues.includes(f.cache)) warnings.push(`File ${file}, function ${f.name}: unsupposed variable for cache : ${f.cache}`);
+      if (f.cache) if (!utils.cacheValues.includes(f.cache)) warnings.push(`File ${file}, function ${f.name}: unsupposed variable for cache : ${f.cache}`);
       if (f.invalidateOn) if (!utils.isValidCron(f.invalidateOn)) warnings.push(`File ${file}, function ${f.name}: unsupposed variable for invalidateOn : ${f.invalidateOn}`);
     }
   }
+  for (const [name, files] of namesInFiles) {
+    if (files.length > 1) warnings.push(`Duplicate names ('${name}'): \n  ${files.join('\n  ')}`);
+  }
   return warnings;
 }
 const sharedLibExternals = {
@@ -519,7 +526,7 @@ function getFuncMetadata(script, fileExtention) {
     if (match) {
       if (!isHeader) isHeader = true;
       const param = match[1];
-      if (param === 'name') data.name = line.match(utils.nameAnnRegex)?.[2];else if (param === 'description') data.description = match[2];else if (param === 'input') {
+      if (param === 'name') data.name = line.match(utils.nameAnnRegex)?.[2]?.toLocaleLowerCase();else if (param === 'description') data.description = match[2];else if (param === 'input') {
         data.inputs.push({
           type: match[2],
           name: match[3]
@@ -539,8 +546,8 @@ function getFuncMetadata(script, fileExtention) {
     }
     if (isHeader) {
       const nm = line.match(utils.nameRegex);
-      if (nm && !match) {
-        data.name = data.name || nm[1];
+      if (nm) data.name = nm[1]?.toLocaleLowerCase();
+      if (data.name && !match) {
         funcData.push(data);
         data = {
           name: '',

package/bin/commands/publish.js

@@ -14,9 +14,9 @@ var _path = _interopRequireDefault(require("path"));
 var _ignoreWalk = _interopRequireDefault(require("ignore-walk"));
 var _jsYaml = _interopRequireDefault(require("js-yaml"));
 var _testUtils = require("../utils/test-utils");
-var _check = require("./check");
 var utils = _interopRequireWildcard(require("../utils/utils"));
 var color = _interopRequireWildcard(require("../utils/color-utils"));
+var _check = require("./check");
 function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
 function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != typeof e && "function" != typeof e) return { default: e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && Object.prototype.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n.default = e, t && t.set(e, n), n; }
 // @ts-ignore
@@ -81,7 +81,8 @@ async function processPackage(debug, rebuild, host, devKey, packageName, suffix)
       rebuild = true;
     }
   }
-  let contentValidationLog = '';
+
+  // let contentValidationLog = '';
   console.log('Starting package checks...');
   const checkStart = Date.now();
   const jsTsFiles = files.filter(f => !f.startsWith('dist/') && (f.endsWith('.js') || f.endsWith('.ts')));
@@ -94,19 +95,19 @@ async function processPackage(debug, rebuild, host, devKey, packageName, suffix)
     const content = _fs.default.readFileSync(webpackConfigPath, {
       encoding: 'utf-8'
     });
-    const externals = (0, _check.extractExternals)(content);
-    if (externals) {
-      const importWarnings = (0, _check.checkImportStatements)(curDir, jsTsFiles, externals);
-      contentValidationLog += importWarnings.join('\n') + (importWarnings.length ? '\n' : '');
-    }
+    // const externals = extractExternals(content);
+    // if (externals) {
+    //   const importWarnings = checkImportStatements(curDir, jsTsFiles, externals);
+    //   contentValidationLog += importWarnings.join('\n') + (importWarnings.length ? '\n' : '');
+    // }
   }
   const funcFiles = jsTsFiles.filter(f => packageFiles.includes(f));
-  const funcWarnings = (0, _check.checkFuncSignatures)(curDir, funcFiles);
-  contentValidationLog += funcWarnings.join('\n') + (funcWarnings.length ? '\n' : '');
-  const packageWarnings = (0, _check.checkPackageFile)(curDir, json);
-  contentValidationLog += packageWarnings.join('\n') + (packageWarnings.length ? '\n' : '');
-  const changelogWarnings = (0, _check.checkChangelog)(curDir, json);
-  contentValidationLog += changelogWarnings.join('\n') + (packageWarnings.length ? '\n' : '');
+  // const funcWarnings = checkFuncSignatures(curDir, funcFiles);
+  // contentValidationLog += funcWarnings.join('\n') + (funcWarnings.length ? '\n' : '');
+  // const packageWarnings = checkPackageFile(curDir, json);
+  // contentValidationLog += packageWarnings.join('\n') + (packageWarnings.length ? '\n' : '');
+  // const changelogWarnings = checkChangelog(curDir, json);
+  // contentValidationLog += changelogWarnings.join('\n') + '';
   console.log(`Checks finished in ${Date.now() - checkStart} ms`);
   const reg = new RegExp(/\${(\w*)}/g);
   const errs = [];
@@ -123,9 +124,10 @@ async function processPackage(debug, rebuild, host, devKey, packageName, suffix)
     }
     if (relativePath.startsWith('upload.keys.json')) return;
     if (relativePath === 'zip') return;
-    if (!utils.checkScriptLocation(canonicalRelativePath)) {
-      contentValidationLog += `Warning: file \`${canonicalRelativePath}\`` + ` should be in directory \`${_path.default.basename(curDir)}/scripts/\`\n`;
-    }
+    // if (!utils.checkScriptLocation(canonicalRelativePath)) {
+    //   contentValidationLog += `Warning: file \`${canonicalRelativePath}\`` +
+    //     ` should be in directory \`${path.basename(curDir)}/scripts/\`\n`;
+    // }
     const t = _fs.default.statSync(fullPath).mtime.toUTCString();
     localTimestamps[canonicalRelativePath] = t;
     if (debug && timestamps[canonicalRelativePath] === t) {
@@ -196,7 +198,7 @@ async function processPackage(debug, rebuild, host, devKey, packageName, suffix)
       return 1;
     } else {
       console.log(log);
-      color.warn(contentValidationLog);
+      // color.warn(contentValidationLog);
     }
   } catch (error) {
     console.error(error);
@@ -205,6 +207,9 @@ async function processPackage(debug, rebuild, host, devKey, packageName, suffix)
   return 0;
 }
 async function publish(args) {
+  if (!args['skip-check']) (0, _check.check)({
+    _: ['check']
+  });
   config = _jsYaml.default.load(_fs.default.readFileSync(confPath, {
     encoding: 'utf-8'
   }));

package/bin/utils/utils.js

@@ -4,7 +4,7 @@ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefau
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.cahceValues = exports.absUrlRegex = exports.TemplateBuilder = void 0;
+exports.cacheValues = exports.absUrlRegex = exports.TemplateBuilder = void 0;
 exports.camelCaseToKebab = camelCaseToKebab;
 exports.checkScriptLocation = checkScriptLocation;
 exports.commentMap = void 0;
@@ -149,7 +149,7 @@ const queryExtension = exports.queryExtension = '.sql';
 const jsExtention = exports.jsExtention = '.js';
 const scriptExtensions = exports.scriptExtensions = ['.jl', '.m', '.py', '.R'];
 function checkScriptLocation(filepath) {
-  if (!(filepath.startsWith('scripts/') || filepath.startsWith('projects/') || filepath.startsWith('dockerfiles/')) && scriptExtensions.some(ext => filepath.endsWith(ext))) return false;
+  if (!(filepath.startsWith('scripts/') || filepath.startsWith('projects/') || filepath.startsWith('dockerfiles/') || filepath.startsWith('python/')) && scriptExtensions.some(ext => filepath.endsWith(ext))) return false;
   return true;
 }
 ;
@@ -165,7 +165,7 @@ function getParam(name, script, comment = '#') {
   return match ? match[1]?.trim() : null;
 }
 ;
-const cahceValues = exports.cahceValues = ['all', 'server', 'client', 'true'];
+const cacheValues = exports.cacheValues = ['all', 'server', 'client', 'true'];
 function isValidCron(cronExpression) {
   const cronRegex = /^(\*|([0-9]|1[0-9]|2[0-9]|3[0-9]|4[0-9]|5[0-9])|\*\/([0-9]|1[0-9]|2[0-9]|3[0-9]|4[0-9]|5[0-9])) (\*|([0-9]|1[0-9]|2[0-3])|\*\/([0-9]|1[0-9]|2[0-3])) (\*|([1-9]|1[0-9]|2[0-9]|3[0-1])|\*\/([1-9]|1[0-9]|2[0-9]|3[0-1])) (\*|([1-9]|1[0-2])|\*\/([1-9]|1[0-2])) (\*|([0-6])|\*\/([0-6]))$/;
   return cronRegex.test(cronExpression);
@@ -183,15 +183,15 @@ const dgToTsTypeMap = exports.dgToTsTypeMap = {
   view: 'DG.View'
 };
 const propertyTypes = exports.propertyTypes = ['bool', 'int', 'double', 'string', 'datetime', 'object', 'column', 'dataframe', 'bitset', 'cell', 'string_list', 'map'];
-const headerTags = exports.headerTags = ['name', 'description', 'help-url', 'input', 'output', 'tags', 'sample', 'language', 'returns', 'test', 'sidebar', 'condition', 'top-menu', 'environment', 'require', 'editor-for', 'schedule', 'reference', 'editor', 'meta'];
+const headerTags = exports.headerTags = ['name', 'description', 'help-url', 'input', 'output', 'tags', 'sample', 'language', 'returns', 'test', 'sidebar', 'condition', 'top-menu', 'environment', 'require', 'editor-for', 'schedule', 'reference', 'editor', 'meta', 'connection', 'friendlyName'];
 const fileParamRegex = exports.fileParamRegex = {
   py: new RegExp(`^\#\\s*((?:${headerTags.join('|')})[^:]*): *([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?[^\\n]*$`),
   ts: new RegExp(`^\/\/\\s*((?:${headerTags.join('|')})[^:]*): *([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?[^\\n]*$`),
   js: new RegExp(`^\/\/\\s*((?:${headerTags.join('|')})[^:]*): *([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?[^\\n]*$`),
   sql: new RegExp(`^--\\s*((?:${headerTags.join('|')})[^:]*): *([^\\s\\[\\{]+) ?([^\\s\\[\\{]+)?[^\\n]*$`)
 };
-const nameAnnRegex = exports.nameAnnRegex = /\/\/\s*(name[^:]*): ([^\n\r\[\{]+)/;
-const nameRegex = exports.nameRegex = /(?:|(?:static)(?:export )(?:async )function )\s+([a-zA-Z_][a-zA-Z0-9_$]*)\s*\((.*?).*/;
+const nameAnnRegex = exports.nameAnnRegex = /\s*(name[^:]*): ([^\n\r\[\{]+)/;
+const nameRegex = exports.nameRegex = /(?:|(?:static)(?:export )(?:async ))\s+function\s+([a-zA-Z_][a-zA-Z0-9_$]*)\s*\((.*?).*/;
 const absUrlRegex = exports.absUrlRegex = new RegExp('^(?:[a-z+]+:)?//', 'i');
 function getScriptOutputType(script, comment = '#') {
   const regex = new RegExp(`${comment}\\s*output:\\s?([a-z_]+)\\s*`);
@@ -245,7 +245,7 @@ async function runScript(script, path, verbose = false) {
     }
   } catch (error) {
     console.error(`Execution failed: ${error.message}`);
-    throw new Error(`Cant run script ${script}`);
+    throw new Error(`Error executing '${script}'. Error message: ${error.message}`);
   }
 }
 function setHost(host, configFile) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "4.14.6",
+  "version": "4.14.8",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {