datagrok-tools 6.0.8 → 6.1.0

package/bin/commands/help.js

@@ -156,13 +156,14 @@ Uploads a package
 Checks for errors before publishing — the package won't be published if there are any.
 
 Options:
-[--all] [--refresh] [--link] [--build] [--release] [--skip-check] [-v | --verbose]
+[--all] [--refresh] [--link] [--build] [--release] [--rebuild-docker] [--skip-check] [-v | --verbose]
 
 --all             Publish all available packages (run in packages directory)
 --refresh         Publish all available already loaded packages (run in packages directory)
 --link            Link the package to local packages
 --build           Builds the package
 --release         Publish package as release version
+--rebuild-docker  Force rebuild Docker images locally before pushing to registry
 --skip-check      Skip check stage
 --verbose         Print detailed output
 

package/bin/commands/publish.js

@@ -23,7 +23,8 @@ function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r
 // @ts-ignore
 
 const {
-  exec
+  exec,
+  execSync
 } = require('child_process');
 const grokDir = _path.default.join(_os.default.homedir(), '.grok');
 const confPath = _path.default.join(grokDir, 'config.yaml');
@@ -35,7 +36,171 @@ let curDir = process.cwd();
 const packDir = _path.default.join(curDir, 'package.json');
 const packageFiles = ['src/package.ts', 'src/detectors.ts', 'src/package.js', 'src/detectors.js', 'src/package-test.ts', 'src/package-test.js', 'package.js', 'detectors.js'];
 let config;
-async function processPackage(debug, rebuild, host, devKey, packageName, dropDb, suffix, hostAlias) {
+const BLEEDING_EDGE_TAG = 'bleeding-edge';
+function discoverDockerfiles(packageName, version) {
+  const dockerfilesDir = _path.default.join(curDir, 'dockerfiles');
+  if (!_fs.default.existsSync(dockerfilesDir)) return [];
+  const results = [];
+  const entries = _fs.default.readdirSync(dockerfilesDir, {
+    withFileTypes: true
+  });
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const dockerfilePath = _path.default.join(dockerfilesDir, entry.name, 'Dockerfile');
+    if (!_fs.default.existsSync(dockerfilePath)) continue;
+    const cleanName = utils.removeScope(packageName).toLowerCase();
+    const imageName = `${cleanName}-${entry.name.toLowerCase()}`;
+    results.push({
+      imageName,
+      imageTag: `${version}.X`,
+      dirName: entry.name,
+      fullLocalName: `${imageName}:${version}.X`
+    });
+  }
+  return results;
+}
+function dockerCommand(args) {
+  return execSync(`docker ${args}`, {
+    encoding: 'utf-8',
+    stdio: ['pipe', 'pipe', 'pipe']
+  }).trim();
+}
+function localImageExists(fullName) {
+  try {
+    const output = dockerCommand(`images --format "{{.Repository}}:{{.Tag}}"`);
+    return output.split('\n').some(line => line.trim() === fullName);
+  } catch {
+    return false;
+  }
+}
+function dockerLogin(registry, devKey) {
+  try {
+    dockerCommand(`login ${registry} -u ${devKey} -p ${devKey}`);
+    return true;
+  } catch (e) {
+    color.warn(`Docker login to ${registry} failed: ${e.message || e}`);
+    return false;
+  }
+}
+function dockerTag(source, target) {
+  try {
+    dockerCommand(`tag ${source} ${target}`);
+    return true;
+  } catch (e) {
+    color.error(`Failed to tag ${source} as ${target}: ${e.message || e}`);
+    return false;
+  }
+}
+function dockerPush(image) {
+  try {
+    dockerCommand(`push ${image}`);
+    return true;
+  } catch (e) {
+    color.error(`Failed to push ${image}: ${e.message || e}`);
+    return false;
+  }
+}
+function dockerBuild(imageName, dockerfilePath, context) {
+  try {
+    color.log(`Building Docker image ${imageName}...`);
+    execSync(`docker build -t ${imageName} -f ${dockerfilePath} ${context}`, {
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe']
+    });
+    return true;
+  } catch (e) {
+    color.error(`Failed to build ${imageName}: ${e.message || e}`);
+    return false;
+  }
+}
+async function resolveLatestCompatible(host, devKey, dockerName) {
+  try {
+    // Login with dev key to get a session token
+    const loginResp = await (0, _nodeFetch.default)(`${host}/users/login/dev/${devKey}`, {
+      method: 'POST'
+    });
+    if (loginResp.status !== 200) return null;
+    const loginData = await loginResp.json();
+    const token = loginData.token;
+    const url = `${host}/docker/images/${encodeURIComponent(dockerName)}/latest-compatible`;
+    const resp = await (0, _nodeFetch.default)(url, {
+      headers: {
+        'Authorization': token
+      }
+    });
+    if (resp.status === 200) return await resp.json();
+    return null;
+  } catch {
+    return null;
+  }
+}
+async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps) {
+  const dockerImages = discoverDockerfiles(packageName, version);
+  if (dockerImages.length === 0) return;
+  color.log(`Found ${dockerImages.length} Dockerfile(s)`);
+  if (registry) dockerLogin(registry, devKey);
+  for (const img of dockerImages) {
+    let result;
+    if (rebuildDocker) {
+      const dockerfileDir = _path.default.join('dockerfiles', img.dirName);
+      const dockerfilePath = _path.default.join(dockerfileDir, 'Dockerfile');
+      if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
+        result = pushImage(img, registry, version);
+        color.success(`Built and tagged ${img.fullLocalName}`);
+      } else {
+        result = await fallbackImage(img, host, devKey, registry);
+      }
+    } else if (localImageExists(img.fullLocalName)) {
+      result = pushImage(img, registry, version);
+      color.success(`Found local image ${img.fullLocalName}`);
+    } else {
+      result = await fallbackImage(img, host, devKey, registry);
+      color.warn(`Local image not found. Expected: ${img.fullLocalName}` + (result.image ? `. Falling back to ${result.image}` : ''));
+      color.log(`  Build it with: docker build -t ${img.fullLocalName} -f dockerfiles/${img.dirName}/Dockerfile dockerfiles/${img.dirName}`);
+    }
+    const imageJsonPath = _path.default.join('dockerfiles', img.dirName, 'image.json');
+    zip.append(JSON.stringify(result, null, 2), {
+      name: imageJsonPath
+    });
+    localTimestamps[imageJsonPath.replace(/\\/g, '/')] = new Date().toUTCString();
+    color.log(`Added ${imageJsonPath}`);
+  }
+}
+function pushImage(img, registry, version) {
+  if (registry) {
+    const remoteTag = `${registry}/${img.imageName}:${version}`;
+    dockerTag(img.fullLocalName, remoteTag);
+    if (dockerPush(remoteTag)) return {
+      image: remoteTag,
+      fallback: false
+    };
+    color.warn(`Push failed, image tagged locally only: ${remoteTag}`);
+    return {
+      image: remoteTag,
+      fallback: false
+    };
+  }
+  color.warn('No registry configured. Image tagged locally only.');
+  return {
+    image: img.fullLocalName,
+    fallback: false
+  };
+}
+async function fallbackImage(img, host, devKey, registry) {
+  const latest = await resolveLatestCompatible(host, devKey, img.imageName);
+  if (latest) return {
+    image: latest.image,
+    fallback: true,
+    requestedVersion: img.imageTag
+  };
+  color.warn('No previous version available. Container will not be available until an image is built.');
+  return {
+    image: null,
+    fallback: true,
+    requestedVersion: img.imageTag
+  };
+}
+async function processPackage(debug, rebuild, host, devKey, packageName, dropDb, suffix, hostAlias, registry, rebuildDocker) {
   // Get the server timestamps
   let timestamps = {};
   let url = `${host}/packages/dev/${devKey}/${packageName}`;
@@ -147,6 +312,10 @@ async function processPackage(debug, rebuild, host, devKey, packageName, dropDb,
     errs.forEach(e => color.error(e));
     return 1;
   }
+
+  // Process Docker images and inject image.json into the ZIP
+  const packageVersion = json.version;
+  await processDockerImages(packageName, packageVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps);
   zip.append(JSON.stringify(localTimestamps), {
     name: 'timestamps.json'
   });
@@ -245,16 +414,22 @@ async function publishPackage(args) {
   if (nArgs === 2) host = args['_'][1];
   let key = '';
   let url = '';
+  let registry;
 
   // The host can be passed either as a URL or an alias
   try {
     url = new URL(host).href;
     if (url.endsWith('/')) url = url.slice(0, -1);
-    if (url in urls) key = config['servers'][urls[url]]['key'];
+    if (url in urls) {
+      const alias = urls[url];
+      key = config['servers'][alias]['key'];
+      registry = config['servers'][alias]['registry'];
+    }
   } catch (error) {
     if (!(host in config.servers)) return color.error(`Unknown server alias. Please add it to ${confPath}`);
     url = config['servers'][host]['url'];
     key = config['servers'][host]['key'];
+    registry = config['servers'][host]['registry'];
   }
 
   // Update the developer key
@@ -285,7 +460,7 @@ async function publishPackage(args) {
         if (!args.suffix && stdout) args.suffix = stdout.toString().substring(0, 8);
       });
       await utils.delay(100);
-      code = await processPackage(!args.release, Boolean(args.rebuild), url, key, packageName, args.dropDb ?? false, args.suffix, host);
+      code = await processPackage(!args.release, Boolean(args.rebuild), url, key, packageName, args.dropDb ?? false, args.suffix, host, registry, args['rebuild-docker']);
     } catch (error) {
       console.error(error);
       code = 1;

package/bin/commands/test.js

@@ -125,9 +125,7 @@ async function test(args) {
     try {
       await testUtils.loadPackages(packagesDir, packageName, args.host, args['skip-publish'], args['skip-build'], args.link);
     } catch (e) {
-      console.error('\n');
-      // @ts-ignore
-      console.error(e.message);
+      console.error(e.message || 'Package build/publish failed with no output. Run with --verbose for details.');
       process.exit(1);
     }
   }
@@ -214,7 +212,7 @@ async function runTesting(args) {
       // Store browser session for potential reuse
       if (r.browserSession) browserSession = r.browserSession;
       if (r.error) {
-        console.log(`\nexecution error:`);
+        color.error(`\nTest execution failed:`);
         console.log(r.error);
         // Close browser on error
         if (browserSession?.browser) await browserSession.browser.close();
@@ -365,9 +363,9 @@ function buildTestArgs(args) {
   return parts;
 }
 function parseTestOutput(stdout) {
-  const passedMatch = stdout.match(/Passed amount:\s*(\d+)/);
-  const failedMatch = stdout.match(/Failed amount:\s*(\d+)/);
-  const skippedMatch = stdout.match(/Skipped amount:\s*(\d+)/);
+  const passedMatch = stdout.match(/Passed (?:amount|tests):\s*(\d+)/);
+  const failedMatch = stdout.match(/Failed (?:amount|tests):\s*(\d+)/);
+  const skippedMatch = stdout.match(/Skipped (?:amount|tests):\s*(\d+)/);
   if (!passedMatch && !failedMatch) return null;
   return {
     passed: passedMatch ? parseInt(passedMatch[1]) : 0,

package/bin/utils/test-utils.js

@@ -217,10 +217,15 @@ const recorderConfig = exports.recorderConfig = {
 async function loadPackage(packageDir, dirName, hostString, skipPublish, skipBuild, linkPackage, release) {
   if (skipPublish != true) {
     process.stdout.write(`Building and publishing ${dirName}...`);
-    await utils.runScript(`npm install`, packageDir);
-    if (linkPackage) await utils.runScript(`grok link`, packageDir);
-    if (skipBuild != true) await utils.runScript(`npm run build`, packageDir);
-    await utils.runScript(`grok publish ${hostString}${release ? ' --release' : ''}`, packageDir);
+    try {
+      await utils.runScript(`npm install`, packageDir);
+      if (linkPackage) await utils.runScript(`grok link`, packageDir);
+      if (skipBuild != true) await utils.runScript(`npm run build`, packageDir);
+      await utils.runScript(`grok publish ${hostString}${release ? ' --release' : ''}`, packageDir);
+    } catch (e) {
+      process.stdout.write(' FAILED\n');
+      throw e;
+    }
     process.stdout.write(` success!\n`);
   }
 }
@@ -365,9 +370,9 @@ function printBrowsersResult(browserResult, verbose = false) {
         }
       }
     }
-    console.log('Passed amount:  ' + browserResult?.passedAmount);
-    console.log('Skipped amount: ' + browserResult?.skippedAmount);
-    console.log('Failed amount:  ' + browserResult?.failedAmount);
+    console.log('Passed tests:  ' + (browserResult?.passedAmount ?? 0));
+    console.log('Skipped tests: ' + (browserResult?.skippedAmount ?? 0));
+    console.log('Failed tests:  ' + (browserResult?.failedAmount ?? 0));
   }
   if (browserResult.failed) {
     if (browserResult.verboseFailed === 'Package not found') color.fail('Tests not found');else color.fail('Tests failed.');
@@ -413,7 +418,9 @@ async function runTests(testParams) {
       if (testParams.params?.skipToTest) testCallParams.skipToTest = testParams.params.skipToTest;
       if (testParams.params?.returnOnFail) testCallParams.returnOnFail = testParams.params.returnOnFail;
     }
-    const df = await window.grok.functions.call(testParams.package + ':test', Object.keys(testCallParams).length > 0 ? testCallParams : undefined);
+    const testFuncName = testParams.package + ':test';
+    const df = await window.grok.functions.call(testFuncName, Object.keys(testCallParams).length > 0 ? testCallParams : undefined);
+    if (df == null) throw new Error(`${testFuncName} returned null instead of a DataFrame. Check that the test function exists and returns a valid result.`);
     if (!df.getCol('flaking')) {
       const flakingCol = window.DG.Column.fromType(window.DG.COLUMN_TYPE.BOOL, 'flaking', df.rowCount);
       df.columns.add(flakingCol);
@@ -460,11 +467,23 @@ async function runTests(testParams) {
       // df: resultDF?.toJson()
     };
   } catch (e) {
-    console.log(`DEBUG: runTests: IN CATCH: ERROR: ${e}`);
+    let stack = '';
+    try {
+      stack = await window.DG.Logger.translateStackTrace(e.stack);
+    } catch (_) {
+      stack = e.stack ?? '';
+    }
     return {
       failed: true,
       retrySupported: false,
-      error: `${e}, ${await window.DG.Logger.translateStackTrace(e.stack)}`
+      verbosePassed: verbosePassed,
+      verboseSkipped: verboseSkipped,
+      verboseFailed: verboseFailed,
+      passedAmount: countPassed,
+      skippedAmount: countSkipped,
+      failedAmount: countFailed,
+      csv: '',
+      error: `${e}` + (stack ? `, ${stack}` : '')
     };
   }
 
@@ -813,12 +832,11 @@ async function runBrowser(testExecutionData, browserOptions, browsersId, testInv
             verbosePassed: '',
             verboseSkipped: '',
             verboseFailed: '',
-            error: JSON.stringify(e),
+            error: (e?.message ?? '') + (e?.stack ? '\n' + e.stack : '') || JSON.stringify(e),
             passedAmount: 0,
             skippedAmount: 0,
             failedAmount: 1,
-            csv: '',
-            df: undefined
+            csv: ''
           });
         });
 

package/bin/utils/utils.js

@@ -277,7 +277,7 @@ async function runScript(script, path, verbose = false) {
     if (stdout && verbose) console.log(`Output: ${stdout}`);
   } catch (error) {
     const output = [error.stdout, error.stderr].filter(Boolean).join('\n');
-    throw new Error(output);
+    throw new Error(output || `Command failed: ${script} (exit code ${error.code ?? 'unknown'})`);
   }
 }
 function setHost(host, configFile, quiet = false) {

package/config-template.yaml

@@ -2,9 +2,11 @@ servers:
   dev:
     url: 'https://dev.datagrok.ai/api'
     key: ''
+    registry: 'registry.dev.datagrok.ai'
   public:
     url: 'https://public.datagrok.ai/api'
     key: ''
+    registry: 'registry.public.datagrok.ai'
   local:
     url: 'http://127.0.0.1:8080/api'
     key: ''

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "6.0.8",
+  "version": "6.1.0",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {