datagrok-tools 6.1.1 → 6.1.4

package/bin/commands/help.js

@@ -158,16 +158,17 @@ Uploads a package
 Checks for errors before publishing — the package won't be published if there are any.
 
 Options:
-[--all] [--refresh] [--link] [--build] [--release] [--rebuild-docker] [--skip-check] [-v | --verbose]
-
---all             Publish all available packages (run in packages directory)
---refresh         Publish all available already loaded packages (run in packages directory)
---link            Link the package to local packages
---build           Builds the package
---release         Publish package as release version
---rebuild-docker  Force rebuild Docker images locally before pushing to registry
---skip-check      Skip check stage
---verbose         Print detailed output
+[--all] [--refresh] [--link] [--build] [--release] [--rebuild-docker] [--skip-docker-rebuild] [--skip-check] [-v | --verbose]
+
+--all                  Publish all available packages (run in packages directory)
+--refresh              Publish all available already loaded packages (run in packages directory)
+--link                 Link the package to local packages
+--build                Builds the package
+--release              Publish package as release version
+--rebuild-docker       Force rebuild Docker images locally before pushing to registry
+--skip-docker-rebuild  Skip auto-rebuild when Dockerfile folder has changed
+--skip-check           Skip check stage
+--verbose              Print detailed output
 
 Running \`grok publish\` is the same as running \`grok publish defaultHost --build --debug\`
 `;

package/bin/commands/publish.js

@@ -7,6 +7,7 @@ Object.defineProperty(exports, "__esModule", {
 exports.processPackage = processPackage;
 exports.publish = publish;
 var _archiverPromise = _interopRequireDefault(require("archiver-promise"));
+var _crypto = _interopRequireDefault(require("crypto"));
 var _fs = _interopRequireDefault(require("fs"));
 var _nodeFetch = _interopRequireDefault(require("node-fetch"));
 var _os = _interopRequireDefault(require("os"));
@@ -49,7 +50,7 @@ function discoverDockerfiles(packageName, version, debug) {
     if (!entry.isDirectory()) continue;
     const dockerfilePath = _path.default.join(dockerfilesDir, entry.name, 'Dockerfile');
     if (!_fs.default.existsSync(dockerfilePath)) continue;
-    const cleanName = utils.removeScope(packageName).replace(/-/g, '').toLowerCase();
+    const cleanName = utils.removeScope(packageName).toLowerCase();
     const imageName = `${cleanName}-${entry.name.toLowerCase()}`;
     const imageTag = debug ? version : `${version}.X`;
     results.push({
@@ -95,7 +96,10 @@ function dockerTag(source, target) {
 }
 function dockerPush(image) {
   try {
-    dockerCommand(`push ${image}`);
+    execSync(`docker push ${image}`, {
+      encoding: 'utf-8',
+      stdio: 'inherit'
+    });
     return true;
   } catch (e) {
     color.error(`Failed to push ${image}: ${e.message || e}`);
@@ -115,6 +119,50 @@ function dockerBuild(imageName, dockerfilePath, context) {
     return false;
   }
 }
+function calculateFolderHash(dirPath) {
+  const hash = _crypto.default.createHash('sha256');
+  const entries = listRecursive(dirPath, '');
+  entries.sort((a, b) => a.relPath < b.relPath ? -1 : a.relPath > b.relPath ? 1 : 0);
+  for (const entry of entries) {
+    if (entry.isDir) {
+      color.log(`  Hash entry: dir:${entry.relPath}:`);
+      hash.update(`dir:${entry.relPath}:`);
+    } else {
+      // Normalize CRLF to LF to match server-side storage
+      const raw = _fs.default.readFileSync(entry.fullPath);
+      const content = raw.includes(0x0d) && !raw.includes(0x00) ? Buffer.from(raw.toString('utf8').replace(/\r\n/g, '\n'), 'utf8') : raw;
+      color.log(`  Hash entry: file:${entry.relPath}: (${content.length} bytes)`);
+      hash.update(`file:${entry.relPath}:`);
+      hash.update(content);
+    }
+  }
+  const result = hash.digest('hex');
+  color.log(`  Folder hash: ${result}`);
+  return result;
+}
+function listRecursive(basePath, rel) {
+  const results = [];
+  const fullDir = rel ? _path.default.join(basePath, rel) : basePath;
+  for (const entry of _fs.default.readdirSync(fullDir, {
+    withFileTypes: true
+  })) {
+    const relPath = rel ? `${rel}/${entry.name}` : entry.name;
+    const fullPath = _path.default.join(fullDir, entry.name);
+    if (entry.isDirectory()) {
+      results.push({
+        relPath,
+        fullPath,
+        isDir: true
+      });
+      results.push(...listRecursive(basePath, relPath));
+    } else results.push({
+      relPath,
+      fullPath,
+      isDir: false
+    });
+  }
+  return results;
+}
 async function getUserLogin(host, devKey) {
   let loginResp;
   try {
@@ -144,14 +192,18 @@ async function getUserLogin(host, devKey) {
     return null;
   }
 }
-async function resolveLatestCompatible(host, devKey, dockerName) {
+async function resolveLatestCompatible(host, devKey, dockerName, version, contentHash) {
   const userInfo = await getUserLogin(host, devKey);
   if (!userInfo) return {
     found: null,
     serverError: `Authentication failed. Check your developer key.`
   };
   try {
-    const url = `${host}/docker/images/${encodeURIComponent(dockerName)}/latest-compatible`;
+    let url = `${host}/docker/images/${encodeURIComponent(dockerName)}/latest-compatible`;
+    const params = [];
+    if (version) params.push(`version=${encodeURIComponent(version)}`);
+    if (contentHash) params.push(`contentHash=${encodeURIComponent(contentHash)}`);
+    if (params.length > 0) url += '?' + params.join('&');
     const resp = await (0, _nodeFetch.default)(url, {
       headers: {
         'Authorization': userInfo.token
@@ -176,29 +228,31 @@ async function resolveLatestCompatible(host, devKey, dockerName) {
     };
   }
 }
-async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps, debug) {
+async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps, debug, skipDockerRebuild = false) {
   const dockerImages = discoverDockerfiles(packageName, version, debug);
   if (dockerImages.length === 0) return;
   color.log(`Found ${dockerImages.length} Dockerfile(s)`);
   if (registry) dockerLogin(registry, devKey);
   for (const img of dockerImages) {
+    color.info(`Processing docker image ${img.fullLocalName}...`);
     let result;
+    const dockerfileDir = _path.default.join('dockerfiles', img.dirName);
+    const dockerfilePath = _path.default.join(dockerfileDir, 'Dockerfile');
+    const contentHash = calculateFolderHash(_path.default.join(curDir, dockerfileDir));
     if (rebuildDocker) {
-      const dockerfileDir = _path.default.join('dockerfiles', img.dirName);
-      const dockerfilePath = _path.default.join(dockerfileDir, 'Dockerfile');
       if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
         result = pushImage(img, registry);
         color.success(`Built and tagged ${img.fullLocalName}`);
       } else {
-        result = await fallbackImage(img, host, devKey, registry);
+        result = await fallbackImage(img, host, devKey, registry, version, contentHash);
       }
     } else if (localImageExists(img.fullLocalName)) {
-      result = pushImage(img, registry);
       color.success(`Found local image ${img.fullLocalName}`);
+      result = pushImage(img, registry);
     } else {
       color.warn(`Local image not found. Expected: ${img.fullLocalName}`);
-      color.log(`  Build it with: docker build -t ${img.fullLocalName} -f dockerfiles/${img.dirName}/Dockerfile dockerfiles/${img.dirName}`);
-      const fallback = await fallbackImage(img, host, devKey, registry);
+      color.log(`  Build it with: docker build -t ${img.fullLocalName} -f ${dockerfilePath} ${dockerfileDir}`);
+      const fallback = await fallbackImage(img, host, devKey, registry, version, contentHash);
       if (fallback.serverError) {
         color.error(`Cannot resolve fallback: ${fallback.serverError}`);
         result = {
@@ -206,14 +260,25 @@ async function processDockerImages(packageName, version, registry, devKey, host,
           fallback: true,
           requestedVersion: img.imageTag
         };
-      } else if (fallback.image) {
+      } else if (fallback.image && fallback.hashMatch === true) {
         result = fallback;
-        color.warn(`Falling back to ${fallback.image}`);
+        color.success(`Falling back to ${fallback.image} (dockerfile unchanged)`);
+      } else if (fallback.image && fallback.hashMatch === false && !skipDockerRebuild) {
+        color.warn(`Dockerfile folder has changed. Rebuilding image...`);
+        if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
+          result = pushImage(img, registry);
+          color.success(`Built and tagged ${img.fullLocalName}`);
+        } else {
+          result = {
+            image: fallback.image,
+            fallback: true,
+            requestedVersion: img.imageTag
+          };
+          color.warn(`Build failed. Falling back to ${fallback.image} (hash mismatch)`);
+        }
       } else {
         // No fallback and no local image — must build
         color.warn(`No fallback available. Building ${img.fullLocalName}...`);
-        const dockerfileDir = _path.default.join('dockerfiles', img.dirName);
-        const dockerfilePath = _path.default.join(dockerfileDir, 'Dockerfile');
         if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
           result = pushImage(img, registry);
           color.success(`Built and tagged ${img.fullLocalName}`);
@@ -256,11 +321,11 @@ function pushImage(img, registry) {
     fallback: false
   };
 }
-async function fallbackImage(img, host, devKey, registry) {
+async function fallbackImage(img, host, devKey, registry, version, contentHash) {
   const {
     found,
     serverError
-  } = await resolveLatestCompatible(host, devKey, img.imageName);
+  } = await resolveLatestCompatible(host, devKey, img.imageName, version, contentHash);
   if (serverError) return {
     image: null,
     fallback: true,
@@ -270,7 +335,8 @@ async function fallbackImage(img, host, devKey, registry) {
   if (found) return {
     image: found.image,
     fallback: true,
-    requestedVersion: img.imageTag
+    requestedVersion: img.imageTag,
+    hashMatch: found.hashMatch
   };
   return {
     image: null,
@@ -278,7 +344,7 @@ async function fallbackImage(img, host, devKey, registry) {
     requestedVersion: img.imageTag
   };
 }
-async function processPackage(debug, rebuild, host, devKey, packageName, dropDb, suffix, hostAlias, registry, rebuildDocker) {
+async function processPackage(debug, rebuild, host, devKey, packageName, dropDb, suffix, hostAlias, registry, rebuildDocker, skipDockerRebuild) {
   // Validate server connectivity and dev key
   let timestamps = {};
   let url = `${host}/packages/dev/${devKey}/${packageName}`;
@@ -400,7 +466,7 @@ async function processPackage(debug, rebuild, host, devKey, packageName, dropDb,
     const userInfo = await getUserLogin(host, devKey);
     if (userInfo) dockerVersion = userInfo.login;
   }
-  await processDockerImages(packageName, dockerVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps, debug);
+  await processDockerImages(packageName, dockerVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps, debug, skipDockerRebuild ?? false);
   zip.append(JSON.stringify(localTimestamps), {
     name: 'timestamps.json'
   });
@@ -545,7 +611,7 @@ async function publishPackage(args) {
         if (!args.suffix && stdout) args.suffix = stdout.toString().substring(0, 8);
       });
       await utils.delay(100);
-      code = await processPackage(!args.release, Boolean(args.rebuild), url, key, packageName, args.dropDb ?? false, args.suffix, host, registry, args['rebuild-docker']);
+      code = await processPackage(!args.release, Boolean(args.rebuild), url, key, packageName, args.dropDb ?? false, args.suffix, host, registry, args['rebuild-docker'], args['skip-docker-rebuild']);
     } catch (error) {
       console.error(error);
       code = 1;

package/bin/grok.js

@@ -27,6 +27,8 @@ const commands = {
 const onPackageCommandNames = ['api', 'check', 'link', 'publish', 'test'];
 
 const command = argv['_'][0];
+if (command !== 'test' && command !== 'stresstest')
+  delete argv.dartium;
 if (command in commands) {
   try {
     if (argv['help']) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "6.1.1",
+  "version": "6.1.4",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {