datagrok-tools 6.4.7 → 6.4.8

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Datagrok-tools changelog
 
+## 6.4.8 (2026-07-22)
+
+* GROK-20452: `grok publish` — always `docker build` the tools-generated worker dirs (`dockerfiles/queue`, `dockerfiles/celery`, `dockerfiles/<app>-celery`) instead of reusing a cached local tag. The "found local image" shortcut froze the worker at whatever stock base the daemon had when the tag was first built (CI ran day-old worker code with a fresh base available); the layer cache keeps an unchanged rebuild near-instant.
+
 ## 6.4.7 (2026-07-22)
 
 * GROK-20452: `grok publish` — generate `dockerfiles/celery` / `dockerfiles/queue` **before** gathering files for the zip. They were generated after, so the generated Dockerfile never reached the server: `meta.queue` functions fell back to a server-side container named `<pkg>-queue-celery` while the client had built `<pkg>-queue` — image validation 404'd and the worker container never started ("Container is not started" on every call).

package/bin/commands/docker-gen.js

@@ -11,6 +11,6 @@ async function dockerGen(args) {
   color.setVerbose(args.verbose || args.v || false);
   const packageDir = process.cwd();
   const generated = (0, _pythonCeleryGen.generateCeleryArtifacts)(packageDir);
-  if (generated) color.success('Generated Celery Docker artifacts');else console.log('No python/ directory with annotated functions found');
+  if (generated.length) color.success('Generated Celery Docker artifacts');else console.log('No python/ directory with annotated functions found');
   return true;
 }

package/bin/commands/publish.js

@@ -284,7 +284,7 @@ async function resolveLatestCompatible(host, devKey, dockerName, version, conten
     };
   }
 }
-async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps, debug, skipDockerRebuild = false) {
+async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps, debug, skipDockerRebuild = false, generatedDirs = []) {
   const dockerImages = discoverDockerfiles(packageName, version, debug);
   if (dockerImages.length === 0) return;
   color.log(`Found ${dockerImages.length} Dockerfile(s)`);
@@ -317,6 +317,13 @@ async function processDockerImages(packageName, version, registry, devKey, host,
       dockerRemove(img.fullLocalName);
       if (registry) dockerRemove(`${registry}/datagrok/${remoteFullName}`);
       result = buildAndPush() ?? (await fallbackImage(img, host, devKey, registry, version, contentHash));
+    } else if (generatedDirs.includes(img.dirName)) {
+      // Generated worker dirs are just FROM the stock base: a cached local tag
+      // pins whatever base the daemon had when it was first built (CI served a
+      // day-old worker this way). Always run docker build — the layer cache
+      // makes an unchanged rebuild near-instant, and a refreshed base lands.
+      color.log(`Rebuilding generated image ${img.fullLocalName} against the current base...`);
+      result = buildAndPush() ?? (await fallbackImage(img, host, devKey, registry, version, contentHash));
     } else {
       // Look for registry-qualified image first, then unqualified
       let foundLocalName = null;
@@ -473,8 +480,7 @@ async function processPackage(debug, rebuild, host, devKey, packageName, dropDb,
   // never binds the queue funcs to the client-built image and falls back to a
   // differently-named server-side container that never starts
   // (<pkg>-queue-celery vs the built <pkg>-queue).
-  (0, _pythonCeleryGen.generateCeleryArtifacts)(curDir);
-  (0, _queueWorkerGen.generateQueueArtifacts)(curDir);
+  const generatedDockerDirs = [...(0, _pythonCeleryGen.generateCeleryArtifacts)(curDir), ...(0, _queueWorkerGen.generateQueueArtifacts)(curDir)];
 
   // Gather the files
   const localTimestamps = {};
@@ -572,7 +578,7 @@ async function processPackage(debug, rebuild, host, devKey, packageName, dropDb,
     const userInfo = await getUserLogin(host, devKey);
     if (userInfo) dockerVersion = userInfo.login;
   }
-  await processDockerImages(packageName, dockerVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps, debug, skipDockerRebuild ?? false);
+  await processDockerImages(packageName, dockerVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps, debug, skipDockerRebuild ?? false, generatedDockerDirs);
   zip.append(JSON.stringify(localTimestamps), {
     name: 'timestamps.json'
   });

package/bin/utils/python-celery-gen.js

@@ -230,24 +230,27 @@ function deployFolder(packageDir, folderPath, dockerSubfolder, base) {
   color.log(`Generated Celery Docker artifacts in dockerfiles/${dockerSubfolder}/`);
   return true;
 }
+
+/** Returns the generated dockerfiles/<dir> names (empty when nothing generated). */
 function generateCeleryArtifacts(packageDir) {
   const pythonDir = _path.default.join(packageDir, 'python');
-  if (!_fs.default.existsSync(pythonDir)) return false;
+  if (!_fs.default.existsSync(pythonDir)) return [];
   const packageJson = JSON.parse(_fs.default.readFileSync(_path.default.join(packageDir, 'package.json'), 'utf-8'));
   const base = (0, _utils.removeScope)(packageJson.name).toLowerCase();
   const entries = _fs.default.readdirSync(pythonDir, {
     withFileTypes: true
   });
   const isNested = entries.length > 0 && entries.every(e => e.isDirectory());
-  let generated = false;
+  const generated = [];
   if (isNested) {
     for (const entry of entries) {
       if (!entry.isDirectory()) continue;
       const folderPath = _path.default.join(pythonDir, entry.name);
-      if (deployFolder(packageDir, folderPath, `${entry.name.toLowerCase()}-celery`, base)) generated = true;
+      const subfolder = `${entry.name.toLowerCase()}-celery`;
+      if (deployFolder(packageDir, folderPath, subfolder, base)) generated.push(subfolder);
     }
   } else {
-    if (deployFolder(packageDir, pythonDir, 'celery', base)) generated = true;
+    if (deployFolder(packageDir, pythonDir, 'celery', base)) generated.push('celery');
   }
   return generated;
 }

package/bin/utils/queue-worker-gen.js

@@ -21,11 +21,12 @@ EXPOSE 8000
 /** Generates dockerfiles/queue/ for packages with \`meta.queue: true\` functions so the
  *  standard docker build/push/image.json flow covers the auto-created Node worker
  *  container (mirror of generateCeleryArtifacts for python/). The worker fetches the
- *  package bundle at runtime, so the image is just the stock worker base. */
+ *  package bundle at runtime, so the image is just the stock worker base.
+ *  Returns the generated dockerfiles/<dir> names (empty when nothing generated). */
 function generateQueueArtifacts(packageDir) {
   const candidates = ['package.ts', 'package.g.ts'].map(f => _path.default.join(packageDir, 'src', f)).filter(f => _fs.default.existsSync(f));
   const hasQueueFuncs = candidates.some(f => queueTrueRegex.test(_fs.default.readFileSync(f, 'utf-8')));
-  if (!hasQueueFuncs) return false;
+  if (!hasQueueFuncs) return [];
   const dockerfilesDir = _path.default.join(packageDir, 'dockerfiles', QUEUE_DIR);
   const dockerfilePath = _path.default.join(dockerfilesDir, 'Dockerfile');
   _fs.default.mkdirSync(dockerfilesDir, {
@@ -38,5 +39,5 @@ function generateQueueArtifacts(packageDir) {
   const containerJsonDest = _path.default.join(dockerfilesDir, 'container.json');
   if (_fs.default.existsSync(containerJsonSrc) && !_fs.default.existsSync(containerJsonDest)) _fs.default.copyFileSync(containerJsonSrc, containerJsonDest);
   color.log(`Generated Node worker Docker artifacts in dockerfiles/${QUEUE_DIR}/`);
-  return true;
+  return [QUEUE_DIR];
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "6.4.7",
+  "version": "6.4.8",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {