datagrok-tools 6.4.6 → 6.4.8

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Datagrok-tools changelog
 
+## 6.4.8 (2026-07-22)
+
+* GROK-20452: `grok publish` — always `docker build` the tools-generated worker dirs (`dockerfiles/queue`, `dockerfiles/celery`, `dockerfiles/<app>-celery`) instead of reusing a cached local tag. The "found local image" shortcut froze the worker at whatever stock base the daemon had when the tag was first built (CI ran day-old worker code with a fresh base available); the layer cache keeps an unchanged rebuild near-instant.
+
+## 6.4.7 (2026-07-22)
+
+* GROK-20452: `grok publish` — generate `dockerfiles/celery` / `dockerfiles/queue` **before** gathering files for the zip. They were generated after, so the generated Dockerfile never reached the server: `meta.queue` functions fell back to a server-side container named `<pkg>-queue-celery` while the client had built `<pkg>-queue` — image validation 404'd and the worker container never started ("Container is not started" on every call).
+
 ## 6.4.6 (2026-07-22)
 
 * `grok test` — the whole-run Puppeteer cap (60 min) is now overridable via `GROK_TEST_INVOCATION_TIMEOUT_MS`. When the cap fires mid-pass all collected results are discarded (empty `test-report.csv`, "Passed tests: 0"), which CI reports as "no results produced"; loaded CI agents can now raise the cap instead.

package/bin/commands/docker-gen.js

@@ -11,6 +11,6 @@ async function dockerGen(args) {
   color.setVerbose(args.verbose || args.v || false);
   const packageDir = process.cwd();
   const generated = (0, _pythonCeleryGen.generateCeleryArtifacts)(packageDir);
-  if (generated) color.success('Generated Celery Docker artifacts');else console.log('No python/ directory with annotated functions found');
+  if (generated.length) color.success('Generated Celery Docker artifacts');else console.log('No python/ directory with annotated functions found');
   return true;
 }

package/bin/commands/publish.js

@@ -284,7 +284,7 @@ async function resolveLatestCompatible(host, devKey, dockerName, version, conten
     };
   }
 }
-async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps, debug, skipDockerRebuild = false) {
+async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps, debug, skipDockerRebuild = false, generatedDirs = []) {
   const dockerImages = discoverDockerfiles(packageName, version, debug);
   if (dockerImages.length === 0) return;
   color.log(`Found ${dockerImages.length} Dockerfile(s)`);
@@ -317,6 +317,13 @@ async function processDockerImages(packageName, version, registry, devKey, host,
       dockerRemove(img.fullLocalName);
       if (registry) dockerRemove(`${registry}/datagrok/${remoteFullName}`);
       result = buildAndPush() ?? (await fallbackImage(img, host, devKey, registry, version, contentHash));
+    } else if (generatedDirs.includes(img.dirName)) {
+      // Generated worker dirs are just FROM the stock base: a cached local tag
+      // pins whatever base the daemon had when it was first built (CI served a
+      // day-old worker this way). Always run docker build — the layer cache
+      // makes an unchanged rebuild near-instant, and a refreshed base lands.
+      color.log(`Rebuilding generated image ${img.fullLocalName} against the current base...`);
+      result = buildAndPush() ?? (await fallbackImage(img, host, devKey, registry, version, contentHash));
     } else {
       // Look for registry-qualified image first, then unqualified
       let foundLocalName = null;
@@ -468,6 +475,13 @@ async function processPackage(debug, rebuild, host, devKey, packageName, dropDb,
   const chunks = [];
   zip.on('data', chunk => chunks.push(chunk));
 
+  // Generate Celery / Node-worker Docker artifacts BEFORE gathering files: the
+  // generated dockerfiles/<dir>/Dockerfile must land in the zip, or the server
+  // never binds the queue funcs to the client-built image and falls back to a
+  // differently-named server-side container that never starts
+  // (<pkg>-queue-celery vs the built <pkg>-queue).
+  const generatedDockerDirs = [...(0, _pythonCeleryGen.generateCeleryArtifacts)(curDir), ...(0, _queueWorkerGen.generateQueueArtifacts)(curDir)];
+
   // Gather the files
   const localTimestamps = {};
   const files = await (0, _ignoreWalk.default)({
@@ -558,19 +572,13 @@ async function processPackage(debug, rebuild, host, devKey, packageName, dropDb,
     return 1;
   }
 
-  // Generate Celery Docker artifacts from python/ if present
-  (0, _pythonCeleryGen.generateCeleryArtifacts)(curDir);
-
-  // Generate the Node worker container for meta.queue JS functions if present
-  (0, _queueWorkerGen.generateQueueArtifacts)(curDir);
-
   // Process Docker images and inject image.json into the ZIP
   let dockerVersion = json.version;
   if (debug) {
     const userInfo = await getUserLogin(host, devKey);
     if (userInfo) dockerVersion = userInfo.login;
   }
-  await processDockerImages(packageName, dockerVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps, debug, skipDockerRebuild ?? false);
+  await processDockerImages(packageName, dockerVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps, debug, skipDockerRebuild ?? false, generatedDockerDirs);
   zip.append(JSON.stringify(localTimestamps), {
     name: 'timestamps.json'
   });

package/bin/utils/python-celery-gen.js

@@ -230,24 +230,27 @@ function deployFolder(packageDir, folderPath, dockerSubfolder, base) {
   color.log(`Generated Celery Docker artifacts in dockerfiles/${dockerSubfolder}/`);
   return true;
 }
+
+/** Returns the generated dockerfiles/<dir> names (empty when nothing generated). */
 function generateCeleryArtifacts(packageDir) {
   const pythonDir = _path.default.join(packageDir, 'python');
-  if (!_fs.default.existsSync(pythonDir)) return false;
+  if (!_fs.default.existsSync(pythonDir)) return [];
   const packageJson = JSON.parse(_fs.default.readFileSync(_path.default.join(packageDir, 'package.json'), 'utf-8'));
   const base = (0, _utils.removeScope)(packageJson.name).toLowerCase();
   const entries = _fs.default.readdirSync(pythonDir, {
     withFileTypes: true
   });
   const isNested = entries.length > 0 && entries.every(e => e.isDirectory());
-  let generated = false;
+  const generated = [];
   if (isNested) {
     for (const entry of entries) {
       if (!entry.isDirectory()) continue;
       const folderPath = _path.default.join(pythonDir, entry.name);
-      if (deployFolder(packageDir, folderPath, `${entry.name.toLowerCase()}-celery`, base)) generated = true;
+      const subfolder = `${entry.name.toLowerCase()}-celery`;
+      if (deployFolder(packageDir, folderPath, subfolder, base)) generated.push(subfolder);
     }
   } else {
-    if (deployFolder(packageDir, pythonDir, 'celery', base)) generated = true;
+    if (deployFolder(packageDir, pythonDir, 'celery', base)) generated.push('celery');
   }
   return generated;
 }

package/bin/utils/queue-worker-gen.js

@@ -9,9 +9,10 @@ var _fs = _interopRequireDefault(require("fs"));
 var _path = _interopRequireDefault(require("path"));
 var color = _interopRequireWildcard(require("./color-utils"));
 function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
-// Matches `//meta.queue: true` in package.ts — functions executed as celery tasks
-// in the Node worker (see help/develop/how-to/packages/js-server-functions.md).
-const queueTrueRegex = /^\s*\/\/\s*meta\.queue\s*:\s*true\s*$/m;
+// Matches `//meta.queue: true` (or its alias `//meta.server: true`) in package.ts —
+// functions executed as celery tasks in the Node worker
+// (see help/develop/how-to/packages/js-server-functions.md).
+const queueTrueRegex = /^\s*\/\/\s*meta\.(queue|server)\s*:\s*true\s*$/m;
 const QUEUE_DIR = 'queue';
 const TEMPLATE_NODE_DOCKER = `FROM datagrok/celery_node_worker:bleeding-edge
 EXPOSE 8000
@@ -20,11 +21,12 @@ EXPOSE 8000
 /** Generates dockerfiles/queue/ for packages with \`meta.queue: true\` functions so the
  *  standard docker build/push/image.json flow covers the auto-created Node worker
  *  container (mirror of generateCeleryArtifacts for python/). The worker fetches the
- *  package bundle at runtime, so the image is just the stock worker base. */
+ *  package bundle at runtime, so the image is just the stock worker base.
+ *  Returns the generated dockerfiles/<dir> names (empty when nothing generated). */
 function generateQueueArtifacts(packageDir) {
   const candidates = ['package.ts', 'package.g.ts'].map(f => _path.default.join(packageDir, 'src', f)).filter(f => _fs.default.existsSync(f));
   const hasQueueFuncs = candidates.some(f => queueTrueRegex.test(_fs.default.readFileSync(f, 'utf-8')));
-  if (!hasQueueFuncs) return false;
+  if (!hasQueueFuncs) return [];
   const dockerfilesDir = _path.default.join(packageDir, 'dockerfiles', QUEUE_DIR);
   const dockerfilePath = _path.default.join(dockerfilesDir, 'Dockerfile');
   _fs.default.mkdirSync(dockerfilesDir, {
@@ -37,5 +39,5 @@ function generateQueueArtifacts(packageDir) {
   const containerJsonDest = _path.default.join(dockerfilesDir, 'container.json');
   if (_fs.default.existsSync(containerJsonSrc) && !_fs.default.existsSync(containerJsonDest)) _fs.default.copyFileSync(containerJsonSrc, containerJsonDest);
   color.log(`Generated Node worker Docker artifacts in dockerfiles/${QUEUE_DIR}/`);
-  return true;
+  return [QUEUE_DIR];
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "6.4.6",
+  "version": "6.4.8",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {