datagrok-tools 6.4.4 → 6.4.6

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Datagrok-tools changelog
 
+## 6.4.6 (2026-07-22)
+
+* `grok test` — the whole-run Puppeteer cap (60 min) is now overridable via `GROK_TEST_INVOCATION_TIMEOUT_MS`. When the cap fires mid-pass all collected results are discarded (empty `test-report.csv`, "Passed tests: 0"), which CI reports as "no results produced"; loaded CI agents can now raise the cap instead.
+* `grok test` — `--no-retry` is now honored for Playwright runs. minimist parsed `--no-retry` as `{retry:false}`, so the flag was silently dropped and failed specs were still retried once; normalized so `--retries=0` reaches Playwright.
+
+## 6.4.5 (2026-06-23)
+
+* `grok test` — screen recording (`--record`) is now optional: if `page.screencast()` can't start (e.g. `ffmpeg` is missing on the runner, `spawnSync ffmpeg ENOENT`), the run warns and continues instead of failing the whole Puppeteer pass with 0 tests executed.
+
 ## 6.4.4 (2026-06-22)
 
 * `grok publish` — fixed every publish failing with a silent `exit 1` after a successful upload: a stray `fs.unlinkSync('zip')` threw `ENOENT` (the archive is streamed in-memory, no `zip` file is ever written), and the surrounding `catch` only logged under `--verbose`. The errant unlink is removed and publish errors are now always surfaced.

package/bin/commands/publish.js

@@ -19,6 +19,7 @@ var utils = _interopRequireWildcard(require("../utils/utils"));
 var color = _interopRequireWildcard(require("../utils/color-utils"));
 var _check = require("./check");
 var _pythonCeleryGen = require("../utils/python-celery-gen");
+var _queueWorkerGen = require("../utils/queue-worker-gen");
 function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
 // @ts-ignore
 
@@ -560,6 +561,9 @@ async function processPackage(debug, rebuild, host, devKey, packageName, dropDb,
   // Generate Celery Docker artifacts from python/ if present
   (0, _pythonCeleryGen.generateCeleryArtifacts)(curDir);
 
+  // Generate the Node worker container for meta.queue JS functions if present
+  (0, _queueWorkerGen.generateQueueArtifacts)(curDir);
+
   // Process Docker images and inject image.json into the ZIP
   let dockerVersion = json.version;
   if (debug) {

package/bin/commands/test.js

@@ -25,7 +25,10 @@ function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r
 
 const execAsync = (0, _util.promisify)(_child_process.exec);
 const execFileAsync = (0, _util.promisify)(_child_process.execFile);
-const testInvocationTimeout = 3600000;
+// Whole-run cap for the Puppeteer pass; when it fires, results collected so far
+// are discarded (they live in the page until a pass completes). CI runners on
+// loaded agents can exceed the 1h default — override via env.
+const testInvocationTimeout = parseInt(process.env['GROK_TEST_INVOCATION_TIMEOUT_MS'] ?? '', 10) || 3600000;
 const availableCommandOptions = ['host', 'package', 'csv', 'gui', 'catchUnhandled', 'platform', 'core', 'report', 'skip-build', 'skip-publish', 'path', 'record', 'verbose', 'benchmark', 'category', 'test', 'stress-test', 'link', 'tag', 'ci-cd', 'debug', 'no-retry', 'dartium', 'f', 'params', 'logfailed', 'skip-playwright', 'skip-puppeteer'];
 const curDir = process.cwd();
 

package/bin/grok.js

@@ -3,6 +3,10 @@ const argv = require('minimist')(process.argv.slice(2), {
   alias: {k: 'key', h: 'help', r: 'recursive'},
   boolean: ['dartium'],
 });
+// minimist maps `--no-retry` to `{retry: false}`, so the `args['no-retry']` checks in
+// test.ts / playwright-runner.ts never fired and `--no-retry` was silently ignored
+// (Playwright kept retrying failed specs). Normalize back to the flag the commands read.
+if (argv.retry === false) argv['no-retry'] = true;
 const help = require('./commands/help').help;
 const runAllCommand = require('./utils/utils').runAll;
 

package/bin/utils/queue-worker-gen.js

@@ -0,0 +1,41 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.generateQueueArtifacts = generateQueueArtifacts;
+var _fs = _interopRequireDefault(require("fs"));
+var _path = _interopRequireDefault(require("path"));
+var color = _interopRequireWildcard(require("./color-utils"));
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+// Matches `//meta.queue: true` in package.ts — functions executed as celery tasks
+// in the Node worker (see help/develop/how-to/packages/js-server-functions.md).
+const queueTrueRegex = /^\s*\/\/\s*meta\.queue\s*:\s*true\s*$/m;
+const QUEUE_DIR = 'queue';
+const TEMPLATE_NODE_DOCKER = `FROM datagrok/celery_node_worker:bleeding-edge
+EXPOSE 8000
+`;
+
+/** Generates dockerfiles/queue/ for packages with \`meta.queue: true\` functions so the
+ *  standard docker build/push/image.json flow covers the auto-created Node worker
+ *  container (mirror of generateCeleryArtifacts for python/). The worker fetches the
+ *  package bundle at runtime, so the image is just the stock worker base. */
+function generateQueueArtifacts(packageDir) {
+  const candidates = ['package.ts', 'package.g.ts'].map(f => _path.default.join(packageDir, 'src', f)).filter(f => _fs.default.existsSync(f));
+  const hasQueueFuncs = candidates.some(f => queueTrueRegex.test(_fs.default.readFileSync(f, 'utf-8')));
+  if (!hasQueueFuncs) return false;
+  const dockerfilesDir = _path.default.join(packageDir, 'dockerfiles', QUEUE_DIR);
+  const dockerfilePath = _path.default.join(dockerfilesDir, 'Dockerfile');
+  _fs.default.mkdirSync(dockerfilesDir, {
+    recursive: true
+  });
+  if (!_fs.default.existsSync(dockerfilePath)) _fs.default.writeFileSync(dockerfilePath, TEMPLATE_NODE_DOCKER);
+
+  // Resource config lives in the reserved queue/ package folder
+  const containerJsonSrc = _path.default.join(packageDir, QUEUE_DIR, 'container.json');
+  const containerJsonDest = _path.default.join(dockerfilesDir, 'container.json');
+  if (_fs.default.existsSync(containerJsonSrc) && !_fs.default.existsSync(containerJsonDest)) _fs.default.copyFileSync(containerJsonSrc, containerJsonDest);
+  color.log(`Generated Node worker Docker artifacts in dockerfiles/${QUEUE_DIR}/`);
+  return true;
+}

package/bin/utils/test-utils.js

@@ -46,17 +46,44 @@ const defaultLaunchParameters = exports.defaultLaunchParameters = {
   protocolTimeout: 0
 };
 async function getToken(url, key) {
-  let response;
-  try {
-    response = await fetch(`${url}/users/login/dev/${key}`, {
-      method: 'POST'
-    });
-  } catch (error) {
-    if (utils.isConnectivityError(error)) color.error(`Server is possibly offline: ${url}`);
-    throw error;
+  // Retry transient failures: on a freshly-provisioned CI stack the datlas behind nginx may not
+  // be serving /api as JSON yet, so the dev-key exchange briefly returns an HTML error page
+  // (`Unexpected token '<'` from response.json()) or the node is momentarily unreachable. These
+  // are stack-readiness races, not bad credentials — re-poll for ~45s before giving up. A real
+  // auth failure (valid JSON with isSuccess=false) is returned immediately, no retry.
+  const maxAttempts = 15;
+  const delayMs = 3000;
+  let lastError;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      const response = await fetch(`${url}/users/login/dev/${key}`, {
+        method: 'POST'
+      });
+      const text = await response.text();
+      let json;
+      try {
+        json = JSON.parse(text);
+      } catch {
+        // Non-JSON body (HTML error page) => stack not ready yet; retry.
+        lastError = new Error(`non-JSON response from ${url}/users/login/dev (status ${response.status})`);
+        if (attempt < maxAttempts) {
+          color.warn(`Playwright: token exchange not ready (attempt ${attempt}/${maxAttempts}), retrying...`);
+          await new Promise(r => setTimeout(r, delayMs));
+          continue;
+        }
+        throw lastError;
+      }
+      if (json.isSuccess == true) return json.token;
+      // Valid JSON but login rejected => definitive failure, do not retry.
+      throw new Error('Unable to login to server. Check your dev key');
+    } catch (error) {
+      if (error?.message === 'Unable to login to server. Check your dev key') throw error;
+      lastError = error;
+      if (utils.isConnectivityError(error)) color.warn(`Playwright: server not reachable yet (attempt ${attempt}/${maxAttempts}): ${url}`);
+      if (attempt < maxAttempts) await new Promise(r => setTimeout(r, delayMs));
+    }
   }
-  const json = await response.json();
-  if (json.isSuccess == true) return json.token;else throw new Error('Unable to login to server. Check your dev key');
+  throw lastError ?? new Error(`Unable to exchange dev key for token at ${url}`);
 }
 async function isPackageOnServer(hostKey, packageName) {
   try {
@@ -256,9 +283,14 @@ async function loadTestsList(packages, core = false, record = false) {
       const suffix = process.env.BACKUP_SIZE && process.env.WORKER_ID && process.env.TOTAL_WORKERS ? `_${process.env.BACKUP_SIZE}_${process.env.WORKER_ID}_${process.env.TOTAL_WORKERS}` : '';
       const logsDir = `./load-test-console-output${suffix}.log`;
       const recordDir = `./load-test-record${suffix}.mp4`;
-      recorder = await page.screencast({
-        path: recordDir
-      });
+      try {
+        recorder = await page.screencast({
+          path: recordDir
+        });
+      } catch (e) {
+        recorder = null;
+        color.warn(`Screen recording disabled: ${e?.message || e}`);
+      }
       await page.exposeFunction('addLogsToFile', addLogsToFile);
       _fs.default.writeFileSync(logsDir, ``);
       page.on('console', msg => {
@@ -636,10 +668,17 @@ async function runBrowser(testExecutionData, browserOptions, browsersId, testInv
     const logsDir = `./test-console-output-${currentBrowserNum}.log`;
     const recordDir = `./test-record-${currentBrowserNum}.mp4`;
     if (browserOptions.record && !existingBrowserSession) {
-      // Only set up recording on initial browser creation, not on retry
-      recorder = await page.screencast({
-        path: recordDir
-      });
+      // Only set up recording on initial browser creation, not on retry.
+      // Recording is an optional diagnostic — if it can't start (e.g. ffmpeg is
+      // missing on the runner), warn and keep testing instead of failing the pass.
+      try {
+        recorder = await page.screencast({
+          path: recordDir
+        });
+      } catch (e) {
+        recorder = null;
+        color.warn(`Screen recording disabled: ${e?.message || e}`);
+      }
       await page.exposeFunction('addLogsToFile', addLogsToFile);
       _fs.default.writeFileSync(logsDir, ``);
       page.on('console', msg => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "6.4.4",
+  "version": "6.4.6",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {

package/script-template/node.js

@@ -4,4 +4,4 @@
 //sample: cars.csv
 //input: dataframe table [Data table]
 //output: int count [Number of cells in table]
-const count = table.dim()[0] * table.dim()[1];
+const count = table.rowCount * table.columns.length;