datagrok-tools 6.0.8 → 6.1.1

package/bin/commands/config.js

@@ -19,6 +19,15 @@ const confTemplate = _jsYaml.default.load(_fs.default.readFileSync(confTemplateD
 }));
 const grokDir = _path.default.join(_os.default.homedir(), '.grok');
 const confPath = _path.default.join(grokDir, 'config.yaml');
+function defaultRegistry(url) {
+  try {
+    const hostname = new URL(url).hostname;
+    if (hostname === 'localhost' || /^[\d.]+$/.test(hostname) || hostname === '::1') return '';
+    return `registry.${hostname}`;
+  } catch (e) {
+    return '';
+  }
+}
 function validateKey(key) {
   if (!key || /^([A-Za-z\d-])+$/.test(key)) return true;else return 'Developer key may only include letters, numbers, or hyphens';
 }
@@ -33,7 +42,7 @@ function generateKeyQ(server, url) {
   if (server.startsWith('local')) question.message = `Developer key for ${origin} (press ENTER to skip):`;
   return question;
 }
-async function addNewServer(config) {
+async function addNewServer(config, askRegistry = false) {
   while (true) {
     const addServer = (await _inquirer.default.prompt({
       name: 'add-server',
@@ -53,17 +62,29 @@ async function addNewServer(config) {
         message: 'Enter a URL:'
       }))['server-url'];
       const key = (await _inquirer.default.prompt(generateKeyQ(name, url)))[name];
-      config.servers[name] = {
+      const entry = {
         url,
         key
       };
+      if (askRegistry) {
+        const regDefault = defaultRegistry(url);
+        const reg = (await _inquirer.default.prompt({
+          name: 'registry',
+          type: 'input',
+          message: `Docker registry for ${name}:`,
+          default: regDefault || undefined
+        }))['registry'] || '';
+        if (reg) entry.registry = reg;
+      }
+      config.servers[name] = entry;
     } else break;
   }
 }
 function config(args) {
   const nOptions = Object.keys(args).length - 1;
-  const interactiveMode = args['_'].length === 1 && (nOptions < 1 || nOptions === 1 && args.reset);
-  const hasAddServerCommand = args['_'].length === 2 && args['_'][1] === 'add' && args.server && args.key && args.k && args.alias && (nOptions === 4 || nOptions === 5 && args.default);
+  const askRegistry = args.registry != null;
+  const interactiveMode = args['_'].length === 1 && (nOptions < 1 || nOptions === 1 && (args.reset || askRegistry) || nOptions === 2 && args.reset && askRegistry);
+  const hasAddServerCommand = args['_'].length === 2 && args['_'][1] === 'add' && args.server && args.key && args.k && args.alias && nOptions >= 4 && nOptions <= 6;
   if (!interactiveMode && !hasAddServerCommand) return false;
   if (!_fs.default.existsSync(grokDir)) _fs.default.mkdirSync(grokDir);
   if (!_fs.default.existsSync(confPath) || args.reset) _fs.default.writeFileSync(confPath, _jsYaml.default.dump(confTemplate));
@@ -77,10 +98,15 @@ function config(args) {
       color.error('URL parsing error. Please, provide a valid server URL.');
       return false;
     }
-    config.servers[args.alias] = {
+    const server = {
       url: args.server,
       key: args.key
     };
+    if (args.registry != null) {
+      const registry = typeof args.registry === 'string' ? args.registry : defaultRegistry(args.server);
+      if (registry) server.registry = registry;
+    }
+    config.servers[args.alias] = server;
     color.success(`Successfully added the server to ${confPath}.`);
     console.log(`Use this command to deploy packages: grok publish ${args.alias}`);
     if (args.default) config.default = args.alias;
@@ -110,8 +136,18 @@ function config(args) {
           question.default = config['servers'][server]['key'];
           const devKey = await _inquirer.default.prompt(question);
           config['servers'][server]['key'] = devKey[server];
+          if (askRegistry) {
+            const regDefault = config['servers'][server]['registry'] || defaultRegistry(url);
+            const reg = (await _inquirer.default.prompt({
+              name: 'registry',
+              type: 'input',
+              message: `Docker registry for ${server}:`,
+              default: regDefault || undefined
+            }))['registry'] || '';
+            if (reg) config['servers'][server]['registry'] = reg;else delete config['servers'][server]['registry'];
+          }
         }
-        await addNewServer(config);
+        await addNewServer(config, askRegistry);
         const defaultServer = await _inquirer.default.prompt({
           name: 'default-server',
           type: 'input',

package/bin/commands/docker-gen.js

@@ -0,0 +1,16 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.dockerGen = dockerGen;
+var _pythonCeleryGen = require("../utils/python-celery-gen");
+var color = _interopRequireWildcard(require("../utils/color-utils"));
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function dockerGen(args) {
+  color.setVerbose(args.verbose || args.v || false);
+  const packageDir = process.cwd();
+  const generated = (0, _pythonCeleryGen.generateCeleryArtifacts)(packageDir);
+  if (generated) color.success('Generated Celery Docker artifacts');else console.log('No python/ directory with annotated functions found');
+  return true;
+}

package/bin/commands/help.js

@@ -10,19 +10,20 @@ Usage: grok <command>
 Datagrok's package management tool
 
 Commands:
-    add       Add an object template
-    api       Create wrapper functions
-    build     Build a package or multiple packages
-    check     Check package content (function signatures, etc.)
-    claude    Launch Claude Code in a Datagrok dev container
-    config    Create and manage config files
-    create    Create a package
-    init      Modify a package template
-    link      Link \`datagrok-api\` and libraries for local development
-    publish   Upload a package
-    test      Run package tests
-    testall      Run packages tests
-    migrate   Migrate legacy tags to meta.role
+    add         Add an object template
+    api         Create wrapper functions
+    build       Build a package or multiple packages
+    check       Check package content (function signatures, etc.)
+    claude      Launch Claude Code in a Datagrok dev container
+    config      Create and manage config files
+    create      Create a package
+    docker-gen  Generate Celery Docker artifacts from Python functions
+    init        Modify a package template
+    link        Link \`datagrok-api\` and libraries for local development
+    publish     Upload a package
+    test        Run package tests
+    testall     Run packages tests
+    migrate     Migrate legacy tags to meta.role
 
 To get help on a particular command, use:
     grok <command> --help
@@ -122,13 +123,14 @@ Usage: grok config
 Create or update a configuration file
 
 Options:
-[--reset] [--server] [--alias] [-k | --key]
+[--reset] [--server] [--alias] [-k | --key] [--registry]
 
 --reset     Restore the default config file template
 --server    Use to add a server to the config (\`grok config add --alias alias --server url --key key\`)
 --alias     Use in conjunction with the \`server\` option to set the server name
 --key       Use in conjunction with the \`server\` option to set the developer key
 --default   Use in conjunction with the \`server\` option to set the added server as default
+--registry  Docker registry URL (default: registry.{server hostname})
 `;
 const HELP_CREATE = `
 Usage: grok create [name]
@@ -156,13 +158,14 @@ Uploads a package
 Checks for errors before publishing — the package won't be published if there are any.
 
 Options:
-[--all] [--refresh] [--link] [--build] [--release] [--skip-check] [-v | --verbose]
+[--all] [--refresh] [--link] [--build] [--release] [--rebuild-docker] [--skip-check] [-v | --verbose]
 
 --all             Publish all available packages (run in packages directory)
 --refresh         Publish all available already loaded packages (run in packages directory)
 --link            Link the package to local packages
 --build           Builds the package
 --release         Publish package as release version
+--rebuild-docker  Force rebuild Docker images locally before pushing to registry
 --skip-check      Skip check stage
 --verbose         Print detailed output
 
@@ -266,6 +269,17 @@ Options:
 --verbose           Prints detailed information about linked packages  
 --all               Links all available packages(run in packages directory)
 `;
+const HELP_DOCKER_GEN = `
+Usage: grok docker-gen
+
+Generate Celery Docker artifacts from annotated Python functions in the python/ directory.
+Produces Dockerfile, tasks.yaml, and Celery entry point in dockerfiles/<name>/.
+
+Options:
+[-v | --verbose]
+
+--verbose         Print detailed output
+`;
 const HELP_MIGRATE = `
 Usage: grok migrate
 
@@ -314,6 +328,7 @@ const help = exports.help = {
   claude: HELP_CLAUDE,
   config: HELP_CONFIG,
   create: HELP_CREATE,
+  'docker-gen': HELP_DOCKER_GEN,
   init: HELP_INIT,
   link: HELP_LINK,
   publish: HELP_PUBLISH,

package/bin/commands/publish.js

@@ -17,13 +17,15 @@ var _testUtils = require("../utils/test-utils");
 var utils = _interopRequireWildcard(require("../utils/utils"));
 var color = _interopRequireWildcard(require("../utils/color-utils"));
 var _check = require("./check");
+var _pythonCeleryGen = require("../utils/python-celery-gen");
 function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
 // @ts-ignore
 
 // @ts-ignore
 
 const {
-  exec
+  exec,
+  execSync
 } = require('child_process');
 const grokDir = _path.default.join(_os.default.homedir(), '.grok');
 const confPath = _path.default.join(grokDir, 'config.yaml');
@@ -35,22 +37,263 @@ let curDir = process.cwd();
 const packDir = _path.default.join(curDir, 'package.json');
 const packageFiles = ['src/package.ts', 'src/detectors.ts', 'src/package.js', 'src/detectors.js', 'src/package-test.ts', 'src/package-test.js', 'package.js', 'detectors.js'];
 let config;
-async function processPackage(debug, rebuild, host, devKey, packageName, dropDb, suffix, hostAlias) {
-  // Get the server timestamps
+const BLEEDING_EDGE_TAG = 'bleeding-edge';
+function discoverDockerfiles(packageName, version, debug) {
+  const dockerfilesDir = _path.default.join(curDir, 'dockerfiles');
+  if (!_fs.default.existsSync(dockerfilesDir)) return [];
+  const results = [];
+  const entries = _fs.default.readdirSync(dockerfilesDir, {
+    withFileTypes: true
+  });
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const dockerfilePath = _path.default.join(dockerfilesDir, entry.name, 'Dockerfile');
+    if (!_fs.default.existsSync(dockerfilePath)) continue;
+    const cleanName = utils.removeScope(packageName).replace(/-/g, '').toLowerCase();
+    const imageName = `${cleanName}-${entry.name.toLowerCase()}`;
+    const imageTag = debug ? version : `${version}.X`;
+    results.push({
+      imageName,
+      imageTag,
+      dirName: entry.name,
+      fullLocalName: `${imageName}:${imageTag}`
+    });
+  }
+  return results;
+}
+function dockerCommand(args) {
+  return execSync(`docker ${args}`, {
+    encoding: 'utf-8',
+    stdio: ['pipe', 'pipe', 'pipe']
+  }).trim();
+}
+function localImageExists(fullName) {
+  try {
+    const output = dockerCommand(`images --format "{{.Repository}}:{{.Tag}}"`);
+    return output.split('\n').some(line => line.trim() === fullName);
+  } catch {
+    return false;
+  }
+}
+function dockerLogin(registry, devKey) {
+  try {
+    dockerCommand(`login ${registry} -u ${devKey} -p ${devKey}`);
+    return true;
+  } catch (e) {
+    color.warn(`Docker login to ${registry} failed: ${e.message || e}`);
+    return false;
+  }
+}
+function dockerTag(source, target) {
+  try {
+    dockerCommand(`tag ${source} ${target}`);
+    return true;
+  } catch (e) {
+    color.error(`Failed to tag ${source} as ${target}: ${e.message || e}`);
+    return false;
+  }
+}
+function dockerPush(image) {
+  try {
+    dockerCommand(`push ${image}`);
+    return true;
+  } catch (e) {
+    color.error(`Failed to push ${image}: ${e.message || e}`);
+    return false;
+  }
+}
+function dockerBuild(imageName, dockerfilePath, context) {
+  try {
+    color.log(`Building Docker image ${imageName}...`);
+    execSync(`docker build -t ${imageName} -f ${dockerfilePath} ${context}`, {
+      encoding: 'utf-8',
+      stdio: 'inherit'
+    });
+    return true;
+  } catch (e) {
+    color.error(`Failed to build ${imageName}: ${e.message || e}`);
+    return false;
+  }
+}
+async function getUserLogin(host, devKey) {
+  let loginResp;
+  try {
+    loginResp = await (0, _nodeFetch.default)(`${host}/users/login/dev/${devKey}`, {
+      method: 'POST'
+    });
+  } catch (e) {
+    color.warn(`Cannot reach server ${host}: ${e.message || e}`);
+    return null;
+  }
+  if (loginResp.status !== 200) return null;
+  const loginData = await loginResp.json();
+  const token = loginData.token;
+  try {
+    const userResp = await (0, _nodeFetch.default)(`${host}/users/current`, {
+      headers: {
+        'Authorization': token
+      }
+    });
+    if (userResp.status !== 200) return null;
+    const userData = await userResp.json();
+    return {
+      login: userData.login,
+      token
+    };
+  } catch (e) {
+    return null;
+  }
+}
+async function resolveLatestCompatible(host, devKey, dockerName) {
+  const userInfo = await getUserLogin(host, devKey);
+  if (!userInfo) return {
+    found: null,
+    serverError: `Authentication failed. Check your developer key.`
+  };
+  try {
+    const url = `${host}/docker/images/${encodeURIComponent(dockerName)}/latest-compatible`;
+    const resp = await (0, _nodeFetch.default)(url, {
+      headers: {
+        'Authorization': userInfo.token
+      }
+    });
+    if (resp.status === 200) return {
+      found: await resp.json(),
+      serverError: null
+    };
+    if (resp.status === 404) return {
+      found: null,
+      serverError: null
+    };
+    return {
+      found: null,
+      serverError: `Unexpected response (HTTP ${resp.status}) from latest-compatible endpoint`
+    };
+  } catch (e) {
+    return {
+      found: null,
+      serverError: `Failed to query latest-compatible: ${e.message || e}`
+    };
+  }
+}
+async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps, debug) {
+  const dockerImages = discoverDockerfiles(packageName, version, debug);
+  if (dockerImages.length === 0) return;
+  color.log(`Found ${dockerImages.length} Dockerfile(s)`);
+  if (registry) dockerLogin(registry, devKey);
+  for (const img of dockerImages) {
+    let result;
+    if (rebuildDocker) {
+      const dockerfileDir = _path.default.join('dockerfiles', img.dirName);
+      const dockerfilePath = _path.default.join(dockerfileDir, 'Dockerfile');
+      if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
+        result = pushImage(img, registry);
+        color.success(`Built and tagged ${img.fullLocalName}`);
+      } else {
+        result = await fallbackImage(img, host, devKey, registry);
+      }
+    } else if (localImageExists(img.fullLocalName)) {
+      result = pushImage(img, registry);
+      color.success(`Found local image ${img.fullLocalName}`);
+    } else {
+      color.warn(`Local image not found. Expected: ${img.fullLocalName}`);
+      color.log(`  Build it with: docker build -t ${img.fullLocalName} -f dockerfiles/${img.dirName}/Dockerfile dockerfiles/${img.dirName}`);
+      const fallback = await fallbackImage(img, host, devKey, registry);
+      if (fallback.serverError) {
+        color.error(`Cannot resolve fallback: ${fallback.serverError}`);
+        result = {
+          image: null,
+          fallback: true,
+          requestedVersion: img.imageTag
+        };
+      } else if (fallback.image) {
+        result = fallback;
+        color.warn(`Falling back to ${fallback.image}`);
+      } else {
+        // No fallback and no local image — must build
+        color.warn(`No fallback available. Building ${img.fullLocalName}...`);
+        const dockerfileDir = _path.default.join('dockerfiles', img.dirName);
+        const dockerfilePath = _path.default.join(dockerfileDir, 'Dockerfile');
+        if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
+          result = pushImage(img, registry);
+          color.success(`Built and tagged ${img.fullLocalName}`);
+        } else {
+          result = {
+            image: null,
+            fallback: true,
+            requestedVersion: img.imageTag
+          };
+          color.error(`Failed to build ${img.fullLocalName}. No container will be available.`);
+        }
+      }
+    }
+    const imageJsonPath = _path.default.join('dockerfiles', img.dirName, 'image.json');
+    zip.append(JSON.stringify(result, null, 2), {
+      name: imageJsonPath
+    });
+    localTimestamps[imageJsonPath.replace(/\\/g, '/')] = new Date().toUTCString();
+    color.log(`Added ${imageJsonPath}`);
+  }
+}
+function pushImage(img, registry) {
+  const canonicalImage = `datagrok/${img.fullLocalName}`;
+  if (registry) {
+    const remoteTag = `${registry}/${canonicalImage}`;
+    dockerTag(img.fullLocalName, remoteTag);
+    if (dockerPush(remoteTag)) return {
+      image: canonicalImage,
+      fallback: false
+    };
+    color.warn(`Push failed, image tagged locally only: ${remoteTag}`);
+    return {
+      image: canonicalImage,
+      fallback: false
+    };
+  }
+  color.warn('No registry configured. Image tagged locally only.');
+  return {
+    image: canonicalImage,
+    fallback: false
+  };
+}
+async function fallbackImage(img, host, devKey, registry) {
+  const {
+    found,
+    serverError
+  } = await resolveLatestCompatible(host, devKey, img.imageName);
+  if (serverError) return {
+    image: null,
+    fallback: true,
+    requestedVersion: img.imageTag,
+    serverError
+  };
+  if (found) return {
+    image: found.image,
+    fallback: true,
+    requestedVersion: img.imageTag
+  };
+  return {
+    image: null,
+    fallback: true,
+    requestedVersion: img.imageTag
+  };
+}
+async function processPackage(debug, rebuild, host, devKey, packageName, dropDb, suffix, hostAlias, registry, rebuildDocker) {
+  // Validate server connectivity and dev key
   let timestamps = {};
   let url = `${host}/packages/dev/${devKey}/${packageName}`;
-  if (debug) {
-    try {
-      timestamps = await (await (0, _nodeFetch.default)(url + '/timestamps')).json();
-      if (timestamps['#type'] === 'ApiError') {
-        color.error(timestamps.message);
-        return 1;
-      }
-    } catch (error) {
-      if (utils.isConnectivityError(error)) color.error(`Server is possibly offline: ${host}`);
-      if (color.isVerbose()) console.error(error);
+  try {
+    const checkResp = await (0, _nodeFetch.default)(url + '/timestamps');
+    const checkData = await checkResp.json();
+    if (checkData['#type'] === 'ApiError') {
+      color.error(checkData.message);
       return 1;
     }
+    if (debug) timestamps = checkData;
+  } catch (error) {
+    if (utils.isConnectivityError(error)) color.error(`Server is possibly offline: ${host}`);
+    if (color.isVerbose()) console.error(error);
+    return 1;
   }
   const zip = (0, _archiverPromise.default)('zip', {
     store: false
@@ -147,6 +390,17 @@ async function processPackage(debug, rebuild, host, devKey, packageName, dropDb,
     errs.forEach(e => color.error(e));
     return 1;
   }
+
+  // Generate Celery Docker artifacts from python/ if present
+  (0, _pythonCeleryGen.generateCeleryArtifacts)(curDir);
+
+  // Process Docker images and inject image.json into the ZIP
+  let dockerVersion = json.version;
+  if (debug) {
+    const userInfo = await getUserLogin(host, devKey);
+    if (userInfo) dockerVersion = userInfo.login;
+  }
+  await processDockerImages(packageName, dockerVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps, debug);
   zip.append(JSON.stringify(localTimestamps), {
     name: 'timestamps.json'
   });
@@ -245,16 +499,22 @@ async function publishPackage(args) {
   if (nArgs === 2) host = args['_'][1];
   let key = '';
   let url = '';
+  let registry;
 
   // The host can be passed either as a URL or an alias
   try {
     url = new URL(host).href;
     if (url.endsWith('/')) url = url.slice(0, -1);
-    if (url in urls) key = config['servers'][urls[url]]['key'];
+    if (url in urls) {
+      const alias = urls[url];
+      key = config['servers'][alias]['key'];
+      registry = config['servers'][alias]['registry'];
+    }
   } catch (error) {
     if (!(host in config.servers)) return color.error(`Unknown server alias. Please add it to ${confPath}`);
     url = config['servers'][host]['url'];
     key = config['servers'][host]['key'];
+    registry = config['servers'][host]['registry'];
   }
 
   // Update the developer key
@@ -285,7 +545,7 @@ async function publishPackage(args) {
         if (!args.suffix && stdout) args.suffix = stdout.toString().substring(0, 8);
       });
       await utils.delay(100);
-      code = await processPackage(!args.release, Boolean(args.rebuild), url, key, packageName, args.dropDb ?? false, args.suffix, host);
+      code = await processPackage(!args.release, Boolean(args.rebuild), url, key, packageName, args.dropDb ?? false, args.suffix, host, registry, args['rebuild-docker']);
     } catch (error) {
       console.error(error);
       code = 1;