datagrok-tools 6.1.0 → 6.1.1

package/bin/commands/config.js

@@ -19,6 +19,15 @@ const confTemplate = _jsYaml.default.load(_fs.default.readFileSync(confTemplateD
 }));
 const grokDir = _path.default.join(_os.default.homedir(), '.grok');
 const confPath = _path.default.join(grokDir, 'config.yaml');
+function defaultRegistry(url) {
+  try {
+    const hostname = new URL(url).hostname;
+    if (hostname === 'localhost' || /^[\d.]+$/.test(hostname) || hostname === '::1') return '';
+    return `registry.${hostname}`;
+  } catch (e) {
+    return '';
+  }
+}
 function validateKey(key) {
   if (!key || /^([A-Za-z\d-])+$/.test(key)) return true;else return 'Developer key may only include letters, numbers, or hyphens';
 }
@@ -33,7 +42,7 @@ function generateKeyQ(server, url) {
   if (server.startsWith('local')) question.message = `Developer key for ${origin} (press ENTER to skip):`;
   return question;
 }
-async function addNewServer(config) {
+async function addNewServer(config, askRegistry = false) {
   while (true) {
     const addServer = (await _inquirer.default.prompt({
       name: 'add-server',
@@ -53,17 +62,29 @@ async function addNewServer(config) {
         message: 'Enter a URL:'
       }))['server-url'];
       const key = (await _inquirer.default.prompt(generateKeyQ(name, url)))[name];
-      config.servers[name] = {
+      const entry = {
         url,
         key
       };
+      if (askRegistry) {
+        const regDefault = defaultRegistry(url);
+        const reg = (await _inquirer.default.prompt({
+          name: 'registry',
+          type: 'input',
+          message: `Docker registry for ${name}:`,
+          default: regDefault || undefined
+        }))['registry'] || '';
+        if (reg) entry.registry = reg;
+      }
+      config.servers[name] = entry;
     } else break;
   }
 }
 function config(args) {
   const nOptions = Object.keys(args).length - 1;
-  const interactiveMode = args['_'].length === 1 && (nOptions < 1 || nOptions === 1 && args.reset);
-  const hasAddServerCommand = args['_'].length === 2 && args['_'][1] === 'add' && args.server && args.key && args.k && args.alias && (nOptions === 4 || nOptions === 5 && args.default);
+  const askRegistry = args.registry != null;
+  const interactiveMode = args['_'].length === 1 && (nOptions < 1 || nOptions === 1 && (args.reset || askRegistry) || nOptions === 2 && args.reset && askRegistry);
+  const hasAddServerCommand = args['_'].length === 2 && args['_'][1] === 'add' && args.server && args.key && args.k && args.alias && nOptions >= 4 && nOptions <= 6;
   if (!interactiveMode && !hasAddServerCommand) return false;
   if (!_fs.default.existsSync(grokDir)) _fs.default.mkdirSync(grokDir);
   if (!_fs.default.existsSync(confPath) || args.reset) _fs.default.writeFileSync(confPath, _jsYaml.default.dump(confTemplate));
@@ -77,10 +98,15 @@ function config(args) {
       color.error('URL parsing error. Please, provide a valid server URL.');
       return false;
     }
-    config.servers[args.alias] = {
+    const server = {
       url: args.server,
       key: args.key
     };
+    if (args.registry != null) {
+      const registry = typeof args.registry === 'string' ? args.registry : defaultRegistry(args.server);
+      if (registry) server.registry = registry;
+    }
+    config.servers[args.alias] = server;
     color.success(`Successfully added the server to ${confPath}.`);
     console.log(`Use this command to deploy packages: grok publish ${args.alias}`);
     if (args.default) config.default = args.alias;
@@ -110,8 +136,18 @@ function config(args) {
           question.default = config['servers'][server]['key'];
           const devKey = await _inquirer.default.prompt(question);
           config['servers'][server]['key'] = devKey[server];
+          if (askRegistry) {
+            const regDefault = config['servers'][server]['registry'] || defaultRegistry(url);
+            const reg = (await _inquirer.default.prompt({
+              name: 'registry',
+              type: 'input',
+              message: `Docker registry for ${server}:`,
+              default: regDefault || undefined
+            }))['registry'] || '';
+            if (reg) config['servers'][server]['registry'] = reg;else delete config['servers'][server]['registry'];
+          }
         }
-        await addNewServer(config);
+        await addNewServer(config, askRegistry);
         const defaultServer = await _inquirer.default.prompt({
           name: 'default-server',
           type: 'input',

package/bin/commands/docker-gen.js

@@ -0,0 +1,16 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.dockerGen = dockerGen;
+var _pythonCeleryGen = require("../utils/python-celery-gen");
+var color = _interopRequireWildcard(require("../utils/color-utils"));
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function dockerGen(args) {
+  color.setVerbose(args.verbose || args.v || false);
+  const packageDir = process.cwd();
+  const generated = (0, _pythonCeleryGen.generateCeleryArtifacts)(packageDir);
+  if (generated) color.success('Generated Celery Docker artifacts');else console.log('No python/ directory with annotated functions found');
+  return true;
+}

package/bin/commands/help.js

@@ -10,19 +10,20 @@ Usage: grok <command>
 Datagrok's package management tool
 
 Commands:
-    add       Add an object template
-    api       Create wrapper functions
-    build     Build a package or multiple packages
-    check     Check package content (function signatures, etc.)
-    claude    Launch Claude Code in a Datagrok dev container
-    config    Create and manage config files
-    create    Create a package
-    init      Modify a package template
-    link      Link \`datagrok-api\` and libraries for local development
-    publish   Upload a package
-    test      Run package tests
-    testall      Run packages tests
-    migrate   Migrate legacy tags to meta.role
+    add         Add an object template
+    api         Create wrapper functions
+    build       Build a package or multiple packages
+    check       Check package content (function signatures, etc.)
+    claude      Launch Claude Code in a Datagrok dev container
+    config      Create and manage config files
+    create      Create a package
+    docker-gen  Generate Celery Docker artifacts from Python functions
+    init        Modify a package template
+    link        Link \`datagrok-api\` and libraries for local development
+    publish     Upload a package
+    test        Run package tests
+    testall     Run packages tests
+    migrate     Migrate legacy tags to meta.role
 
 To get help on a particular command, use:
     grok <command> --help
@@ -122,13 +123,14 @@ Usage: grok config
 Create or update a configuration file
 
 Options:
-[--reset] [--server] [--alias] [-k | --key]
+[--reset] [--server] [--alias] [-k | --key] [--registry]
 
 --reset     Restore the default config file template
 --server    Use to add a server to the config (\`grok config add --alias alias --server url --key key\`)
 --alias     Use in conjunction with the \`server\` option to set the server name
 --key       Use in conjunction with the \`server\` option to set the developer key
 --default   Use in conjunction with the \`server\` option to set the added server as default
+--registry  Docker registry URL (default: registry.{server hostname})
 `;
 const HELP_CREATE = `
 Usage: grok create [name]
@@ -267,6 +269,17 @@ Options:
 --verbose           Prints detailed information about linked packages  
 --all               Links all available packages(run in packages directory)
 `;
+const HELP_DOCKER_GEN = `
+Usage: grok docker-gen
+
+Generate Celery Docker artifacts from annotated Python functions in the python/ directory.
+Produces Dockerfile, tasks.yaml, and Celery entry point in dockerfiles/<name>/.
+
+Options:
+[-v | --verbose]
+
+--verbose         Print detailed output
+`;
 const HELP_MIGRATE = `
 Usage: grok migrate
 
@@ -315,6 +328,7 @@ const help = exports.help = {
   claude: HELP_CLAUDE,
   config: HELP_CONFIG,
   create: HELP_CREATE,
+  'docker-gen': HELP_DOCKER_GEN,
   init: HELP_INIT,
   link: HELP_LINK,
   publish: HELP_PUBLISH,

package/bin/commands/publish.js

@@ -17,6 +17,7 @@ var _testUtils = require("../utils/test-utils");
 var utils = _interopRequireWildcard(require("../utils/utils"));
 var color = _interopRequireWildcard(require("../utils/color-utils"));
 var _check = require("./check");
+var _pythonCeleryGen = require("../utils/python-celery-gen");
 function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
 // @ts-ignore
 
@@ -37,7 +38,7 @@ const packDir = _path.default.join(curDir, 'package.json');
 const packageFiles = ['src/package.ts', 'src/detectors.ts', 'src/package.js', 'src/detectors.js', 'src/package-test.ts', 'src/package-test.js', 'package.js', 'detectors.js'];
 let config;
 const BLEEDING_EDGE_TAG = 'bleeding-edge';
-function discoverDockerfiles(packageName, version) {
+function discoverDockerfiles(packageName, version, debug) {
   const dockerfilesDir = _path.default.join(curDir, 'dockerfiles');
   if (!_fs.default.existsSync(dockerfilesDir)) return [];
   const results = [];
@@ -48,13 +49,14 @@ function discoverDockerfiles(packageName, version) {
     if (!entry.isDirectory()) continue;
     const dockerfilePath = _path.default.join(dockerfilesDir, entry.name, 'Dockerfile');
     if (!_fs.default.existsSync(dockerfilePath)) continue;
-    const cleanName = utils.removeScope(packageName).toLowerCase();
+    const cleanName = utils.removeScope(packageName).replace(/-/g, '').toLowerCase();
     const imageName = `${cleanName}-${entry.name.toLowerCase()}`;
+    const imageTag = debug ? version : `${version}.X`;
     results.push({
       imageName,
-      imageTag: `${version}.X`,
+      imageTag,
       dirName: entry.name,
-      fullLocalName: `${imageName}:${version}.X`
+      fullLocalName: `${imageName}:${imageTag}`
     });
   }
   return results;
@@ -105,7 +107,7 @@ function dockerBuild(imageName, dockerfilePath, context) {
     color.log(`Building Docker image ${imageName}...`);
     execSync(`docker build -t ${imageName} -f ${dockerfilePath} ${context}`, {
       encoding: 'utf-8',
-      stdio: ['pipe', 'pipe', 'pipe']
+      stdio: 'inherit'
     });
     return true;
   } catch (e) {
@@ -113,29 +115,69 @@ function dockerBuild(imageName, dockerfilePath, context) {
     return false;
   }
 }
-async function resolveLatestCompatible(host, devKey, dockerName) {
+async function getUserLogin(host, devKey) {
+  let loginResp;
   try {
-    // Login with dev key to get a session token
-    const loginResp = await (0, _nodeFetch.default)(`${host}/users/login/dev/${devKey}`, {
+    loginResp = await (0, _nodeFetch.default)(`${host}/users/login/dev/${devKey}`, {
       method: 'POST'
     });
-    if (loginResp.status !== 200) return null;
-    const loginData = await loginResp.json();
-    const token = loginData.token;
-    const url = `${host}/docker/images/${encodeURIComponent(dockerName)}/latest-compatible`;
-    const resp = await (0, _nodeFetch.default)(url, {
+  } catch (e) {
+    color.warn(`Cannot reach server ${host}: ${e.message || e}`);
+    return null;
+  }
+  if (loginResp.status !== 200) return null;
+  const loginData = await loginResp.json();
+  const token = loginData.token;
+  try {
+    const userResp = await (0, _nodeFetch.default)(`${host}/users/current`, {
       headers: {
         'Authorization': token
       }
     });
-    if (resp.status === 200) return await resp.json();
-    return null;
-  } catch {
+    if (userResp.status !== 200) return null;
+    const userData = await userResp.json();
+    return {
+      login: userData.login,
+      token
+    };
+  } catch (e) {
     return null;
   }
 }
-async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps) {
-  const dockerImages = discoverDockerfiles(packageName, version);
+async function resolveLatestCompatible(host, devKey, dockerName) {
+  const userInfo = await getUserLogin(host, devKey);
+  if (!userInfo) return {
+    found: null,
+    serverError: `Authentication failed. Check your developer key.`
+  };
+  try {
+    const url = `${host}/docker/images/${encodeURIComponent(dockerName)}/latest-compatible`;
+    const resp = await (0, _nodeFetch.default)(url, {
+      headers: {
+        'Authorization': userInfo.token
+      }
+    });
+    if (resp.status === 200) return {
+      found: await resp.json(),
+      serverError: null
+    };
+    if (resp.status === 404) return {
+      found: null,
+      serverError: null
+    };
+    return {
+      found: null,
+      serverError: `Unexpected response (HTTP ${resp.status}) from latest-compatible endpoint`
+    };
+  } catch (e) {
+    return {
+      found: null,
+      serverError: `Failed to query latest-compatible: ${e.message || e}`
+    };
+  }
+}
+async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps, debug) {
+  const dockerImages = discoverDockerfiles(packageName, version, debug);
   if (dockerImages.length === 0) return;
   color.log(`Found ${dockerImages.length} Dockerfile(s)`);
   if (registry) dockerLogin(registry, devKey);
@@ -145,18 +187,45 @@ async function processDockerImages(packageName, version, registry, devKey, host,
       const dockerfileDir = _path.default.join('dockerfiles', img.dirName);
       const dockerfilePath = _path.default.join(dockerfileDir, 'Dockerfile');
       if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
-        result = pushImage(img, registry, version);
+        result = pushImage(img, registry);
         color.success(`Built and tagged ${img.fullLocalName}`);
       } else {
         result = await fallbackImage(img, host, devKey, registry);
       }
     } else if (localImageExists(img.fullLocalName)) {
-      result = pushImage(img, registry, version);
+      result = pushImage(img, registry);
       color.success(`Found local image ${img.fullLocalName}`);
     } else {
-      result = await fallbackImage(img, host, devKey, registry);
-      color.warn(`Local image not found. Expected: ${img.fullLocalName}` + (result.image ? `. Falling back to ${result.image}` : ''));
+      color.warn(`Local image not found. Expected: ${img.fullLocalName}`);
       color.log(`  Build it with: docker build -t ${img.fullLocalName} -f dockerfiles/${img.dirName}/Dockerfile dockerfiles/${img.dirName}`);
+      const fallback = await fallbackImage(img, host, devKey, registry);
+      if (fallback.serverError) {
+        color.error(`Cannot resolve fallback: ${fallback.serverError}`);
+        result = {
+          image: null,
+          fallback: true,
+          requestedVersion: img.imageTag
+        };
+      } else if (fallback.image) {
+        result = fallback;
+        color.warn(`Falling back to ${fallback.image}`);
+      } else {
+        // No fallback and no local image — must build
+        color.warn(`No fallback available. Building ${img.fullLocalName}...`);
+        const dockerfileDir = _path.default.join('dockerfiles', img.dirName);
+        const dockerfilePath = _path.default.join(dockerfileDir, 'Dockerfile');
+        if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
+          result = pushImage(img, registry);
+          color.success(`Built and tagged ${img.fullLocalName}`);
+        } else {
+          result = {
+            image: null,
+            fallback: true,
+            requestedVersion: img.imageTag
+          };
+          color.error(`Failed to build ${img.fullLocalName}. No container will be available.`);
+        }
+      }
     }
     const imageJsonPath = _path.default.join('dockerfiles', img.dirName, 'image.json');
     zip.append(JSON.stringify(result, null, 2), {
@@ -166,34 +235,43 @@ async function processDockerImages(packageName, version, registry, devKey, host,
     color.log(`Added ${imageJsonPath}`);
   }
 }
-function pushImage(img, registry, version) {
+function pushImage(img, registry) {
+  const canonicalImage = `datagrok/${img.fullLocalName}`;
   if (registry) {
-    const remoteTag = `${registry}/${img.imageName}:${version}`;
+    const remoteTag = `${registry}/${canonicalImage}`;
     dockerTag(img.fullLocalName, remoteTag);
     if (dockerPush(remoteTag)) return {
-      image: remoteTag,
+      image: canonicalImage,
       fallback: false
     };
     color.warn(`Push failed, image tagged locally only: ${remoteTag}`);
     return {
-      image: remoteTag,
+      image: canonicalImage,
       fallback: false
     };
   }
   color.warn('No registry configured. Image tagged locally only.');
   return {
-    image: img.fullLocalName,
+    image: canonicalImage,
     fallback: false
   };
 }
 async function fallbackImage(img, host, devKey, registry) {
-  const latest = await resolveLatestCompatible(host, devKey, img.imageName);
-  if (latest) return {
-    image: latest.image,
+  const {
+    found,
+    serverError
+  } = await resolveLatestCompatible(host, devKey, img.imageName);
+  if (serverError) return {
+    image: null,
+    fallback: true,
+    requestedVersion: img.imageTag,
+    serverError
+  };
+  if (found) return {
+    image: found.image,
     fallback: true,
     requestedVersion: img.imageTag
   };
-  color.warn('No previous version available. Container will not be available until an image is built.');
   return {
     image: null,
     fallback: true,
@@ -201,21 +279,21 @@ async function fallbackImage(img, host, devKey, registry) {
   };
 }
 async function processPackage(debug, rebuild, host, devKey, packageName, dropDb, suffix, hostAlias, registry, rebuildDocker) {
-  // Get the server timestamps
+  // Validate server connectivity and dev key
   let timestamps = {};
   let url = `${host}/packages/dev/${devKey}/${packageName}`;
-  if (debug) {
-    try {
-      timestamps = await (await (0, _nodeFetch.default)(url + '/timestamps')).json();
-      if (timestamps['#type'] === 'ApiError') {
-        color.error(timestamps.message);
-        return 1;
-      }
-    } catch (error) {
-      if (utils.isConnectivityError(error)) color.error(`Server is possibly offline: ${host}`);
-      if (color.isVerbose()) console.error(error);
+  try {
+    const checkResp = await (0, _nodeFetch.default)(url + '/timestamps');
+    const checkData = await checkResp.json();
+    if (checkData['#type'] === 'ApiError') {
+      color.error(checkData.message);
       return 1;
     }
+    if (debug) timestamps = checkData;
+  } catch (error) {
+    if (utils.isConnectivityError(error)) color.error(`Server is possibly offline: ${host}`);
+    if (color.isVerbose()) console.error(error);
+    return 1;
   }
   const zip = (0, _archiverPromise.default)('zip', {
     store: false
@@ -313,9 +391,16 @@ async function processPackage(debug, rebuild, host, devKey, packageName, dropDb,
     return 1;
   }
 
+  // Generate Celery Docker artifacts from python/ if present
+  (0, _pythonCeleryGen.generateCeleryArtifacts)(curDir);
+
   // Process Docker images and inject image.json into the ZIP
-  const packageVersion = json.version;
-  await processDockerImages(packageName, packageVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps);
+  let dockerVersion = json.version;
+  if (debug) {
+    const userInfo = await getUserLogin(host, devKey);
+    if (userInfo) dockerVersion = userInfo.login;
+  }
+  await processDockerImages(packageName, dockerVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps, debug);
   zip.append(JSON.stringify(localTimestamps), {
     name: 'timestamps.json'
   });