datagrok-tools 6.1.0 → 6.1.1

package/bin/utils/python-celery-gen.js

@@ -0,0 +1,247 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.generateCeleryArtifacts = generateCeleryArtifacts;
+var _fs = _interopRequireDefault(require("fs"));
+var _path = _interopRequireDefault(require("path"));
+var color = _interopRequireWildcard(require("./color-utils"));
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+// Header tags recognized in Python function metadata comments (ported from ScriptParser.headerTags)
+const headerTags = ['name', 'description', 'help-url', 'input', 'output', 'tags', 'sample', 'language', 'endpoint', 'requiresServer', 'param-csrfmiddlewaretoken', 'returns', 'test', 'sidebar', 'condition', 'top-menu', 'environment', 'require', 'editor-for', 'schedule', 'schedule.runAs', 'reference', 'editor'];
+const paramRegex = new RegExp(`(#\\s*(${headerTags.join('|')}|meta\\.[^:]*):)\\s*(\\S+)\\s?(\\S+)?`);
+const defRegex = /^\s*def\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(/;
+const TASKS_FILE = 'tasks.yaml';
+const TEMPLATE_PIP_DOCKER = `FROM python:3.11-slim-bookworm
+
+RUN apt-get update && apt-get install -y --no-install-recommends \\
+    gcc \\
+    libc-dev \\
+    && apt-get clean \\
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+RUN useradd -m datagrok
+
+WORKDIR /app
+COPY . /app
+ENV PYTHONPATH="/app:\${PYTHONPATH}"
+RUN chown -R datagrok:datagrok /app
+
+# Install Python dependencies using uv
+RUN uv pip install --system celery datagrok-celery-task pyyaml && \\
+    if [ -f requirements.in ]; then uv pip install --system --no-cache -r requirements.in; \\
+    elif [ -f requirements.txt ]; then uv pip install --system --no-cache -r requirements.txt; fi
+
+USER datagrok
+
+EXPOSE 8000
+
+CMD celery -A \$DATAGROK_CELERY_NAME worker \\
+    --loglevel=info \\
+    --hostname=\$CELERY_HOSTNAME \\
+    --concurrency=\$WORKERS_PROCESSES \\
+    -Q \$TASK_QUEUE_NAME
+`;
+const TEMPLATE_CONDA_DOCKER = `FROM mambaorg/micromamba:latest as builder
+
+USER root
+
+ENV MAMBA_ROOT_PREFIX=/opt/conda \\
+    MAMBA_DOCKERFILE_ACTIVATE=1 \\
+    DEFAULT_ENV_NAME=myenv
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+WORKDIR /tmp/app
+
+COPY environment.yaml environment.yml* ./
+
+RUN if [ -f environment.yaml ]; then sed -i 's/\\r\$//' environment.yaml; fi && \\
+    if [ -f environment.yml ]; then sed -i 's/\\r\$//' environment.yml; fi
+
+RUN ENV_FILE=\$( [ -f environment.yaml ] && echo environment.yaml || echo environment.yml ) && \\
+    ENV_NAME=\$(grep '^name:' \$ENV_FILE | cut -d' ' -f2 || echo "myenv") && \\
+    micromamba create -y -n \$ENV_NAME -f \$ENV_FILE && \\
+    uv pip install --python /opt/conda/envs/\$ENV_NAME/bin/python celery datagrok-celery-task pyyaml && \\
+    echo "\$ENV_NAME" > /tmp/env_name.txt
+
+RUN micromamba clean --all --yes
+
+FROM mambaorg/micromamba:latest
+
+USER root
+
+ENV MAMBA_ROOT_PREFIX=/opt/conda \\
+    MAMBA_DOCKERFILE_ACTIVATE=1
+
+# Create non-root user
+RUN useradd -m -s /bin/bash grok
+
+WORKDIR /app
+
+COPY --from=builder /opt/conda /opt/conda
+COPY --from=builder /tmp/env_name.txt /opt/env_name.txt
+
+COPY . /app
+
+RUN chown -R grok:grok /opt/conda /app /opt/env_name.txt
+EXPOSE 8000
+USER grok
+SHELL ["/bin/bash", "-c"]
+
+CMD micromamba run -n \$(cat /opt/env_name.txt) \\
+    celery -A \$DATAGROK_CELERY_NAME worker \\
+        --loglevel=info \\
+        --hostname=\$CELERY_HOSTNAME \\
+        --concurrency=\$WORKERS_PROCESSES \\
+        -Q \$TASK_QUEUE_NAME
+`;
+const TEMPLATE_PYTHON_ENTRY = `import os
+import importlib
+import logging
+
+import yaml
+from celery import Celery
+from datagrok_celery_task import DatagrokTask, Settings
+
+import sys
+sys.path.insert(0, os.getcwd())
+
+settings = Settings(log_level=logging.DEBUG)
+app = Celery(settings.celery_name, broker=settings.broker_url)
+
+with open("${TASKS_FILE}") as f:
+    tasks = yaml.safe_load(f)
+
+if not isinstance(tasks, dict) or not isinstance(tasks['tasks'], list):
+    raise RuntimeError('Incorrect format for tasks.yaml. It should contain tasks field.')
+
+for task_def in tasks['tasks']:
+    mod = importlib.import_module(task_def["module"])
+    fn = getattr(mod, task_def["name"])
+    app.task(name=task_def["name"], base=DatagrokTask)(fn)
+`;
+function getAllPyFiles(dir) {
+  const results = [];
+  const entries = _fs.default.readdirSync(dir, {
+    withFileTypes: true
+  });
+  for (const entry of entries) {
+    const fullPath = _path.default.join(dir, entry.name);
+    if (entry.isDirectory()) results.push(...getAllPyFiles(fullPath));else if (entry.name.endsWith('.py')) results.push(fullPath);
+  }
+  return results;
+}
+function scanPythonFunctions(pythonDir, root) {
+  const pyFiles = getAllPyFiles(pythonDir);
+  const tasks = [];
+  for (const pyFile of pyFiles) {
+    const content = _fs.default.readFileSync(pyFile, 'utf-8');
+    const lines = content.split('\n');
+    let header = [];
+    let isHeader = false;
+    for (const line of lines) {
+      if (line.length > 0 && paramRegex.test(line)) {
+        if (!isHeader) isHeader = true;
+        header.push(line);
+      } else {
+        if (isHeader && defRegex.test(line)) {
+          const match = defRegex.exec(line);
+          const name = match[1];
+          let module = _path.default.basename(pyFile, '.py');
+          const fileDir = _path.default.dirname(pyFile);
+          if (fileDir !== root) {
+            const relDir = _path.default.relative(root, fileDir).replace(/[\\/]+/g, '.');
+            if (relDir && relDir !== '.') module = relDir + '.' + module;
+          }
+          tasks.push({
+            name,
+            module
+          });
+          isHeader = false;
+          header = [];
+        } else if (isHeader) {
+          isHeader = false;
+          header = [];
+        }
+      }
+    }
+  }
+  return tasks;
+}
+function copyDirContents(src, dest) {
+  const entries = _fs.default.readdirSync(src, {
+    withFileTypes: true
+  });
+  for (const entry of entries) {
+    const srcPath = _path.default.join(src, entry.name);
+    const destPath = _path.default.join(dest, entry.name);
+    if (entry.isDirectory()) {
+      _fs.default.mkdirSync(destPath, {
+        recursive: true
+      });
+      copyDirContents(srcPath, destPath);
+    } else _fs.default.copyFileSync(srcPath, destPath);
+  }
+}
+function useConda(dir) {
+  return _fs.default.existsSync(_path.default.join(dir, 'environment.yaml')) || _fs.default.existsSync(_path.default.join(dir, 'environment.yml'));
+}
+function deployFolder(packageDir, folderPath, dirName) {
+  const tasks = scanPythonFunctions(folderPath, folderPath);
+  if (tasks.length === 0) {
+    color.log(`No annotated Python functions found in ${dirName}`);
+    return false;
+  }
+  const dockerfilesDir = _path.default.join(packageDir, 'dockerfiles', dirName);
+  _fs.default.mkdirSync(dockerfilesDir, {
+    recursive: true
+  });
+
+  // Generate Dockerfile
+  const dockerfile = useConda(folderPath) ? TEMPLATE_CONDA_DOCKER : TEMPLATE_PIP_DOCKER;
+  _fs.default.writeFileSync(_path.default.join(dockerfilesDir, 'Dockerfile'), dockerfile);
+
+  // Generate tasks.yaml (JSON-encoded, matching server behavior)
+  _fs.default.writeFileSync(_path.default.join(dockerfilesDir, TASKS_FILE), JSON.stringify({
+    tasks
+  }, null, 2));
+
+  // Generate Celery entry point
+  const celeryName = dirName.replace(/-/g, '_');
+  _fs.default.writeFileSync(_path.default.join(dockerfilesDir, celeryName + '.py'), TEMPLATE_PYTHON_ENTRY);
+
+  // Copy Python source files
+  copyDirContents(folderPath, dockerfilesDir);
+
+  // Copy container.json if present
+  const containerJsonSrc = _path.default.join(folderPath, 'container.json');
+  const containerJsonDest = _path.default.join(dockerfilesDir, 'container.json');
+  if (_fs.default.existsSync(containerJsonSrc) && !_fs.default.existsSync(containerJsonDest)) _fs.default.copyFileSync(containerJsonSrc, containerJsonDest);
+  color.log(`Generated Celery Docker artifacts in dockerfiles/${dirName}/`);
+  return true;
+}
+function generateCeleryArtifacts(packageDir) {
+  const pythonDir = _path.default.join(packageDir, 'python');
+  if (!_fs.default.existsSync(pythonDir)) return false;
+  const entries = _fs.default.readdirSync(pythonDir, {
+    withFileTypes: true
+  });
+  const isNested = entries.length > 0 && entries.every(e => e.isDirectory);
+  let generated = false;
+  if (isNested) {
+    for (const entry of entries) {
+      if (!entry.isDirectory()) continue;
+      const folderPath = _path.default.join(pythonDir, entry.name);
+      if (deployFolder(packageDir, folderPath, entry.name)) generated = true;
+    }
+  } else {
+    const dirName = _path.default.basename(packageDir).toLowerCase();
+    if (deployFolder(packageDir, pythonDir, dirName)) generated = true;
+  }
+  return generated;
+}

package/bin/utils/test-utils.js

@@ -114,7 +114,11 @@ function getDevKey(hostKey) {
     key
   };
 }
-async function getBrowserPage(puppeteer, params = defaultLaunchParameters) {
+function appendUrlParams(url, extraParams) {
+  if (!extraParams) return url;
+  return url + (url.includes('?') ? '&' : '?') + extraParams;
+}
+async function getBrowserPage(puppeteer, params = defaultLaunchParameters, urlParams) {
   let url = process.env.HOST ?? '';
   const cfg = getDevKey(url);
   url = cfg.url;
@@ -153,7 +157,7 @@ async function getBrowserPage(puppeteer, params = defaultLaunchParameters) {
   await page.evaluate(token => {
     window.localStorage.setItem('auth', token);
   }, token);
-  await page.goto(url);
+  await page.goto(appendUrlParams(url, urlParams));
   try {
     //    await page.waitForSelector('.grok-preloader', { timeout: 1800000 });
     await page.waitForFunction(() => document.querySelector('.grok-preloader') == null, {
@@ -625,7 +629,7 @@ async function runBrowser(testExecutionData, browserOptions, browsersId, testInv
 
       // Remove old console listeners before refresh to avoid stale state references
       page.removeAllListeners('console');
-      await page.goto(webUrl);
+      await page.goto(appendUrlParams(webUrl, browserOptions.urlParams));
       try {
         await page.waitForFunction(() => document.querySelector('.grok-preloader') == null, {
           timeout: 3600000
@@ -639,7 +643,7 @@ async function runBrowser(testExecutionData, browserOptions, browsersId, testInv
         devtools: browserOptions.debug
       }, defaultLaunchParameters);
       if (browserOptions.gui) params['headless'] = false;
-      const out = await getBrowserPage(_puppeteer.default, params);
+      const out = await getBrowserPage(_puppeteer.default, params, browserOptions.urlParams);
       browser = out.browser;
       page = out.page;
       webUrl = await getWebUrlFromPage(page);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "6.1.0",
+  "version": "6.1.1",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {