datagrok-tools 6.0.1 → 6.0.2

package/.claude/settings.local.json

@@ -0,0 +1,20 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(find /c/repos/Datagrok -type f \\\\\\( -name \"Dockerfile*\" -o -name \"docker-compose*\" \\\\\\) 2>/dev/null | grep -v node_modules | head -20)",
+      "Bash(npm run:*)",
+      "Bash(docker build:*)",
+      "Bash(cd C:/repos/awesome && node C:/repos/Datagrok/public/tools/bin/grok.js claude destroy GROK-PLAYGROUND 2>&1)",
+      "Bash(cd C:/repos/awesome && node C:/repos/Datagrok/public/tools/bin/grok.js claude GROK-PLAYGROUND --in-place --keep --prompt \"echo hello and exit\" 2>&1)",
+      "Bash(cd C:/repos/awesome && node C:/repos/Datagrok/public/tools/bin/grok.js claude GROK-PLAYGROUND --in-place --keep --prompt \"echo done\" 2>&1)",
+      "Bash(git -C C:/repos/awesome rev-parse --show-toplevel 2>&1)",
+      "Bash(cd C:/repos/awesome && node C:/repos/Datagrok/public/tools/bin/grok.js claude GROK-PLAYGROUND 2>&1)",
+      "Bash(cd C:/repos/awesome && node C:/repos/Datagrok/public/tools/bin/grok.js claude GROK-PLAYGROUND --keep 2>&1)",
+      "Bash(MSYS_NO_PATHCONV=1 docker exec dg-pkg-grok-playground-tools-dev-1 ls /workspace/datagrok/ 2>&1)",
+      "Bash(MSYS_NO_PATHCONV=1 docker exec dg-pkg-grok-playground-tools-dev-1 ls -la /workspace/datagrok/packages/awesome 2>&1)",
+      "Bash(MSYS_NO_PATHCONV=1 docker exec dg-pkg-grok-playground-tools-dev-1 ls /workspace/datagrok/packages/awesome/ 2>&1)",
+      "Bash(MSYS_NO_PATHCONV=1 docker exec dg-pkg-grok-playground-tools-dev-1 bash -c 'echo \"=== Symlink ===\" && ls -la /workspace/datagrok/packages/awesome && echo \"=== Package contents ===\" && ls /workspace/datagrok/packages/awesome/ && echo \"=== Public repo root ===\" && ls /workspace/datagrok/js-api /workspace/datagrok/packages /workspace/datagrok/CLAUDE.md 2>&1 | head -5' 2>&1)",
+      "Bash(grep -r \"tools-dev\" /c/repos/Datagrok --include=\"*\" 2>/dev/null | head -20)"
+    ]
+  }
+}

package/.devcontainer/.gitattributes

@@ -0,0 +1 @@
+*.sh text eol=lf

package/.devcontainer/entrypoint.sh

@@ -1,39 +1,23 @@
 #!/bin/bash
 set -e
 
-# Resolve the actual version when DG_VERSION is "latest" — needed for git branch matching.
-resolve_version() {
+# Map DG_VERSION to a git branch name for cloning the public repo.
+# Docker tags like "latest" or "bleeding-edge" aren't git branches — use master.
+# Explicit version numbers (e.g. "0.151.9") are tried as-is with a master fallback.
+resolve_branch() {
   local ver="${1:-latest}"
-  if [ "$ver" = "latest" ]; then
-    echo "[tools-dev] Resolving latest Datagrok version from Docker Hub..." >&2
-    local resolved
-    resolved=$(curl -sf "https://hub.docker.com/v2/repositories/datagrok/datagrok/tags?page_size=100&ordering=-name" \
-      | node -e "
-        const data = JSON.parse(require('fs').readFileSync('/dev/stdin','utf8'));
-        const tags = (data.results||[]).map(t=>t.name).filter(n=>/^\d+\.\d+\.\d+$/.test(n));
-        tags.sort((a,b)=>{
-          const pa=a.split('.').map(Number), pb=b.split('.').map(Number);
-          return pb[0]-pa[0]||pb[1]-pa[1]||pb[2]-pa[2];
-        });
-        if(tags[0]) process.stdout.write(tags[0]);
-      " 2>/dev/null) || true
-    if [ -n "$resolved" ]; then
-      echo "[tools-dev] Resolved latest version: $resolved" >&2
-      echo "$resolved"
-    else
-      echo "[tools-dev] Could not resolve latest version, falling back to master" >&2
-      echo "master"
-    fi
-  else
-    echo "$ver"
-  fi
+  case "$ver" in
+    latest|bleeding-edge) echo "master" ;;
+    *) echo "$ver" ;;
+  esac
 }
 
-PUBLIC_DIR="/workspace/public"
-# Detect if /workspace/repo IS the public repo (has js-api/ at root)
-if [ -d "/workspace/repo/js-api" ]; then
+PUBLIC_DIR="${DG_PUBLIC_DIR:-/workspace/datagrok}"
+# Detect if /workspace/repo IS the public repo.
+# Require .git to avoid false positives from packages that have public/js-api via npm link.
+if [ -e "/workspace/repo/.git" ] && [ -d "/workspace/repo/js-api" ]; then
   echo "[tools-dev] Public repo detected at /workspace/repo — skipping clone."
-elif [ -d "/workspace/repo/public/js-api" ]; then
+elif [ -e "/workspace/repo/.git" ] && [ -d "/workspace/repo/public/js-api" ]; then
   echo "[tools-dev] Monorepo detected — public repo at /workspace/repo/public."
 elif [ -d "$PUBLIC_DIR/js-api" ]; then
   echo "[tools-dev] Public repo already at $PUBLIC_DIR."
@@ -43,31 +27,45 @@ else
   if [ -n "$DG_PUBLIC_BRANCH" ]; then
     BRANCH="$DG_PUBLIC_BRANCH"
   else
-    BRANCH=$(resolve_version "${DG_VERSION:-latest}")
+    BRANCH=$(resolve_branch "${DG_VERSION:-latest}")
   fi
-  echo "[tools-dev] Cloning public repo ($BRANCH) into $PUBLIC_DIR..."
-  git clone --depth 1 --branch "$BRANCH" "$REPO" "$PUBLIC_DIR" 2>/dev/null \
+  # Sparse checkout: only fetch dirs needed for package context (js-api, libraries, help, packages)
+  # plus root files (CLAUDE.md, .claude/, etc.). Much faster than a full clone.
+  sparse_clone() {
+    local branch="$1"
+    git clone --depth 1 --branch "$branch" --filter=blob:none --no-checkout "$REPO" "$PUBLIC_DIR" \
+      && git -C "$PUBLIC_DIR" sparse-checkout set --no-cone \
+           '/*' '!connectors/' '!docker/' '!docusaurus/' '!docusaurus-static/' \
+           '!environments/' '!hooks/' '!misc/' '!python-api/' '!datagrok-celery-task/' \
+           '/js-api/**' '/libraries/**' '/help/**' '/packages/**' '/tools/**' \
+      && git -C "$PUBLIC_DIR" checkout
+  }
+  echo "[tools-dev] Cloning public repo ($BRANCH, sparse) into $PUBLIC_DIR..."
+  sparse_clone "$BRANCH" \
     || { echo "[tools-dev] Branch '$BRANCH' not found, falling back to master."
-         git clone --depth 1 --branch master "$REPO" "$PUBLIC_DIR"; }
+         rm -rf "$PUBLIC_DIR"
+         sparse_clone "master"; }
   echo "[tools-dev] Public repo ready at $PUBLIC_DIR (branch: $(git -C "$PUBLIC_DIR" branch --show-current))."
 fi
 
 # ── Mount workspace inside public repo for non-public workspaces ──
 # When TASK_KEY is set and workspace is not the public repo, link it into packages/
 # so Claude Code walks up to find all public repo context (CLAUDE.md, .claude/, js-api/, etc.)
-if [ -d "/workspace/repo/js-api" ]; then
+if [ -e "/workspace/repo/.git" ] && [ -d "/workspace/repo/js-api" ]; then
   echo "[tools-dev] Public repo IS the workspace — no linking needed."
-elif [ -d "/workspace/repo/public/js-api" ]; then
+elif [ -e "/workspace/repo/.git" ] && [ -d "/workspace/repo/public/js-api" ]; then
   echo "[tools-dev] Monorepo detected — public context at /workspace/repo/public/."
 elif [ -d "$PUBLIC_DIR/js-api" ]; then
   # Cloned public repo — link workspace into packages/ and expose context at /workspace/
-  if [ -n "$TASK_KEY" ]; then
+  LINK_NAME="${FOLDER_NAME:-$TASK_KEY}"
+  if [ -n "$LINK_NAME" ]; then
     mkdir -p "$PUBLIC_DIR/packages" 2>/dev/null || true
-    [ ! -e "$PUBLIC_DIR/packages/$TASK_KEY" ] && ln -s /workspace/repo "$PUBLIC_DIR/packages/$TASK_KEY"
-    echo "[tools-dev] Workspace linked at $PUBLIC_DIR/packages/$TASK_KEY"
+    [ ! -e "$PUBLIC_DIR/packages/$LINK_NAME" ] && ln -s /workspace/repo "$PUBLIC_DIR/packages/$LINK_NAME"
+    echo "[tools-dev] Workspace linked at $PUBLIC_DIR/packages/$LINK_NAME"
   fi
-  [ ! -e /workspace/.claude ] && ln -s public/.claude /workspace/.claude
-  [ ! -e /workspace/CLAUDE.md ] && ln -s public/CLAUDE.md /workspace/CLAUDE.md
+  PUBLIC_BASENAME=$(basename "$PUBLIC_DIR")
+  [ ! -e /workspace/.claude ] && ln -s "$PUBLIC_BASENAME/.claude" /workspace/.claude
+  [ ! -e /workspace/CLAUDE.md ] && ln -s "$PUBLIC_BASENAME/CLAUDE.md" /workspace/CLAUDE.md
   echo "[tools-dev] Linked public repo context at /workspace/"
 fi
 

package/CLAUDE.md

@@ -14,7 +14,7 @@ npm run build                    # Transpile TypeScript to JavaScript using Babe
 npm run debug-source-map         # Build with source maps for debugging
 ```
 
-The build process uses Babel with `@babel/preset-typescript` to transpile TypeScript files from `bin/` (TypeScript source) to `bin/` (JavaScript output). The source TypeScript files are transpiled in-place.
+The build process uses Babel with `@babel/preset-typescript` to transpile TypeScript files from `bin/` to `bin/` (in-place). **Important:** `.ts` source and `.js` output coexist in the same `bin/` directory — `grok.js` requires the transpiled `.js` files, not the `.ts` sources. After editing any `.ts` file, you must run `npm run build` before testing.
 
 ### Link for Local Development
 ```bash
@@ -42,13 +42,18 @@ The CLI uses a modular command pattern. Each command is a separate module that:
 - `add.ts` - Add entities (functions, scripts, queries, etc.) to packages
 - `publish.ts` - Upload and deploy packages to Datagrok servers
 - `check.ts` - Validate package structure, signatures, imports
+- `build.ts` - Build one package or recursively build all packages in a directory
 - `test.ts` - Run Puppeteer-based tests for a single package
 - `test-all.ts` - Run tests across multiple packages
+- `stress-tests.ts` - Run stress tests (must be run from ApiTests package)
 - `api.ts` - Auto-generate TypeScript wrappers for scripts/queries
 - `link.ts` - Link libraries for plugin development
+- `claude.ts` - Launch a Dockerized dev environment with Datagrok + Claude Code
 - `migrate.ts` - Update legacy packages
 - `init.ts` - Apply configuration to existing packages
 
+The commands `api`, `check`, `link`, `publish`, and `test` support the `--all` flag to run recursively across all packages in the current directory.
+
 ### Template System
 
 **Package Templates** (`package-template/`)
@@ -132,6 +137,29 @@ Tests use Puppeteer for headless browser automation:
 - `--catchUnhandled` - Catch unhandled exceptions
 - `--debug` - Debug breakpoints (requires `--gui`)
 
+### `grok build` Command
+
+Builds packages with `npm install` + `npm run build`. Supports:
+- Single package: `grok build` (from package directory)
+- Recursive: `grok build --recursive` (discovers and builds all packages in subdirectories)
+- `--filter "name:Chem"` - Filter packages by package.json fields (supports regex, `&&` for multiple conditions)
+- `--parallel N` - Max parallel build jobs (default 4)
+- `--no-incremental` - Force full rebuild (default uses `--env incremental`)
+
+### `grok claude` Command
+
+Launches a full Dockerized development environment (Datagrok + PostgreSQL + RabbitMQ + tools-dev container with Claude Code):
+```bash
+grok claude <project-name>              # Create worktree + start containers + launch Claude
+grok claude <project-name> --in-place   # Use current directory (no worktree)
+grok claude <project-name> --keep       # Leave containers running on exit
+grok claude <project-name> --profile full  # Include spawner, JKG, demo DBs
+grok claude destroy <project-name>      # Tear down containers + worktree
+grok claude destroy-all                 # Destroy all known projects
+```
+
+Creates a git worktree at `~/pkg-worktrees/<project-name>`, writes Docker Compose files to `$TMPDIR/dg-pkg-<project-name>`, and auto-detects the Datagrok version (`bleeding-edge` for public repo, `latest` otherwise). Mounts `~/.claude` into the container for credentials.
+
 ## Key Patterns and Conventions
 
 ### Naming Conventions

package/bin/commands/claude.js

@@ -248,11 +248,13 @@ services:
       DG_VERSION: \${DG_VERSION:-latest}
       DG_PUBLIC_REPO: \${DG_PUBLIC_REPO:-https://github.com/datagrok-ai/public.git}
       DG_PUBLIC_BRANCH: \${DG_PUBLIC_BRANCH:-}
+      DG_PUBLIC_DIR: \${DG_PUBLIC_DIR:-/workspace/datagrok}
       JIRA_URL: \${JIRA_URL:-https://reddata.atlassian.net}
       JIRA_USERNAME: \${JIRA_USERNAME:-}
       JIRA_TOKEN: \${JIRA_TOKEN:-}
       GITHUB_TOKEN: \${GITHUB_TOKEN:-}
       TASK_KEY: \${TASK_KEY:-}
+      FOLDER_NAME: \${FOLDER_NAME:-}
     working_dir: /workspace/repo
     networks:
       dg:
@@ -332,7 +334,7 @@ function writeProjectFiles(taskKey, args, worktreeRoot, dgPort) {
   const dockerSock = process.platform === 'win32' ? '//var/run/docker.sock' : '/var/run/docker.sock';
 
   // Generate .env — all paths use forward slashes for Docker compatibility
-  const envLines = [`WORKTREE_PATH=${toDockerPath(worktreeRoot)}`, `DG_PORT=${dgPort}`, `TASK_KEY=${taskKey.toLowerCase()}`, `DOCKER_SOCK=${dockerSock}`, `DATAGROK_VERSION=${args.version || 'latest'}`, `DG_VERSION=${args.version || 'latest'}`, `GROK_CONNECT_VERSION=${args['grok-connect-version'] || 'latest'}`, `GROK_SPAWNER_VERSION=${args['grok-spawner-version'] || 'latest'}`, `JKG_VERSION=${args['jkg-version'] || 'latest'}`, `TOOLS_DEV_VERSION=${args['tools-dev-version'] || 'latest'}`];
+  const envLines = [`WORKTREE_PATH=${toDockerPath(worktreeRoot)}`, `DG_PORT=${dgPort}`, `TASK_KEY=${taskKey.toLowerCase()}`, `DOCKER_SOCK=${dockerSock}`, `DATAGROK_VERSION=${args.version || 'latest'}`, `DG_VERSION=${args.version || 'latest'}`, `GROK_CONNECT_VERSION=${args['grok-connect-version'] || 'latest'}`, `GROK_SPAWNER_VERSION=${args['grok-spawner-version'] || 'latest'}`, `JKG_VERSION=${args['jkg-version'] || 'latest'}`, `TOOLS_DEV_VERSION=${args['tools-dev-version'] || 'latest'}`, `FOLDER_NAME=${_path.default.basename(worktreeRoot)}`];
   for (const env of ['ANTHROPIC_API_KEY', 'DG_PUBLIC_BRANCH', 'JIRA_URL', 'JIRA_USERNAME', 'JIRA_TOKEN', 'GITHUB_TOKEN']) if (process.env[env]) envLines.push(`${env}=${process.env[env]}`);
   _fs.default.writeFileSync(_path.default.join(projectDir, '.env'), envLines.join('\n') + '\n');
 
@@ -540,61 +542,37 @@ async function claude(args) {
     return false;
   }
 
-  // Wait for Datagrok to be healthy
-  const healthy = await waitForDatagrok(dgPort);
-  if (!healthy) color.warn('Datagrok may not be ready. Proceeding anyway...');
-
-  // Determine whether this is a public repo and compute Claude working directory
-  const isPublic = repoRoot ? isPublicRepo(repoRoot) : false;
-  const claudeWorkDir = isPublic ? '/workspace/repo/public' : `/workspace/public/packages/${taskKey}`;
-
-  // Fix ownership on bind-mounted directories (host UID may differ from container node user)
+  // Determine Claude working directory based on repo type
+  // Only trust public-repo markers when we are inside a real git repo — packages can have
+  // public/js-api via node_modules symlinks, which would cause a false positive.
   const containerName = `dg-pkg-${taskKey.toLowerCase()}-tools-dev-1`;
-  (0, _child_process.spawnSync)('docker', ['exec', '-u', 'root', containerName, 'bash', '-c', 'chown node:node /workspace 2>/dev/null; ' + 'for d in /home/node/.claude /home/node/.npm; do ' + 'mkdir -p "$d" 2>/dev/null; chown -R node:node "$d" 2>/dev/null; done; ' + '[ -f /home/node/.claude.json ] && chown node:node /home/node/.claude.json 2>/dev/null; true'], {
-    stdio: 'inherit'
-  });
-
-  // Copy host grok config into container and ensure 'local' server exists
-  (0, _child_process.spawnSync)('docker', ['exec', '-u', 'root', containerName, 'bash', '-c', 'mkdir -p /home/node/.grok && chown node:node /home/node/.grok'], {
-    stdio: 'inherit'
-  });
-  const grokConfigPath = _path.default.join(_os.default.homedir(), '.grok', 'config.yaml');
-  if (_fs.default.existsSync(grokConfigPath)) {
-    (0, _child_process.spawnSync)('docker', ['cp', grokConfigPath, `${containerName}:/home/node/.grok/config.yaml`], {
-      stdio: 'inherit'
-    });
-    (0, _child_process.spawnSync)('docker', ['exec', '-u', 'root', containerName, 'chown', 'node:node', '/home/node/.grok/config.yaml'], {
+  let claudeWorkDir;
+  if (repoRoot && _fs.default.existsSync(_path.default.join(repoRoot, 'js-api'))) claudeWorkDir = '/workspace/repo';else if (repoRoot && _fs.default.existsSync(_path.default.join(repoRoot, 'public', 'js-api'))) claudeWorkDir = '/workspace/repo/public';else {
+    // External repo: entrypoint clones public repo and creates symlink — wait for it
+    const folderName = _path.default.basename(worktreeRoot);
+    claudeWorkDir = `/workspace/datagrok/packages/${folderName}`;
+    color.info(`Waiting for workspace at ${claudeWorkDir} (streaming container logs)...`);
+
+    // Stream container logs so the user can see clone progress
+    const logsProc = (0, _child_process.spawn)('docker', ['logs', '-f', containerName], {
       stdio: 'inherit'
     });
-    color.info('Copied grok config into container');
-  }
-
-  // Add/ensure 'local' server pointing to compose datagrok (key=admin matches GROK_PARAMETERS)
-  (0, _child_process.spawnSync)('docker', ['exec', containerName, 'node', '-e', `
-    const fs = require('fs');
-    const p = '/home/node/.grok/config.yaml';
-    let t = '';
-    try { t = fs.readFileSync(p, 'utf8'); } catch {}
-    if (!t.trim()) {
-      t = 'default: local\\nservers:\\n  local:\\n    url: http://datagrok:8080/api\\n    key: admin\\n';
-    } else if (!t.includes('datagrok:8080')) {
-      t = t.replace(/^(servers:)/m, '\\$1\\n  local:\\n    url: http://datagrok:8080/api\\n    key: admin');
-      t = t.replace(/^default:.*/m, 'default: local');
-    } else {
-      t = t.replace(/^default:.*/m, 'default: local');
+    let ready = false;
+    for (let i = 0; i < 120; i++) {
+      const check = (0, _child_process.spawnSync)('docker', ['exec', containerName, 'test', '-d', claudeWorkDir]);
+      if (check.status === 0) {
+        ready = true;
+        break;
+      }
+      await new Promise(r => setTimeout(r, 5000));
     }
-    fs.writeFileSync(p, t);
-  `], {
-    stdio: 'inherit'
-  });
 
-  // Set up workspace context: for non-public repos, bind-mount workspace into public/packages/
-  // so that process.cwd() returns the mount-point path (not the resolved symlink target)
-  // and relative paths like ../../js-api resolve correctly inside the public repo tree.
-  if (!isPublic) {
-    (0, _child_process.spawnSync)('docker', ['exec', '-u', 'root', containerName, 'bash', '-c', 'for i in $(seq 1 60); do [ -d /workspace/public/js-api ] && break; sleep 3; done; ' + 'if [ -d /workspace/public/js-api ]; then ' + '  mkdir -p /workspace/public/packages 2>/dev/null; ' + `  rm -f /workspace/public/packages/${taskKey} 2>/dev/null; ` + `  if ! mountpoint -q /workspace/public/packages/${taskKey} 2>/dev/null; then ` + `    mkdir -p /workspace/public/packages/${taskKey} && ` + `    mount --bind /workspace/repo /workspace/public/packages/${taskKey}; ` + '  fi; ' + `  echo "[grok claude] Workspace at /workspace/public/packages/${taskKey}"; ` + 'else ' + '  echo "[grok claude] Warning: public repo clone not ready"; ' + 'fi'], {
-      stdio: 'inherit'
-    });
+    // Stop streaming logs
+    logsProc.kill();
+    if (!ready) {
+      color.warn(`${claudeWorkDir} not available after 600s — falling back to /workspace/repo`);
+      claudeWorkDir = '/workspace/repo';
+    }
   }
   const claudeArgs = ['--dangerously-skip-permissions'];
   if (args.prompt) claudeArgs.push('-p', args.prompt);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "6.0.1",
+  "version": "6.0.2",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {