datagrok-tools 6.0.2 → 6.0.4

package/.claude/settings.local.json

@@ -14,7 +14,9 @@
       "Bash(MSYS_NO_PATHCONV=1 docker exec dg-pkg-grok-playground-tools-dev-1 ls -la /workspace/datagrok/packages/awesome 2>&1)",
       "Bash(MSYS_NO_PATHCONV=1 docker exec dg-pkg-grok-playground-tools-dev-1 ls /workspace/datagrok/packages/awesome/ 2>&1)",
       "Bash(MSYS_NO_PATHCONV=1 docker exec dg-pkg-grok-playground-tools-dev-1 bash -c 'echo \"=== Symlink ===\" && ls -la /workspace/datagrok/packages/awesome && echo \"=== Package contents ===\" && ls /workspace/datagrok/packages/awesome/ && echo \"=== Public repo root ===\" && ls /workspace/datagrok/js-api /workspace/datagrok/packages /workspace/datagrok/CLAUDE.md 2>&1 | head -5' 2>&1)",
-      "Bash(grep -r \"tools-dev\" /c/repos/Datagrok --include=\"*\" 2>/dev/null | head -20)"
+      "Bash(grep -r \"tools-dev\" /c/repos/Datagrok --include=\"*\" 2>/dev/null | head -20)",
+      "Bash(find /c/repos/Datagrok/public/tools/.devcontainer -type f -name \".*\" 2>/dev/null | sort)",
+      "Bash(grep -r \"tools-dev\" /c/repos/Datagrok/public/tools/bin --include=\"*.ts\" 2>/dev/null | head -20)"
     ]
   }
 }

package/.devcontainer/entrypoint.sh

@@ -33,17 +33,24 @@ else
   # plus root files (CLAUDE.md, .claude/, etc.). Much faster than a full clone.
   sparse_clone() {
     local branch="$1"
-    git clone --depth 1 --branch "$branch" --filter=blob:none --no-checkout "$REPO" "$PUBLIC_DIR" \
+    # Use init+fetch instead of clone to handle pre-existing directories (e.g. mount points)
+    git init "$PUBLIC_DIR" \
+      && git -C "$PUBLIC_DIR" remote add origin "$REPO" \
       && git -C "$PUBLIC_DIR" sparse-checkout set --no-cone \
            '/*' '!connectors/' '!docker/' '!docusaurus/' '!docusaurus-static/' \
-           '!environments/' '!hooks/' '!misc/' '!python-api/' '!datagrok-celery-task/' \
-           '/js-api/**' '/libraries/**' '/help/**' '/packages/**' '/tools/**' \
-      && git -C "$PUBLIC_DIR" checkout
+           '!environments/' '!hooks/' '!misc/' 'python-api/' '!datagrok-celery-task/' \
+           '/js-api/**' '/libraries/**' '/help/**/*.md*' '/packages/**' '!help/uploads/**' \
+      && git -C "$PUBLIC_DIR" fetch --depth 1 --filter=blob:none origin "$branch" \
+      && git -C "$PUBLIC_DIR" checkout -B "$branch" FETCH_HEAD
+  }
+  # Clear directory contents without removing the dir itself (may be a mount point)
+  clear_dir() {
+    find "$1" -mindepth 1 -delete 2>/dev/null || rm -rf "$1"/* "$1"/.[!.]* 2>/dev/null || true
   }
   echo "[tools-dev] Cloning public repo ($BRANCH, sparse) into $PUBLIC_DIR..."
   sparse_clone "$BRANCH" \
-    || { echo "[tools-dev] Branch '$BRANCH' not found, falling back to master."
-         rm -rf "$PUBLIC_DIR"
+    || { echo "[tools-dev] Branch '$BRANCH' clone failed, falling back to master."
+         clear_dir "$PUBLIC_DIR"
          sparse_clone "master"; }
   echo "[tools-dev] Public repo ready at $PUBLIC_DIR (branch: $(git -C "$PUBLIC_DIR" branch --show-current))."
 fi

package/CLAUDE.md

@@ -148,17 +148,78 @@ Builds packages with `npm install` + `npm run build`. Supports:
 
 ### `grok claude` Command
 
-Launches a full Dockerized development environment (Datagrok + PostgreSQL + RabbitMQ + tools-dev container with Claude Code):
+Launches a full Dockerized development environment with Datagrok + Claude Code. The compose configuration is embedded in `claude.ts` — no external files needed.
+
 ```bash
-grok claude <project-name>              # Create worktree + start containers + launch Claude
-grok claude <project-name> --in-place   # Use current directory (no worktree)
-grok claude <project-name> --keep       # Leave containers running on exit
-grok claude <project-name> --profile full  # Include spawner, JKG, demo DBs
-grok claude destroy <project-name>      # Tear down containers + worktree
-grok claude destroy-all                 # Destroy all known projects
+grok claude <project>                      # Create worktree + start containers + launch Claude
+grok claude <project> --in-place           # Use current directory (no worktree)
+grok claude <project> --keep               # Leave containers running on exit
+grok claude <project> --version 1.22.0     # Pin Datagrok version
+grok claude <project> --profile full       # Include spawner, JKG, demo DBs
+grok claude <project> --profile scripting  # Include JKG for Python/R/Julia
+grok claude <project> --port 8080          # Fix Datagrok port (default: random free port)
+grok claude <project> --prompt "fix bug"   # Pass prompt to Claude Code (-p flag)
+grok claude destroy <project>              # Tear down containers + worktree + temp files
+grok claude destroy-all                    # Destroy all known projects
 ```
 
-Creates a git worktree at `~/pkg-worktrees/<project-name>`, writes Docker Compose files to `$TMPDIR/dg-pkg-<project-name>`, and auto-detects the Datagrok version (`bleeding-edge` for public repo, `latest` otherwise). Mounts `~/.claude` into the container for credentials.
+**Project name restrictions:** `master` and `main` are rejected.
+
+**Lifecycle:**
+1. Creates git worktree at `~/pkg-worktrees/<project>` (unless `--in-place` or not in a git repo)
+2. Writes `docker-compose.yaml` + `.env` + optional `docker-compose.override.yaml` to `$TMPDIR/dg-pkg-<project>`
+3. Runs `docker compose up -d --wait`
+4. Detects Claude working directory based on repo type (see below)
+5. Launches `claude --dangerously-skip-permissions` inside the `tools-dev` container
+6. On exit: stops containers (unless `--keep`)
+
+**Version resolution:** `--version` flag > `bleeding-edge` (if inside public repo) > `latest`
+
+**Compose services (embedded template):**
+- Always started: `postgres` (pgvector:pg17), `rabbitmq`, `grok_pipe`, `datagrok`, `grok_connect`, `tools-dev`
+- Profile `full`: adds `grok_spawner`
+- Profile `scripting`/`full`: adds `jupyter_kernel_gateway`
+- Profile `demo`/`full`: adds `world`, `test_db`, `northwind` demo databases
+
+**Note:** The embedded template in `claude.ts` differs from `.devcontainer/docker-compose.yaml`:
+- Embedded uses separate version vars per service (`DATAGROK_VERSION`, `GROK_CONNECT_VERSION`, `GROK_SPAWNER_VERSION`, `JKG_VERSION`, `TOOLS_DEV_VERSION`); `.devcontainer/` uses a single `DG_VERSION` for all
+- Embedded mounts workspace at `/workspace/repo`; `.devcontainer/` mounts at `/workspace`
+- Embedded has `grok_connect` always-on (no profile); `.devcontainer/` puts it under `profiles: ["full"]`
+- Embedded uses `grok_pipe:latest`; `.devcontainer/` uses `grok_pipe:${DG_VERSION}`
+- Embedded doesn't mount `~/.grok` from host; `.devcontainer/` does
+
+**Host config override:** If `~/.claude` (or `CLAUDE_HOME`) is found on the host, a `docker-compose.override.yaml` is generated to bind-mount it into the container. Also mounts `~/.claude.json` if present.
+
+**Working directory detection inside container:**
+- Public repo root (has `js-api/`): `/workspace/repo`
+- Monorepo (has `public/js-api/`): `/workspace/repo/public`
+- External repo: `/workspace/datagrok/packages/<folder-name>` (waits up to 600s for entrypoint to clone public repo)
+
+#### tools-dev Container (`Dockerfile.pkg_dev`)
+
+Based on `node:22-bookworm-slim`. Pre-installed:
+- Google Chrome stable (for Puppeteer), Playwright + Chromium
+- `datagrok-tools` (grok CLI) and `@anthropic-ai/claude-code` (global npm)
+- git, curl, jq, docker CLI
+- Runs as `node` user (UID 1000, added to `docker` group)
+
+#### Entrypoint (`entrypoint.sh`)
+
+1. **Repo detection:** checks if `/workspace/repo` is the public repo (has `.git` + `js-api/`) or monorepo (`public/js-api/`)
+2. **Auto-clone:** if workspace is not the public repo, sparse-clones it to `$DG_PUBLIC_DIR` (default `/workspace/datagrok`) — excludes `connectors/`, `docker/`, `environments/`, `python-api/`, etc. for speed. Branch resolved as: `DG_PUBLIC_BRANCH` > `DG_VERSION` mapped to branch > `master` fallback
+3. **Workspace linking:** for non-public repos, symlinks `/workspace/repo` into the cloned repo's `packages/` dir and links `.claude`/`CLAUDE.md` at `/workspace/` for context discovery
+4. **Grok config:** auto-creates `~/.grok/config.yaml` pointing to `http://datagrok:8080/api` with key `admin` (only if config doesn't already exist)
+
+#### Profiles
+
+| Profile | Additional services |
+|---------|-------------------|
+| (none) | postgres, rabbitmq, grok_pipe, datagrok, grok_connect, tools-dev |
+| `scripting` | + jupyter_kernel_gateway |
+| `demo` | + world, test_db, northwind |
+| `full` | + grok_spawner, JKG, demo DBs |
+
+See `.devcontainer/PACKAGES_DEV.md` for detailed usage docs, architecture diagram, MCP plugin setup (Jira/GitHub), and troubleshooting.
 
 ## Key Patterns and Conventions
 

package/bin/commands/claude.js

@@ -553,22 +553,24 @@ async function claude(args) {
     claudeWorkDir = `/workspace/datagrok/packages/${folderName}`;
     color.info(`Waiting for workspace at ${claudeWorkDir} (streaming container logs)...`);
 
-    // Stream container logs so the user can see clone progress
+    // Stream container logs so the user can see clone progress (don't inherit stdin)
     const logsProc = (0, _child_process.spawn)('docker', ['logs', '-f', containerName], {
-      stdio: 'inherit'
+      stdio: ['ignore', 'inherit', 'inherit']
     });
     let ready = false;
+    const waitStart = Date.now();
     for (let i = 0; i < 120; i++) {
       const check = (0, _child_process.spawnSync)('docker', ['exec', containerName, 'test', '-d', claudeWorkDir]);
       if (check.status === 0) {
         ready = true;
         break;
       }
+      if (i > 0 && i % 12 === 0) color.info(`Still waiting for workspace... (${Math.round((Date.now() - waitStart) / 1000)}s elapsed)`);
       await new Promise(r => setTimeout(r, 5000));
     }
 
     // Stop streaming logs
-    logsProc.kill();
+    logsProc.kill('SIGKILL');
     if (!ready) {
       color.warn(`${claudeWorkDir} not available after 600s — falling back to /workspace/repo`);
       claudeWorkDir = '/workspace/repo';

package/package-template/package.json

@@ -7,7 +7,7 @@
     "datagrok-api": "^1.26.0",
     "cash-dom": "^8.1.5",
     "dayjs": "^1.11.13",
-    "@datagrok-libraries/utils": "^4.6.5"
+    "@datagrok-libraries/test": "^1.1.0"
   },
   "devDependencies": {
     "datagrok-tools": "^5.1.5",

package/package-template/src/package-test.ts

@@ -1,4 +1,4 @@
-import { runTests, tests, TestContext , initAutoTests as initTests } from '@datagrok-libraries/utils/src/test';
+import { runTests, tests, TestContext , initAutoTests as initTests } from '@datagrok-libraries/test/src/test';
 import * as DG from 'datagrok-api/dg';
 
 export let _package = new DG.Package();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "6.0.2",
+  "version": "6.0.4",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {