datagrok-tools 5.1.8 → 6.0.0

package/.devcontainer/docker-compose.yaml

@@ -0,0 +1,236 @@
+# Packages Dev — lightweight JS/TS package development environment
+# Usage: docker compose up -d
+#
+# All images are pre-built on Docker Hub. No local build step needed.
+# The Dockerfile.pkg_dev in this directory is the build recipe for the
+# datagrok/tools-dev image — used by CI, not by end users.
+#
+# On Windows, use `grok claude` instead — it handles path conversion
+# and Docker socket mapping automatically.
+
+services:
+  postgres:
+    image: pgvector/pgvector:pg17
+    command: postgres -c 'max_connections=1000'
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    shm_size: 2gb
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      retries: 10
+    networks:
+      dg:
+        aliases:
+          - database
+          - postgres
+    restart: unless-stopped
+
+  rabbitmq:
+    image: rabbitmq:4.0.5-management
+    platform: linux/amd64
+    environment:
+      RABBITMQ_DEFAULT_USER: guest
+      RABBITMQ_DEFAULT_PASS: guest
+    networks:
+      dg:
+        aliases:
+          - rabbitmq
+    restart: unless-stopped
+
+  grok_pipe:
+    image: datagrok/grok_pipe:${DG_VERSION:-latest}
+    platform: linux/amd64
+    environment:
+      API_KEY: test-key
+    networks:
+      dg:
+        aliases:
+          - grok_pipe
+    restart: unless-stopped
+
+  datagrok:
+    image: datagrok/datagrok:${DG_VERSION:-latest}
+    depends_on:
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_started
+    environment:
+      GROK_PARAMETERS: |
+        {
+          "dbServer": "database",
+          "db": "datagrok",
+          "dbLogin": "dg",
+          "dbPassword": "dg",
+          "dbAdminLogin": "postgres",
+          "dbAdminPassword": "postgres",
+          "adminPassword": "admin",
+          "adminDevKey": "admin",
+          "webRoot": "http://datagrok:8080",
+          "apiRoot": "http://datagrok:8080/api",
+          "deployDemo": true,
+          "deployTestDemo": false
+        }
+    ports:
+      - "${DG_PORT:-0}:8080"
+    volumes:
+      - datagrok_data:/home/grok/data
+      - datagrok_cfg:/home/grok/cfg
+    networks:
+      dg:
+        aliases:
+          - datagrok
+    restart: on-failure
+
+  # ── Optional services (use --profile to enable) ──────────────
+
+  grok_connect:
+    image: datagrok/grok_connect:${DG_VERSION:-latest}
+    platform: linux/amd64
+    environment:
+      GROK_CONNECT_PORT: 1234
+    networks:
+      dg:
+        aliases:
+          - grok_connect
+    restart: unless-stopped
+    profiles: ["full"]
+
+  grok_spawner:
+    image: datagrok/grok_spawner:${DG_VERSION:-latest}
+    platform: linux/amd64
+    user: root
+    environment:
+      X_API_KEY: test-x-api-key
+      GROK_SPAWNER_ENVIRONMENT: ${COMPOSE_PROJECT_NAME:-dg-pkg}
+      GROK_SPAWNER_CORE_MODE: "false"
+      GROK_SPAWNER_FORCE_DOCKER_TYPE_HOST: "true"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      dg:
+        aliases:
+          - grok_spawner
+    restart: unless-stopped
+    profiles: ["full"]
+
+  jupyter_kernel_gateway:
+    image: datagrok/jupyter_kernel_gateway:${DG_VERSION:-latest}
+    platform: linux/amd64
+    depends_on:
+      rabbitmq:
+        condition: service_started
+    environment:
+      GROK_PARAMETERS: |
+        {
+          "queueSettings": {
+            "useQueue": true,
+            "amqpHost": "rabbitmq",
+            "amqpUser": "guest",
+            "amqpPassword": "guest",
+            "amqpPort": 5672,
+            "pipeHost": "grok_pipe",
+            "pipePort": 3000,
+            "pipeKey": "test-key",
+            "maxConcurrentCalls": 16
+          }
+        }
+    networks:
+      dg:
+        aliases:
+          - jupyter_kernel_gateway
+    restart: unless-stopped
+    profiles: ["scripting", "full"]
+
+  world:
+    image: datagrok/demo_db_postgres:12-1.1.0
+    environment:
+      POSTGRES_DB: world
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    volumes:
+      - demo_world:/var/lib/postgresql/data
+    networks:
+      dg:
+        aliases:
+          - world
+    restart: on-failure
+    profiles: ["demo", "full"]
+
+  test_db:
+    image: datagrok/demo_db_postgres:12-1.1.0
+    environment:
+      POSTGRES_DB: test
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    volumes:
+      - demo_test:/var/lib/postgresql/data
+    networks:
+      dg:
+        aliases:
+          - test_db
+    restart: on-failure
+    profiles: ["demo", "full"]
+
+  northwind:
+    image: datagrok/demo_db_postgres:12-1.1.0
+    environment:
+      POSTGRES_DB: northwind
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    volumes:
+      - demo_northwind:/var/lib/postgresql/data
+    networks:
+      dg:
+        aliases:
+          - northwind
+    restart: on-failure
+    profiles: ["demo", "full"]
+
+  # ── Package dev container (pre-built image from Docker Hub) ──
+
+  tools-dev:
+    image: datagrok/tools-dev:${DG_TOOLS_DEV_VERSION:-latest}
+    cap_add:
+      - SYS_ADMIN
+    security_opt:
+      - seccomp=unconfined
+    volumes:
+      - ${WORKTREE_PATH:-.}:/workspace
+      - ${HOST_HOME:-${HOME}}/.claude:/home/node/.claude
+      - ${HOST_HOME:-${HOME}}/.grok:/home/node/.grok
+      - /var/run/docker.sock:/var/run/docker.sock
+      - npm_cache:/home/node/.npm
+    environment:
+      ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY}
+      DG_VERSION: ${DG_VERSION:-latest}
+      DG_PUBLIC_REPO: ${DG_PUBLIC_REPO:-https://github.com/datagrok-ai/public.git}
+      DG_PUBLIC_BRANCH: ${DG_PUBLIC_BRANCH:-}
+      JIRA_URL: ${JIRA_URL:-https://reddata.atlassian.net}
+      JIRA_USERNAME: ${JIRA_USERNAME}
+      JIRA_TOKEN: ${JIRA_TOKEN}
+      GITHUB_TOKEN: ${GITHUB_TOKEN}
+    networks:
+      dg:
+        aliases:
+          - tools-dev
+    stdin_open: true
+    tty: true
+    restart: unless-stopped
+
+volumes:
+  pgdata:
+  datagrok_data:
+  datagrok_cfg:
+  demo_world:
+  demo_test:
+  demo_northwind:
+  npm_cache:
+
+networks:
+  dg:
+    name: dg-pkg-${TASK_KEY:-default}-net

package/.devcontainer/entrypoint.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+set -e
+
+# Resolve the actual version when DG_VERSION is "latest" — needed for git branch matching.
+resolve_version() {
+  local ver="${1:-latest}"
+  if [ "$ver" = "latest" ]; then
+    echo "[tools-dev] Resolving latest Datagrok version from Docker Hub..." >&2
+    local resolved
+    resolved=$(curl -sf "https://hub.docker.com/v2/repositories/datagrok/datagrok/tags?page_size=100&ordering=-name" \
+      | node -e "
+        const data = JSON.parse(require('fs').readFileSync('/dev/stdin','utf8'));
+        const tags = (data.results||[]).map(t=>t.name).filter(n=>/^\d+\.\d+\.\d+$/.test(n));
+        tags.sort((a,b)=>{
+          const pa=a.split('.').map(Number), pb=b.split('.').map(Number);
+          return pb[0]-pa[0]||pb[1]-pa[1]||pb[2]-pa[2];
+        });
+        if(tags[0]) process.stdout.write(tags[0]);
+      " 2>/dev/null) || true
+    if [ -n "$resolved" ]; then
+      echo "[tools-dev] Resolved latest version: $resolved" >&2
+      echo "$resolved"
+    else
+      echo "[tools-dev] Could not resolve latest version, falling back to master" >&2
+      echo "master"
+    fi
+  else
+    echo "$ver"
+  fi
+}
+
+PUBLIC_DIR="/workspace/public"
+# Detect if /workspace/repo IS the public repo (has js-api/ at root)
+if [ -d "/workspace/repo/js-api" ]; then
+  echo "[tools-dev] Public repo detected at /workspace/repo — skipping clone."
+elif [ -d "/workspace/repo/public/js-api" ]; then
+  echo "[tools-dev] Monorepo detected — public repo at /workspace/repo/public."
+elif [ -d "$PUBLIC_DIR/js-api" ]; then
+  echo "[tools-dev] Public repo already at $PUBLIC_DIR."
+else
+  REPO="${DG_PUBLIC_REPO:-https://github.com/datagrok-ai/public.git}"
+  # Resolve branch: explicit DG_PUBLIC_BRANCH > resolved DG_VERSION > master
+  if [ -n "$DG_PUBLIC_BRANCH" ]; then
+    BRANCH="$DG_PUBLIC_BRANCH"
+  else
+    BRANCH=$(resolve_version "${DG_VERSION:-latest}")
+  fi
+  echo "[tools-dev] Cloning public repo ($BRANCH) into $PUBLIC_DIR..."
+  git clone --depth 1 --branch "$BRANCH" "$REPO" "$PUBLIC_DIR" 2>/dev/null \
+    || { echo "[tools-dev] Branch '$BRANCH' not found, falling back to master."
+         git clone --depth 1 --branch master "$REPO" "$PUBLIC_DIR"; }
+  echo "[tools-dev] Public repo ready at $PUBLIC_DIR (branch: $(git -C "$PUBLIC_DIR" branch --show-current))."
+fi
+
+# ── Mount workspace inside public repo for non-public workspaces ──
+# When TASK_KEY is set and workspace is not the public repo, link it into packages/
+# so Claude Code walks up to find all public repo context (CLAUDE.md, .claude/, js-api/, etc.)
+if [ -d "/workspace/repo/js-api" ]; then
+  echo "[tools-dev] Public repo IS the workspace — no linking needed."
+elif [ -d "/workspace/repo/public/js-api" ]; then
+  echo "[tools-dev] Monorepo detected — public context at /workspace/repo/public/."
+elif [ -d "$PUBLIC_DIR/js-api" ]; then
+  # Cloned public repo — link workspace into packages/ and expose context at /workspace/
+  if [ -n "$TASK_KEY" ]; then
+    mkdir -p "$PUBLIC_DIR/packages" 2>/dev/null || true
+    [ ! -e "$PUBLIC_DIR/packages/$TASK_KEY" ] && ln -s /workspace/repo "$PUBLIC_DIR/packages/$TASK_KEY"
+    echo "[tools-dev] Workspace linked at $PUBLIC_DIR/packages/$TASK_KEY"
+  fi
+  [ ! -e /workspace/.claude ] && ln -s public/.claude /workspace/.claude
+  [ ! -e /workspace/CLAUDE.md ] && ln -s public/CLAUDE.md /workspace/CLAUDE.md
+  echo "[tools-dev] Linked public repo context at /workspace/"
+fi
+
+# Auto-configure grok CLI to point to the local Datagrok instance.
+# Only creates the config if it doesn't already exist (preserves host config).
+GROK_DIR="/home/node/.grok"
+GROK_CFG="$GROK_DIR/config.yaml"
+if [ ! -f "$GROK_CFG" ] || [ ! -s "$GROK_CFG" ]; then
+  mkdir -p "$GROK_DIR" 2>/dev/null || true
+  cat > "$GROK_CFG" <<'YAML' 2>/dev/null || true
+default: local
+servers:
+  local:
+    url: http://datagrok:8080/api
+    key: admin
+YAML
+  [ -s "$GROK_CFG" ] && echo "[tools-dev] Created grok config at $GROK_CFG"
+elif ! grep -q "datagrok:8080" "$GROK_CFG" 2>/dev/null; then
+  echo "[tools-dev] Note: existing grok config found. Add 'local' server with:"
+  echo "  grok config add --alias local --server http://datagrok:8080/api --key admin --default"
+fi
+
+exec "$@"