@dsalazardev/ai-hub-cli 0.0.0-stage → 0.1.0

package/skills/shared/modal/references/images-gpu-cookbook.md

@@ -0,0 +1,160 @@
+# Images, GPU & state cookbook
+
+Deep recipes for `modal.Image` builds, GPU selection, and Volume/Secret-backed state.
+All forms are Modal 1.0+ (modal 1.4.3). GPU is always a string; `modal.gpu.X()` is removed.
+
+## Image recipes
+
+### Torch + CUDA inference
+
+```python
+import modal
+
+image = (
+    modal.Image.debian_slim(python_version="3.12")
+    .uv_pip_install(
+        "torch==2.5.1",
+        "transformers==4.46.0",
+        "accelerate==1.1.1",
+    )
+)
+```
+
+`debian_slim` already ships a CUDA-capable userspace for the GPU you attach — you do not
+install CUDA yourself. Pin every wheel so a rebuild is byte-for-byte reproducible.
+
+### vLLM server (own HTTP server via web_server)
+
+```python
+vllm_image = (
+    modal.Image.debian_slim(python_version="3.12")
+    .uv_pip_install("vllm==0.6.4", "fastapi[standard]")
+)
+
+
+@app.function(image=vllm_image, gpu="H100", scaledown_window=300, timeout=600)
+@modal.concurrent(max_inputs=32)
+@modal.web_server(port=8000, startup_timeout=300)
+def serve():
+    import subprocess
+
+    # vLLM exposes its own OpenAI-compatible server on :8000; web_server proxies it.
+    subprocess.Popen(
+        "vllm serve meta-llama/Llama-3.1-8B-Instruct --port 8000",
+        shell=True,
+    )
+```
+
+`web_server` is the right decorator when the process *is* the server (vLLM, TGI, a custom
+uvicorn). Use `startup_timeout` to allow weight loading before health checks begin.
+
+### Diffusers image generation
+
+```python
+diff_image = modal.Image.debian_slim(python_version="3.12").uv_pip_install(
+    "diffusers==0.31.0", "torch==2.5.1", "transformers==4.46.0", "accelerate==1.1.1"
+)
+```
+
+### System binaries (ffmpeg, git)
+
+```python
+image = (
+    modal.Image.debian_slim()
+    .apt_install("ffmpeg", "git")     # system layer first — rarely changes
+    .uv_pip_install("imageio-ffmpeg==0.5.1")
+)
+```
+
+### Custom base image
+
+```python
+image = modal.Image.from_registry(
+    "nvidia/cuda:12.4.0-devel-ubuntu22.04", add_python="3.12"
+).uv_pip_install("xformers==0.0.28.post3")
+```
+
+Reach for `from_registry` only when `debian_slim` can't express what you need (a specific CUDA
+devel toolchain, a vendor image). It's heavier and slower to pull.
+
+## GPU selection matrix
+
+| GPU string | Rough fit | Notes |
+|---|---|---|
+| `"T4"` | Cheap small-model inference, embeddings | 16GB; cheapest |
+| `"L4"` | Efficient inference, light vision | 24GB |
+| `"A10"` | Mid-range inference | 24GB |
+| `"L40S"` | Mid/large inference, some training | 48GB |
+| `"A100-40GB"` / `"A100-80GB"` | Training, 7–70B serving | colon for count: `"A100:4"` |
+| `"H100"` | Fast large-model training/serving | 80GB |
+| `"H200"` | Larger memory frontier work | 141GB |
+| `"B200"` | Frontier-scale | newest |
+| `"RTX-PRO-6000"` | Workstation-class | |
+
+Multi-GPU: `gpu="A100:8"`. Fallbacks for capacity: `gpu=["H100", "A100", "any"]` — Modal grabs
+the first available so a job isn't stuck waiting for one exact type. Always start with the
+smallest GPU your model fits in; step up only when you OOM.
+
+## Volume-backed weight cache (download once, mount everywhere)
+
+The pattern that makes GPU cold starts cheap: download weights into a Volume once, then mount
+the same Volume read-mostly into every serving container.
+
+```python
+import modal
+
+app = modal.App("cached-llm")
+cache = modal.Volume.from_name("hf-cache", create_if_missing=True)
+image = modal.Image.debian_slim(python_version="3.12").uv_pip_install(
+    "huggingface_hub==0.26.2", "transformers==4.46.0", "torch==2.5.1"
+)
+CACHE_DIR = "/cache"
+
+
+@app.function(
+    image=image,
+    volumes={CACHE_DIR: cache},
+    secrets=[modal.Secret.from_name("hf-token")],
+    timeout=1800,
+)
+def warm_cache(repo: str = "meta-llama/Llama-3.1-8B-Instruct"):
+    import os
+    from huggingface_hub import snapshot_download
+
+    snapshot_download(repo, cache_dir=CACHE_DIR, token=os.environ["HF_TOKEN"])
+    cache.commit()  # persist the download so serving containers can read it
+
+
+@app.function(image=image, gpu="H100", volumes={CACHE_DIR: cache})
+def generate(prompt: str):
+    cache.reload()  # see what warm_cache committed
+    import os
+
+    os.environ["HF_HOME"] = CACHE_DIR  # transformers reads weights from the mounted cache
+    # ... load model from CACHE_DIR, no re-download ...
+```
+
+Run `modal run app.py::warm_cache` once, then deploy the serving function. Without
+`cache.commit()` the download evaporates with the container; without `cache.reload()` the
+serving container may not see it yet.
+
+## Secret patterns
+
+```python
+# Reference an existing named Secret (created via dashboard or `modal secret create`).
+hf = modal.Secret.from_name("hf-token")       # injects HF_TOKEN env var
+openai = modal.Secret.from_name("openai")     # injects OPENAI_API_KEY
+
+# Ad-hoc inline (for local dev only; prefer named Secrets in prod).
+inline = modal.Secret.from_dict({"MY_FLAG": "1"})
+
+
+@app.function(secrets=[hf, openai])
+def call():
+    import os
+
+    return os.environ["OPENAI_API_KEY"][:6]
+```
+
+Secrets become environment variables inside the container. Never put them in the image, in
+`run_commands`, or in source — those persist in layer history and in git.

package/skills/shared/modal/references/web-and-scaling.md

@@ -0,0 +1,138 @@
+# Web endpoints & scaling — deep dive
+
+The four web decorators, concurrency/batching, and the cold-start vs cost tradeoff.
+All Modal 1.0+ (modal 1.4.3). Stack order is always: `@app.function` outermost, then optional
+`@modal.concurrent`, then the web decorator innermost.
+
+## The four web decorators
+
+### `@modal.fastapi_endpoint()` — one function, one URL
+
+The lightest option: turn a single function into a GET/POST endpoint. Needs
+`fastapi[standard]` in the image.
+
+```python
+image = modal.Image.debian_slim().uv_pip_install("fastapi[standard]")
+
+
+@app.function(image=image)
+@modal.fastapi_endpoint(method="POST", docs=True)
+def predict(payload: dict) -> dict:
+    return {"echo": payload}
+```
+
+Pass `method=` for the verb and `docs=True` to expose the OpenAPI docs page. Good for a quick
+inference URL; reach for `asgi_app` when you need routing.
+
+### `@modal.asgi_app()` — a full FastAPI / Starlette app
+
+Return an ASGI app object. Use when you want multiple routes, middleware, or a real API surface.
+
+```python
+@app.function(image=image)
+@modal.concurrent(max_inputs=20)
+@modal.asgi_app()
+def api():
+    from fastapi import FastAPI
+
+    web = FastAPI()
+
+    @web.get("/items/{item_id}")
+    def read(item_id: int):
+        return {"item_id": item_id}
+
+    return web
+```
+
+The *design* of this FastAPI app (Pydantic models, dependencies, error envelopes) is the
+`fastapi` sibling's job — Modal only hosts it.
+
+### `@modal.wsgi_app()` — Flask / Django
+
+Return a WSGI callable. Same stacking; install the framework in the image.
+
+```python
+@app.function(image=modal.Image.debian_slim().uv_pip_install("flask==3.1.0"))
+@modal.wsgi_app()
+def flask_app():
+    from flask import Flask
+
+    web = Flask(__name__)
+
+    @web.route("/")
+    def index():
+        return "ok"
+
+    return web
+```
+
+### `@modal.web_server(port=...)` — run your own server
+
+When the workload spawns its own HTTP server (vLLM, TGI, a uvicorn process), use `web_server`
+and let Modal proxy the port.
+
+```python
+@app.function(image=vllm_image, gpu="H100")
+@modal.web_server(port=8000, startup_timeout=300)
+def serve():
+    import subprocess
+
+    subprocess.Popen("vllm serve <model> --port 8000", shell=True)
+```
+
+`startup_timeout` covers slow model loads before health checks begin.
+
+## Concurrency: `@modal.concurrent`
+
+One container, many simultaneous inputs. Replaces the removed `allow_concurrent_inputs=` arg.
+
+```python
+@app.function(image=image)
+@modal.concurrent(max_inputs=100)
+def handler(x): ...
+```
+
+Set `max_inputs` to what a single container can handle concurrently without resource
+contention — for I/O-bound work it can be high; for GPU inference it's bounded by VRAM and
+batch capacity. Too high → OOM / latency spikes; too low → you pay for more containers than
+needed.
+
+## Batching: `@modal.batched`
+
+Coalesce individual calls into batches server-side (great for GPU inference throughput).
+
+```python
+@app.function(image=image, gpu="A100")
+@modal.batched(max_batch_size=16, wait_ms=50)
+def embed(texts: list[str]) -> list[list[float]]:
+    # Modal collects up to 16 calls (or waits 50ms) and passes them as one list.
+    ...
+```
+
+`max_batch_size` caps the batch; `wait_ms` is how long to wait to fill it. Trade a little
+latency for much higher GPU utilization.
+
+## Scaling & cold-start tradeoff
+
+| Param | Lower value | Higher value |
+|---|---|---|
+| `min_containers` | Scale fully to zero — cheapest, cold starts on first hit | Warm pool always ready — no cold start, pays idle 24/7 |
+| `buffer_containers` | No pre-warm buffer | Absorbs traffic bursts before autoscale catches up |
+| `scaledown_window` | Containers die fast after idle — cheaper, more cold starts | Stay warm longer — fewer cold starts, more idle cost |
+
+Tuning order for a latency-sensitive endpoint: first widen `scaledown_window` so nearby
+requests reuse a hot container; only set `min_containers >= 1` if you truly need zero cold
+starts (e.g. a customer-facing API with an SLA). For a GPU endpoint, even `min_containers=1`
+is a real ongoing cost — measure before pinning it.
+
+## Custom domains, auth, proxy tokens
+
+- **Custom domain:** map your domain to the deployed app in the Modal dashboard; the
+  deployment URL is stable after `modal deploy`.
+- **Endpoint auth:** protect a `fastapi_endpoint`/`asgi_app` with Modal proxy auth tokens so
+  only callers with the token reach it — configure tokens on the function and pass them as
+  request headers. Prefer this over hand-rolling auth inside the handler for machine-to-machine
+  endpoints.
+- **Dev vs prod:** iterate with `modal serve` (hot reload, ephemeral URL); ship with
+  `modal deploy` (persistent named app, stable URL). Never rely on a `modal serve` URL in
+  production — it dies when you stop the process.

package/skills/shared/modal/scripts/verify.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+#
+# verify.sh — static correctness lint for an emitted Modal app (*.py).
+#
+# Usage:
+#   ./scripts/verify.sh [TARGET_PATH]   # default: .
+#
+# Pure read-only static checks over the Modal Python files under TARGET. Requires NO Modal
+# credentials and never executes the app. Checks, per file that looks like a Modal app:
+#   - requires `modal.App(`                                  (it is a Modal app at all)
+#   - FAILS if the removed `modal.gpu.<X>(` object form is present (use the string form)
+#   - if a web decorator (@modal.fastapi_endpoint/asgi_app/wsgi_app/web_server) is present,
+#     checks an `@app.function` decorator appears somewhere above it (stack-order heuristic)
+#   - if `modal.Volume.from_name(` is present, checks `create_if_missing` appears nearby
+# Best-effort: runs `python -c "import modal"` ONLY if a python with modal installed is found,
+# otherwise SKIPs (never FAILs on a missing interpreter/package).
+#
+# On an empty/clean target (no Modal *.py at all) it prints a SKIP and exits 0 — no false fail.
+# Compatible with stock macOS bash 3.2: no mapfile, no associative arrays, no globstar.
+
+set -euo pipefail
+
+TARGET="${1:-.}"
+
+if [ -t 1 ]; then
+  RED=$'\033[0;31m'; GREEN=$'\033[0;32m'; YELLOW=$'\033[0;33m'; RESET=$'\033[0m'
+else
+  RED=''; GREEN=''; YELLOW=''; RESET=''
+fi
+warn() { printf '%s%s%s\n' "$YELLOW" "$*" "$RESET"; }
+ok()   { printf '%s%s%s\n' "$GREEN" "$*" "$RESET"; }
+fail() { printf '%s%s%s\n' "$RED" "$*" "$RESET"; }
+
+have() { command -v "$1" >/dev/null 2>&1; }
+
+PASSED=0; FAILED=0; SKIPPED=0
+
+if [ ! -e "$TARGET" ]; then
+  warn "SKIP: target '${TARGET}' does not exist (nothing to verify)"
+  ok "verify.sh: ok (nothing to verify)"
+  exit 0
+fi
+
+# Collect candidate Modal app files: any *.py under TARGET that imports/uses modal.
+# bash 3.2 friendly: feed find output through a while-read loop, NUL-separated.
+APP_FILES=""
+while IFS= read -r -d '' f; do
+  if grep -lqE 'import[[:space:]]+modal|modal\.App\(' "$f" 2>/dev/null; then
+    APP_FILES="${APP_FILES}${f}"$'\n'
+  fi
+done < <(find "$TARGET" -type f -name '*.py' -print0 2>/dev/null)
+
+# Strip trailing newline; if empty, there is nothing to verify.
+APP_FILES="$(printf '%s' "$APP_FILES")"
+if [ -z "$APP_FILES" ]; then
+  warn "SKIP: no Modal *.py found under '${TARGET}' (no 'import modal' / 'modal.App(')"
+  ok "verify.sh: ok (nothing to verify)"
+  exit 0
+fi
+
+check_file() {
+  file="$1"
+  printf '==> %s\n' "$file"
+
+  # 1) Must declare a modal.App
+  if grep -qE 'modal\.App\(' "$file"; then
+    ok "  PASS: declares modal.App(...)"; PASSED=$((PASSED + 1))
+  else
+    fail "  FAIL: no modal.App(...) found (imports modal but never builds an App)"
+    FAILED=$((FAILED + 1))
+  fi
+
+  # 2) Removed GPU object form -> FAIL. Match modal.gpu.<Word>( e.g. modal.gpu.H100(
+  if grep -qE 'modal\.gpu\.[A-Za-z0-9_]+\(' "$file"; then
+    fail "  FAIL: removed API 'modal.gpu.X()' — use the string form, e.g. gpu=\"H100\""
+    FAILED=$((FAILED + 1))
+  else
+    ok "  PASS: no removed modal.gpu.X() object form"; PASSED=$((PASSED + 1))
+  fi
+
+  # 3) Web decorator present -> expect an @app.function above it (stack-order heuristic).
+  if grep -qE '@modal\.(fastapi_endpoint|asgi_app|wsgi_app|web_server)' "$file"; then
+    if grep -qE '@app\.function|@[A-Za-z_][A-Za-z0-9_]*\.function' "$file"; then
+      ok "  PASS: web decorator paired with an @app.function"; PASSED=$((PASSED + 1))
+    else
+      fail "  FAIL: web decorator without an @app.function (stack order: @app.function outermost)"
+      FAILED=$((FAILED + 1))
+    fi
+  fi
+
+  # 4) Volume.from_name -> expect create_if_missing somewhere in the file.
+  if grep -qE 'Volume\.from_name\(' "$file"; then
+    if grep -qE 'create_if_missing' "$file"; then
+      ok "  PASS: Volume.from_name uses create_if_missing"; PASSED=$((PASSED + 1))
+    else
+      fail "  FAIL: Volume.from_name without create_if_missing (will error if the volume is absent)"
+      FAILED=$((FAILED + 1))
+    fi
+  fi
+}
+
+while IFS= read -r f; do
+  [ -n "$f" ] && check_file "$f"
+done <<EOF
+$APP_FILES
+EOF
+
+# 5) Best-effort import smoke test, only if a python with modal is available.
+PYBIN=""
+if have python3; then PYBIN="python3"; elif have python; then PYBIN="python"; fi
+if [ -n "$PYBIN" ] && "$PYBIN" -c "import modal" >/dev/null 2>&1; then
+  printf '==> import modal smoke test\n'
+  if "$PYBIN" -c "import modal" >/dev/null 2>&1; then
+    ok "  PASS: 'import modal' works ($PYBIN)"; PASSED=$((PASSED + 1))
+  fi
+else
+  warn "SKIP: 'import modal' smoke test (modal not installed in this env)"
+  SKIPPED=$((SKIPPED + 1))
+fi
+
+printf '\n%d passed, %d skipped, %d failed\n' "$PASSED" "$SKIPPED" "$FAILED"
+if [ "$FAILED" -gt 0 ]; then
+  fail "verify.sh: failures detected"
+  exit 1
+fi
+ok "verify.sh: ok"
+exit 0