@gen3/workspaces 0.12.64 → 0.12.66

package/dist/jupyterlite-builds/scripts/build-jupyterlite.sh

@@ -1,18 +1,56 @@
 #!/usr/bin/env bash
+# =============================================================================
+# build-jupyterlite.sh — Build a JupyterLite site (free or remote tier)
+#
+# Sources setup-env.sh so the venv, Python version, and tooling are identical
+# to those used by build-gen3sdk-lite.sh.
+#
+# Package delivery model (see jupyter_lite_config.json):
+#   - PyodideAddon vendors the full Pyodide distribution into static/pyodide,
+#     so the runtime and its ~300 wheels are served from our own origin.
+#   - PyodideLockAddon is DISABLED. In jupyterlite-pyodide-kernel 0.8.5 its
+#     post_build_lock() copies the lock away from the wheels it references,
+#     so a vendored local distribution fails with Pep508UrlError. The
+#     distribution's own pyodide-lock.json is shipped untouched instead.
+#   - Our wheels reach the browser via PipliteAddon, fed by the
+#     --piplite-wheels flags below.
+#
+# Usage:
+#   ./build-jupyterlite.sh <tier> [build_dir]
+#     tier: 'free' or 'remote'
+# =============================================================================
 set -euo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
 usage() {
   echo "Usage: $0 <tier> [build_dir]"
-  echo "  tier: 'free' or 'remote'"
+  echo "  tier:  'free' or 'remote'"
   exit 1
 }
 
+# ── Parse flags (strip before positional args are consumed) ──────────────────
+POSITIONAL_ARGS=()
+for _arg in "$@"; do
+  case "$_arg" in
+    # Accepted and ignored for backwards compatibility. This used to skip
+    # downloading CDN wheels and patching pyodide-lock.json; that step is gone
+    # because the Pyodide distribution is now vendored at build time.
+    --no-extras)
+      echo "[WARN]  --no-extras is obsolete and has no effect; remove it from package.json"
+      ;;
+    *) POSITIONAL_ARGS+=("$_arg") ;;
+  esac
+done
+set -- "${POSITIONAL_ARGS[@]+"${POSITIONAL_ARGS[@]}"}"
+
 TIER="${1:-}"
 if [[ "$TIER" != "free" && "$TIER" != "remote" ]]; then
   usage
 fi
 
-ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+# ── Path constants (self-contained, mirrors build-gen3sdk-lite layout) ───────
+ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
 
 if [[ "$TIER" == "free" ]]; then
   SRC_DIR="$ROOT_DIR/free-private"
@@ -20,47 +58,126 @@ else
   SRC_DIR="$ROOT_DIR/remote-private"
 fi
 
+_resolve_against_init_cwd() {
+  local path="$1"
+  local base="${INIT_CWD:-$(pwd)}"
+  case "$path" in
+    /*) printf '%s' "$path" ;;
+    *)  printf '%s/%s' "$base" "$path" ;;
+  esac
+}
+
+BUILD_DIR="$(_resolve_against_init_cwd "${2:-builds}")"
+mkdir -p "$BUILD_DIR"
+BUILD_DIR="$(cd "$BUILD_DIR" && pwd)"
+
+VENV_DIR="$BUILD_DIR/.venv"
+OUTPUT_DIR="$BUILD_DIR/pypi"
+
+export BUILD_DIR   # setup-env.sh reads this
+
+# ── Colours / helpers (available before setup-env.sh) ────────────────────────
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+GREEN='\033[0;32m'
+NC='\033[0m'
+info()    { echo -e "${CYAN}[INFO]${NC}  $*"; }
+success() { echo -e "${GREEN}[OK]${NC}    $*"; }
+warn()    { echo -e "${YELLOW}[WARN]${NC}  $*"; }
+fail()    { echo -e "${RED}[FAIL]${NC}  $*"; exit 1; }
+
+# ── Source shared environment (creates/reuses the venv) ─────────────────────
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/setup-env.sh"
+
+# ── JupyterLite-specific setup ──────────────────────────────────────────────
 CONFIG_FILE="$SRC_DIR/jupyter_lite_config.json"
 REQUIREMENTS_FILE="$SRC_DIR/requirements.txt"
-PYTHON_BIN="${PYTHON_BIN:-$(command -v python3)}"
-BUILD_DIR="${2:-$SRC_DIR/build}"
-VENV_DIR="$BUILD_DIR/.venv"
-VENV_PYTHON="$VENV_DIR/bin/python"
 VENV_JUPYTER_LITE="$VENV_DIR/bin/jupyter-lite"
-
-if [[ -z "$PYTHON_BIN" ]] || ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then
-  echo "python3 executable not found. Set PYTHON_BIN to a valid Python interpreter."
-  exit 1
+PYPI="$OUTPUT_DIR"   # wheels built by build-gen3sdk-lite land here
+
+[[ -f "$REQUIREMENTS_FILE" ]] || fail "Missing requirements file: $REQUIREMENTS_FILE"
+[[ -f "$CONFIG_FILE" ]]       || fail "Missing config file: $CONFIG_FILE"
+
+# For the free tier, skip reinstalling requirements when jupyter-lite is already
+# present to preserve exact package versions set up by build-gen3sdk-lite.sh.
+# For the remote tier, always install because its requirements.txt includes
+# jupyterlite-remote-server which the free-tier build never installs.
+#
+if [[ "$TIER" == "free" && -x "$VENV_JUPYTER_LITE" ]]; then
+  info "Existing venv already has jupyter-lite — skipping requirements install"
+else
+  info "Installing JupyterLite requirements …"
+  uv pip install --verbose -r "$REQUIREMENTS_FILE" 2>&1 | tail -5
+  success "JupyterLite requirements installed"
 fi
 
-if [[ ! -f "$REQUIREMENTS_FILE" ]]; then
-  echo "Missing requirements file: $REQUIREMENTS_FILE"
-  exit 1
-fi
+[[ -x "$VENV_JUPYTER_LITE" ]] || \
+  fail "jupyter-lite executable not found in $VENV_DIR after dependency install."
 
-mkdir -p "$BUILD_DIR"
-
-"$PYTHON_BIN" -m venv "$VENV_DIR"
-
-"$VENV_PYTHON" -m pip install --upgrade pip
-"$VENV_PYTHON" -m pip install -r "$REQUIREMENTS_FILE"
-
-if [[ ! -x "$VENV_JUPYTER_LITE" ]]; then
-  echo "jupyter-lite executable not found in $VENV_DIR after dependency install."
-  exit 1
+# ── Collect our own wheels for PipliteAddon ─────────────────────────────────
+# With PyodideLockAddon disabled these flags are the ONLY route by which our
+# wheels reach the browser. This only applies to the free tier
+WHEEL_ARGS=()
+WHEEL_COUNT=0
+if [[ "$TIER" == "free" ]]; then
+  if [[ -d "$PYPI" ]]; then
+    info "Looking for wheels in $PYPI"
+    while IFS= read -r -d '' whl; do
+      WHEEL_ARGS+=(--piplite-wheels "$whl")
+      ((WHEEL_COUNT++)) || true
+    done < <(find "$PYPI" -maxdepth 1 -name "*.whl" -print0 | sort -z)
+  fi
+
+  if (( WHEEL_COUNT == 0 )); then
+    fail "No wheels found in $PYPI — run build-gen3sdk-lite.sh first."
+  fi
+else
+  info "Skipping custom wheel collection for $TIER tier"
 fi
 
-
-echo "$VENV_JUPYTER_LITE" build \
-       --config "$CONFIG_FILE" \
-       --lite-dir "$SRC_DIR" \
-       --output-dir "$BUILD_DIR/$TIER"
-
+# ── Build ───────────────────────────────────────────────────────────────────
+info "Running: jupyter-lite build ($TIER tier, ${WHEEL_COUNT} custom wheels)"
 "$VENV_JUPYTER_LITE" build \
   --config "$CONFIG_FILE" \
   --lite-dir "$SRC_DIR" \
-  --output-dir "$BUILD_DIR/$TIER"
-
+  --output-dir "$BUILD_DIR/$TIER" \
+  ${WHEEL_ARGS[@]+"${WHEEL_ARGS[@]}"}
+
+# ── Verify the site is self-contained ───────────────────────────────────────
+# The whole point of vendoring the distribution is that nothing is fetched from
+# a CDN at runtime. Check it here rather than discovering it as a browser hang.
+LOCK_FILE="$BUILD_DIR/$TIER/static/pyodide/pyodide-lock.json"
+if [[ -f "$LOCK_FILE" ]]; then
+  REMOTE_COUNT="$(python3 - "$LOCK_FILE" <<'PYEOF'
+import json, sys
+lock = json.load(open(sys.argv[1]))
+remote = [
+    m.get("file_name", "")
+    for m in lock.get("packages", {}).values()
+    if "://" in m.get("file_name", "")
+]
+print(len(remote))
+for r in remote[:10]:
+    print(f"  {r}", file=sys.stderr)
+PYEOF
+)"
+  if [[ "$REMOTE_COUNT" != "0" ]]; then
+    warn "$REMOTE_COUNT package(s) in pyodide-lock.json point at a remote host;"
+    warn "  the browser will fetch these at runtime (see the sample above)."
+  else
+    success "pyodide-lock.json is fully local ($(ls -1 "$BUILD_DIR/$TIER/static/pyodide"/*.whl 2>/dev/null | wc -l | tr -d ' ') wheels)"
+  fi
+else
+  warn "No pyodide-lock.json at $LOCK_FILE — is PyodideAddon.pyodide_url set?"
+fi
 
+PIPLITE_DIR="$BUILD_DIR/$TIER/static/pypi"
+if [[ -d "$PIPLITE_DIR" ]]; then
+  success "piplite index: $(ls -1 "$PIPLITE_DIR"/*.whl 2>/dev/null | wc -l | tr -d ' ') wheels"
+else
+  warn "No piplite index at $PIPLITE_DIR — our wheels did not reach the site."
+fi
 
-echo "$TIER-tier JupyterLite build complete: $BUILD_DIR/$TIER (venv: $VENV_DIR)"
+success "$TIER-tier JupyterLite build complete: $BUILD_DIR/$TIER"

package/dist/jupyterlite-builds/scripts/setup-env.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# =============================================================================
+# setup-env.sh — Shared build-environment bootstrap
+#
+# Guarantees both build-gen3sdk-lite and build-jupyterlite use the same
+# Python interpreter and virtual environment.
+#
+# BUILD_DIR resolution (first match wins):
+#   1. Pre-set BUILD_DIR environment variable  (export BUILD_DIR=…)
+#   2. First positional argument               (source setup-env.sh [/tmp/mybuild | ./mybuild])
+#   3. Default: "builds"
+#
+# Usage:
+#   source setup-env.sh [build_dir]
+#
+# After sourcing, the venv at $BUILD_DIR/.venv is activated in the calling
+# shell, and VENV_DIR is exported.
+# =============================================================================
+
+# ── Configuration ────────────────────────────────────────────────────────────
+PYTHON_VERSION="3.14.2"
+
+# Resolves a possibly-relative path against INIT_CWD (or $PWD) rather than
+# whatever directory happens to be current when this runs.
+_resolve_against_init_cwd() {
+  local path="$1"
+  local base="${INIT_CWD:-$(pwd)}"
+  case "$path" in
+    /*) printf '%s' "$path" ;;
+    *)  printf '%s/%s' "$base" "$path" ;;
+  esac
+}
+
+BUILD_DIR="${BUILD_DIR:-$(_resolve_against_init_cwd "${1:-builds}")}"
+
+mkdir -p "$BUILD_DIR"
+BUILD_DIR="$(cd "$BUILD_DIR" && pwd)"
+
+VENV_DIR="$BUILD_DIR/.venv"
+export BUILD_DIR VENV_DIR PYTHON_VERSION
+
+# ── Colours / helpers ────────────────────────────────────────────────────────
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+info()    { echo -e "${CYAN}[INFO]${NC}  $*"; }
+success() { echo -e "${GREEN}[OK]${NC}    $*"; }
+warn()    { echo -e "${YELLOW}[WARN]${NC}  $*"; }
+fail()    { echo -e "${RED}[FAIL]${NC}  $*"; return 1 2>/dev/null || exit 1; }
+
+# Export helpers so functions in the calling script can use them.
+export -f info success warn fail
+
+# ── uv ───────────────────────────────────────────────────────────────────────
+if ! command -v uv &>/dev/null; then
+  fail "uv is not installed. Install it first: https://docs.astral.sh/uv/getting-started/installation/"
+fi
+success "uv found: $(uv --version)"
+
+# ── Python (via uv) ─────────────────────────────────────────────────────────
+info "Ensuring Python ${PYTHON_VERSION} is available via uv …"
+uv python install "${PYTHON_VERSION}" 2>&1 | tail -1
+success "Python ${PYTHON_VERSION} ready"
+
+# ── Virtual environment (create once, reuse everywhere) ──────────────────────
+if [[ ! -d "${VENV_DIR}" ]]; then
+  info "Creating virtual environment at ${VENV_DIR} …"
+  uv venv --python "${PYTHON_VERSION}" "${VENV_DIR}"
+fi
+
+# shellcheck disable=SC1091
+source "${VENV_DIR}/bin/activate"
+success "Activated venv ($(python --version))"
+
+# ── Core Python tooling (idempotent — uv skips already-installed) ────────────
+info "Ensuring core Python build tooling …"
+uv pip install --upgrade pip build 2>&1 | tail -3
+success "Core tooling ready"

package/dist/jupyterlite-builds/scripts/sync-jupyter-assets-into-package.sh

@@ -1,12 +1,23 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-
-BUILD_SRC="${1:-}"
+_resolve_against_init_cwd() {
+  local path="$1"
+  local base="${INIT_CWD:-$(pwd)}"
+  case "$path" in
+    /*) printf '%s' "$path" ;;
+    *)  printf '%s/%s' "$base" "$path" ;;
+  esac
+}
+
+BUILD_SRC="$(_resolve_against_init_cwd "${1:-builds}")"
 if [[ -z "$BUILD_SRC" ]]; then
   usage "set to the output of the juypterlite install"
 fi
 
+
+echo "Build directory is $BUILD_SRC"
+
 PKG_DIR="${2:-jupyter-workspaces}"
 echo "assets directory is: $PKG_DIR"
 
@@ -16,6 +27,7 @@ REMOTE_SRC="$BUILD_SRC/remote"
 FREE_DST="$PKG_DIR/assets/free"
 REMOTE_DST="$PKG_DIR/assets/remote"
 VALIDATE_SCRIPT="$SCRIPT_DIR/scripts/validate-jupyterlite-assets.sh"
+source "${SCRIPT_DIR}/scripts/setup-env.sh"
 
 if [[ ! -d "$FREE_SRC" ]]; then
   echo "Missing free JupyterLite build output: $FREE_SRC"
@@ -29,6 +41,11 @@ if [[ ! -d "$REMOTE_SRC" ]]; then
   exit 1
 fi
 
+echo $VALIDATE_SCRIPT
+echo $FREE_SRC
+echo $REMOTE_SRC
+
+
 bash "$VALIDATE_SCRIPT" --free-dir "$FREE_SRC" --remote-dir "$REMOTE_SRC"
 
 rm -rf "$FREE_DST" "$REMOTE_DST"

package/dist/jupyterlite-builds/scripts/validate-jupyterlite-assets.sh

@@ -48,6 +48,9 @@ fi
 FREE_JSON="$FREE_DIR/jupyter-lite.json"
 REMOTE_JSON="$REMOTE_DIR/jupyter-lite.json"
 
+
+echo "Free $FREE_DIR, Remote $REMOTE_DIR"
+
 if [[ ! -f "$FREE_JSON" ]]; then
   echo "Missing free config: $FREE_JSON"
   exit 1

package/jupyterlite-builds/free-private/jupyter_lite_config.json

@@ -3,24 +3,10 @@
     "apps": ["lab"],
     "output_dir": "site"
   },
-  "LiteServerConfig": {
-    "contents": ["./notebooks"],
-    "settings_overrides": {
-      "@jupyterlite/piplite-extension:plugin": {
-        "pipliteDeps": [
-          "numpy",
-          "pandas",
-          "scipy",
-          "matplotlib",
-          "seaborn",
-          "biopython",
-          "scanpy",
-          "anndata",
-          "pysam",
-          "boto3",
-          "s3fs"
-        ]
-      }
-    }
+  "PyodideAddon": {
+    "pyodide_url": "https://github.com/pyodide/pyodide/releases/download/314.0.5/pyodide-314.0.5.tar.bz2"
+  },
+  "PyodideLockAddon": {
+    "enabled": false
   }
 }

package/jupyterlite-builds/free-private/requirements.txt

@@ -1,3 +1,3 @@
 jupyterlite-core>=0.8.1
 jupyterlab>=4.6.0
-jupyterlite-pyodide-kernel
+jupyterlite-pyodide-kernel[lock]

package/jupyterlite-builds/remote-private/overrides.json

@@ -22,5 +22,21 @@
         "disabled": true
       }
     ]
+  },
+  "@jupyterlab/application-extension:context-menu": {
+    "contextMenu": [
+      {
+        "command": "filebrowser:download",
+        "selector": ".jp-DirListing-item[data-isdir=\"false\"]",
+        "rank": 9,
+        "disabled": true
+      },
+      {
+        "command": "filebrowser:copy-download-link",
+        "selector": ".jp-DirListing-item[data-isdir=\"false\"]",
+        "rank": 13,
+        "disabled": true
+      }
+    ]
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gen3/workspaces",
-  "version": "0.12.64",
+  "version": "0.12.66",
   "description": "Workspaces for Gen3",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -28,6 +28,8 @@
     "build": "npm run compile && npm run types && npm run rollup",
     "build:clean": "npm run clean && npm run build",
     "dev": "echo no dev tasks",
+    "build:jupyterlite:setupenv": "bash scripts/build-gen3sdk-lite.sh free",
+    "build:jupyterlite:gen3sdk": "bash scripts/build-gen3sdk-lite.sh free",
     "build:jupyterlite:free": "bash scripts/build-jupyterlite.sh free",
     "build:jupyterlite:remote": "bash scripts/build-jupyterlite.sh remote",
     "build:jupyterlite:all": "npm run build:jupyterlite:free && npm run build:jupyterlite:remote && npm run validate:jupyterlite:assets",
@@ -52,8 +54,8 @@
     "./jupyterlite-builds/*": "./dist/jupyterlite-builds/*"
   },
   "dependencies": {
-    "@gen3/core": "^0.12.64",
-    "@gen3/frontend": "^0.12.64",
+    "@gen3/core": "^0.12.66",
+    "@gen3/frontend": "^0.12.66",
     "camelcase-keys": "^10.0.2"
   },
   "peerDependencies": {