@gen3/workspaces 0.12.64 → 0.12.66

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/TODO.md +6 -7
  2. package/dist/jupyterlite-builds/free-private/jupyter_lite_config.json +5 -19
  3. package/dist/jupyterlite-builds/free-private/requirements.txt +1 -1
  4. package/dist/jupyterlite-builds/remote-private/overrides.json +16 -0
  5. package/dist/jupyterlite-builds/scripts/build-gen3sdk-lite.sh +428 -0
  6. package/dist/jupyterlite-builds/scripts/build-jupyterlite.sh +150 -33
  7. package/dist/jupyterlite-builds/scripts/setup-env.sh +81 -0
  8. package/dist/jupyterlite-builds/scripts/sync-jupyter-assets-into-package.sh +19 -2
  9. package/dist/jupyterlite-builds/scripts/validate-jupyterlite-assets.sh +3 -0
  10. package/jupyterlite-builds/free-private/jupyter_lite_config.json +5 -19
  11. package/jupyterlite-builds/free-private/requirements.txt +1 -1
  12. package/jupyterlite-builds/remote-private/overrides.json +16 -0
  13. package/package.json +5 -3
  14. package/scripts/build-gen3sdk-lite.sh +428 -0
  15. package/scripts/build-jupyterlite.sh +150 -33
  16. package/scripts/patch-pyidode-lock.py +100 -0
  17. package/scripts/setup-env.sh +81 -0
  18. package/scripts/sync-jupyter-assets-into-package.sh +19 -2
  19. package/scripts/validate-jupyterlite-assets.sh +3 -0
  20. package/.jupyterlite.doit.db +0 -0
  21. package/dist/jupyterlite-builds/scripts/build-gen3sdk-lite_v2.sh +0 -310
  22. package/dist/jupyterlite-builds/scripts/build-jupyterlite_v2.sh +0 -111
  23. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/aiofiles-25.1.0-py3-none-any.whl +0 -0
  24. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/backoff-2.2.1-py3-none-any.whl +0 -0
  25. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/cdislogging-1.2.0-py3-none-any.whl +0 -0
  26. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/comm-0.2.3-py3-none-any.whl +0 -0
  27. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/dataclasses_json-0.5.9-py3-none-any.whl +0 -0
  28. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/dictionaryutils-4.0.0-py3-none-any.whl +0 -0
  29. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/drsclient-0.4.0-py3-none-any.whl +0 -0
  30. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/fastavro-1.12.2-py3-none-any.whl +0 -0
  31. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/gen3-4.28.3-py3-none-any.whl +0 -0
  32. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/gen3dictionary-3.0.0-py3-none-any.whl +0 -0
  33. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/gen3users-1.1.1-py3-none-any.whl +0 -0
  34. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/humanfriendly-10.0-py2.py3-none-any.whl +0 -0
  35. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/importlib_metadata-8.9.0-py3-none-any.whl +0 -0
  36. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/indexclient-6.3.0-py3-none-any.whl +0 -0
  37. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/ipykernel-6.9.2-py3-none-any.whl +0 -0
  38. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/ipython_pygments_lexers-1.1.1-py3-none-any.whl +0 -0
  39. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/marshmallow-3.26.2-py3-none-any.whl +0 -0
  40. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/marshmallow_enum-1.5.1-py2.py3-none-any.whl +0 -0
  41. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/mypy_extensions-1.1.0-py3-none-any.whl +0 -0
  42. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/piplite-0.8.5-py3-none-any.whl +0 -0
  43. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/pyodide_http-0.2.2-py3-none-any.whl +0 -0
  44. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/pyodide_kernel-0.8.5-py3-none-any.whl +0 -0
  45. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/pypfb-0.6.2-py3-none-any.whl +0 -0
  46. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/python_dateutil-2.9.0.post0-py2.py3-none-any.whl +0 -0
  47. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/python_json_logger-4.2.0-py3-none-any.whl +0 -0
  48. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/typing_inspect-0.9.0-py3-none-any.whl +0 -0
  49. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/xmltodict-0.13.0-py2.py3-none-any.whl +0 -0
  50. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_uv_wheels/zipp-4.1.0-py3-none-any.whl +0 -0
  51. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_work/constraints.txt +0 -354
  52. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_work/excludes.txt +0 -3
  53. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_work/pylock.toml +0 -347
  54. package/jupyterlite-builds/free-private/.cache/pyodide-lock/_work/requirements.in +0 -26
  55. package/jupyterlite-builds/free-private/.cache/pyodide-lock/patched-pyodide-lock.json +0 -1
  56. package/jupyterlite-builds/free-private/.cache/pyodide-lock/pyodide-lock.json +0 -1
  57. package/jupyterlite-builds/free-private/.cache/pyodide-lock/remote-pyodide-lock.json +0 -1
  58. package/jupyterlite-builds/free-private/.cache/wheels/aiofiles-25.1.0-py3-none-any.whl.meta.json +0 -24
  59. package/jupyterlite-builds/free-private/.cache/wheels/backoff-2.2.1-py3-none-any.whl.meta.json +0 -24
  60. package/jupyterlite-builds/free-private/.cache/wheels/cdislogging-1.2.0-py3-none-any.whl.meta.json +0 -24
  61. package/jupyterlite-builds/free-private/.cache/wheels/dataclasses_json-0.5.9-py3-none-any.whl.meta.json +0 -24
  62. package/jupyterlite-builds/free-private/.cache/wheels/dictionaryutils-4.0.0-py3-none-any.whl.meta.json +0 -24
  63. package/jupyterlite-builds/free-private/.cache/wheels/drsclient-0.4.0-py3-none-any.whl.meta.json +0 -24
  64. package/jupyterlite-builds/free-private/.cache/wheels/fastavro-1.12.2-py3-none-any.whl.meta.json +0 -24
  65. package/jupyterlite-builds/free-private/.cache/wheels/gen3-4.28.3-py3-none-any.whl.meta.json +0 -24
  66. package/jupyterlite-builds/free-private/.cache/wheels/gen3dictionary-3.0.0-py3-none-any.whl.meta.json +0 -24
  67. package/jupyterlite-builds/free-private/.cache/wheels/gen3users-1.1.1-py3-none-any.whl.meta.json +0 -24
  68. package/jupyterlite-builds/free-private/.cache/wheels/humanfriendly-10.0-py2.py3-none-any.whl.meta.json +0 -24
  69. package/jupyterlite-builds/free-private/.cache/wheels/importlib_metadata-8.9.0-py3-none-any.whl.meta.json +0 -24
  70. package/jupyterlite-builds/free-private/.cache/wheels/indexclient-6.3.0-py3-none-any.whl.meta.json +0 -24
  71. package/jupyterlite-builds/free-private/.cache/wheels/marshmallow-3.26.2-py3-none-any.whl.meta.json +0 -24
  72. package/jupyterlite-builds/free-private/.cache/wheels/marshmallow_enum-1.5.1-py2.py3-none-any.whl.meta.json +0 -24
  73. package/jupyterlite-builds/free-private/.cache/wheels/mypy_extensions-1.1.0-py3-none-any.whl.meta.json +0 -24
  74. package/jupyterlite-builds/free-private/.cache/wheels/pypfb-0.6.2-py3-none-any.whl.meta.json +0 -24
  75. package/jupyterlite-builds/free-private/.cache/wheels/python_dateutil-2.9.0.post0-py2.py3-none-any.whl.meta.json +0 -24
  76. package/jupyterlite-builds/free-private/.cache/wheels/python_json_logger-4.2.0-py3-none-any.whl.meta.json +0 -24
  77. package/jupyterlite-builds/free-private/.cache/wheels/typing_inspect-0.9.0-py3-none-any.whl.meta.json +0 -24
  78. package/jupyterlite-builds/free-private/.cache/wheels/xmltodict-0.13.0-py2.py3-none-any.whl.meta.json +0 -24
  79. package/jupyterlite-builds/free-private/.cache/wheels/zipp-4.1.0-py3-none-any.whl.meta.json +0 -24
  80. package/jupyterlite-builds/free-private/cockle-config.json +0 -71
  81. package/jupyterlite-builds/free-private/cockle_wasm_env/.mambarc +0 -1
  82. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/coreutils.js +0 -5421
  83. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/coreutils.wasm +0 -0
  84. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/fs.d.ts +0 -375
  85. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/fs.js +0 -3401
  86. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/fs.wasm +0 -0
  87. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/grep.js +0 -3839
  88. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/grep.wasm +0 -0
  89. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/less.data +0 -0
  90. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/less.js +0 -5060
  91. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/less.wasm +0 -0
  92. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/sed.js +0 -3843
  93. package/jupyterlite-builds/free-private/cockle_wasm_env/bin/sed.wasm +0 -0
  94. package/jupyterlite-builds/free-private/cockle_wasm_env/conda-meta/cockle_fs-0.3.0-h27506cd_2.json +0 -60
  95. package/jupyterlite-builds/free-private/cockle_wasm_env/conda-meta/coreutils-9.10-h072c4ef_3.json +0 -52
  96. package/jupyterlite-builds/free-private/cockle_wasm_env/conda-meta/emscripten-abi-4.0.9-h267e887_9.json +0 -31
  97. package/jupyterlite-builds/free-private/cockle_wasm_env/conda-meta/grep-3.12-h8b79025_0.json +0 -52
  98. package/jupyterlite-builds/free-private/cockle_wasm_env/conda-meta/history +0 -10
  99. package/jupyterlite-builds/free-private/cockle_wasm_env/conda-meta/less-693-hf259948_0.json +0 -60
  100. package/jupyterlite-builds/free-private/cockle_wasm_env/conda-meta/sed-4.9-h072c4ef_0.json +0 -52
  101. package/scripts/aiohttp-3.13.5-cp314-cp314-pyemscripten_2026_0_wasm32.whl +0 -0
  102. package/scripts/build-gen3sdk-lite_v2.sh +0 -310
  103. package/scripts/build-jupyterlite_v2.sh +0 -111
@@ -0,0 +1,100 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ Localize CDN wheels referenced by a JupyterLite pyodide-lock.json.
4
+
5
+ Replaces the inline heredoc in build-jupyterlite.sh. Two fixes over the
6
+ original:
7
+
8
+ 1. Writes ``file_name`` (the only field Pyodide reads) instead of ``url``,
9
+ which is not part of the pyodide-lock schema and is silently ignored.
10
+ 2. Writes a bare filename. ``file_name`` is resolved relative to the
11
+ directory holding pyodide-lock.json, which is where we download to,
12
+ so no "../../static/..." prefix is needed.
13
+
14
+ Fails loudly if any CDN reference survives, so a no-op patch can never be
15
+ reported as success.
16
+
17
+ Usage:
18
+ LOCK_FILE=... LOCK_OUT=... python3 patch-pyodide-lock.py
19
+ """
20
+
21
+ import json
22
+ import os
23
+ import sys
24
+ import urllib.request
25
+
26
+ CDN_HOSTS = ("cdn.jsdelivr.net", "pypi.org", "files.pythonhosted.org")
27
+
28
+ lock_file = os.environ["LOCK_FILE"]
29
+ lock_out = os.environ["LOCK_OUT"]
30
+
31
+ with open(lock_file) as fh:
32
+ lock = json.load(fh)
33
+
34
+ packages = lock.get("packages", {})
35
+ localized = 0
36
+ failed = []
37
+
38
+ for name, meta in packages.items():
39
+ ref = meta.get("file_name", "")
40
+ if not any(host in ref for host in CDN_HOSTS):
41
+ continue
42
+
43
+ fname = ref.split("/")[-1].split("?")[0]
44
+ dest = os.path.join(lock_out, fname)
45
+
46
+ if not os.path.exists(dest):
47
+ try:
48
+ urllib.request.urlretrieve(ref, dest)
49
+ print(f" downloaded {fname}", flush=True)
50
+ except Exception as exc:
51
+ failed.append((name, fname, exc))
52
+ print(f" WARN: {fname}: {exc}", file=sys.stderr, flush=True)
53
+ continue
54
+
55
+ # The only field Pyodide reads. sha256 is unchanged because it is the
56
+ # same artifact, so integrity checks still pass.
57
+ meta["file_name"] = fname
58
+ meta.pop("url", None) # drop the stale no-op key if a prior run added it
59
+ localized += 1
60
+
61
+ with open(lock_file, "w") as fh:
62
+ json.dump(lock, fh)
63
+
64
+ # ---- verification: no CDN reference may survive -----------------------------
65
+ with open(lock_file) as fh:
66
+ reloaded = json.load(fh)
67
+
68
+ survivors = [
69
+ (n, m.get("file_name", ""))
70
+ for n, m in reloaded.get("packages", {}).items()
71
+ if any(host in m.get("file_name", "") for host in CDN_HOSTS)
72
+ ]
73
+
74
+ print(f"Lock patched: {localized} localized, {len(failed)} download failures")
75
+
76
+ if survivors:
77
+ print(
78
+ f"ERROR: {len(survivors)} package(s) still point at a remote host; "
79
+ "the browser will fetch these at import time and may hang:",
80
+ file=sys.stderr,
81
+ )
82
+ for n, ref in survivors[:20]:
83
+ print(f" {n}: {ref}", file=sys.stderr)
84
+ sys.exit(1)
85
+
86
+ # Every localized reference must actually exist on disk next to the lock.
87
+ missing = [
88
+ (n, m["file_name"])
89
+ for n, m in reloaded.get("packages", {}).items()
90
+ if m.get("file_name")
91
+ and "/" not in m["file_name"]
92
+ and not os.path.exists(os.path.join(lock_out, m["file_name"]))
93
+ ]
94
+ if missing:
95
+ print(f"ERROR: {len(missing)} lock entries reference missing files:", file=sys.stderr)
96
+ for n, f in missing[:20]:
97
+ print(f" {n}: {f}", file=sys.stderr)
98
+ sys.exit(1)
99
+
100
+ print("Verified: all package references are local and present.")
@@ -0,0 +1,81 @@
1
+ #!/usr/bin/env bash
2
+ # =============================================================================
3
+ # setup-env.sh — Shared build-environment bootstrap
4
+ #
5
+ # Guarantees both build-gen3sdk-lite and build-jupyterlite use the same
6
+ # Python interpreter and virtual environment.
7
+ #
8
+ # BUILD_DIR resolution (first match wins):
9
+ # 1. Pre-set BUILD_DIR environment variable (export BUILD_DIR=…)
10
+ # 2. First positional argument (source setup-env.sh [/tmp/mybuild | ./mybuild])
11
+ # 3. Default: "builds"
12
+ #
13
+ # Usage:
14
+ # source setup-env.sh [build_dir]
15
+ #
16
+ # After sourcing, the venv at $BUILD_DIR/.venv is activated in the calling
17
+ # shell, and VENV_DIR is exported.
18
+ # =============================================================================
19
+
20
+ # ── Configuration ────────────────────────────────────────────────────────────
21
+ PYTHON_VERSION="3.14.2"
22
+
23
+ # Resolves a possibly-relative path against INIT_CWD (or $PWD) rather than
24
+ # whatever directory happens to be current when this runs.
25
+ _resolve_against_init_cwd() {
26
+ local path="$1"
27
+ local base="${INIT_CWD:-$(pwd)}"
28
+ case "$path" in
29
+ /*) printf '%s' "$path" ;;
30
+ *) printf '%s/%s' "$base" "$path" ;;
31
+ esac
32
+ }
33
+
34
+ BUILD_DIR="${BUILD_DIR:-$(_resolve_against_init_cwd "${1:-builds}")}"
35
+
36
+ mkdir -p "$BUILD_DIR"
37
+ BUILD_DIR="$(cd "$BUILD_DIR" && pwd)"
38
+
39
+ VENV_DIR="$BUILD_DIR/.venv"
40
+ export BUILD_DIR VENV_DIR PYTHON_VERSION
41
+
42
+ # ── Colours / helpers ────────────────────────────────────────────────────────
43
+ RED='\033[0;31m'
44
+ GREEN='\033[0;32m'
45
+ YELLOW='\033[1;33m'
46
+ CYAN='\033[0;36m'
47
+ NC='\033[0m'
48
+
49
+ info() { echo -e "${CYAN}[INFO]${NC} $*"; }
50
+ success() { echo -e "${GREEN}[OK]${NC} $*"; }
51
+ warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
52
+ fail() { echo -e "${RED}[FAIL]${NC} $*"; return 1 2>/dev/null || exit 1; }
53
+
54
+ # Export helpers so functions in the calling script can use them.
55
+ export -f info success warn fail
56
+
57
+ # ── uv ───────────────────────────────────────────────────────────────────────
58
+ if ! command -v uv &>/dev/null; then
59
+ fail "uv is not installed. Install it first: https://docs.astral.sh/uv/getting-started/installation/"
60
+ fi
61
+ success "uv found: $(uv --version)"
62
+
63
+ # ── Python (via uv) ─────────────────────────────────────────────────────────
64
+ info "Ensuring Python ${PYTHON_VERSION} is available via uv …"
65
+ uv python install "${PYTHON_VERSION}" 2>&1 | tail -1
66
+ success "Python ${PYTHON_VERSION} ready"
67
+
68
+ # ── Virtual environment (create once, reuse everywhere) ──────────────────────
69
+ if [[ ! -d "${VENV_DIR}" ]]; then
70
+ info "Creating virtual environment at ${VENV_DIR} …"
71
+ uv venv --python "${PYTHON_VERSION}" "${VENV_DIR}"
72
+ fi
73
+
74
+ # shellcheck disable=SC1091
75
+ source "${VENV_DIR}/bin/activate"
76
+ success "Activated venv ($(python --version))"
77
+
78
+ # ── Core Python tooling (idempotent — uv skips already-installed) ────────────
79
+ info "Ensuring core Python build tooling …"
80
+ uv pip install --upgrade pip build 2>&1 | tail -3
81
+ success "Core tooling ready"
@@ -1,12 +1,23 @@
1
1
  #!/usr/bin/env bash
2
2
  set -euo pipefail
3
3
 
4
-
5
- BUILD_SRC="${1:-}"
4
+ _resolve_against_init_cwd() {
5
+ local path="$1"
6
+ local base="${INIT_CWD:-$(pwd)}"
7
+ case "$path" in
8
+ /*) printf '%s' "$path" ;;
9
+ *) printf '%s/%s' "$base" "$path" ;;
10
+ esac
11
+ }
12
+
13
+ BUILD_SRC="$(_resolve_against_init_cwd "${1:-builds}")"
6
14
  if [[ -z "$BUILD_SRC" ]]; then
7
15
  usage "set to the output of the juypterlite install"
8
16
  fi
9
17
 
18
+
19
+ echo "Build directory is $BUILD_SRC"
20
+
10
21
  PKG_DIR="${2:-jupyter-workspaces}"
11
22
  echo "assets directory is: $PKG_DIR"
12
23
 
@@ -16,6 +27,7 @@ REMOTE_SRC="$BUILD_SRC/remote"
16
27
  FREE_DST="$PKG_DIR/assets/free"
17
28
  REMOTE_DST="$PKG_DIR/assets/remote"
18
29
  VALIDATE_SCRIPT="$SCRIPT_DIR/scripts/validate-jupyterlite-assets.sh"
30
+ source "${SCRIPT_DIR}/scripts/setup-env.sh"
19
31
 
20
32
  if [[ ! -d "$FREE_SRC" ]]; then
21
33
  echo "Missing free JupyterLite build output: $FREE_SRC"
@@ -29,6 +41,11 @@ if [[ ! -d "$REMOTE_SRC" ]]; then
29
41
  exit 1
30
42
  fi
31
43
 
44
+ echo $VALIDATE_SCRIPT
45
+ echo $FREE_SRC
46
+ echo $REMOTE_SRC
47
+
48
+
32
49
  bash "$VALIDATE_SCRIPT" --free-dir "$FREE_SRC" --remote-dir "$REMOTE_SRC"
33
50
 
34
51
  rm -rf "$FREE_DST" "$REMOTE_DST"
@@ -48,6 +48,9 @@ fi
48
48
  FREE_JSON="$FREE_DIR/jupyter-lite.json"
49
49
  REMOTE_JSON="$REMOTE_DIR/jupyter-lite.json"
50
50
 
51
+
52
+ echo "Free $FREE_DIR, Remote $REMOTE_DIR"
53
+
51
54
  if [[ ! -f "$FREE_JSON" ]]; then
52
55
  echo "Missing free config: $FREE_JSON"
53
56
  exit 1
Binary file
@@ -1,310 +0,0 @@
1
- #!/usr/bin/env bash
2
- # =============================================================================
3
- # build-gen3sdk-lite.sh — Build Gen3 SDK and dependencies as WASM wheels
4
- #
5
- # Prerequisites:
6
- # - uv (https://docs.astral.sh/uv/getting-started/installation/)
7
- # - git
8
- # - cmake, make, and a C compiler (for emsdk bootstrap)
9
- #
10
- # Usage:
11
- # ./build-gen3sdk-lite.sh # full build (setup + all packages)
12
- # ./build-gen3sdk-lite.sh --setup # environment setup only
13
- # ./build-gen3sdk-lite.sh --build # build wheels only (assumes setup done)
14
- # ./build-gen3sdk-lite.sh --clean # remove cloned repos, build artifacts
15
- # =============================================================================
16
- set -euo pipefail
17
-
18
- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
19
-
20
- # ── Configuration ────────────────────────────────────────────────────────────
21
- # Repositories to build (order matters — independent packages first)
22
- # Format: "name|git_url|subdir" (subdir is optional)
23
- REPOS=(
24
- "cdislogging|https://github.com/uc-cdis/cdislogging.git"
25
- "python-json-logger|https://github.com/nhairs/python-json-logger.git"
26
- "fastavro|https://github.com/fastavro/fastavro.git"
27
- "drsclient|https://github.com/uc-cdis/drsclient.git"
28
- "indexd|https://github.com/uc-cdis/indexd.git|indexclient"
29
- "dictionaryutils|https://github.com/uc-cdis/dictionaryutils.git"
30
- "pypfb|https://github.com/uc-cdis/pypfb.git"
31
- "gen3sdk-python|https://github.com/uc-cdis/gen3sdk-python.git"
32
- "gen3users|git@github.com:uc-cdis/gen3users.git"
33
- )
34
-
35
- # ── Source shared environment ────────────────────────────────────────────────
36
- # setup-env.sh exports: ROOT_DIR, SRC_DIR, BUILD_DIR, VENV_DIR, WORK_DIR,
37
- # EMSDK_DIR, SOURCES_DIR, OUTPUT_DIR, LOG_DIR and helper functions.
38
- # It also creates + activates the venv and installs core tooling.
39
- source_shared_env() {
40
- # shellcheck disable=SC1091
41
- source "${SCRIPT_DIR}/setup-env.sh"
42
- }
43
-
44
- # ── Emscripten SDK ───────────────────────────────────────────────────────────
45
- setup_emsdk() {
46
- info "Setting up Emscripten SDK …"
47
-
48
- if [[ ! -d "${EMSDK_DIR}" ]]; then
49
- git clone https://github.com/emscripten-core/emsdk.git "${EMSDK_DIR}"
50
- fi
51
-
52
- local em_version
53
- em_version="$(pyodide config get emscripten_version)"
54
- info "Pyodide requires Emscripten ${em_version}"
55
-
56
- pushd "${EMSDK_DIR}" > /dev/null
57
- ./emsdk install "${em_version}"
58
- ./emsdk activate "${em_version}"
59
- # shellcheck disable=SC1091
60
- source emsdk_env.sh
61
- popd > /dev/null
62
-
63
- success "Emscripten $(emcc --version | head -1) activated"
64
- }
65
-
66
- # ── Full environment setup ───────────────────────────────────────────────────
67
- setup_environment() {
68
- source_shared_env
69
- mkdir -p "${WORK_DIR}" "${SOURCES_DIR}" "${OUTPUT_DIR}" "${LOG_DIR}"
70
-
71
- # WASM-specific tooling (on top of the shared base)
72
- info "Installing pyodide-build …"
73
- uv pip install pyodide-build 2>&1 | tail -3
74
- success "pyodide-build $(pyodide --version 2>/dev/null || echo '(installed)')"
75
-
76
- setup_emsdk
77
- }
78
-
79
- # ── Clone / update sources ──────────────────────────────────────────────────
80
- clone_repos() {
81
- info "Cloning / updating repositories …"
82
- for entry in "${REPOS[@]}"; do
83
- IFS='|' read -r name url _subdir <<< "${entry}"
84
- local dest="${SOURCES_DIR}/${name}"
85
- if [[ -d "${dest}/.git" ]]; then
86
- info " ${name}: pulling latest …"
87
- git -C "${dest}" pull --ff-only 2>&1 | tail -1
88
- else
89
- info " ${name}: cloning …"
90
- git clone "${url}" "${dest}" 2>&1 | tail -1
91
- fi
92
- done
93
- success "All repositories ready"
94
- }
95
-
96
- # ── Build WASM wheels ───────────────────────────────────────────────────────
97
- build_wheels() {
98
- # Make sure emsdk env is active in this shell
99
- if [[ -f "${EMSDK_DIR}/emsdk_env.sh" ]]; then
100
- # shellcheck disable=SC1091
101
- source "${EMSDK_DIR}/emsdk_env.sh" 2>/dev/null
102
- fi
103
- # shellcheck disable=SC1091
104
- source "${VENV_DIR}/bin/activate"
105
-
106
- local total=${#REPOS[@]}
107
- local built=0
108
- local failed=0
109
- local failed_names=()
110
-
111
- # ── Source patches ─────────────────────────────────────────────────────
112
-
113
- # gen3users: old poetry.masonry.api → poetry.core.masonry.api
114
- local gen3users_pyproject="${SOURCES_DIR}/gen3users/pyproject.toml"
115
- if [[ -f "${gen3users_pyproject}" ]]; then
116
- sed -i '' 's|build-backend = "poetry.masonry.api"|build-backend = "poetry.core.masonry.api"|' "${gen3users_pyproject}"
117
- info "Patched gen3users: build-backend updated to poetry.core.masonry.api"
118
- fi
119
-
120
- # gen3sdk-python: relax indexclient pin
121
- local gen3sdk_pyproject="${SOURCES_DIR}/gen3sdk-python/pyproject.toml"
122
- if [[ -f "${gen3sdk_pyproject}" ]]; then
123
- sed -i '' 's/indexclient = "\^2\.3\.0"/indexclient = ">=2.3.0"/' "${gen3sdk_pyproject}"
124
- info "Patched gen3sdk-python: indexclient constraint relaxed to >=2.3.0"
125
- fi
126
-
127
- # dictionaryutils: relax jsonschema pin
128
- local dictionaryutils_pyproject="${SOURCES_DIR}/dictionaryutils/pyproject.toml"
129
- if [[ -f "${dictionaryutils_pyproject}" ]]; then
130
- sed -i '' 's/jsonschema = "<=4\.23\.0"/jsonschema = ">=4.0.0"/' "${dictionaryutils_pyproject}"
131
- info "Patched dictionaryutils: jsonschema constraint relaxed to >=4.0.0"
132
- fi
133
-
134
- # psqlgraph: multiple fixes for WASM compatibility
135
- local psqlgraph_pyproject="${SOURCES_DIR}/psqlgraph/pyproject.toml"
136
- if [[ -f "${psqlgraph_pyproject}" ]]; then
137
- sed -i '' 's/"sqlalchemy~=1\.4"/"sqlalchemy>=1.4"/' "${psqlgraph_pyproject}"
138
- sed -i '' 's/, "versionista>=1\.1\.0"//' "${psqlgraph_pyproject}"
139
- sed -i '' 's/local_scheme = "versionista-local-format"/local_scheme = "no-local-version"/' "${psqlgraph_pyproject}"
140
- sed -i '' 's/version_scheme = "versionista-format"/version_scheme = "guess-next-dev"/' "${psqlgraph_pyproject}"
141
- sed -i '' '/"psycopg2"/d' "${psqlgraph_pyproject}"
142
- sed -i '' '/"xlocal"/d' "${psqlgraph_pyproject}"
143
- info "Patched psqlgraph: sqlalchemy >=1.4, removed versionista + psycopg2 + xlocal"
144
-
145
- local psql_py="${SOURCES_DIR}/psqlgraph/src/psqlgraph/psql.py"
146
- if [[ -f "${psql_py}" ]]; then
147
- python3 - "${psql_py}" <<'PYEOF'
148
- import sys, re
149
-
150
- path = sys.argv[1]
151
- text = open(path).read()
152
- stub = '''\
153
- try:
154
- import xlocal
155
- except ImportError:
156
- class _XLocalInst:
157
- class _Ctx:
158
- def __init__(self, parent, **kw):
159
- self._parent, self._kw = parent, kw
160
- def __enter__(self):
161
- for k, v in self._kw.items():
162
- setattr(self._parent, k, v)
163
- return self._parent
164
- def __exit__(self, *a):
165
- for k in self._kw:
166
- self._parent.__dict__.pop(k, None)
167
- def __call__(self, **kw):
168
- return self._Ctx(self, **kw)
169
- class _xlocal_module:
170
- xlocal = _XLocalInst
171
- xlocal = _xlocal_module()'''
172
- text = re.sub(r'^import xlocal$', stub, text, flags=re.MULTILINE)
173
- open(path, 'w').write(text)
174
- PYEOF
175
- info "Patched psqlgraph/psql.py: xlocal import wrapped in try/except stub"
176
- fi
177
- fi
178
-
179
- # ── Build loop ─────────────────────────────────────────────────────────
180
- info "Building ${total} packages …"
181
- echo ""
182
-
183
- for entry in "${REPOS[@]}"; do
184
- IFS='|' read -r name _url subdir <<< "${entry}"
185
- local src="${SOURCES_DIR}/${name}"
186
- local build_dir="${src}"
187
- [[ -n "${subdir}" ]] && build_dir="${src}/${subdir}"
188
- local logfile="${LOG_DIR}/${name}.log"
189
- local start_ts
190
- start_ts=$(date +%s)
191
-
192
- local label="${name}"
193
- [[ -n "${subdir}" ]] && label="${name}/${subdir}"
194
- printf " [%d/%d] %-25s " $((built + failed + 1)) "${total}" "${label}"
195
-
196
- if (cd "${build_dir}" && pyodide build) > "${logfile}" 2>&1; then
197
- local end_ts
198
- end_ts=$(date +%s)
199
- echo -e "${GREEN}OK${NC} ($(elapsed $((end_ts - start_ts))))"
200
- ((built++))
201
- else
202
- local end_ts
203
- end_ts=$(date +%s)
204
- echo -e "${RED}FAILED${NC} ($(elapsed $((end_ts - start_ts)))) — see ${logfile}"
205
- ((failed++))
206
- failed_names+=("${name}")
207
- fi
208
- done
209
-
210
- echo ""
211
-
212
- # ── Collect wheels ─────────────────────────────────────────────────────
213
- info "Collecting .whl files into ${OUTPUT_DIR} …"
214
- local count=0
215
- for entry in "${REPOS[@]}"; do
216
- IFS='|' read -r name _url subdir <<< "${entry}"
217
- local src="${SOURCES_DIR}/${name}"
218
- local build_dir="${src}"
219
- [[ -n "${subdir}" ]] && build_dir="${src}/${subdir}"
220
- if compgen -G "${build_dir}/dist/*.whl" > /dev/null 2>&1; then
221
- cp "${build_dir}"/dist/*.whl "${OUTPUT_DIR}/"
222
- ((count += $(ls -1 "${build_dir}"/dist/*.whl 2>/dev/null | wc -l)))
223
- fi
224
- done
225
-
226
- # ── Normalise platform tags ────────────────────────────────────────────
227
- for whl in "${OUTPUT_DIR}"/*p313-cp313-pyemscripten_2025_0_wasm32.whl; do
228
- [[ -f "${whl}" ]] || continue
229
- new_whl="${whl/p313-cp313-pyemscripten_2025_0_wasm32/py3-none-any}"
230
- mv "${whl}" "${new_whl}"
231
- info "Renamed $(basename "${whl}") → $(basename "${new_whl}")"
232
- done
233
-
234
- # ── Summary ────────────────────────────────────────────────────────────
235
- echo ""
236
- echo "========================================="
237
- echo " Build Summary"
238
- echo "========================================="
239
- echo " Succeeded : ${built}"
240
- echo " Failed : ${failed}"
241
- echo " Wheels : ${count} file(s) in ${OUTPUT_DIR}/"
242
- if (( failed > 0 )); then
243
- echo ""
244
- echo " Failed packages:"
245
- for n in "${failed_names[@]}"; do
246
- echo " - ${n} (log: ${LOG_DIR}/${n}.log)"
247
- done
248
- fi
249
- echo "========================================="
250
- echo ""
251
-
252
- if (( failed > 0 )); then
253
- warn "Some packages failed. Check the logs above."
254
- return 1
255
- fi
256
-
257
- success "All wheels built and collected in ${OUTPUT_DIR}/"
258
- echo ""
259
- ls -lh "${OUTPUT_DIR}"/*.whl
260
- }
261
-
262
- # ── Clean ────────────────────────────────────────────────────────────────────
263
- clean() {
264
- # Need the path variables but don't need the full env setup
265
- ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
266
- SRC_DIR="$ROOT_DIR/free-private"
267
- BUILD_DIR="${2:-$SRC_DIR/build}"
268
- WORK_DIR="$BUILD_DIR/work"
269
- OUTPUT_DIR="$BUILD_DIR/pypi"
270
- LOG_DIR="$BUILD_DIR/logs"
271
-
272
- echo -e '\033[0;36m[INFO]\033[0m Cleaning build artifacts …'
273
- rm -rf "${WORK_DIR}" "${OUTPUT_DIR}" "${LOG_DIR}"
274
- echo -e '\033[0;32m[OK]\033[0m Clean complete'
275
- }
276
-
277
- # ── Main ─────────────────────────────────────────────────────────────────────
278
- main() {
279
- local mode="${1:-all}"
280
-
281
- case "${mode}" in
282
- --setup)
283
- setup_environment
284
- ;;
285
- --build)
286
- source_shared_env
287
- clone_repos
288
- build_wheels
289
- ;;
290
- --clean)
291
- clean
292
- ;;
293
- --help|-h)
294
- echo "Usage: $0 [--setup | --build | --clean | --help]"
295
- echo ""
296
- echo " (no flag) Full run: setup environment, clone repos, build wheels"
297
- echo " --setup Set up Python, venv, pyodide-build, and emsdk"
298
- echo " --build Clone/update repos and build WASM wheels (requires prior --setup)"
299
- echo " --clean Remove work/, dist/, and logs/"
300
- echo " --help Show this message"
301
- ;;
302
- *)
303
- setup_environment
304
- clone_repos
305
- build_wheels
306
- ;;
307
- esac
308
- }
309
-
310
- main "$@"
@@ -1,111 +0,0 @@
1
- #!/usr/bin/env bash
2
- # =============================================================================
3
- # build-jupyterlite.sh — Build a JupyterLite site (free or remote tier)
4
- #
5
- # Sources setup-env.sh so the venv, Python version, and tooling are identical
6
- # to those used by build-gen3sdk-lite.sh.
7
- #
8
- # Usage:
9
- # ./build-jupyterlite.sh <tier> [build_dir]
10
- # tier: 'free' or 'remote'
11
- # =============================================================================
12
- set -euo pipefail
13
-
14
- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
15
-
16
- usage() {
17
- echo "Usage: $0 <tier> [build_dir]"
18
- echo " tier: 'free' or 'remote'"
19
- exit 1
20
- }
21
-
22
- TIER="${1:-}"
23
- if [[ "$TIER" != "free" && "$TIER" != "remote" ]]; then
24
- usage
25
- fi
26
-
27
- # ── Derive SRC_DIR before sourcing shared env ────────────────────────────────
28
- # setup-env.sh respects pre-set ROOT_DIR / SRC_DIR / BUILD_DIR.
29
- export ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
30
-
31
- if [[ "$TIER" == "free" ]]; then
32
- export SRC_DIR="$ROOT_DIR/free-private"
33
- else
34
- export SRC_DIR="$ROOT_DIR/remote-private"
35
- fi
36
-
37
- # Allow caller to override BUILD_DIR via the second positional arg.
38
- if [[ -n "${2:-}" ]]; then
39
- export BUILD_DIR="$2"
40
- fi
41
-
42
- # ── Source shared environment (creates/reuses the venv) ──────────────────────
43
- # After this: VENV_DIR, OUTPUT_DIR, BUILD_DIR, etc. are set and the venv is
44
- # active with the same Python version + core tooling as build-gen3sdk-lite.
45
- # shellcheck disable=SC1091
46
- source "${SCRIPT_DIR}/setup-env.sh"
47
-
48
- # ── JupyterLite-specific tooling ────────────────────────────────────────────
49
- CONFIG_FILE="$SRC_DIR/jupyter_lite_config.json"
50
- REQUIREMENTS_FILE="$SRC_DIR/requirements.txt"
51
- VENV_JUPYTER_LITE="$VENV_DIR/bin/jupyter-lite"
52
- PYPI="$OUTPUT_DIR" # wheels built by build-gen3sdk-lite land here
53
-
54
- if [[ ! -f "$REQUIREMENTS_FILE" ]]; then
55
- echo "Missing requirements file: $REQUIREMENTS_FILE"
56
- exit 1
57
- fi
58
-
59
- # Install JupyterLite requirements into the shared venv (uv for speed).
60
- info "Installing JupyterLite requirements …"
61
- uv pip install -r "$REQUIREMENTS_FILE" 2>&1 | tail -5
62
- success "JupyterLite requirements installed"
63
-
64
- if [[ ! -x "$VENV_JUPYTER_LITE" ]]; then
65
- echo "jupyter-lite executable not found in $VENV_DIR after dependency install."
66
- exit 1
67
- fi
68
-
69
- # ── Build ────────────────────────────────────────────────────────────────────
70
- if [[ "$TIER" == "free" ]]; then
71
-
72
- # Patch config to use the absolute pypi path so jupyter-lite can find the
73
- # custom wheels built by build-gen3sdk-lite.
74
- PATCHED_CONFIG="$BUILD_DIR/jupyter_lite_config.json"
75
- python - <<PYEOF
76
- import json
77
-
78
- with open('$CONFIG_FILE') as fh:
79
- cfg = json.load(fh)
80
-
81
- addon = cfg.get('PyodideLockAddon', {})
82
- if 'wheels' in addon:
83
- addon['wheels'] = ['$PYPI/']
84
- cfg['PyodideLockAddon'] = addon
85
-
86
- with open('$PATCHED_CONFIG', 'w') as fh:
87
- json.dump(cfg, fh, indent=2)
88
- PYEOF
89
-
90
- info "Looking for wheels in $PYPI"
91
- WHEEL_ARGS=()
92
- if [[ -d "$PYPI" ]]; then
93
- while IFS= read -r -d '' whl; do
94
- WHEEL_ARGS+=(--piplite-wheels "$whl")
95
- done < <(find "$PYPI" -maxdepth 1 -name "*.whl" -print0 | sort -z)
96
- fi
97
-
98
- info "Running: jupyter-lite build (free tier, ${#WHEEL_ARGS[@]} custom wheels)"
99
- "$VENV_JUPYTER_LITE" build \
100
- --config "$PATCHED_CONFIG" \
101
- --lite-dir "$SRC_DIR" ${WHEEL_ARGS[@]+"${WHEEL_ARGS[@]}"} \
102
- --output-dir "$BUILD_DIR/$TIER"
103
- else
104
- info "Running: jupyter-lite build (remote tier)"
105
- "$VENV_JUPYTER_LITE" build \
106
- --config "$CONFIG_FILE" \
107
- --lite-dir "$SRC_DIR" \
108
- --output-dir "$BUILD_DIR/$TIER"
109
- fi
110
-
111
- success "$TIER-tier JupyterLite build complete: $BUILD_DIR/$TIER"