@music-league-eras/local-runner 0.1.0

package/README.md

@@ -0,0 +1,71 @@
+# Music League Eras Local Runner (npx)
+
+This package runs the Music League scraping flow **locally** (opens a real browser on your machine), then uploads the
+extracted `manifest.json` to the Music League Eras API using your short-lived `sync_token`.
+
+## Prerequisites (macOS MVP)
+
+- Node.js `>=18.17`
+- Python `>=3.11` available as `python3`
+
+On first run, Playwright may download Chromium (network required).
+
+## Usage
+
+Run the commands shown in the web app (recommended), or run manually:
+
+### Step A (bootstrap, no secrets)
+
+```bash
+npx @music-league-eras/local-runner@latest bootstrap
+```
+
+### Step B (runner, safe default)
+
+```bash
+npx @music-league-eras/local-runner@latest local-sync \\
+  --api-base-url https://<deployment-host> \\
+  --sync-session-id <sync_session_id>
+```
+
+The runner will prompt you to paste the sync token (input hidden).
+
+### Step B (power users / CI)
+
+Option A: environment variable (recommended for CI)
+
+```bash
+export ML_ERAS_SYNC_TOKEN='<sync_token>'
+npx @music-league-eras/local-runner@latest local-sync \\
+  --api-base-url https://<deployment-host> \\
+  --sync-session-id <sync_session_id>
+```
+
+Option B: CLI flag (advanced; not recommended)
+
+```bash
+npx @music-league-eras/local-runner@latest local-sync \\
+  --api-base-url https://<deployment-host> \\
+  --sync-session-id <sync_session_id> \\
+  --sync-token <sync_token>
+```
+
+Optional flags:
+
+- `--music-league-base-url https://app.musicleague.com`
+- `--capture-headless` / `--no-capture-headless`
+- `--scrape-headless` / `--no-scrape-headless`
+- `--out-dir /path/to/artifacts`
+- `--timeout-s 60`
+
+## Runner home directory
+
+By default, the runner stores its Python virtualenv under:
+
+- `~/.music-league-eras/local-runner`
+
+Override with:
+
+- `ML_ERAS_LOCAL_RUNNER_HOME=/some/dir`
+
+To reset the runner environment, delete that folder.

package/bin/ml-eras-local-runner.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+import { main } from "../src/cli.js";
+
+const exitCode = await main(process.argv.slice(2));
+process.exit(exitCode);
+

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@music-league-eras/local-runner",
+  "version": "0.1.0",
+  "description": "Music League Eras local runner (npx wrapper around the Python scraper runner).",
+  "type": "module",
+  "license": "UNLICENSED",
+  "engines": {
+    "node": ">=18.17.0"
+  },
+  "bin": {
+    "ml-eras-local-runner": "./bin/ml-eras-local-runner.js"
+  },
+  "files": [
+    "bin/",
+    "src/",
+    "scripts/",
+    "vendor/",
+    "README.md"
+  ],
+  "scripts": {
+    "prepack": "node ./scripts/prepack.mjs",
+    "test": "node --test"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}
+

package/scripts/prepack.mjs

@@ -0,0 +1,54 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+function ensureDir(dir) {
+  fs.mkdirSync(dir, { recursive: true });
+}
+
+function copyFile(src, dst) {
+  ensureDir(path.dirname(dst));
+  fs.copyFileSync(src, dst);
+}
+
+function assertExists(filePath) {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`Missing expected file: ${filePath}`);
+  }
+}
+
+const packageRoot = path.dirname(fileURLToPath(import.meta.url));
+const pkg = path.resolve(packageRoot, "..");
+const repoRoot = path.resolve(pkg, "..", "..");
+
+const destRoot = path.join(pkg, "vendor", "python", "app");
+fs.rmSync(destRoot, { recursive: true, force: true });
+ensureDir(destRoot);
+
+const files = [
+  // package markers
+  { src: "services/scraper/app/__init__.py", dst: "vendor/python/app/__init__.py" },
+  { src: "services/scraper/app/browser/__init__.py", dst: "vendor/python/app/browser/__init__.py" },
+  { src: "services/scraper/app/services/__init__.py", dst: "vendor/python/app/services/__init__.py" },
+
+  // entrypoint + runner
+  { src: "services/scraper/app/local_runner_cli.py", dst: "vendor/python/app/local_runner_cli.py" },
+  { src: "services/scraper/app/local_sync_runner.py", dst: "vendor/python/app/local_sync_runner.py" },
+
+  // browser capture
+  { src: "services/scraper/app/browser/session.py", dst: "vendor/python/app/browser/session.py" },
+
+  // scraping
+  { src: "services/scraper/app/services/scrape_manifest.py", dst: "vendor/python/app/services/scrape_manifest.py" },
+  { src: "services/scraper/app/services/viewer.py", dst: "vendor/python/app/services/viewer.py" }
+];
+
+for (const file of files) {
+  const srcAbs = path.join(repoRoot, file.src);
+  const dstAbs = path.join(pkg, file.dst);
+  assertExists(srcAbs);
+  copyFile(srcAbs, dstAbs);
+}
+
+console.log(`Vendored ${files.length} Python files into ${path.relative(repoRoot, path.join(pkg, "vendor", "python"))}`);
+

package/src/args.js

@@ -0,0 +1,64 @@
+const BOOLEAN_FLAGS = new Set(["--capture-headless", "--no-capture-headless", "--scrape-headless", "--no-scrape-headless"]);
+
+export function parseArgs(argv) {
+  const args = Array.isArray(argv) ? argv : [];
+  const nonFlags = args.filter((a) => typeof a === "string" && !a.startsWith("-"));
+  const firstNonFlag = nonFlags[0];
+
+  const command = firstNonFlag === "bootstrap" ? "bootstrap" : "local-sync";
+  const tokens = firstNonFlag === "bootstrap" || firstNonFlag === "local-sync" ? args.slice(1) : args.slice(0);
+
+  if (tokens.includes("--help") || tokens.includes("-h")) {
+    return { command, help: true, error: null, flags: {} };
+  }
+
+  if (command === "bootstrap") {
+    if (tokens.length) {
+      return { command, help: false, error: `Unexpected argument: ${tokens[0]}`, flags: {} };
+    }
+    return { command, help: false, error: null, flags: {} };
+  }
+
+  const flags = {};
+  for (let i = 0; i < tokens.length; i += 1) {
+    const token = tokens[i];
+    if (!token || typeof token !== "string") continue;
+    if (!token.startsWith("--")) {
+      return { command, help: false, error: `Unexpected argument: ${token}`, flags: {} };
+    }
+    if (BOOLEAN_FLAGS.has(token)) {
+      if (token === "--capture-headless") flags.captureHeadless = true;
+      if (token === "--no-capture-headless") flags.captureHeadless = false;
+      if (token === "--scrape-headless") flags.scrapeHeadless = true;
+      if (token === "--no-scrape-headless") flags.scrapeHeadless = false;
+      continue;
+    }
+    const value = tokens[i + 1];
+    if (!value || typeof value !== "string" || value.startsWith("--")) {
+      return { command, help: false, error: `Missing value for ${token}`, flags: {} };
+    }
+    i += 1;
+    if (token === "--api-base-url") flags.apiBaseUrl = value;
+    else if (token === "--sync-session-id") flags.syncSessionId = value;
+    else if (token === "--sync-token") flags.syncToken = value;
+    else if (token === "--music-league-base-url") flags.musicLeagueBaseUrl = value;
+    else if (token === "--out-dir") flags.outDir = value;
+    else if (token === "--timeout-s") flags.timeoutS = value;
+    else return { command, help: false, error: `Unknown flag: ${token}`, flags: {} };
+  }
+
+  const missing = [];
+  if (!flags.apiBaseUrl) missing.push("--api-base-url");
+  if (!flags.syncSessionId) missing.push("--sync-session-id");
+  if (missing.length) {
+    return { command, help: false, error: `Missing required flags: ${missing.join(", ")}`, flags: {} };
+  }
+
+  const timeoutS = flags.timeoutS ? Number(flags.timeoutS) : undefined;
+  if (flags.timeoutS && (!Number.isFinite(timeoutS) || timeoutS <= 0)) {
+    return { command, help: false, error: `Invalid --timeout-s: ${flags.timeoutS}`, flags: {} };
+  }
+  if (timeoutS) flags.timeoutS = timeoutS;
+
+  return { command, help: false, error: null, flags };
+}

package/src/cli.js

@@ -0,0 +1,99 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { parseArgs } from "./args.js";
+import { createLogger, redactSecret } from "./log.js";
+import { bootstrapPythonRunner, resolveRunnerHome } from "./python/bootstrap.js";
+import { runPythonLocalSync } from "./python/run.js";
+import { resolveSyncToken } from "./syncToken.js";
+
+function usageBootstrap() {
+  return [
+    "Usage:",
+    "  ml-eras-local-runner bootstrap",
+    "",
+    "Example:",
+    "  npx @music-league-eras/local-runner@latest bootstrap"
+  ].join("\n");
+}
+
+function usageLocalSync() {
+  return [
+    "Usage:",
+    "  ml-eras-local-runner local-sync --api-base-url <url> --sync-session-id <id>",
+    "",
+    "Example:",
+    "  npx @music-league-eras/local-runner@latest local-sync \\",
+    "    --api-base-url https://<deployment-host> \\",
+    "    --sync-session-id <sync_session_id>",
+    "",
+    "Token input:",
+    "  - default: prompt (safe, input hidden)",
+    "  - or: export ML_ERAS_SYNC_TOKEN=... (advanced/CI)",
+    "  - or: --sync-token ... (advanced/CI; not recommended)"
+  ].join("\n");
+}
+
+export async function main(argv, { stdout, stderr } = {}) {
+  const log = createLogger({ stdout, stderr });
+  const parsed = parseArgs(argv);
+  if (parsed.help) {
+    if (parsed.command === "bootstrap") log.info(usageBootstrap());
+    else log.info(usageLocalSync());
+    return 0;
+  }
+  if (parsed.error) {
+    log.error(parsed.error);
+    log.error("");
+    if (parsed.command === "bootstrap") log.error(usageBootstrap());
+    else log.error(usageLocalSync());
+    return 1;
+  }
+
+  if (parsed.command === "bootstrap") {
+    log.info("Bootstrapping local runner (Python + Playwright)...");
+    log.info(`Runner home: ${resolveRunnerHome()}`);
+
+    const packageRoot = path.dirname(fileURLToPath(new URL("../package.json", import.meta.url)));
+    const runnerHome = resolveRunnerHome();
+    const requirementsPath = path.join(packageRoot, "vendor", "python", "requirements.txt");
+
+    const boot = bootstrapPythonRunner({ packageRoot, runnerHome, requirementsPath, log });
+    if (!boot.ok) return boot.exitCode;
+
+    log.info("");
+    log.info("✅ Prereqs installed. Return to the web app and run Step B (local-sync).");
+    return 0;
+  }
+
+  const flags = parsed.flags;
+  const tokenResult = await resolveSyncToken({
+    flagToken: flags.syncToken,
+    env: process.env,
+    stdin: process.stdin,
+    stdout: stdout ?? process.stdout,
+    stderr: stderr ?? process.stderr
+  });
+  if (!tokenResult.ok) {
+    log.error(tokenResult.error);
+    return 1;
+  }
+  flags.syncToken = tokenResult.token;
+
+  log.info("Bootstrapping local runner (Python + Playwright)...");
+  log.info(`Runner home: ${resolveRunnerHome()}`);
+  log.info(
+    `Sync session: ${flags.syncSessionId}, token: ${redactSecret(flags.syncToken)} (not printed), API: ${flags.apiBaseUrl}`
+  );
+
+  const packageRoot = path.dirname(fileURLToPath(new URL("../package.json", import.meta.url)));
+  const runnerHome = resolveRunnerHome();
+  const requirementsPath = path.join(packageRoot, "vendor", "python", "requirements.txt");
+
+  const boot = bootstrapPythonRunner({ packageRoot, runnerHome, requirementsPath, log });
+  if (!boot.ok) return boot.exitCode;
+
+  log.info("Starting local sync (browser will open)...");
+  const code = runPythonLocalSync({ venvPython: boot.venvPython, pythonPath: boot.pythonPath, flags });
+  return code;
+}

package/src/log.js

@@ -0,0 +1,16 @@
+export function createLogger({ stdout = process.stdout, stderr = process.stderr } = {}) {
+  return {
+    info(message) {
+      stdout.write(`${message}\n`);
+    },
+    error(message) {
+      stderr.write(`${message}\n`);
+    }
+  };
+}
+
+export function redactSecret(value) {
+  if (!value) return "<redacted>";
+  return "<redacted>";
+}
+

package/src/python/bootstrap.js

@@ -0,0 +1,127 @@
+import os from "node:os";
+import path from "node:path";
+import fs from "node:fs";
+import { spawnSync as nodeSpawnSync } from "node:child_process";
+
+function _defaultRunnerHome() {
+  return path.join(os.homedir(), ".music-league-eras", "local-runner");
+}
+
+export function resolveRunnerHome(env = process.env) {
+  const override = env.ML_ERAS_LOCAL_RUNNER_HOME;
+  return override && override.trim() ? override.trim() : _defaultRunnerHome();
+}
+
+export function resolveVenvPython(runnerHome, platform = process.platform) {
+  if (platform === "win32") return path.win32.join(runnerHome, "venv", "Scripts", "python.exe");
+  return path.join(runnerHome, "venv", "bin", "python");
+}
+
+function _spawn(spawnSync, file, args, opts) {
+  const result = spawnSync(file, args, opts);
+  if (result.error) throw result.error;
+  return result;
+}
+
+function _pythonCandidates(platform = process.platform) {
+  if (platform === "win32") {
+    // Prefer the official Python Launcher (`py -3`) if present.
+    return [
+      { file: "py", prefixArgs: ["-3"] },
+      { file: "python", prefixArgs: [] },
+      { file: "python3", prefixArgs: [] }
+    ];
+  }
+  return [{ file: "python3", prefixArgs: [] }, { file: "python", prefixArgs: [] }];
+}
+
+function _pythonNotFoundMessage(platform = process.platform) {
+  if (platform === "win32") {
+    return [
+      "python3 not found.",
+      "Install Python 3.11+ (recommended: `winget install Python.Python.3.11`),",
+      "then re-run: `npx @music-league-eras/local-runner@latest bootstrap`."
+    ].join(" ");
+  }
+  return "python3 not found. Install Python 3.11+ (e.g. `brew install python@3.11`).";
+}
+
+export function ensurePythonAvailable({ spawnSync = nodeSpawnSync, platform = process.platform } = {}) {
+  const versionSnippet = "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}')";
+  const candidates = _pythonCandidates(platform);
+  let chosen = null;
+  let versionText = null;
+
+  for (const c of candidates) {
+    const result = _spawn(spawnSync, c.file, [...c.prefixArgs, "-c", versionSnippet], { encoding: "utf-8" });
+    if (result.status === 0) {
+      chosen = c;
+      versionText = String(result.stdout || "").trim();
+      break;
+    }
+  }
+
+  if (!chosen) {
+    return { ok: false, error: _pythonNotFoundMessage(platform) };
+  }
+  const text = String(versionText || "").trim();
+  const parts = text.split(".").map((p) => Number(p));
+  if (parts.length < 2 || !Number.isFinite(parts[0]) || !Number.isFinite(parts[1])) {
+    return { ok: false, error: `Unable to parse python3 version: ${text}` };
+  }
+  const [major, minor] = parts;
+  if (major < 3 || (major === 3 && minor < 11)) {
+    return { ok: false, error: `python3 >= 3.11 required (found ${text}).` };
+  }
+  return { ok: true, version: text, python: chosen };
+}
+
+export function bootstrapPythonRunner({
+  packageRoot,
+  runnerHome,
+  requirementsPath,
+  spawnSync = nodeSpawnSync,
+  log,
+  platform = process.platform,
+  existsSync = fs.existsSync
+}) {
+  const pythonCheck = ensurePythonAvailable({ spawnSync, platform });
+  if (!pythonCheck.ok) {
+    log.error(pythonCheck.error);
+    return { ok: false, exitCode: 2 };
+  }
+  log.info(`Using python3 ${pythonCheck.version}`);
+
+  const venvDir = platform === "win32" ? path.win32.join(runnerHome, "venv") : path.join(runnerHome, "venv");
+  const venvPython = resolveVenvPython(runnerHome, platform);
+
+  // Create venv if missing.
+  if (!existsSync(venvPython)) {
+    log.info(`Creating venv at ${venvDir}`);
+    const created = _spawn(spawnSync, pythonCheck.python.file, [...pythonCheck.python.prefixArgs, "-m", "venv", venvDir], {
+      stdio: "inherit"
+    });
+    if (created.status !== 0) return { ok: false, exitCode: created.status ?? 1 };
+  }
+
+  log.info("Installing Python dependencies (pip)");
+  const pipUpgrade = _spawn(spawnSync, venvPython, ["-m", "pip", "install", "--upgrade", "pip"], {
+    stdio: "inherit"
+  });
+  if (pipUpgrade.status !== 0) return { ok: false, exitCode: pipUpgrade.status ?? 1 };
+
+  const pipInstall = _spawn(spawnSync, venvPython, ["-m", "pip", "install", "-r", requirementsPath], {
+    stdio: "inherit"
+  });
+  if (pipInstall.status !== 0) return { ok: false, exitCode: pipInstall.status ?? 1 };
+
+  log.info("Ensuring Playwright Chromium is installed");
+  const pwInstall = _spawn(spawnSync, venvPython, ["-m", "playwright", "install", "chromium"], { stdio: "inherit" });
+  if (pwInstall.status !== 0) return { ok: false, exitCode: pwInstall.status ?? 1 };
+
+  return {
+    ok: true,
+    venvPython,
+    pythonPath: path.join(packageRoot, "vendor", "python")
+  };
+}

package/src/python/run.js

@@ -0,0 +1,50 @@
+import path from "node:path";
+import { spawnSync as nodeSpawnSync } from "node:child_process";
+
+function _spawn(spawnSync, file, args, opts) {
+  const result = spawnSync(file, args, opts);
+  if (result.error) throw result.error;
+  return result;
+}
+
+export function runPythonLocalSync({
+  venvPython,
+  pythonPath,
+  flags,
+  spawnSync = nodeSpawnSync
+}) {
+  const args = [
+    "-m",
+    "app.local_runner_cli",
+    "local-sync",
+    "--api-base-url",
+    flags.apiBaseUrl,
+    "--sync-session-id",
+    flags.syncSessionId
+  ];
+  if (flags.musicLeagueBaseUrl) {
+    args.push("--music-league-base-url", flags.musicLeagueBaseUrl);
+  }
+  if (typeof flags.captureHeadless === "boolean") {
+    args.push(flags.captureHeadless ? "--capture-headless" : "--no-capture-headless");
+  }
+  if (typeof flags.scrapeHeadless === "boolean") {
+    args.push(flags.scrapeHeadless ? "--scrape-headless" : "--no-scrape-headless");
+  }
+  if (flags.outDir) {
+    args.push("--out-dir", flags.outDir);
+  }
+  if (flags.timeoutS) {
+    args.push("--timeout-s", String(flags.timeoutS));
+  }
+
+  const env = { ...process.env };
+  if (flags.syncToken) {
+    env.ML_ERAS_SYNC_TOKEN = flags.syncToken;
+  }
+  const existing = env.PYTHONPATH ? String(env.PYTHONPATH) : "";
+  env.PYTHONPATH = existing ? `${pythonPath}${path.delimiter}${existing}` : pythonPath;
+
+  const result = _spawn(spawnSync, venvPython, args, { stdio: "inherit", env });
+  return typeof result.status === "number" ? result.status : 1;
+}

package/src/syncToken.js

@@ -0,0 +1,37 @@
+import { promptSecret } from "./ttyPrompt.js";
+
+function _trimOrNull(value) {
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim();
+  return trimmed ? trimmed : null;
+}
+
+export async function resolveSyncToken({
+  flagToken,
+  env,
+  stdin,
+  stdout,
+  stderr,
+  promptSecretFn = promptSecret
+}) {
+  const fromFlag = _trimOrNull(flagToken);
+  if (fromFlag) return { ok: true, token: fromFlag, source: "flag" };
+
+  const fromEnv = _trimOrNull(env?.ML_ERAS_SYNC_TOKEN);
+  if (fromEnv) return { ok: true, token: fromEnv, source: "env" };
+
+  if (!stdin?.isTTY) {
+    return {
+      ok: false,
+      error:
+        "Missing sync token. Re-run with ML_ERAS_SYNC_TOKEN set, or (advanced) pass --sync-token."
+    };
+  }
+
+  stderr?.write("Paste the sync token from the web app (input hidden), then press Enter:\n");
+  const prompted = _trimOrNull(await promptSecretFn("Sync token: ", { stdin, stdout }));
+  if (!prompted) {
+    return { ok: false, error: "Sync token was empty. Re-run and paste the token from the web app." };
+  }
+  return { ok: true, token: prompted, source: "prompt" };
+}

package/src/ttyPrompt.js

@@ -0,0 +1,50 @@
+export async function promptSecret(prompt, { stdin = process.stdin, stdout = process.stdout } = {}) {
+  if (!stdin.isTTY) {
+    throw new Error("promptSecret requires a TTY stdin.");
+  }
+
+  stdout.write(prompt);
+
+  stdin.setEncoding("utf8");
+  stdin.setRawMode(true);
+  stdin.resume();
+
+  let value = "";
+
+  return await new Promise((resolve, reject) => {
+    function cleanup() {
+      try {
+        stdin.setRawMode(false);
+      } catch {
+        // ignore
+      }
+      stdin.pause();
+      stdin.off("data", onData);
+    }
+
+    function onData(chunk) {
+      const input = String(chunk);
+      for (const char of input) {
+        if (char === "\r" || char === "\n") {
+          cleanup();
+          stdout.write("\n");
+          resolve(value);
+          return;
+        }
+        if (char === "\u0003") {
+          cleanup();
+          reject(new Error("Cancelled."));
+          return;
+        }
+        if (char === "\u007f" || char === "\b") {
+          value = value.slice(0, -1);
+          continue;
+        }
+        value += char;
+      }
+    }
+
+    stdin.on("data", onData);
+  });
+}
+