deaf-intelligence 1.0.1

package/dist/scraper.js

@@ -0,0 +1,2072 @@
+/**
+ * Spotify GraphQL scraper — pure Node.js, minimal dependencies.
+ *
+ * Uses playwright-core (4MB) + system Chrome. No Python. No browser download.
+ *
+ * VERIFIED 2026-03-22:
+ * - KAYO: 38 songs, 982K streams, 16,429 ML, 535 followers, top 5 cities
+ * - ABDE: 65 songs, 49.8M streams, 99K ML, solo/ft/feat roles
+ *
+ * GOTCHAS:
+ * - get_access_token returns 403 from pure HTTP (TLS fingerprint check)
+ *   → SOLUTION: playwright-core launches system Chrome, captures token from network
+ * - Track pages don't show play counts without login
+ *   → SOLUTION: queryAlbumTracks GraphQL endpoint returns them
+ * - Rate limiting: ~50 fast calls triggers 429
+ *   → SOLUTION: 300ms delay between album queries, retry with 5s backoff
+ * - GraphQL hashes may change when Spotify updates their client
+ *   → SOLUTION: hashes from Spicetify community, monitor for failures
+ */
+import { chromium } from "playwright-core";
+import Database from "./db.js";
+import ExcelJS from "exceljs";
+import * as fs from "fs";
+import { constrainedDisaggregate, roundWithAnchor } from "./constrained-disagg.js";
+import { validateS4AData, printValidationReport } from "./validation.js";
+import * as path from "path";
+// GraphQL operation hashes (from Spotify web client / Spicetify)
+const HASHES = {
+    queryArtistOverview: "35648a112beb1794e39ab931365f6ae4a8d45e65396d641eeda94e4003d41497",
+    queryArtistDiscographyAll: "9380995a9d4663cbcb5113fef3c6aabf70ae6d407ba61793fd01e2a1dd6929b0",
+    queryAlbumTracks: "3ea563e1d68f486d8df30f69de9dcedae74c77e684b889ba7408c589d30f7f2e",
+};
+const API_BASE = "https://api-partner.spotify.com/pathfinder/v1/query";
+// Find system Chrome/Edge
+function findChrome() {
+    const paths = [
+        "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+        "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
+        "/Applications/Chromium.app/Contents/MacOS/Chromium",
+        "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
+        "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
+        "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
+        "/usr/bin/google-chrome",
+        "/usr/bin/google-chrome-stable",
+        "/usr/bin/chromium-browser",
+        "/usr/bin/chromium",
+    ];
+    for (const p of paths) {
+        if (fs.existsSync(p))
+            return p;
+    }
+    throw new Error("Chrome/Edge not found. Install Chrome or set CHROME_PATH env var.");
+}
+// ─── Token ───
+export async function getAnonymousToken(artistId) {
+    const chromePath = process.env.CHROME_PATH || findChrome();
+    const browser = await chromium.launch({ executablePath: chromePath, headless: true });
+    const ctx = await browser.newContext({
+        userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    });
+    const page = await ctx.newPage();
+    let token = null;
+    page.on("response", async (resp) => {
+        if (token)
+            return;
+        try {
+            const body = await resp.text();
+            const m = body.match(/"accessToken":"([^"]+)"/);
+            if (m)
+                token = m[1];
+        }
+        catch { }
+    });
+    await page.goto(`https://open.spotify.com/artist/${artistId}`, { waitUntil: "networkidle", timeout: 30000 });
+    await new Promise((r) => setTimeout(r, 3000));
+    await browser.close();
+    if (!token)
+        throw new Error("Failed to capture Spotify token");
+    return token;
+}
+// ─── GraphQL ───
+async function gql(token, operation, variables) {
+    const params = new URLSearchParams({
+        operationName: operation,
+        variables: JSON.stringify(variables),
+        extensions: JSON.stringify({ persistedQuery: { version: 1, sha256Hash: HASHES[operation] } }),
+    });
+    const resp = await fetch(`${API_BASE}?${params}`, {
+        headers: {
+            Authorization: `Bearer ${token}`,
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+        },
+    });
+    if (resp.status === 429) {
+        // Rate limited — wait and retry once
+        await new Promise((r) => setTimeout(r, 5000));
+        const retry = await fetch(`${API_BASE}?${params}`, {
+            headers: {
+                Authorization: `Bearer ${token}`,
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+            },
+        });
+        if (!retry.ok)
+            throw new Error(`GraphQL ${operation}: ${retry.status}`);
+        return retry.json();
+    }
+    if (!resp.ok)
+        throw new Error(`GraphQL ${operation}: ${resp.status}`);
+    return resp.json();
+}
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+export async function getArtistOverview(token, artistId) {
+    const result = await gql(token, "queryArtistOverview", {
+        uri: `spotify:artist:${artistId}`,
+        locale: "en",
+        includePrerelease: true,
+    });
+    const artist = result.data.artistUnion;
+    const stats = artist.stats || {};
+    const profile = artist.profile || {};
+    const visuals = artist.visuals || {};
+    const disco = artist.discography || {};
+    const related = artist.relatedContent || {};
+    // Images: avatar (largest first), header banner, full gallery
+    const avatarSources = visuals.avatarImage?.sources || [];
+    const profileImage = avatarSources.sort((a, b) => (b.width || 0) - (a.width || 0))[0]?.url || null;
+    const headerImage = visuals.headerImage?.sources?.[0]?.url || null;
+    const galleryImages = (visuals.gallery?.items || [])
+        .map((item) => item.sources?.[0]?.url)
+        .filter(Boolean);
+    const brandColor = visuals.avatarImage?.extractedColors?.colorRaw?.hex
+        || visuals.headerImage?.extractedColors?.colorRaw?.hex
+        || null;
+    return {
+        artist_id: artistId,
+        name: profile.name || "",
+        verified: profile.verified || false,
+        bio: profile.biography?.text || null,
+        profile_image: profileImage,
+        header_image: headerImage,
+        gallery_images: galleryImages,
+        brand_color: brandColor,
+        monthly_listeners: Number(stats.monthlyListeners) || 0,
+        followers: Number(stats.followers) || 0,
+        world_rank: Number(stats.worldRank) || 0,
+        top_cities: (stats.topCities?.items || []).map((c) => ({
+            city: c.city, country: c.country, listeners: Number(c.numberOfListeners) || 0,
+        })),
+        top_tracks: (disco.topTracks?.items || []).map((item) => {
+            const t = item.track || {};
+            return {
+                track_id: (t.uri || "").split(":").pop() || "",
+                title: t.name || "",
+                playcount: parseInt(t.playcount || "0", 10),
+                artists: (t.artists?.items || []).map((a) => a.profile?.name || "?"),
+            };
+        }),
+        related_artists: (related.relatedArtists?.items || []).map((item) => {
+            const a = item.artist || {};
+            return { name: a.profile?.name || "?", id: a.id || "", followers: Number(a.stats?.followers) || 0 };
+        }).filter((a) => a.name !== "?"),
+        discovered_on: (related.discoveredOnV2?.items || [])
+            .filter((item) => item.data?.__typename === "Playlist")
+            .map((item) => {
+            const pl = item.data || {};
+            return { name: pl.name || "", owner: pl.ownerV2?.data?.name || "" };
+        })
+            .filter((d) => d.name),
+        external_links: (profile.externalLinks?.items || []).map((l) => ({ name: l.name, url: l.url })),
+        latest_release: disco.latest ? {
+            name: disco.latest.name || "",
+            type: disco.latest.type || "",
+            label: disco.latest.label || "",
+            date: disco.latest.date?.isoString || String(disco.latest.date?.year || ""),
+        } : null,
+        singles_count: disco.singles?.totalCount || 0,
+        albums_count: disco.albums?.totalCount || 0,
+    };
+}
+export async function getAllTracks(token, artistId) {
+    const result = await gql(token, "queryArtistDiscographyAll", {
+        uri: `spotify:artist:${artistId}`, offset: 0, limit: 100,
+    });
+    const albums = [];
+    for (const item of result.data.artistUnion.discography.all.items) {
+        for (const r of item.releases?.items || []) {
+            albums.push({
+                id: r.id,
+                name: r.name,
+                type: r.type || "SINGLE",
+                date: r.date?.isoString || String(r.date?.year || ""),
+                cover: r.coverArt?.sources?.[0]?.url || null,
+            });
+        }
+    }
+    const allTracks = [];
+    const seen = new Set();
+    for (const album of albums) {
+        await sleep(300); // Rate limit courtesy
+        try {
+            const res = await gql(token, "queryAlbumTracks", {
+                uri: `spotify:album:${album.id}`, offset: 0, limit: 300,
+            });
+            for (const tItem of res.data.album.tracks.items) {
+                const track = tItem.track || {};
+                const tid = (track.uri || "").split(":").pop() || "";
+                if (!tid || seen.has(tid))
+                    continue;
+                seen.add(tid);
+                const artists = (track.artists?.items || []).map((a) => a.profile?.name || "?");
+                const artistIds = (track.artists?.items || []).map((a) => (a.uri || "").split(":").pop() || "");
+                const isPrimary = artistIds[0] === artistId;
+                const isSolo = artists.length === 1 && isPrimary;
+                const isFeat = artistIds.includes(artistId) && !isPrimary;
+                let role;
+                if (isSolo)
+                    role = "SOLO";
+                else if (isPrimary) {
+                    const featNames = artists.filter((_, i) => artistIds[i] !== artistId);
+                    role = `ft. ${featNames.join(" & ")}`;
+                }
+                else if (isFeat)
+                    role = `FEAT on ${artists[0]}`;
+                else
+                    role = "OTHER";
+                allTracks.push({
+                    track_id: tid, title: track.name || "", playcount: parseInt(track.playcount || "0", 10),
+                    popularity: 0, // populated by batch Web API call below
+                    artists, artist_ids: artistIds, role,
+                    album_name: album.name, album_id: album.id,
+                    release_date: album.date, release_type: album.type, cover_art: album.cover,
+                });
+            }
+        }
+        catch (e) {
+            // Skip failed albums silently, continue with rest
+        }
+    }
+    // Batch fetch popularity scores from Spotify Web API (50 per call)
+    const trackIds = allTracks.map(t => t.track_id);
+    for (let i = 0; i < trackIds.length; i += 50) {
+        const batch = trackIds.slice(i, i + 50);
+        try {
+            const resp = await fetch(`https://api.spotify.com/v1/tracks?ids=${batch.join(",")}`, {
+                headers: { Authorization: `Bearer ${token}` },
+            });
+            if (resp.ok) {
+                const data = await resp.json();
+                for (const t of data.tracks || []) {
+                    if (!t)
+                        continue;
+                    const match = allTracks.find(at => at.track_id === t.id);
+                    if (match)
+                        match.popularity = Number(t.popularity) || 0;
+                }
+            }
+            await sleep(300);
+        }
+        catch { }
+    }
+    return allTracks;
+}
+// ─── SQLite Storage ───
+const SCHEMA = `
+PRAGMA journal_mode = DELETE;
+
+CREATE TABLE IF NOT EXISTS artist (
+  spotify_id TEXT PRIMARY KEY, name TEXT NOT NULL, verified INTEGER DEFAULT 0,
+  bio TEXT, brand_color TEXT, created_at INTEGER DEFAULT (unixepoch())
+);
+CREATE TABLE IF NOT EXISTS daily_stats (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, artist_id TEXT NOT NULL,
+  metric_name TEXT NOT NULL, metric_value REAL NOT NULL,
+  delta_value REAL, delta_pct REAL, scraped_at INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_daily ON daily_stats(artist_id, metric_name, scraped_at);
+
+CREATE TABLE IF NOT EXISTS song_snapshots (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, artist_id TEXT NOT NULL,
+  track_id TEXT, track_title TEXT NOT NULL, play_count INTEGER NOT NULL,
+  popularity INTEGER DEFAULT 0,
+  delta_value INTEGER, delta_pct REAL, scraped_at INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_songs ON song_snapshots(artist_id, track_title, scraped_at);
+
+CREATE TABLE IF NOT EXISTS tracks (
+  spotify_id TEXT, artist_id TEXT NOT NULL, title TEXT NOT NULL,
+  release_type TEXT, release_date TEXT, release_year INTEGER,
+  cover_art_url TEXT, role TEXT, UNIQUE(artist_id, title)
+);
+CREATE TABLE IF NOT EXISTS locations (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, artist_id TEXT NOT NULL,
+  location_type TEXT NOT NULL, location_name TEXT NOT NULL,
+  listener_count INTEGER, rank INTEGER, scraped_at INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS related_artists (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, artist_id TEXT NOT NULL,
+  related_name TEXT NOT NULL, related_spotify_id TEXT, scraped_at INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS discovered_on (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, artist_id TEXT NOT NULL,
+  playlist_name TEXT NOT NULL, playlist_owner TEXT, scraped_at INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS external_links (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, artist_id TEXT NOT NULL,
+  link_name TEXT NOT NULL, link_url TEXT NOT NULL, scraped_at INTEGER NOT NULL,
+  UNIQUE(artist_id, link_name, link_url)
+);
+`;
+export function storeData(dbPath, overview, tracks) {
+    const db = new Database(dbPath);
+    db.exec(SCHEMA);
+    // Migrate: add columns that may not exist in older DBs
+    const cols = db.prepare("PRAGMA table_info(artist)").all();
+    const colNames = new Set(cols.map((c) => c.name));
+    if (!colNames.has("brand_color"))
+        db.exec("ALTER TABLE artist ADD COLUMN brand_color TEXT");
+    const snapCols = db.prepare("PRAGMA table_info(song_snapshots)").all();
+    const snapColNames = new Set(snapCols.map((c) => c.name));
+    if (!snapColNames.has("popularity"))
+        db.exec("ALTER TABLE song_snapshots ADD COLUMN popularity INTEGER DEFAULT 0");
+    const ts = Math.floor(Date.now() / 1000);
+    const aid = overview.artist_id;
+    // Artist (including bio + brand_color)
+    db.prepare("INSERT OR REPLACE INTO artist (spotify_id, name, verified, bio, brand_color) VALUES (?, ?, ?, ?, ?)")
+        .run(aid, overview.name, overview.verified ? 1 : 0, overview.bio, overview.brand_color);
+    // Daily stats with delta
+    const prevStmt = db.prepare("SELECT metric_value FROM daily_stats WHERE artist_id=? AND metric_name=? ORDER BY scraped_at DESC LIMIT 1");
+    const insertStat = db.prepare("INSERT INTO daily_stats (artist_id, metric_name, metric_value, delta_value, delta_pct, scraped_at) VALUES (?,?,?,?,?,?)");
+    for (const [metric, value] of [
+        ["monthly_listeners", overview.monthly_listeners],
+        ["followers", overview.followers],
+        ["world_rank", overview.world_rank],
+    ]) {
+        if (value == null)
+            continue;
+        const prev = prevStmt.get(aid, metric);
+        let delta = null, deltaPct = null;
+        if (prev?.metric_value > 0) {
+            delta = value - prev.metric_value;
+            deltaPct = (delta / prev.metric_value) * 100;
+        }
+        insertStat.run(aid, metric, value, delta, deltaPct, ts);
+    }
+    // Cities
+    const insertCity = db.prepare("INSERT INTO locations (artist_id, location_type, location_name, listener_count, rank, scraped_at) VALUES (?,?,?,?,?,?)");
+    overview.top_cities.forEach((c, i) => insertCity.run(aid, "city", `${c.city}, ${c.country}`, c.listeners, i + 1, ts));
+    // Related
+    const insertRelated = db.prepare("INSERT INTO related_artists (artist_id, related_name, related_spotify_id, scraped_at) VALUES (?,?,?,?)");
+    for (const ra of overview.related_artists) {
+        if (ra.name)
+            insertRelated.run(aid, ra.name, ra.id, ts);
+    }
+    // Discovered on playlists
+    const insertDiscovered = db.prepare("INSERT INTO discovered_on (artist_id, playlist_name, playlist_owner, scraped_at) VALUES (?,?,?,?)");
+    for (const d of overview.discovered_on) {
+        if (d.name)
+            insertDiscovered.run(aid, d.name, d.owner, ts);
+    }
+    // External links (INSERT OR IGNORE — only add new ones)
+    const insertLink = db.prepare("INSERT OR IGNORE INTO external_links (artist_id, link_name, link_url, scraped_at) VALUES (?,?,?,?)");
+    for (const l of overview.external_links) {
+        if (l.url)
+            insertLink.run(aid, l.name, l.url, ts);
+    }
+    // Tracks + snapshots
+    const upsertTrack = db.prepare("INSERT OR REPLACE INTO tracks (spotify_id, artist_id, title, release_type, release_date, release_year, cover_art_url, role) VALUES (?,?,?,?,?,?,?,?)");
+    const prevPlay = db.prepare("SELECT play_count FROM song_snapshots WHERE artist_id=? AND track_title=? ORDER BY scraped_at DESC LIMIT 1");
+    const insertSnap = db.prepare("INSERT INTO song_snapshots (artist_id, track_id, track_title, play_count, popularity, delta_value, delta_pct, scraped_at) VALUES (?,?,?,?,?,?,?,?)");
+    const insertAll = db.transaction(() => {
+        for (const t of tracks) {
+            const year = t.release_date?.length >= 4 ? parseInt(t.release_date.substring(0, 4), 10) : null;
+            upsertTrack.run(t.track_id, aid, t.title, t.release_type, t.release_date, year, t.cover_art, t.role);
+            const prev = prevPlay.get(aid, t.title);
+            let delta = null, deltaPct = null;
+            if (prev?.play_count > 0) {
+                delta = t.playcount - prev.play_count;
+                deltaPct = (delta / prev.play_count) * 100;
+            }
+            insertSnap.run(aid, t.track_id, t.title, t.playcount, t.popularity || 0, delta, deltaPct, ts);
+        }
+    });
+    insertAll();
+    db.close();
+}
+// ─── Image Download ───
+async function downloadImage(url, destPath) {
+    try {
+        const resp = await fetch(url);
+        if (!resp.ok)
+            return false;
+        const buffer = Buffer.from(await resp.arrayBuffer());
+        fs.mkdirSync(path.dirname(destPath), { recursive: true });
+        fs.writeFileSync(destPath, buffer);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// Extract Spotify's unique image ID from CDN URL.
+// URL format: https://i.scdn.co/image/ab6761610000e5ebd3bb05bb2490318ab924b02a
+// Falls back to sha256 hash of full URL if format changes.
+function imageIdFromUrl(url) {
+    const match = url.match(/\/image\/([a-f0-9]+)/);
+    if (match)
+        return match[1];
+    const crypto = require("crypto");
+    return crypto.createHash("sha256").update(url).digest("hex").slice(0, 24);
+}
+function readManifest(manifestPath) {
+    try {
+        return JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
+    }
+    catch {
+        return { gallery_urls: [], cover_urls: {} };
+    }
+}
+function writeManifest(manifestPath, manifest) {
+    fs.mkdirSync(path.dirname(manifestPath), { recursive: true });
+    fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
+}
+// Artist folder: ~/.artist-os/{Artist Name}/ — everything customer sees lives here
+function artistDir(dataDir, artistName) {
+    return path.join(dataDir, artistName);
+}
+export async function downloadImages(artistId, overview, tracks, dataDir) {
+    const imgDir = path.join(artistDir(dataDir, overview.name), "images");
+    const coversDir = path.join(imgDir, "covers");
+    const galleryDir = path.join(imgDir, "gallery");
+    const manifestPath = path.join(imgDir, "manifest.json");
+    const manifest = readManifest(manifestPath);
+    let downloaded = 0, skipped = 0;
+    // --- Profile image ---
+    // Always profile.jpg = latest. If URL changed, archive old as profile_{image_id}.jpg
+    if (overview.profile_image) {
+        const dest = path.join(imgDir, "profile.jpg");
+        if (manifest.profile_url && manifest.profile_url !== overview.profile_image && fs.existsSync(dest)) {
+            // URL changed — archive the old one, then download new
+            const oldId = imageIdFromUrl(manifest.profile_url);
+            fs.renameSync(dest, path.join(imgDir, `profile_${oldId}.jpg`));
+            if (await downloadImage(overview.profile_image, dest))
+                downloaded++;
+        }
+        else if (!fs.existsSync(dest)) {
+            if (await downloadImage(overview.profile_image, dest))
+                downloaded++;
+        }
+        else
+            skipped++;
+        manifest.profile_url = overview.profile_image;
+    }
+    // --- Header banner ---
+    // Same logic: header.jpg = latest, archive old on change
+    if (overview.header_image) {
+        const dest = path.join(imgDir, "header.jpg");
+        if (manifest.header_url && manifest.header_url !== overview.header_image && fs.existsSync(dest)) {
+            const oldId = imageIdFromUrl(manifest.header_url);
+            fs.renameSync(dest, path.join(imgDir, `header_${oldId}.jpg`));
+            if (await downloadImage(overview.header_image, dest))
+                downloaded++;
+        }
+        else if (!fs.existsSync(dest)) {
+            if (await downloadImage(overview.header_image, dest))
+                downloaded++;
+        }
+        else
+            skipped++;
+        manifest.header_url = overview.header_image;
+    }
+    // --- Gallery ---
+    // Use Spotify image ID as filename. Photos accumulate — never deleted.
+    // Added photo → new file. Removed photo → old file stays. Reordered → no change.
+    for (const url of overview.gallery_images) {
+        const id = imageIdFromUrl(url);
+        const dest = path.join(galleryDir, `${id}.jpg`);
+        if (fs.existsSync(dest)) {
+            skipped++;
+            continue;
+        }
+        if (await downloadImage(url, dest))
+            downloaded++;
+        await sleep(100);
+    }
+    manifest.gallery_urls = overview.gallery_images;
+    // --- Cover arts ---
+    // Keyed by track_id — covers don't change after release.
+    for (const t of tracks) {
+        if (!t.cover_art)
+            continue;
+        const dest = path.join(coversDir, `${t.track_id}.jpg`);
+        if (fs.existsSync(dest)) {
+            skipped++;
+            continue;
+        }
+        if (await downloadImage(t.cover_art, dest))
+            downloaded++;
+        await sleep(100);
+    }
+    writeManifest(manifestPath, manifest);
+    return { downloaded, skipped };
+}
+// ─── Load all historical S4A dumps for an artist ───
+function loadAllDumps(artistId) {
+    const dumpRoot = path.join(process.env.HOME || "~", ".artist-os", "s4a-dumps");
+    if (!fs.existsSync(dumpRoot))
+        return [];
+    const dirs = fs.readdirSync(dumpRoot).filter((d) => /^\d{4}-\d{2}-\d{2}$/.test(d)).sort();
+    const dumps = [];
+    for (const dir of dirs) {
+        const file = path.join(dumpRoot, dir, `${artistId}_s4a.json`);
+        if (!fs.existsSync(file))
+            continue;
+        try {
+            const data = JSON.parse(fs.readFileSync(file, "utf-8"));
+            if (data.perSong && Object.keys(data.perSong).length > 0)
+                dumps.push({ date: dir, data });
+        }
+        catch { }
+    }
+    return dumps;
+}
+// ─── Master XLSX: one workbook per artist, grows forever ───
+export async function generateWorkbook(dbPath, dataDir, s4a) {
+    const db = new Database(dbPath, { readonly: true });
+    const artist = db.prepare("SELECT name, spotify_id, verified, bio FROM artist LIMIT 1").get();
+    try {
+        const bc = db.prepare("SELECT brand_color FROM artist LIMIT 1").get();
+        if (bc)
+            artist.brand_color = bc.brand_color;
+    }
+    catch { }
+    if (!artist) {
+        db.close();
+        return;
+    }
+    const aid = artist.spotify_id;
+    const artDir = artistDir(dataDir, artist.name);
+    fs.mkdirSync(artDir, { recursive: true });
+    const xlsxPath = path.join(artDir, `${artist.name}.xlsx`);
+    // JEDEN timestamp per scrape — nikdy nepřepočítávat Date.now() znovu
+    const scrapeTs = Math.floor(Date.now() / 1000);
+    const now = scrapeTs;
+    // today = datum SCRAPU (metadata), NE datum data pointu!
+    const today = new Date(scrapeTs * 1000).toISOString().slice(0, 10);
+    const wb = new ExcelJS.Workbook();
+    wb.creator = "Artist-OS";
+    wb.created = new Date();
+    const hdrDark = {
+        font: { bold: true, color: { argb: "FFFFFFFF" }, size: 11 },
+        fill: { type: "pattern", pattern: "solid", fgColor: { argb: "FF1a1a2e" } },
+        alignment: { horizontal: "center" },
+    };
+    const hdrGreen = {
+        font: { bold: true, color: { argb: "FFFFFFFF" }, size: 11 },
+        fill: { type: "pattern", pattern: "solid", fgColor: { argb: "FF1B5E20" } },
+        alignment: { horizontal: "center" },
+    };
+    const estFill = { type: "pattern", pattern: "solid", fgColor: { argb: "FFF5F5DC" } };
+    const projFill = { type: "pattern", pattern: "solid", fgColor: { argb: "FFE8EAF6" } };
+    const numFmt = "#,##0";
+    const applyHdr = (ws, style) => ws.getRow(1).eachCell(c => { Object.assign(c, { style }); });
+    const sectionStyle = {
+        font: { bold: true, size: 12 },
+        fill: { type: "pattern", pattern: "solid", fgColor: { argb: "FFE8EAF6" } },
+        alignment: { horizontal: "center", vertical: "middle" },
+    };
+    const labelStyle = {
+        font: { bold: true, color: { argb: "FF333333" } },
+        alignment: { horizontal: "center", vertical: "middle" },
+    };
+    const metaStyle = {
+        font: { italic: true, color: { argb: "FF666666" }, size: 9 },
+        alignment: { horizontal: "center", vertical: "middle" },
+    };
+    const addSection = (ws, name) => {
+        const row = ws.addRow([`═══ ${name} ═══`, "", ""]);
+        row.eachCell(c => { Object.assign(c, { style: sectionStyle }); });
+    };
+    const addField = (ws, field, value, since) => {
+        const row = ws.addRow([field, value ?? "", since]);
+        row.getCell(1).style = labelStyle;
+        row.getCell(2).alignment = { horizontal: "center", vertical: "middle", wrapText: true };
+        row.getCell(3).alignment = { horizontal: "center", vertical: "middle" };
+    };
+    const addHistoryNote = (ws, label) => {
+        const row = ws.addRow(["  HISTORY", label, ""]);
+        row.eachCell(c => { Object.assign(c, { style: metaStyle }); });
+    };
+    const addSpacer = (ws) => ws.addRow(["", "", ""]);
+    const scrapeDate = today;
+    // ═══════════════════════════════════════════
+    //  SHEET 1: PROFILE (v2 — ACTUAL + HISTORY)
+    // ═══════════════════════════════════════════
+    const profileSheet = wb.addWorksheet("Profile", { properties: { tabColor: { argb: "FF2E7D32" } } });
+    profileSheet.columns = [
+        { header: "Field", width: 22 },
+        { header: "Value", width: 55 },
+        { header: "Since", width: 14 },
+    ];
+    applyHdr(profileSheet, hdrDark);
+    let links = [];
+    try {
+        links = db.prepare("SELECT link_name, link_url FROM external_links WHERE artist_id = ?").all(aid);
+    }
+    catch { }
+    const trackCountTotal = db.prepare("SELECT COUNT(*) as c FROM tracks WHERE artist_id = ?").get(aid)?.c || 0;
+    const latestRel = db.prepare("SELECT title, release_type, release_date FROM tracks WHERE artist_id = ? ORDER BY release_date DESC LIMIT 1").get(aid);
+    // Identity
+    addSection(profileSheet, "IDENTITY");
+    addField(profileSheet, "Spotify ID", aid, "—");
+    addField(profileSheet, "Name", artist.name, scrapeDate);
+    addHistoryNote(profileSheet, "(no changes)");
+    addSpacer(profileSheet);
+    addField(profileSheet, "Verified", artist.verified ? "Yes" : "No", scrapeDate);
+    addHistoryNote(profileSheet, "(no changes)");
+    addSpacer(profileSheet);
+    addField(profileSheet, "Bio", artist.bio || "(empty)", scrapeDate);
+    addHistoryNote(profileSheet, "(no changes)");
+    addSpacer(profileSheet);
+    addField(profileSheet, "Brand Color", artist.brand_color || "(not captured)", scrapeDate);
+    addHistoryNote(profileSheet, "(no changes)");
+    addSpacer(profileSheet);
+    // Images
+    addSection(profileSheet, "IMAGES");
+    const imgDir = path.join(artDir, "images");
+    const profileImgPath = path.join(imgDir, "profile.jpg");
+    if (fs.existsSync(profileImgPath)) {
+        addField(profileSheet, "Profile Image", "See below", scrapeDate);
+        const pImgId = wb.addImage({ filename: profileImgPath, extension: "jpeg" });
+        const pImgRow = profileSheet.rowCount + 1;
+        profileSheet.addRow(["", "", ""]);
+        profileSheet.getRow(pImgRow).height = 120;
+        profileSheet.addImage(pImgId, { tl: { col: 1, row: pImgRow - 1 }, ext: { width: 120, height: 120 } });
+    }
+    else {
+        addField(profileSheet, "Profile Image", "(not downloaded)", scrapeDate);
+    }
+    addHistoryNote(profileSheet, "(no changes)");
+    addSpacer(profileSheet);
+    const headerImgPath = path.join(imgDir, "header.jpg");
+    if (fs.existsSync(headerImgPath)) {
+        addField(profileSheet, "Header Image", "See below", scrapeDate);
+        const hImgId = wb.addImage({ filename: headerImgPath, extension: "jpeg" });
+        const hImgRow = profileSheet.rowCount + 1;
+        profileSheet.addRow(["", "", ""]);
+        profileSheet.getRow(hImgRow).height = 80;
+        profileSheet.addImage(hImgId, { tl: { col: 1, row: hImgRow - 1 }, ext: { width: 300, height: 80 } });
+    }
+    else {
+        addField(profileSheet, "Header Image", "(not downloaded)", scrapeDate);
+    }
+    addHistoryNote(profileSheet, "(no changes)");
+    addSpacer(profileSheet);
+    const galleryDirPath = path.join(imgDir, "gallery");
+    const galFiles = fs.existsSync(galleryDirPath)
+        ? fs.readdirSync(galleryDirPath).filter(f => f.endsWith(".jpg")).slice(0, 8) : [];
+    addField(profileSheet, "Gallery", `${galFiles.length} photos`, scrapeDate);
+    if (galFiles.length > 0) {
+        for (let i = 0; i < galFiles.length; i++) {
+            const col = profileSheet.getColumn(2 + i);
+            if ((col.width || 0) < 16)
+                col.width = 16;
+        }
+        const galRow = profileSheet.rowCount + 1;
+        profileSheet.addRow(["", "", ""]);
+        profileSheet.getRow(galRow).height = 100;
+        for (let i = 0; i < galFiles.length; i++) {
+            const gId = wb.addImage({ filename: path.join(galleryDirPath, galFiles[i]), extension: "jpeg" });
+            profileSheet.addImage(gId, { tl: { col: 1 + i, row: galRow - 1 }, ext: { width: 100, height: 100 } });
+        }
+    }
+    addHistoryNote(profileSheet, "(no changes)");
+    addSpacer(profileSheet);
+    // External Links
+    addSection(profileSheet, "EXTERNAL LINKS");
+    if (links.length) {
+        for (const l of links)
+            addField(profileSheet, l.link_name, l.link_url, scrapeDate);
+    }
+    else {
+        addField(profileSheet, "(none captured)", "", "");
+    }
+    addHistoryNote(profileSheet, "(no changes)");
+    addSpacer(profileSheet);
+    // Release Info
+    addSection(profileSheet, "RELEASE INFO");
+    if (latestRel) {
+        addField(profileSheet, "Latest Release", `${latestRel.title} / ${latestRel.release_type} / ${latestRel.release_date?.slice(0, 10)}`, scrapeDate);
+    }
+    addField(profileSheet, "Label", "—", scrapeDate);
+    addField(profileSheet, "Singles Count", trackCountTotal, scrapeDate);
+    addField(profileSheet, "Albums Count", 0, scrapeDate);
+    addHistoryNote(profileSheet, "(no changes)");
+    addSpacer(profileSheet);
+    // S4A section — fields 14-21 per SPEC-02_DATA.md (from REAL S4A data)
+    addSection(profileSheet, "S4A");
+    if (s4a) {
+        // 14: Account Owner
+        const owner = s4a.accountOwner;
+        addField(profileSheet, "Account Owner", owner?.name ? `${owner.name} (${owner.email || ""})` : "Connected", scrapeDate);
+        addHistoryNote(profileSheet, "(no changes)");
+        addSpacer(profileSheet);
+        // 15: Permissions
+        const perms = s4a.permissions;
+        addField(profileSheet, "Permissions", perms ? JSON.stringify(perms).slice(0, 100) : "(not captured)", scrapeDate);
+        addHistoryNote(profileSheet, "(no changes)");
+        addSpacer(profileSheet);
+        // 16: Canvas Permissions
+        const canvas = s4a.canvasPermissions;
+        addField(profileSheet, "Canvas Permissions", canvas ? JSON.stringify(canvas).slice(0, 100) : "(not captured)", scrapeDate);
+        addHistoryNote(profileSheet, "(no changes)");
+        addSpacer(profileSheet);
+        // 17: Campaign Eligibility
+        const campaign = s4a.campaignEligibility;
+        if (campaign) {
+            const parts = [];
+            if (campaign.marquee)
+                parts.push(`Marquee: ${campaign.marquee.eligible ? "YES" : "NO"}${campaign.marquee.meetsThreshold ? " (threshold: YES)" : ""}`);
+            if (campaign.showcase)
+                parts.push(`Showcase: ${campaign.showcase.eligible ? "YES" : "NO"}`);
+            addField(profileSheet, "Campaign Eligibility", parts.join(" / ") || JSON.stringify(campaign).slice(0, 100), scrapeDate);
+        }
+        else {
+            addField(profileSheet, "Campaign Eligibility", "(not captured)", scrapeDate);
+        }
+        addHistoryNote(profileSheet, "(no changes)");
+        addSpacer(profileSheet);
+        // 18: Team Membership
+        const team = s4a.teamMembership;
+        addField(profileSheet, "Team Membership", team?.isArtistTeamMember != null ? (team.isArtistTeamMember ? "Yes" : "No") : "(not captured)", scrapeDate);
+        addHistoryNote(profileSheet, "(no changes)");
+        addSpacer(profileSheet);
+        // 19: Upcoming Release — from /music/upcoming page (captured in upcoming field)
+        addField(profileSheet, "Upcoming Release", s4a.upcoming ? JSON.stringify(s4a.upcoming).slice(0, 200) : "(none or not captured)", scrapeDate);
+        addHistoryNote(profileSheet, "(no changes)");
+        addSpacer(profileSheet);
+        // 20: Pitch Status — part of upcoming data
+        addField(profileSheet, "Pitch Status", "(see Upcoming Release)", scrapeDate);
+        addSpacer(profileSheet);
+        // 21: Rights Access — from perSong data
+        const allRights = new Set();
+        if (s4a.perSong) {
+            for (const tData of Object.values(s4a.perSong)) {
+                if (tData.rights_access)
+                    allRights.add(tData.rights_access);
+            }
+        }
+        addField(profileSheet, "Rights Access", allRights.size > 0 ? Array.from(allRights).join(", ") : "(not captured)", scrapeDate);
+        addHistoryNote(profileSheet, "(no changes)");
+    }
+    else {
+        // S4A not connected
+        addField(profileSheet, "Status", hasS4ASession() ? "Session expired — run connect_s4a" : "Not connected — run connect_s4a for full data", "—");
+        addSpacer(profileSheet);
+        for (const f of ["Account Owner", "Permissions", "Canvas Permissions",
+            "Campaign Eligibility", "Team Membership", "Upcoming Release",
+            "Pitch Status", "Rights Access"]) {
+            addField(profileSheet, f, "NULL", "—");
+        }
+    }
+    // ═══════════════════════════════════════════
+    //  SHEET 2: STREAMS (v2 — per track per day)
+    // ═══════════════════════════════════════════
+    const latestTracksForStreams = db.prepare(`
+    SELECT t.title, t.spotify_id as track_id, t.release_date, t.role, t.release_type, s.play_count
+    FROM tracks t
+    JOIN song_snapshots s ON s.track_id = t.spotify_id AND s.artist_id = t.artist_id
+    WHERE t.artist_id = ? AND s.scraped_at = (SELECT MAX(scraped_at) FROM song_snapshots WHERE artist_id = ?)
+    ORDER BY s.play_count DESC
+  `).all(aid, aid);
+    const streamTrackNames = latestTracksForStreams.map((t) => t.title);
+    const s4aFill = { type: "pattern", pattern: "solid", fgColor: { argb: "FFE8F5E9" } };
+    const dayNames = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
+    // Build track ID → name mapping for S4A perSong lookup
+    const trackIdToName = {};
+    const trackNameToId = {};
+    for (const t of latestTracksForStreams) {
+        trackIdToName[t.track_id] = t.title;
+        trackNameToId[t.title] = t.track_id;
+    }
+    // Scraped cumulative play counts by date (from DB — only for Streams sheet)
+    const allSnaps = db.prepare("SELECT date(s.scraped_at, 'unixepoch') as date, s.track_title, s.play_count FROM song_snapshots s WHERE s.artist_id = ? ORDER BY s.scraped_at ASC").all(aid);
+    const streamsByDate = {};
+    for (const r of allSnaps) {
+        if (!streamsByDate[r.date])
+            streamsByDate[r.date] = {};
+        streamsByDate[r.date][r.track_title] = r.play_count;
+    }
+    // Velocity for estimation (streams only)
+    const vel = {};
+    for (const t of latestTracksForStreams) {
+        if (!t.release_date)
+            continue;
+        const days = Math.max(1, Math.floor((now - new Date(t.release_date).getTime() / 1000) / 86400));
+        vel[t.title] = t.play_count / days;
+    }
+    const sReleaseDates = latestTracksForStreams.map((t) => t.release_date?.slice(0, 10)).filter(Boolean).sort();
+    const sEarliestRelease = sReleaseDates[0];
+    // ── Helper: generate a metric sheet (365d daily + S4A_PREV + YoY + summary) ──
+    const metricSheets = [
+        { name: "Streams", key: "streams", color: "FFff6b6b", numFormat: "#,##0", hasEst: true },
+        { name: "Listeners", key: "listeners", color: "FF42A5F5", numFormat: "#,##0", hasEst: false },
+        { name: "Saves", key: "saves", color: "FF66BB6A", numFormat: "#,##0", hasEst: false },
+        { name: "Playlist Adds", key: "playlist_adds", color: "FFFFCA28", numFormat: "#,##0", hasEst: false },
+        { name: "Streams per Listener", key: "streams_per_listener", color: "FFAB47BC", numFormat: "#,##0.00", hasEst: false },
+    ];
+    for (const metric of metricSheets) {
+        const ws = wb.addWorksheet(metric.name, { properties: { tabColor: { argb: metric.color } } });
+        ws.columns = [
+            { header: "Date", width: 12 }, { header: "Day", width: 5 }, { header: "Source", width: 8 },
+            ...streamTrackNames.map((n) => ({ header: n, width: 13 })),
+            { header: "TOTAL", width: 14 },
+        ];
+        applyHdr(ws, hdrDark);
+        // Metadata rows
+        const relRow = ws.addRow(["Release", "", "", ...streamTrackNames.map((n) => {
+                const t = latestTracksForStreams.find((x) => x.title === n);
+                return t?.release_date?.slice(0, 10) || "";
+            }), ""]);
+        relRow.eachCell(c => { Object.assign(c, { style: metaStyle }); });
+        const roleRow = ws.addRow(["Role", "", "", ...streamTrackNames.map((n) => {
+                const t = latestTracksForStreams.find((x) => x.title === n);
+                return t?.role || "";
+            }), ""]);
+        roleRow.eachCell(c => { Object.assign(c, { style: metaStyle }); });
+        ws.addRow([""]);
+        // Extract S4A 365d daily timeseries for this metric
+        const s4aMetricDaily = {};
+        if (s4a?.perSong) {
+            for (const [tid, tData] of Object.entries(s4a.perSong)) {
+                const name = trackIdToName[tid] || tData.metadata?.name;
+                if (!name)
+                    continue;
+                const ts = tData[metric.key]?.current_period_timeseries;
+                if (!ts || !Array.isArray(ts))
+                    continue;
+                for (const pt of ts) {
+                    if (!pt.x)
+                        continue;
+                    if (!s4aMetricDaily[pt.x])
+                        s4aMetricDaily[pt.x] = {};
+                    s4aMetricDaily[pt.x][name] = metric.key === "streams_per_listener"
+                        ? parseFloat(pt.y) || 0
+                        : parseInt(pt.y) || 0;
+                }
+            }
+        }
+        const s4aDates = Object.keys(s4aMetricDaily).sort();
+        // 1+2. EST + S4A_PREV — constrained disaggregation from release to S4A start
+        // Uses constrainedDisaggregate() for smooth, sum-preserving daily estimates.
+        const s4aPrevFill = { type: "pattern", pattern: "solid", fgColor: { argb: "FFFFF3E0" } };
+        if (s4aDates.length > 0) {
+            const s4aStartDate = new Date(s4aDates[0]);
+            const s4aPrevBoundary = new Date(s4aStartDate.getTime() - 365 * 86400000);
+            // UTC! new Date(y,m,1) is LOCAL timezone — April 1 CET = March 31 UTC,
+            // causing off-by-one in day indices vs ISO string comparisons.
+            const _earliest = new Date(sEarliestRelease || s4aDates[0]);
+            const preS4aStartDate = new Date(Date.UTC(_earliest.getUTCFullYear(), _earliest.getUTCMonth(), 1));
+            const totalPreS4aDays = Math.max(0, Math.floor((s4aStartDate.getTime() - preS4aStartDate.getTime()) / 86400000));
+            if (totalPreS4aDays > 0 && sEarliestRelease) {
+                const boundaryDayIdx = Math.floor((s4aPrevBoundary.getTime() - preS4aStartDate.getTime()) / 86400000);
+                // Per-track disaggregated daily values
+                const perTrackDaily = {};
+                for (const t of latestTracksForStreams) {
+                    if (!t.release_date)
+                        continue;
+                    const tid = trackNameToId[t.title];
+                    const prevAgg = Number(tid && s4a?.perSong && s4a.perSong[tid]?.[metric.key]?.previous_period_agg) || 0;
+                    const curAgg = Number(tid && s4a?.perSong && s4a.perSong[tid]?.[metric.key]?.current_period_agg) || 0;
+                    let estTotal;
+                    if (metric.key === "streams") {
+                        estTotal = Math.max(0, t.play_count - curAgg - prevAgg);
+                    }
+                    else {
+                        estTotal = 0;
+                    }
+                    const firstS4aVal = Number(s4aMetricDaily[s4aDates[0]]?.[t.title]) || 0;
+                    // Compute tailAvg from last 14 S4A data points for this track
+                    const last14 = s4aDates.slice(-14);
+                    let tailSum = 0, tailCount = 0;
+                    for (const d of last14) {
+                        const v = Number(s4aMetricDaily[d]?.[t.title]) || 0;
+                        if (v > 0) {
+                            tailSum += v;
+                            tailCount++;
+                        }
+                    }
+                    const tailAvg = tailCount > 0 ? tailSum / tailCount : (prevAgg / Math.max(1, totalPreS4aDays - boundaryDayIdx));
+                    // streams_per_listener: prev_agg is cumulative period ratio (total streams / unique listeners)
+                    // NOT comparable to daily ratio — skip EST/S4A_PREV timeline entirely.
+                    // The prev_agg is shown as summary info row in the summary section below.
+                    if (metric.key === "streams_per_listener") {
+                        continue;
+                    }
+                    // Per-metric spike params
+                    let spikeA, spikeLambda;
+                    switch (metric.key) {
+                        case "streams":
+                            spikeA = tailAvg * 3;
+                            spikeLambda = 0.06;
+                            break;
+                        case "listeners":
+                            spikeA = tailAvg * 2.5;
+                            spikeLambda = 0.08;
+                            break;
+                        case "saves":
+                            spikeA = tailAvg * 5;
+                            spikeLambda = 0.12;
+                            break;
+                        case "playlist_adds":
+                            spikeA = tailAvg * 8;
+                            spikeLambda = 0.50;
+                            break;
+                        default:
+                            spikeA = tailAvg * 3;
+                            spikeLambda = 0.06;
+                    }
+                    const releaseDayIdx = Math.floor((new Date(t.release_date).getTime() - preS4aStartDate.getTime()) / 86400000);
+                    // Listeners: prev_agg = UNIQUE listeners (not sum of daily values).
+                    // Daily listener values overlap (same person on multiple days), so
+                    // sum(daily) >> unique_count. Compute overlap ratio from S4A current
+                    // period and adjust prevAgg to represent listener-DAYS for disagg.
+                    let disaggPrevAgg = prevAgg;
+                    if (metric.key === "listeners" && prevAgg > 0 && curAgg > 0) {
+                        let dailyListenerSum = 0;
+                        for (const d of s4aDates) {
+                            dailyListenerSum += Number(s4aMetricDaily[d]?.[t.title]) || 0;
+                        }
+                        const overlapRatio = dailyListenerSum / curAgg;
+                        if (overlapRatio > 1) {
+                            disaggPrevAgg = Math.round(prevAgg * overlapRatio);
+                        }
+                    }
+                    const result = constrainedDisaggregate({
+                        totalDays: totalPreS4aDays,
+                        boundaryIdx: boundaryDayIdx,
+                        estTotal,
+                        prevAgg: disaggPrevAgg,
+                        firstS4aVal,
+                        releaseDayIdx,
+                        spikeA,
+                        spikeLambda,
+                        baseline: tailAvg,
+                    });
+                    // Round with anchor: Largest Remainder (Hamilton) with pinned endpoint.
+                    // Pass explicit target (estTotal + disaggPrevAgg) so rounding hits the
+                    // KNOWN total, not the float sum which may differ due to blend absorption.
+                    const anchorIdx = totalPreS4aDays - 1;
+                    const intTarget = Math.round(estTotal + disaggPrevAgg);
+                    const rounded = roundWithAnchor(result.daily, anchorIdx, Math.round(firstS4aVal), intTarget);
+                    // Zero out days before release (shape already has zeros, but safety net)
+                    const releaseStr = t.release_date.slice(0, 10);
+                    for (let day = 0; day < totalPreS4aDays; day++) {
+                        const dd = new Date(preS4aStartDate.getTime() + day * 86400000);
+                        if (dd.toISOString().slice(0, 10) < releaseStr)
+                            rounded[day] = 0;
+                    }
+                    perTrackDaily[t.title] = rounded;
+                }
+                // Write daily rows — EST and S4A_PREV with appropriate colors
+                for (let day = 0; day < totalPreS4aDays; day++) {
+                    const dd = new Date(preS4aStartDate.getTime() + day * 86400000);
+                    const d = dd.toISOString().slice(0, 10);
+                    const dow = dayNames[dd.getDay()];
+                    const isS4aPrev = day >= boundaryDayIdx;
+                    if (!isS4aPrev && !metric.hasEst)
+                        continue;
+                    const source = isS4aPrev ? "S4A_PREV" : "EST";
+                    const fill = isS4aPrev ? s4aPrevFill : estFill;
+                    const values = [];
+                    for (const t of latestTracksForStreams) {
+                        const daily = perTrackDaily[t.title];
+                        if (!daily || !t.release_date || d < t.release_date.slice(0, 10)) {
+                            values.push(null);
+                            continue;
+                        }
+                        const v = daily[day];
+                        // Pre-release: null (track didn't exist). Post-release: 0 is a real value.
+                        if (metric.key === "streams_per_listener") {
+                            values.push(v > 0 ? Math.round(v * 100) / 100 : (v === 0 ? 0 : null));
+                        }
+                        else {
+                            values.push(v != null ? v : null);
+                        }
+                    }
+                    const total = values.reduce((s, v) => s + (v || 0), 0);
+                    const r = ws.addRow([d, dow, source, ...values, total]);
+                    r.eachCell(c => { if (typeof c.value === "number") {
+                        c.numFmt = metric.numFormat;
+                        c.fill = fill;
+                    } });
+                }
+            }
+        }
+        // 3. S4A rows — 365d daily
+        for (const d of s4aDates) {
+            const dow = dayNames[new Date(d).getDay()];
+            const values = [];
+            let total = 0;
+            for (const t of latestTracksForStreams) {
+                const v = s4aMetricDaily[d]?.[t.title];
+                if (v != null) {
+                    values.push(v);
+                    total += v;
+                }
+                else {
+                    values.push(null);
+                }
+            }
+            const row = ws.addRow([d, dow, "S4A", ...values, metric.key === "streams_per_listener" ? "" : total]);
+            row.eachCell(c => { if (typeof c.value === "number")
+                c.numFmt = metric.numFormat; c.fill = s4aFill; });
+        }
+        // 4. SCR rows (only Streams — cumulative play counts from GraphQL)
+        // Offset rule: SCR date = scrape date - 1 day (S4A data ends yesterday)
+        if (metric.key === "streams") {
+            for (const [scrapeDate, tData] of Object.entries(streamsByDate).sort()) {
+                const offsetDate = new Date(new Date(scrapeDate).getTime() - 86400000).toISOString().slice(0, 10);
+                const dow = dayNames[new Date(offsetDate).getDay()];
+                const values = streamTrackNames.map((n) => tData[n] ?? null);
+                const total = values.reduce((s, v) => s + (v || 0), 0);
+                const row = ws.addRow([offsetDate, dow, "SCR", ...values, total]);
+                row.eachCell(c => { if (typeof c.value === "number")
+                    c.numFmt = metric.numFormat; });
+            }
+        }
+        // 5. PROJ rows (6 months forward projection)
+        // Uses YoY% (period_change_pct) as long-term trend + seasonal patterns from 365d data.
+        // - YoY% = verified 12m growth rate, immune to short-term dips/spikes
+        // - Weekday pattern = Mon-Sun factors from 365d average (e.g. Fri release day spike)
+        // - Monthly pattern = Jan-Dec factors from 365d average (summer vs winter)
+        // Track age < 60d: decay curve from initial spike (YoY not meaningful)
+        // Streams/Listener: no projection (ratio, not estimable)
+        if (s4aDates.length > 0 && metric.key !== "streams_per_listener") {
+            const projStart = new Date(s4aDates[s4aDates.length - 1]);
+            projStart.setDate(projStart.getDate() + 1);
+            // Extract seasonal patterns from 365d S4A data (across all tracks combined)
+            // Weekday factors: average streams per weekday relative to overall mean
+            const dowSums = [0, 0, 0, 0, 0, 0, 0]; // Sun-Sat
+            const dowCounts = [0, 0, 0, 0, 0, 0, 0];
+            // Monthly factors: average streams per month relative to overall mean
+            const monthSums = new Array(12).fill(0);
+            const monthCounts = new Array(12).fill(0);
+            let grandTotal = 0, grandCount = 0;
+            for (const d of s4aDates) {
+                const dt = new Date(d + "T12:00:00Z");
+                const dow = dt.getUTCDay();
+                const month = dt.getUTCMonth();
+                let daySum = 0, dayN = 0;
+                for (const t of latestTracksForStreams) {
+                    const v = s4aMetricDaily[d]?.[t.title];
+                    if (v != null && v > 0) {
+                        daySum += v;
+                        dayN++;
+                    }
+                }
+                if (dayN > 0) {
+                    dowSums[dow] += daySum;
+                    dowCounts[dow]++;
+                    monthSums[month] += daySum;
+                    monthCounts[month]++;
+                    grandTotal += daySum;
+                    grandCount++;
+                }
+            }
+            const grandAvg = grandCount > 0 ? grandTotal / grandCount : 1;
+            // Weekday factor: ratio of weekday avg to grand avg (1.0 = average day)
+            const dowFactors = dowSums.map((s, i) => dowCounts[i] > 0 ? (s / dowCounts[i]) / grandAvg : 1);
+            // Monthly factor: ratio of month avg to grand avg
+            const monthFactors = monthSums.map((s, i) => monthCounts[i] > 0 ? (s / monthCounts[i]) / grandAvg : 1);
+            const projParams = {};
+            for (const t of latestTracksForStreams) {
+                if (!t.release_date)
+                    continue;
+                const daysSinceRelease = Math.max(1, Math.floor((now - new Date(t.release_date).getTime() / 1000) / 86400));
+                const tid = trackNameToId[t.title];
+                const vals = [];
+                for (const d of s4aDates) {
+                    const v = s4aMetricDaily[d]?.[t.title];
+                    if (v != null)
+                        vals.push(v);
+                }
+                if (vals.length < 7)
+                    continue;
+                const lastVal = vals[vals.length - 1];
+                if (daysSinceRelease < 60) {
+                    // Young track: still decaying — use exponential decay from spike
+                    const last90 = vals.slice(-Math.min(90, vals.length));
+                    const B = last90.reduce((a, b) => a + b, 0) / last90.length;
+                    const early = vals.slice(0, 14);
+                    const earlyAvg = early.reduce((a, b) => a + b, 0) / early.length;
+                    const A = Math.max(0, earlyAvg - B);
+                    const lambda = A > 0 ? 0.04 : 0;
+                    projParams[t.title] = { lastVal, dailyRate: 0, isDecay: true, A, lambda, tOffset: daysSinceRelease, B };
+                }
+                else {
+                    // Determine trend source: YoY vs median slope
+                    let dailyRate = 0;
+                    const prevAggVal = Number(tid && s4a.perSong[tid]?.[metric.key]?.previous_period_agg) || 0;
+                    const yoyPct = Number(tid && s4a.perSong[tid]?.[metric.key]?.period_change_pct) || 0;
+                    // YoY is trustworthy only when: track > 15 months AND prev_period_agg is substantial
+                    const yoyReliable = daysSinceRelease > 450 && prevAggVal > 10;
+                    if (yoyReliable) {
+                        // Mature track with reliable YoY: cap to [-50%, +200%] annually
+                        const cappedYoy = Math.max(-50, Math.min(200, yoyPct));
+                        const annualMultiplier = 1 + cappedYoy / 100;
+                        dailyRate = annualMultiplier > 0 ? Math.pow(annualMultiplier, 1 / 365) - 1 : 0;
+                    }
+                    else {
+                        // Track 60d-450d OR unreliable YoY: use median-based slope from last 90d
+                        // Median slope is robust to spikes, bot removals, and seasonal noise
+                        const window = vals.slice(-Math.min(90, vals.length));
+                        if (window.length >= 14) {
+                            // Split into two halves, compare medians for robust trend
+                            const half = Math.floor(window.length / 2);
+                            const firstHalf = [...window.slice(0, half)].sort((a, b) => a - b);
+                            const secondHalf = [...window.slice(half)].sort((a, b) => a - b);
+                            const medFirst = firstHalf[Math.floor(firstHalf.length / 2)];
+                            const medSecond = secondHalf[Math.floor(secondHalf.length / 2)];
+                            if (medFirst > 0) {
+                                const slopePct = (medSecond - medFirst) / medFirst;
+                                // Annualize the half-window slope, then cap
+                                const halfDays = window.length / 2;
+                                const annualized = Math.pow(1 + slopePct, 365 / halfDays) - 1;
+                                const capped = Math.max(-0.5, Math.min(2.0, annualized));
+                                dailyRate = Math.pow(1 + capped, 1 / 365) - 1;
+                            }
+                        }
+                    }
+                    projParams[t.title] = { lastVal, dailyRate, isDecay: false, A: 0, lambda: 0, tOffset: 0, B: lastVal };
+                }
+            }
+            for (let day = 0; day < 180; day++) {
+                const d = new Date(projStart.getTime() + day * 86400000);
+                const dateStr = d.toISOString().slice(0, 10);
+                const dow = dayNames[d.getDay()];
+                const dowIdx = d.getDay();
+                const monthIdx = d.getMonth();
+                const values = [];
+                let total = 0;
+                for (const t of latestTracksForStreams) {
+                    const p = projParams[t.title];
+                    if (!p) {
+                        values.push(null);
+                        continue;
+                    }
+                    let proj;
+                    if (p.isDecay) {
+                        // Young track: decay curve continuing
+                        proj = p.A * Math.exp(-p.lambda * (p.tOffset + day)) + p.B;
+                    }
+                    else {
+                        // Mature track: compound growth × seasonal factors
+                        const trendVal = p.lastVal * Math.pow(1 + p.dailyRate, day + 1);
+                        proj = trendVal * dowFactors[dowIdx] * monthFactors[monthIdx];
+                        // Guard rails: floor at 10% of last value, ceiling at 5× last value
+                        proj = Math.max(p.lastVal * 0.1, Math.min(p.lastVal * 5, proj));
+                    }
+                    const rounded = Math.round(proj);
+                    values.push(rounded);
+                    total += rounded;
+                }
+                const row = ws.addRow([dateStr, dow, "PROJ", ...values, total]);
+                row.eachCell(c => {
+                    if (typeof c.value === "number")
+                        c.numFmt = metric.numFormat;
+                    c.fill = projFill;
+                });
+            }
+        }
+        // ── Summary section ──
+        ws.addRow([""]);
+        // YoY %
+        if (s4a?.perSong) {
+            const yoyHdr = ws.addRow(["YoY %", "", "", ...streamTrackNames.map(() => ""), ""]);
+            yoyHdr.eachCell(c => { Object.assign(c, { style: sectionStyle }); });
+            const yoyVals = streamTrackNames.map((n) => {
+                const tid = trackNameToId[n];
+                const pct = Number(tid && s4a.perSong[tid]?.[metric.key]?.period_change_pct);
+                return !isNaN(pct) ? Math.round(pct * 10) / 10 : null;
+            });
+            ws.addRow(["YoY change", "", "", ...yoyVals, ""]).eachCell(c => { if (typeof c.value === "number")
+                c.numFmt = "+0.0\"%\";-0.0\"%\""; });
+            // Prev + Cur 12m totals
+            const prevVals = streamTrackNames.map((n) => {
+                const tid = trackNameToId[n];
+                const v = Number(tid && s4a.perSong[tid]?.[metric.key]?.previous_period_agg) || 0;
+                return v != null ? (metric.key === "streams_per_listener" ? Math.round(v * 100) / 100 : Math.round(v)) : null;
+            });
+            const prevLabel = metric.key === "streams_per_listener" ? "Prev 12m avg ratio" : "Prev 12m total";
+            ws.addRow([prevLabel, "", "", ...prevVals, ""]).eachCell(c => { if (typeof c.value === "number")
+                c.numFmt = metric.numFormat; });
+            const curVals = streamTrackNames.map((n) => {
+                const tid = trackNameToId[n];
+                const v = Number(tid && s4a.perSong[tid]?.[metric.key]?.current_period_agg) || 0;
+                return v > 0 ? (metric.key === "streams_per_listener" ? Math.round(v * 100) / 100 : Math.round(v)) : null;
+            });
+            const curLabel = metric.key === "streams_per_listener" ? "Cur 12m avg ratio" : "Cur 12m total";
+            ws.addRow([curLabel, "", "", ...curVals, ""]).eachCell(c => { if (typeof c.value === "number")
+                c.numFmt = metric.numFormat; });
+        }
+        // Velocity + Share % (only Streams)
+        if (metric.key === "streams") {
+            ws.addRow([""]);
+            const vHdr = ws.addRow(["VELOCITY (streams/day)", "", "", ...streamTrackNames.map(() => ""), ""]);
+            vHdr.eachCell(c => { Object.assign(c, { style: sectionStyle }); });
+            const vVals = streamTrackNames.map((n) => vel[n] ? Math.round(vel[n] * 10) / 10 : null);
+            ws.addRow(["Lifetime avg", "", "", ...vVals, ""]).eachCell(c => { if (typeof c.value === "number")
+                c.numFmt = "#,##0.0"; });
+            ws.addRow([""]);
+            const shHdr = ws.addRow(["SHARE %", "", "", ...streamTrackNames.map(() => ""), ""]);
+            shHdr.eachCell(c => { Object.assign(c, { style: sectionStyle }); });
+            const totStr = latestTracksForStreams.reduce((s, t) => s + (t.play_count || 0), 0);
+            const shVals = streamTrackNames.map((n) => {
+                const t = latestTracksForStreams.find((x) => x.title === n);
+                return t?.play_count && totStr ? Math.round((t.play_count / totStr) * 1000) / 10 : null;
+            });
+            ws.addRow(["Current", "", "", ...shVals, "100%"]).eachCell(c => { if (typeof c.value === "number")
+                c.numFmt = "0.0\"%\""; });
+            // Cumulative running total (Streams only — daily values summed over time)
+            ws.addRow([""]);
+            const cumHdr = ws.addRow(["CUMULATIVE TOTAL", "", "", ...streamTrackNames.map(() => ""), ""]);
+            cumHdr.eachCell(c => { Object.assign(c, { style: sectionStyle }); });
+            // Latest SCR value = current known cumulative total per track
+            const latestTotal = {};
+            for (const t of latestTracksForStreams)
+                latestTotal[t.title] = t.play_count;
+            const cumRow = ws.addRow(["Latest (SCR)", "", "", ...streamTrackNames.map((n) => latestTotal[n] || null),
+                Object.values(latestTotal).reduce((a, b) => a + b, 0)]);
+            cumRow.eachCell(c => { if (typeof c.value === "number")
+                c.numFmt = numFmt; });
+            // Total stream count from S4A (all-time per track)
+            if (s4a?.perSong) {
+                const s4aTotals = streamTrackNames.map((n) => {
+                    const tid = trackNameToId[n];
+                    const v = tid && s4a.perSong[tid]?.total_stream_count;
+                    return v ? parseInt(v) : null;
+                });
+                ws.addRow(["S4A all-time", "", "", ...s4aTotals, ""]).eachCell(c => { if (typeof c.value === "number")
+                    c.numFmt = numFmt; });
+            }
+        }
+    } // end metricSheets loop
+    // ─────────────────────────────────────────────
+    //   D8: Countries — daily timeline (tracks side by side)
+    // ─────────────────────────────────────────────
+    // Merges ALL historical dumps to build complete picture:
+    // - Timeline countries = UNION of all countries ever in top 3 (across all dumps)
+    // - 28d snapshot shares = latest available per country
+    // - Daily values: TIMELINE (exact) for known countries, DISAGG (estimated) for rest
+    // Columns grow over time as new dumps add countries to timeline or snapshots.
+    if (s4a?.perSong) {
+        const ws8 = wb.addWorksheet("Countries", { properties: { tabColor: { argb: "FF26A69A" } } });
+        const dn8 = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
+        const disaggFill = { type: "pattern", pattern: "solid", fgColor: { argb: "FFF5F5DC" } };
+        // Load ALL historical dumps
+        const allDumps = loadAllDumps(aid);
+        // Build merged data per track across ALL dumps
+        const trackMap = new Map();
+        const allDates8 = new Set();
+        // Process each dump (oldest first → newest overwrites)
+        for (const dump of allDumps) {
+            if (!dump.data.perSong)
+                continue;
+            for (const [tid, d] of Object.entries(dump.data.perSong)) {
+                const nm = d.metadata?.name || d.name || tid;
+                if (!trackMap.has(tid)) {
+                    trackMap.set(tid, {
+                        name: nm, allCodes: [], timelineCodes: new Set(),
+                        timelinePts: {}, snapShares: {}, dailyTotals: {},
+                    });
+                }
+                const t = trackMap.get(tid);
+                // Merge timeline data (UNION of countries, latest daily values win)
+                for (const ct of (d.countryTimelines || [])) {
+                    const cc = ct.countryCode || "??";
+                    t.timelineCodes.add(cc); // once in top 3, always a timeline column
+                    if (!t.timelinePts[cc])
+                        t.timelinePts[cc] = {};
+                    for (const p of (ct.timelinePoint || [])) {
+                        t.timelinePts[cc][p.date] = Number(p.num) || 0;
+                        allDates8.add(p.date);
+                    }
+                }
+                // Merge 28d snapshot (latest wins)
+                let raw = d.perSongCountries || [];
+                if (!raw.length && d.geography?.length && d.geography[0]?.localized_country)
+                    raw = d.geography;
+                if (raw.length > 0) {
+                    const snap = {};
+                    for (const c of raw)
+                        snap[c.name] = Number(c.num) || 0;
+                    const tlTotal = [...t.timelineCodes].reduce((s, cc) => s + (snap[cc] || 0), 0);
+                    const nonTlTotal = Object.entries(snap).filter(([cc]) => !t.timelineCodes.has(cc)).reduce((s, [, v]) => s + v, 0);
+                    for (const [cc, v] of Object.entries(snap)) {
+                        if (!t.timelineCodes.has(cc) && nonTlTotal > 0)
+                            t.snapShares[cc] = v / nonTlTotal;
+                    }
+                }
+                // Merge daily totals (latest timeseries wins)
+                const streamTs = d.streams?.current_period_timeseries;
+                if (Array.isArray(streamTs)) {
+                    for (const pt of streamTs) {
+                        if (pt.x)
+                            t.dailyTotals[pt.x] = Number(pt.y) || 0;
+                    }
+                }
+            }
+        }
+        // Build allCodes per track: timeline countries first, then snapshot countries, sorted by total desc
+        const tracks8 = [];
+        for (const [, t] of [...trackMap.entries()].sort((a, b) => {
+            const totA = Object.values(a[1].dailyTotals).reduce((s, v) => s + v, 0);
+            const totB = Object.values(b[1].dailyTotals).reduce((s, v) => s + v, 0);
+            return totB - totA;
+        })) {
+            // Timeline codes sorted by total timeline streams desc
+            const tlSorted = [...t.timelineCodes].sort((a, b) => {
+                const sa = Object.values(t.timelinePts[a] || {}).reduce((s, v) => s + v, 0);
+                const sb = Object.values(t.timelinePts[b] || {}).reduce((s, v) => s + v, 0);
+                return sb - sa;
+            });
+            // Non-timeline codes sorted by share desc
+            const nonTlSorted = Object.entries(t.snapShares)
+                .filter(([cc]) => !t.timelineCodes.has(cc))
+                .sort((a, b) => b[1] - a[1])
+                .map(([cc]) => cc);
+            t.allCodes = [...tlSorted, ...nonTlSorted];
+            if (t.allCodes.length > 0)
+                tracks8.push(t);
+        }
+        const dates8 = [...allDates8].sort();
+        // Row 1: track names
+        const r1 = ["", ""];
+        for (const t of tracks8) {
+            r1.push(t.name);
+            for (let i = 1; i < t.allCodes.length; i++)
+                r1.push("");
+            r1.push("");
+        }
+        const row1 = ws8.addRow(r1);
+        let ci8 = 3;
+        for (const t of tracks8) {
+            row1.getCell(ci8).font = { bold: true, size: 11 };
+            ci8 += t.allCodes.length + 1;
+        }
+        // Row 2: country codes + TOTAL
+        const r2 = ["Date", "Day"];
+        for (const t of tracks8)
+            r2.push(...t.allCodes, "TOTAL");
+        ws8.addRow(r2).eachCell(c => { Object.assign(c, { style: hdrDark }); });
+        // Row 3: source indicator (S4A = timeline exact, DISAGG = estimated)
+        const r3 = ["Source", ""];
+        for (const t of tracks8) {
+            r3.push(...t.allCodes.map(cc => t.timelineCodes.has(cc) ? "S4A" : "DISAGG"), "");
+        }
+        ws8.addRow(r3).eachCell(c => { c.font = { size: 8, italic: true, color: { argb: "FF999999" } }; });
+        // 365d daily rows
+        for (const date of dates8) {
+            const d = new Date(date);
+            const row = [date, dn8[d.getUTCDay()]];
+            for (const t of tracks8) {
+                const timelineSum = [...t.timelineCodes].reduce((s, cc) => s + (t.timelinePts[cc]?.[date] || 0), 0);
+                const dailyTotal = t.dailyTotals[date] || 0;
+                const remainder = Math.max(0, dailyTotal - timelineSum);
+                const vals = [];
+                for (const cc of t.allCodes) {
+                    if (t.timelineCodes.has(cc)) {
+                        vals.push(t.timelinePts[cc]?.[date] || 0);
+                    }
+                    else {
+                        vals.push(Math.round(remainder * (t.snapShares[cc] || 0)));
+                    }
+                }
+                row.push(...vals, vals.reduce((a, b) => a + b, 0));
+            }
+            const xlRow = ws8.addRow(row);
+            xlRow.eachCell((c, colNumber) => {
+                if (typeof c.value === "number") {
+                    c.numFmt = numFmt;
+                    if (colNumber > 2) {
+                        let off = colNumber - 2;
+                        for (const t of tracks8) {
+                            if (off <= t.allCodes.length) {
+                                const cc = t.allCodes[off - 1];
+                                if (cc && !t.timelineCodes.has(cc))
+                                    c.fill = disaggFill;
+                                break;
+                            }
+                            off -= t.allCodes.length + 1;
+                        }
+                    }
+                }
+            });
+        }
+        // Summary
+        const totR = ["TOTAL (12m)", ""];
+        for (const t of tracks8) {
+            const sums = t.allCodes.map(cc => {
+                if (t.timelineCodes.has(cc))
+                    return Object.values(t.timelinePts[cc] || {}).reduce((a, b) => a + b, 0);
+                let s = 0;
+                for (const date of dates8) {
+                    const tlSum = [...t.timelineCodes].reduce((s2, cc2) => s2 + (t.timelinePts[cc2]?.[date] || 0), 0);
+                    s += Math.round(Math.max(0, (t.dailyTotals[date] || 0) - tlSum) * (t.snapShares[cc] || 0));
+                }
+                return s;
+            });
+            totR.push(...sums, sums.reduce((a, b) => a + b, 0));
+        }
+        ws8.addRow(totR).eachCell(c => { c.font = { bold: true }; if (typeof c.value === "number")
+            c.numFmt = numFmt; });
+        ws8.getColumn(1).width = 12;
+        ws8.getColumn(2).width = 5;
+        const tc8 = 2 + tracks8.reduce((s, t) => s + t.allCodes.length + 1, 0);
+        for (let i = 3; i <= tc8; i++)
+            ws8.getColumn(i).width = 10;
+        ws8.views = [{ state: "frozen", xSplit: 2, ySplit: 3, topLeftCell: "C4", activeCell: "C4" }];
+    }
+    // ─────────────────────────────────────────────
+    //   D8b: Country Snapshots (accumulated across ALL scrapes)
+    // ─────────────────────────────────────────────
+    // Each scrape date = one block of rows. Grows with every new dump.
+    // Layout: Scrape Date | Country | Track1 streams | Track2 streams | ...
+    // This is the append-only log that feeds delta computation.
+    {
+        const wsSn = wb.addWorksheet("Country Snapshots", { properties: { tabColor: { argb: "FF80CBC4" } } });
+        const allDumps = loadAllDumps(aid);
+        // Collect track names (sorted by total streams in latest dump)
+        const trackOrder = [];
+        if (s4a?.perSong) {
+            for (const [tid, d] of Object.entries(s4a.perSong)
+                .sort((a, b) => Number(b[1].total_stream_count || 0) - Number(a[1].total_stream_count || 0))) {
+                trackOrder.push({ tid, name: d.metadata?.name || d.name || tid });
+            }
+        }
+        // Header
+        const h1 = ["Scrape Date", "Country"];
+        for (const t of trackOrder)
+            h1.push(t.name);
+        wsSn.addRow(h1);
+        for (let i = 3; i <= h1.length; i++)
+            wsSn.getRow(1).getCell(i).font = { bold: true, size: 10 };
+        const h2 = ["", ""];
+        for (const t of trackOrder)
+            h2.push("28d Streams");
+        wsSn.addRow(h2).eachCell(c => { Object.assign(c, { style: hdrDark }); });
+        // One block per dump date
+        for (const dump of allDumps) {
+            if (!dump.data.perSong)
+                continue;
+            // Collect all countries from this dump
+            const countriesInDump = new Set();
+            const snapByTrack = {};
+            for (const [tid, d] of Object.entries(dump.data.perSong)) {
+                let raw = d.perSongCountries || [];
+                if (!raw.length && d.geography?.length && d.geography[0]?.localized_country)
+                    raw = d.geography;
+                snapByTrack[tid] = {};
+                for (const c of raw) {
+                    const code = c.name;
+                    countriesInDump.add(code);
+                    snapByTrack[tid][code] = Number(c.num) || 0;
+                }
+            }
+            const countries = [...countriesInDump].sort();
+            for (const code of countries) {
+                const row = [dump.date, code];
+                for (const t of trackOrder) {
+                    row.push(snapByTrack[t.tid]?.[code] ?? null);
+                }
+                wsSn.addRow(row).eachCell(c => { if (typeof c.value === "number")
+                    c.numFmt = numFmt; });
+            }
+            // Separator between dump dates
+            wsSn.addRow([]);
+        }
+        wsSn.getColumn(1).width = 12;
+        wsSn.getColumn(2).width = 8;
+        for (let i = 3; i <= 2 + trackOrder.length; i++)
+            wsSn.getColumn(i).width = 10;
+        wsSn.views = [{ state: "frozen", xSplit: 2, ySplit: 2, topLeftCell: "C3", activeCell: "C3" }];
+    }
+    // ─────────────────────────────────────────────
+    //   D9: Cities Snapshots (28d per scrape, per-song)
+    // ─────────────────────────────────────────────
+    // Same concept as Country Snapshots but for cities.
+    if (s4a?.perSong) {
+        const wsCi = wb.addWorksheet("City Snapshots", { properties: { tabColor: { argb: "FF7986CB" } } });
+        const sortedTracks = Object.entries(s4a.perSong)
+            .sort((a, b) => Number(b[1].total_stream_count || 0) - Number(a[1].total_stream_count || 0));
+        // Collect all unique cities
+        const allCities = new Map();
+        const trackCitySnaps = [];
+        for (const [, d] of sortedTracks) {
+            const nm = d.metadata?.name || d.name || "?";
+            let raw = d.perSongCities || [];
+            if (!raw.length && d.geography?.length && d.geography[0]?.region)
+                raw = d.geography;
+            const map = {};
+            for (const x of raw) {
+                const key = `${x.name}|${x.country || ""}`;
+                allCities.set(key, { country: x.country || "", region: x.region || "" });
+                map[key] = Number(x.num) || 0;
+            }
+            trackCitySnaps.push({ name: nm, data: map });
+        }
+        // Sort cities by total streams across all tracks
+        const cityTotals = {};
+        for (const t of trackCitySnaps)
+            for (const [k, v] of Object.entries(t.data))
+                cityTotals[k] = (cityTotals[k] || 0) + v;
+        const citiesSorted = Object.entries(cityTotals).sort((a, b) => b[1] - a[1]).map(([k]) => k);
+        // Header 1: track names
+        const h1 = ["City", "Country"];
+        for (const t of trackCitySnaps)
+            h1.push(t.name);
+        wsCi.addRow(h1);
+        for (let i = 3; i <= h1.length; i++)
+            wsCi.getRow(1).getCell(i).font = { bold: true, size: 10 };
+        // Header 2
+        const h2 = ["", ""];
+        for (const t of trackCitySnaps)
+            h2.push("Streams");
+        wsCi.addRow(h2).eachCell(c => { Object.assign(c, { style: hdrDark }); });
+        // One row per city
+        for (const key of citiesSorted) {
+            const [cityName] = key.split("|");
+            const meta = allCities.get(key);
+            const row = [cityName, meta.country];
+            for (const t of trackCitySnaps)
+                row.push(t.data[key] || null);
+            wsCi.addRow(row).eachCell(c => { if (typeof c.value === "number")
+                c.numFmt = numFmt; });
+        }
+        wsCi.getColumn(1).width = 20;
+        wsCi.getColumn(2).width = 6;
+        for (let i = 3; i <= 2 + trackCitySnaps.length; i++)
+            wsCi.getColumn(i).width = 10;
+        wsCi.views = [{ state: "frozen", xSplit: 2, ySplit: 2, topLeftCell: "C3", activeCell: "C3" }];
+    }
+    await wb.xlsx.writeFile(xlsxPath);
+    db.close();
+}
+// ─── S4A Authentication ───
+const S4A_SESSION_DIR = path.join(process.env.HOME || "~", ".artist-os", ".s4a-session");
+export async function connectS4A(artistId) {
+    const chromePath = process.env.CHROME_PATH || findChrome();
+    const CDP_PORT = 9222;
+    const CHROME_DEBUG_DIR = path.join(process.env.HOME || "~", ".artist-os", ".chrome-debug");
+    const systemChromeProfile = path.join(process.env.HOME || "~", "Library", "Application Support", "Google", "Chrome");
+    fs.mkdirSync(S4A_SESSION_DIR, { recursive: true });
+    // Step 1: Copy Chrome profile (one-time) — has saved passwords, Google account, autofill
+    if (!fs.existsSync(path.join(CHROME_DEBUG_DIR, "Default"))) {
+        const { execSync } = require("child_process");
+        fs.mkdirSync(CHROME_DEBUG_DIR, { recursive: true });
+        execSync(`cp -r "${path.join(systemChromeProfile, "Default")}" "${CHROME_DEBUG_DIR}/Default"`, { stdio: "ignore" });
+        try {
+            execSync(`cp "${path.join(systemChromeProfile, "Local State")}" "${CHROME_DEBUG_DIR}/"`, { stdio: "ignore" });
+        }
+        catch { }
+    }
+    // Step 2: Clean crash state in debug profile
+    const defaultDir = path.join(CHROME_DEBUG_DIR, "Default");
+    for (const f of ["Last Session", "Last Tabs", "Current Session", "Current Tabs"]) {
+        try {
+            fs.unlinkSync(path.join(defaultDir, f));
+        }
+        catch { }
+    }
+    try {
+        const prefsPath = path.join(defaultDir, "Preferences");
+        const p = JSON.parse(fs.readFileSync(prefsPath, "utf-8"));
+        if (!p.profile)
+            p.profile = {};
+        p.profile.exit_type = "Normal";
+        p.profile.exited_cleanly = true;
+        fs.writeFileSync(prefsPath, JSON.stringify(p));
+    }
+    catch { }
+    // Step 3: Kill Chrome, relaunch with debug profile + debugging port
+    const { execSync, spawn } = require("child_process");
+    try {
+        execSync("pkill -f 'Google Chrome'", { stdio: "ignore" });
+    }
+    catch { }
+    await new Promise(r => setTimeout(r, 2000));
+    const chromeProc = spawn(chromePath, [
+        `--user-data-dir=${CHROME_DEBUG_DIR}`,
+        `--remote-debugging-port=${CDP_PORT}`,
+        "--no-first-run",
+        `https://artists.spotify.com/c/artist/${artistId}/home`,
+    ], { detached: true, stdio: "ignore" });
+    chromeProc.unref();
+    await new Promise(r => setTimeout(r, 5000));
+    // Step 4: Connect via CDP
+    const browser = await chromium.connectOverCDP(`http://localhost:${CDP_PORT}`);
+    const context = browser.contexts()[0];
+    // Step 5: Wait for S4A dashboard (max 5 min — user may need to enter SMS code)
+    const deadline = Date.now() + 5 * 60 * 1000;
+    let loggedIn = false;
+    while (Date.now() < deadline) {
+        const pages = context.pages();
+        const s4aPage = pages.find(p => {
+            const u = p.url();
+            return u.includes("artists.spotify.com") && u.includes("/home") &&
+                !u.includes("accounts.spotify.com") && !u.includes("challenge.spotify.com");
+        });
+        if (s4aPage) {
+            loggedIn = true;
+            break;
+        }
+        await new Promise(r => setTimeout(r, 2000));
+    }
+    if (!loggedIn) {
+        await browser.close();
+        return { success: false, error: "Login timed out after 5 minutes. Run connect_s4a again." };
+    }
+    // Step 6: Extract and save cookies
+    let name = artistId;
+    try {
+        const cookies = await context.cookies("https://artists.spotify.com");
+        fs.writeFileSync(path.join(S4A_SESSION_DIR, "cookies.json"), JSON.stringify(cookies, null, 2));
+        const spDc = cookies.find((c) => c.name === "sp_dc");
+        if (spDc)
+            fs.writeFileSync(path.join(S4A_SESSION_DIR, "sp_dc.txt"), spDc.value);
+    }
+    catch { }
+    // Step 7: Close only S4A tabs, Chrome stays open for the user
+    try {
+        const s4aPages = context.pages().filter(p => p.url().includes("artists.spotify.com"));
+        for (const p of s4aPages)
+            await p.close();
+    }
+    catch { }
+    await browser.close(); // Disconnect CDP — Chrome stays running
+    return { success: true, name };
+}
+export function hasS4ASession() {
+    return fs.existsSync(path.join(S4A_SESSION_DIR, "sp_dc.txt"));
+}
+function getSpDc() {
+    try {
+        return fs.readFileSync(path.join(S4A_SESSION_DIR, "sp_dc.txt"), "utf-8").trim();
+    }
+    catch {
+        return null;
+    }
+}
+// Get S4A bearer token from sp_dc cookie (valid ~1 hour)
+async function getS4ABearerToken(spDc) {
+    const resp = await fetch("https://generic.wg.spotify.com/creator-auth-proxy/v1/web/token", {
+        headers: {
+            "Cookie": `sp_dc=${spDc}`,
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+        },
+    });
+    if (!resp.ok)
+        throw new Error(`S4A token failed: ${resp.status}`);
+    const data = await resp.json();
+    return data.accessToken || data.access_token;
+}
+// S4A API call helper
+async function s4aApi(token, endpoint) {
+    const resp = await fetch(`https://generic.wg.spotify.com/${endpoint}`, {
+        headers: {
+            "Authorization": `Bearer ${token}`,
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+            "Accept": "application/json",
+        },
+    });
+    if (!resp.ok)
+        return null;
+    return resp.json();
+}
+// Scrape S4A via headless browser (Spotify blocks direct API calls via TLS fingerprinting)
+export async function scrapeS4ADirect(artistId) {
+    const spDc = getSpDc();
+    if (!spDc)
+        return null;
+    // Launch headless browser with injected cookies (proven working approach)
+    const chromePath = process.env.CHROME_PATH || findChrome();
+    const browser = await chromium.launch({ executablePath: chromePath, headless: true });
+    const context = await browser.newContext({
+        userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    });
+    // Inject all saved cookies
+    try {
+        const savedCookies = JSON.parse(fs.readFileSync(path.join(S4A_SESSION_DIR, "cookies.json"), "utf-8"));
+        const validCookies = savedCookies.filter((c) => c.name && c.value && c.domain).map((c) => ({
+            name: c.name, value: c.value, domain: c.domain, path: c.path || "/",
+            httpOnly: c.httpOnly ?? true, secure: c.secure ?? true, sameSite: "None",
+        }));
+        await context.addCookies(validCookies);
+    }
+    catch {
+        // Fallback: inject just sp_dc
+        await context.addCookies([
+            { name: "sp_dc", value: spDc, domain: ".spotify.com", path: "/", httpOnly: true, secure: true, sameSite: "None" },
+        ]);
+    }
+    const data = {};
+    const aid = artistId;
+    const page = await context.newPage();
+    const captured = {};
+    // Intercept API responses
+    page.on("response", async (resp) => {
+        const url = resp.url();
+        if (!url.includes("spotify.com") || resp.status() !== 200)
+            return;
+        const patterns = ["audience-engagement", "catalog-view", "fanatic-audience", "s4x-insights-api", "song-stats", "entity-comparison", "source-of-stream", "s4x-me", "canvaz-view", "buyerxp-campaign-view", "app-manager", "artist-videos"];
+        if (!patterns.some(p => url.includes(p)))
+            return;
+        try {
+            captured[url] = await resp.json();
+        }
+        catch { }
+    });
+    // RULE: tyto datumy jsou pro URL REQUEST, ne pro ukládání.
+    // Skutečné datumy data pointů bereme Z RESPONSE (timeseries `x` pole).
+    // S4A vrací data končící VČERA, ne dnes.
+    const requestToDate = new Date().toISOString().slice(0, 10);
+    const requestFromDate12m = new Date(Date.now() - 365 * 86400000).toISOString().slice(0, 10);
+    // Artist-level sections
+    // NOTE: audience/stats, demographic, location — NO custom date params!
+    // S4A API uses from_date/to_date (snake_case) internally. Passing fromDate (camelCase)
+    // in the PAGE URL causes the internal API call to fail with 400.
+    // Let S4A choose its own date range — it defaults to 28d which is what we get.
+    const sections = [
+        `/c/artist/${aid}/home`,
+        `/c/artist/${aid}/audience/stats`,
+        `/c/artist/${aid}/audience/demographic`,
+        `/c/artist/${aid}/audience/location`,
+        `/c/artist/${aid}/audience/segments`,
+        `/c/artist/${aid}/music/songs?time-filter=last12months`,
+        `/c/artist/${aid}/music/playlists?time-filter=last12months`,
+        `/c/artist/${aid}/music/upcoming`,
+        `/c/artist/${aid}/music/videos`,
+    ];
+    for (const section of sections) {
+        try {
+            await page.goto(`https://artists.spotify.com${section}`, { waitUntil: "networkidle", timeout: 25000 });
+            await sleep(2000);
+        }
+        catch { }
+    }
+    // Parse captured responses into data object (helper — called multiple times)
+    const parseCaptured = () => {
+        data.perSong = data.perSong || {};
+        for (const [url, resp] of Object.entries(captured)) {
+            if (url.includes("gender-by-age"))
+                data.genderByAge = resp;
+            else if (url.includes("/gender") && !url.includes("gender-by-age"))
+                data.gender = resp;
+            else if (url.includes("top-cities") && !url.includes("recording"))
+                data.topCities = resp;
+            else if (url.includes("/locations") && !url.includes("recording"))
+                data.locations = resp;
+            else if (url.includes("fanatic-audience-segments"))
+                data.segments = resp;
+            else if (url.includes("audience-engagement") && url.includes("stats") && !url.includes("recording"))
+                data.audienceStats = resp;
+            else if (url.includes("catalog-view") && url.includes("songs"))
+                data.catalog = resp;
+            else if (url.includes("playlists/curated")) {
+                if (!data.playlists)
+                    data.playlists = {};
+                data.playlists.curated = resp;
+            }
+            else if (url.includes("playlists/listener")) {
+                if (!data.playlists)
+                    data.playlists = {};
+                data.playlists.listener = resp;
+            }
+            else if (url.includes("playlists/personalized")) {
+                if (!data.playlists)
+                    data.playlists = {};
+                data.playlists.personalized = resp;
+            }
+            else if (url.includes("artist-videos"))
+                data.videos = resp;
+            // Profile fields 14-21
+            else if (url.includes("s4x-me") && url.includes("/me") && !url.includes("recent"))
+                data.accountOwner = resp;
+            else if (url.includes("s4x-me") && url.includes("artists/"))
+                data.permissions = resp;
+            else if (url.includes("canvaz-view") && url.includes("permissions"))
+                data.canvasPermissions = resp;
+            else if (url.includes("buyerxp-campaign-view") && url.includes("eligibility"))
+                data.campaignEligibility = resp;
+            else if (url.includes("app-manager") && url.includes("banner-metadata"))
+                data.teamMembership = resp;
+            // Per-song location data (top-cities and locations per recording)
+            else if (url.includes("top-cities") && url.includes("recording")) {
+                const m = url.match(/recording[_/]([a-zA-Z0-9]+)/);
+                if (m) {
+                    if (!data.perSong[m[1]])
+                        data.perSong[m[1]] = {};
+                    data.perSong[m[1]].topCities = resp;
+                }
+            }
+            else if (url.includes("/locations") && url.includes("recording")) {
+                const m = url.match(/recording[_/]([a-zA-Z0-9]+)/);
+                if (m) {
+                    if (!data.perSong[m[1]])
+                        data.perSong[m[1]] = {};
+                    data.perSong[m[1]].topCountries = resp;
+                }
+            }
+            // Per-song demographics (gender, gender-by-age per recording)
+            else if (url.includes("gender-by-age") && url.includes("recording")) {
+                const m = url.match(/recording[_/]([a-zA-Z0-9]+)/);
+                if (m) {
+                    if (!data.perSong[m[1]])
+                        data.perSong[m[1]] = {};
+                    data.perSong[m[1]].perSongGenderByAge = resp;
+                }
+            }
+            else if (url.includes("/gender") && !url.includes("gender-by-age") && url.includes("recording")) {
+                const m = url.match(/recording[_/]([a-zA-Z0-9]+)/);
+                if (m) {
+                    if (!data.perSong[m[1]])
+                        data.perSong[m[1]] = {};
+                    data.perSong[m[1]].perSongGender = resp;
+                }
+            }
+            // Per-song source of streams (28d aggregate — editorial/algorithmic/user/other)
+            else if (url.includes("source-of-stream") && url.includes("recording")) {
+                const m = url.match(/recording[_/]([a-zA-Z0-9]+)/);
+                if (m) {
+                    if (!data.perSong[m[1]])
+                        data.perSong[m[1]] = {};
+                    data.perSong[m[1]].sourceOfStreams = resp;
+                }
+            }
+            // Per-song country comparison timeseries (365d daily per selected country)
+            else if (url.includes("audience-engagement") && url.includes("recording") && url.includes("stats")) {
+                const m = url.match(/recording[_/]([a-zA-Z0-9]+)/);
+                if (m) {
+                    if (!data.perSong[m[1]])
+                        data.perSong[m[1]] = {};
+                    if (!data.perSong[m[1]].countryTimeline)
+                        data.perSong[m[1]].countryTimeline = {};
+                    // Store keyed by country code if available in response, or accumulate
+                    data.perSong[m[1]].countryTimeline._raw = resp;
+                }
+            }
+            else if (url.includes("song-stats")) {
+                const m = url.match(/recording[_/]([a-zA-Z0-9]+)/);
+                if (m) {
+                    if (!data.perSong[m[1]])
+                        data.perSong[m[1]] = {};
+                    const existing = data.perSong[m[1]];
+                    // Before merge: protect geography data by saving cities vs countries separately
+                    if (resp.geography && Array.isArray(resp.geography) && resp.geography.length > 0) {
+                        const first = resp.geography[0];
+                        if (first.region) {
+                            // Has region = cities data (e.g. Prague, CZ, region=10)
+                            existing.perSongCities = resp.geography;
+                        }
+                        else if (first.localized_country) {
+                            // Has localized_country = countries data (e.g. CZ, Czech Republic)
+                            existing.perSongCountries = resp.geography;
+                        }
+                    }
+                    // Protect countryTimelines from being overwritten
+                    if (resp.countryTimelines && Array.isArray(resp.countryTimelines)) {
+                        existing.countryTimelines = resp.countryTimelines;
+                    }
+                    // Prefer longer timeseries — 365d response must not be overwritten by 28d
+                    const existingLen = existing.streams?.current_period_timeseries?.length || 0;
+                    const newLen = resp.streams?.current_period_timeseries?.length || 0;
+                    if (newLen > 0 && existingLen > newLen) {
+                        // Keep existing longer timeseries, only merge non-timeseries fields
+                        const { streams, listeners, saves, playlist_adds, streams_per_listener, geography, countryTimelines, ...rest } = resp;
+                        Object.assign(existing, rest);
+                    }
+                    else {
+                        const { geography, countryTimelines, ...rest } = resp;
+                        Object.assign(existing, rest);
+                        // Only merge timeseries metrics if new data is longer
+                        if (resp.streams)
+                            existing.streams = resp.streams;
+                        if (resp.listeners)
+                            existing.listeners = resp.listeners;
+                        if (resp.saves)
+                            existing.saves = resp.saves;
+                        if (resp.playlist_adds)
+                            existing.playlist_adds = resp.playlist_adds;
+                        if (resp.streams_per_listener)
+                            existing.streams_per_listener = resp.streams_per_listener;
+                    }
+                }
+            }
+        }
+    };
+    // Parse artist-level data BEFORE per-song loop so we have trackIds
+    parseCaptured();
+    // Per-song pages — each track × each metric tab = 12 months of daily data
+    const trackIds = [];
+    if (data.catalog?.songs) {
+        for (const s of data.catalog.songs)
+            trackIds.push(s.id);
+    }
+    if (trackIds.length === 0) {
+        console.error(`[S4A] WARNING: No track IDs from catalog (catalog songs: ${data.catalog?.songs?.length ?? 'none'}). Per-song scrape skipped.`);
+    }
+    else {
+        console.log(`[S4A] Scraping per-song data for ${trackIds.length} tracks...`);
+    }
+    // Per-song tabs:
+    // 1. Overview (stats) — ONE call returns ALL 5 metrics (365 days daily each)
+    // 2. Source of stream — user/personalized/catalog/editorial/network/other
+    // 3. Location — top countries + cities + country comparison timeline
+    // 4. Demographics — gender + gender×age per track
+    // 5. Playlists — top playlists per track
+    // 6. SongDNA — TODO (new feature)
+    // NOTE: URL is "source-of-stream" (singular!), not "source-of-streams"
+    // Per-song: navigate to stats, click "12 months" → S4A picks correct date range.
+    // Don't guess toDate — Spotify updates when it wants. Let S4A decide.
+    // ONE stats call returns ALL 5 metrics × 365d daily + previous_period_agg.
+    // Then source-of-stream and playlists tabs for additional data.
+    for (let i = 0; i < trackIds.length; i++) {
+        const tid = trackIds[i];
+        console.log(`[S4A] Track ${i + 1}/${trackIds.length}: ${tid}`);
+        // Stats — load default, click "12 months"
+        try {
+            await page.goto(`https://artists.spotify.com/c/artist/${aid}/song/${tid}/stats`, { waitUntil: "networkidle", timeout: 20000 });
+            await sleep(1500);
+            await page.evaluate(`(() => {
+        const b = Array.from(document.querySelectorAll("button")).find(b => b.textContent?.trim() === "12 months");
+        if (b) b.click();
+      })()`);
+            await sleep(3000);
+        }
+        catch { }
+        // Source of streams
+        try {
+            await page.goto(`https://artists.spotify.com/c/artist/${aid}/song/${tid}/source-of-stream`, { waitUntil: "networkidle", timeout: 20000 });
+            await sleep(1500);
+        }
+        catch { }
+        // Location — top countries (28d, all countries) + top cities (28d, top 50) + country comparison timeline (365d, default top 3)
+        try {
+            await page.goto(`https://artists.spotify.com/c/artist/${aid}/song/${tid}/location`, { waitUntil: "networkidle", timeout: 20000 });
+            await sleep(1500);
+            // Click "12 months" for country comparison timeline (365d daily data)
+            await page.evaluate(`(() => {
+        const b = Array.from(document.querySelectorAll("button")).find(b => b.textContent?.trim() === "12 months");
+        if (b) b.click();
+      })()`);
+            await sleep(2000);
+        }
+        catch { }
+        // Demographics — per-song gender + gender×age breakdown
+        try {
+            await page.goto(`https://artists.spotify.com/c/artist/${aid}/song/${tid}/demographic`, { waitUntil: "networkidle", timeout: 20000 });
+            await sleep(1500);
+        }
+        catch { }
+        // Playlists
+        try {
+            await page.goto(`https://artists.spotify.com/c/artist/${aid}/song/${tid}/playlists`, { waitUntil: "networkidle", timeout: 20000 });
+            await sleep(1500);
+        }
+        catch { }
+    }
+    await browser.close();
+    // Final parse — picks up per-song responses
+    parseCaptured();
+    // Save raw dump
+    const dumpDir = path.join(process.env.HOME || "~", ".artist-os", "s4a-dumps", new Date().toISOString().slice(0, 10));
+    fs.mkdirSync(dumpDir, { recursive: true });
+    fs.writeFileSync(path.join(dumpDir, `${aid}_s4a.json`), JSON.stringify(data, null, 2));
+    return Object.keys(data).length > 0 ? data : null;
+}
+export async function scrapeS4A(artistId, outputDir) {
+    if (!hasS4ASession())
+        throw new Error("Not connected to S4A. Use connect_s4a first.");
+    const chromePath = process.env.CHROME_PATH || findChrome();
+    const browser = await chromium.launchPersistentContext(S4A_SESSION_DIR, {
+        executablePath: chromePath,
+        headless: true,
+    });
+    const captured = {};
+    const page = browser.pages()[0] || await browser.newPage();
+    // Intercept API responses
+    page.on("response", async (resp) => {
+        const url = resp.url();
+        if (!url.includes("spotify.com") || resp.status() !== 200)
+            return;
+        const interesting = ["audience-engagement", "catalog-view", "s4x-insights", "fanatic",
+            "song-stats", "entity-comparison", "canvaz-view", "buyerxp", "app-manager", "s4x-me"];
+        if (!interesting.some(k => url.includes(k)))
+            return;
+        try {
+            const data = await resp.json();
+            const key = url.split("spotify.com/")[1]?.replace(/[/?&=]/g, "_").slice(0, 120) || "unknown";
+            captured[key] = { url, data };
+        }
+        catch { }
+    });
+    // Navigate through S4A sections
+    const sections = [
+        `/c/artist/${artistId}/home`,
+        `/c/artist/${artistId}/audience/stats`,
+        `/c/artist/${artistId}/audience/demographic`,
+        `/c/artist/${artistId}/audience/location`,
+        `/c/artist/${artistId}/audience/segments`,
+        `/c/artist/${artistId}/music/songs`,
+        `/c/artist/${artistId}/music/releases`,
+        `/c/artist/${artistId}/music/upcoming`,
+        `/c/artist/${artistId}/music/playlists`,
+    ];
+    for (const section of sections) {
+        try {
+            await page.goto(`https://artists.spotify.com${section}`, { waitUntil: "networkidle", timeout: 30000 });
+            await new Promise(r => setTimeout(r, 2000));
+        }
+        catch { }
+    }
+    await browser.close();
+    // Save captured data
+    fs.mkdirSync(outputDir, { recursive: true });
+    let count = 0;
+    for (const [key, entry] of Object.entries(captured)) {
+        const filepath = path.join(outputDir, `${key}.json`);
+        fs.writeFileSync(filepath, JSON.stringify(entry.data, null, 2));
+        count++;
+    }
+    // Save manifest
+    fs.writeFileSync(path.join(outputDir, "_manifest.json"), JSON.stringify({
+        artist_id: artistId,
+        scraped_at: new Date().toISOString(),
+        files: count,
+    }, null, 2));
+    return { files: count };
+}
+export async function fullScrape(artistId, dbPath) {
+    const token = await getAnonymousToken(artistId);
+    const overview = await getArtistOverview(token, artistId);
+    const tracks = await getAllTracks(token, artistId);
+    const total_streams = tracks.reduce((sum, t) => sum + t.playcount, 0);
+    const dataDir = dbPath ? path.dirname(dbPath) : path.join(process.env.HOME || "~", ".artist-os");
+    // S4A scrape — automatic if session exists
+    let s4a = null;
+    if (hasS4ASession()) {
+        try {
+            s4a = await scrapeS4ADirect(artistId);
+        }
+        catch { }
+    }
+    if (dbPath) {
+        storeData(dbPath, overview, tracks);
+    }
+    const images = await downloadImages(artistId, overview, tracks, dataDir);
+    // Validate S4A data before workbook generation
+    if (s4a) {
+        const report = validateS4AData(s4a);
+        printValidationReport(report);
+        // Save report alongside dump
+        const dumpDir = path.join(process.env.HOME || "~", ".artist-os", "s4a-dumps", new Date().toISOString().slice(0, 10));
+        if (fs.existsSync(dumpDir)) {
+            fs.writeFileSync(path.join(dumpDir, `${artistId}_validation.json`), JSON.stringify(report, null, 2));
+        }
+    }
+    if (dbPath) {
+        await generateWorkbook(dbPath, dataDir, s4a);
+    }
+    return { overview, tracks, total_streams, images, s4a };
+}