@mogulmoretti/skrape 0.1.0

package/dist/fetch/chromeSetup.d.ts

@@ -0,0 +1,54 @@
+/**
+ * All the real, non-pure IO Playwright's Chrome install needs: checking a
+ * cheap on-disk marker, probing whether Chrome is actually launchable, and
+ * running the installer. Kept separate from `../tui/chromeSetup.ts`, which
+ * holds the decision logic and is unit-tested against fakes of the
+ * functions here — none of *this* file is meaningfully unit-testable
+ * (it launches a real browser / spawns a real child process / hits disk).
+ */
+/**
+ * True when a past run confirmed Chrome installed *and* the executable path
+ * recorded then still exists now. The marker stores that resolved path
+ * (see `markChromeInstalled`) rather than a bare flag, so an uninstall,
+ * move, or OS package-manager removal after the marker was written is
+ * caught here — cheaply, via `existsSync`, never by launching a browser.
+ * A missing/unreadable/empty marker is just treated as "not installed".
+ */
+export declare function isChromeMarkedInstalled(markerPath: string): boolean;
+/** Records the confirmed Chrome executable's path, so later runs can skip
+ *  the probe by cheaply checking that path still exists on disk. */
+export declare function markChromeInstalled(markerPath: string, executablePath: string): Promise<void>;
+/**
+ * Real "is Chrome actually launchable" probe: Playwright exposes no public
+ * API to ask "is the 'chrome' channel installed" without launching it —
+ * `chromium.executablePath()` takes no channel argument and always returns
+ * the bundled Chromium's default path regardless, whether or not anything
+ * is actually installed there. So the reliable check is a real launch.
+ *
+ * This launches a throwaway, headless Chrome via `browserType.launchServer()`
+ * (no persistent context, no profile directory at all) and closes it
+ * immediately — it can never touch the user's real login profile, and nothing
+ * survives the call either way. `launchServer()` (unlike `launch()`) exposes
+ * the spawned process via `.process()`, whose Node `spawnfile` is the actual
+ * resolved Chrome executable path Playwright launched — that's what gets
+ * returned so callers can persist a marker that verifies something real,
+ * rather than a bare boolean. Playwright itself is imported dynamically so
+ * a user who already has a confirmed marker (the common case after the first
+ * run) never pays Playwright's module-load cost at all.
+ */
+export declare function probeChromeLaunchable(): Promise<string | undefined>;
+/**
+ * Runs the equivalent of `npx playwright install chrome` in-process, by
+ * spawning Playwright's own bundled CLI script directly. Preferred over
+ * `npx playwright install chrome` because the CLI path is resolved from
+ * node_modules up front (no npm package resolution / registry round trip
+ * just to find the command), and preferred over trying to call an in-process
+ * "install" function because Playwright doesn't export one from its public
+ * API surface — the installer lives inside its bundled, unexported CLI.
+ *
+ * `onProgress` fires roughly once a second with the elapsed seconds so
+ * callers can render a simple "still working…" indicator — Playwright's own
+ * download progress bars are written directly to the child's TTY and aren't
+ * easily piped through as structured progress.
+ */
+export declare function installChromeViaCli(onProgress?: (elapsedSeconds: number) => void): Promise<void>;

package/dist/fetch/chromeSetup.js

@@ -0,0 +1,126 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { writeFile } from 'node:fs/promises';
+import { spawn } from 'node:child_process';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+/**
+ * All the real, non-pure IO Playwright's Chrome install needs: checking a
+ * cheap on-disk marker, probing whether Chrome is actually launchable, and
+ * running the installer. Kept separate from `../tui/chromeSetup.ts`, which
+ * holds the decision logic and is unit-tested against fakes of the
+ * functions here — none of *this* file is meaningfully unit-testable
+ * (it launches a real browser / spawns a real child process / hits disk).
+ */
+/**
+ * True when a past run confirmed Chrome installed *and* the executable path
+ * recorded then still exists now. The marker stores that resolved path
+ * (see `markChromeInstalled`) rather than a bare flag, so an uninstall,
+ * move, or OS package-manager removal after the marker was written is
+ * caught here — cheaply, via `existsSync`, never by launching a browser.
+ * A missing/unreadable/empty marker is just treated as "not installed".
+ */
+export function isChromeMarkedInstalled(markerPath) {
+    if (!existsSync(markerPath))
+        return false;
+    let storedPath;
+    try {
+        storedPath = readFileSync(markerPath, 'utf8').trim();
+    }
+    catch {
+        return false;
+    }
+    return storedPath.length > 0 && existsSync(storedPath);
+}
+/** Records the confirmed Chrome executable's path, so later runs can skip
+ *  the probe by cheaply checking that path still exists on disk. */
+export async function markChromeInstalled(markerPath, executablePath) {
+    await writeFile(markerPath, executablePath, 'utf8');
+}
+/**
+ * Real "is Chrome actually launchable" probe: Playwright exposes no public
+ * API to ask "is the 'chrome' channel installed" without launching it —
+ * `chromium.executablePath()` takes no channel argument and always returns
+ * the bundled Chromium's default path regardless, whether or not anything
+ * is actually installed there. So the reliable check is a real launch.
+ *
+ * This launches a throwaway, headless Chrome via `browserType.launchServer()`
+ * (no persistent context, no profile directory at all) and closes it
+ * immediately — it can never touch the user's real login profile, and nothing
+ * survives the call either way. `launchServer()` (unlike `launch()`) exposes
+ * the spawned process via `.process()`, whose Node `spawnfile` is the actual
+ * resolved Chrome executable path Playwright launched — that's what gets
+ * returned so callers can persist a marker that verifies something real,
+ * rather than a bare boolean. Playwright itself is imported dynamically so
+ * a user who already has a confirmed marker (the common case after the first
+ * run) never pays Playwright's module-load cost at all.
+ */
+export async function probeChromeLaunchable() {
+    try {
+        const { chromium } = await import('playwright');
+        const server = await chromium.launchServer({ channel: 'chrome', headless: true });
+        try {
+            return server.process().spawnfile;
+        }
+        finally {
+            await server.close();
+        }
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Resolves the on-disk path to Playwright's own bundled CLI script, via the
+ *  package's exported `./package.json` subpath (not `./cli.js`, which isn't
+ *  in Playwright's `exports` map and so can't be resolved directly) — this
+ *  needs no network or npm resolution, unlike shelling out to `npx`. */
+async function resolvePlaywrightCliPath() {
+    const pkgJsonUrl = import.meta.resolve('playwright/package.json');
+    return join(dirname(fileURLToPath(pkgJsonUrl)), 'cli.js');
+}
+/**
+ * Runs the equivalent of `npx playwright install chrome` in-process, by
+ * spawning Playwright's own bundled CLI script directly. Preferred over
+ * `npx playwright install chrome` because the CLI path is resolved from
+ * node_modules up front (no npm package resolution / registry round trip
+ * just to find the command), and preferred over trying to call an in-process
+ * "install" function because Playwright doesn't export one from its public
+ * API surface — the installer lives inside its bundled, unexported CLI.
+ *
+ * `onProgress` fires roughly once a second with the elapsed seconds so
+ * callers can render a simple "still working…" indicator — Playwright's own
+ * download progress bars are written directly to the child's TTY and aren't
+ * easily piped through as structured progress.
+ */
+export async function installChromeViaCli(onProgress) {
+    const cliPath = await resolvePlaywrightCliPath();
+    await new Promise((resolve, reject) => {
+        const child = spawn(process.execPath, [cliPath, 'install', 'chrome'], {
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        let stdout = '';
+        let stderr = '';
+        child.stdout?.on('data', (chunk) => {
+            stdout += chunk.toString();
+        });
+        child.stderr?.on('data', (chunk) => {
+            stderr += chunk.toString();
+        });
+        const startedAt = Date.now();
+        const interval = setInterval(() => {
+            onProgress?.(Math.round((Date.now() - startedAt) / 1000));
+        }, 1000);
+        child.on('error', (error) => {
+            clearInterval(interval);
+            reject(error);
+        });
+        child.on('exit', (code) => {
+            clearInterval(interval);
+            if (code === 0) {
+                resolve();
+                return;
+            }
+            const detail = (stderr || stdout).trim().split('\n').slice(-20).join('\n');
+            reject(new Error(`\`playwright install chrome\` exited with code ${code}${detail ? `:\n${detail}` : ''}`));
+        });
+    });
+}

package/dist/fetch/http.d.ts

@@ -0,0 +1,17 @@
+import type { Fetcher } from '../types.js';
+export interface HttpFetcherOptions {
+    cookie?: string;
+    retries?: number;
+}
+export declare class HttpFetcher implements Fetcher {
+    private readonly options;
+    constructor(options?: HttpFetcherOptions);
+    getPage(url: string): Promise<string>;
+    /**
+     * Follows redirects manually, one hop at a time, so the cookie decision is
+     * re-evaluated against the CURRENT hop's hostname rather than the original
+     * URL. This prevents a skool.com -> off-domain redirect from carrying the
+     * session cookie to a non-Skool host.
+     */
+    private fetchFollowingRedirects;
+}

package/dist/fetch/http.js

@@ -0,0 +1,64 @@
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
+    '(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const MAX_REDIRECTS = 5;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+export class HttpFetcher {
+    options;
+    constructor(options = {}) {
+        this.options = options;
+    }
+    async getPage(url) {
+        const retries = this.options.retries ?? 3;
+        let lastError = new Error('no attempt made');
+        for (let attempt = 0; attempt < retries; attempt++) {
+            try {
+                const response = await this.fetchFollowingRedirects(url);
+                if (response.status === 429) {
+                    const retryAfter = Number(response.headers.get('retry-after') ?? 0);
+                    await sleep(retryAfter > 0 ? retryAfter * 1000 : 2 ** attempt * 1000);
+                    lastError = new Error('rate limited (429)');
+                    continue;
+                }
+                if (!response.ok)
+                    throw new Error(`HTTP ${response.status} for ${url}`);
+                return await response.text();
+            }
+            catch (error) {
+                lastError = error;
+                if (attempt < retries - 1)
+                    await sleep(2 ** attempt * 500);
+            }
+        }
+        throw lastError;
+    }
+    /**
+     * Follows redirects manually, one hop at a time, so the cookie decision is
+     * re-evaluated against the CURRENT hop's hostname rather than the original
+     * URL. This prevents a skool.com -> off-domain redirect from carrying the
+     * session cookie to a non-Skool host.
+     */
+    async fetchFollowingRedirects(initialUrl) {
+        let currentUrl = initialUrl;
+        for (let hop = 0; hop < MAX_REDIRECTS; hop++) {
+            const headers = { 'user-agent': UA, accept: 'text/html,*/*' };
+            // Only send the Skool session to Skool. Loom must stay unauthenticated.
+            // Recomputed on every hop: see comment above.
+            const { hostname } = new URL(currentUrl);
+            const isSkool = hostname === 'skool.com' || hostname.endsWith('.skool.com');
+            if (this.options.cookie && isSkool) {
+                headers.cookie = this.options.cookie;
+            }
+            const response = await fetch(currentUrl, { headers, redirect: 'manual' });
+            if (!REDIRECT_STATUSES.has(response.status)) {
+                return response;
+            }
+            const location = response.headers.get('location');
+            if (!location) {
+                throw new Error(`Redirect response (${response.status}) missing Location header for ${currentUrl}`);
+            }
+            currentUrl = new URL(location, currentUrl).toString();
+        }
+        throw new Error(`Too many redirects: exceeded limit of ${MAX_REDIRECTS} hops for ${initialUrl}`);
+    }
+}

package/dist/fetch/nextdata.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Signals that a page did not yield a parseable payload — the trigger for
+ * escalating a route from HTTP to a real browser. Distinct from network errors,
+ * which must NOT trigger escalation.
+ */
+export declare class PayloadParseError extends Error {
+    constructor(message: string);
+}
+export declare function extractNextData(html: string): unknown;

package/dist/fetch/nextdata.js

@@ -0,0 +1,23 @@
+const NEXT_DATA_RE = /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/;
+/**
+ * Signals that a page did not yield a parseable payload — the trigger for
+ * escalating a route from HTTP to a real browser. Distinct from network errors,
+ * which must NOT trigger escalation.
+ */
+export class PayloadParseError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'PayloadParseError';
+    }
+}
+export function extractNextData(html) {
+    const match = html.match(NEXT_DATA_RE);
+    if (!match)
+        throw new PayloadParseError('no __NEXT_DATA__ script tag (session expired or markup changed?)');
+    try {
+        return JSON.parse(match[1]);
+    }
+    catch (error) {
+        throw new PayloadParseError(`malformed __NEXT_DATA__ JSON: ${error.message}`);
+    }
+}

package/dist/fetch/resilient.d.ts

@@ -0,0 +1,12 @@
+import type { Fetcher } from '../types.js';
+export declare class ResilientFetcher implements Fetcher {
+    private readonly http;
+    private readonly makeBrowser;
+    private readonly isAuthenticated?;
+    private readonly escalated;
+    private browserPromise;
+    private browser;
+    constructor(http: Fetcher, makeBrowser: () => Promise<Fetcher>, isAuthenticated?: ((html: string) => boolean) | undefined);
+    get escalatedRoutes(): ReadonlySet<string>;
+    getPage(url: string): Promise<string>;
+}

package/dist/fetch/resilient.js

@@ -0,0 +1,68 @@
+import { extractNextData, PayloadParseError } from './nextdata.js';
+/** Group urls by hostname and first two path segments to distinguish different hosts. */
+function routeKey(url) {
+    try {
+        const parsed = new URL(url);
+        const pathPart = parsed.pathname.split('/').slice(0, 3).join('/');
+        return `${parsed.hostname}${pathPart}`;
+    }
+    catch {
+        return url;
+    }
+}
+function needsPayload(url) {
+    try {
+        return new URL(url).hostname.endsWith('skool.com');
+    }
+    catch {
+        return false;
+    }
+}
+export class ResilientFetcher {
+    http;
+    makeBrowser;
+    isAuthenticated;
+    escalated = new Set();
+    browserPromise = null;
+    browser = null;
+    constructor(http, makeBrowser, isAuthenticated) {
+        this.http = http;
+        this.makeBrowser = makeBrowser;
+        this.isAuthenticated = isAuthenticated;
+    }
+    get escalatedRoutes() {
+        return this.escalated;
+    }
+    async getPage(url) {
+        const route = routeKey(url);
+        if (!this.escalated.has(route)) {
+            const html = await this.http.getPage(url); // network errors propagate, no escalation
+            if (!needsPayload(url))
+                return html;
+            try {
+                extractNextData(html);
+                // The payload parsed structurally, but a logged-out (or redirected) Skool page can
+                // still ship a valid __NEXT_DATA__ blob — Next.js renders one either way. A semantic
+                // check catches that case and escalates to the authenticated browser too.
+                if (!this.isAuthenticated || this.isAuthenticated(html))
+                    return html;
+                this.escalated.add(route);
+            }
+            catch (error) {
+                if (!(error instanceof PayloadParseError))
+                    throw error;
+                this.escalated.add(route);
+            }
+        }
+        try {
+            this.browserPromise ??= this.makeBrowser();
+            this.browser = await this.browserPromise;
+        }
+        catch (error) {
+            // Clear the promise memo on rejection to allow retries
+            this.browserPromise = null;
+            throw error;
+        }
+        return this.browser.getPage(url);
+    }
+}

package/dist/media/index.d.ts

@@ -0,0 +1,4 @@
+import { loomTranscript } from './loom.js';
+import type { Fetcher, TranscriptResult } from '../types.js';
+export declare function getTranscript(videoUrl: string, fetcher: Fetcher): Promise<TranscriptResult>;
+export { loomTranscript };

package/dist/media/index.js

@@ -0,0 +1,11 @@
+import { loomTranscript } from './loom.js';
+/** Host substring -> provider. Add YouTube/Whisper here without touching callers. */
+const PROVIDERS = [['loom.com', loomTranscript]];
+export async function getTranscript(videoUrl, fetcher) {
+    const entry = PROVIDERS.find(([host]) => videoUrl.includes(host));
+    if (!entry) {
+        return { status: 'unavailable', reason: `unsupported video host: ${videoUrl}`, provider: 'none' };
+    }
+    return entry[1](videoUrl, fetcher);
+}
+export { loomTranscript };

package/dist/media/loom.d.ts

@@ -0,0 +1,5 @@
+import type { Fetcher, TranscriptResult } from '../types.js';
+export declare function loomIdFromUrl(url: string): string | null;
+export declare function extractCaptionsUrl(html: string): string | null;
+export declare function extractTranscriptionStatus(html: string): string | null;
+export declare function loomTranscript(videoUrl: string, fetcher: Fetcher): Promise<TranscriptResult>;

package/dist/media/loom.js

@@ -0,0 +1,60 @@
+import { parseVtt } from '../normalize/vtt.js';
+const CAPTIONS_RE = /https:\/\/cdn\.loom\.com\/mediametadata\/captions\/[^"'\\\s]+(?:\\u0026[^"'\\\s]+)*/;
+const STATUS_RE = /transcription_status\\?"\s*:\s*\\?"([a-z_]+)/;
+const MIN_WORDS = 50;
+export function loomIdFromUrl(url) {
+    if (!url.includes('loom.com'))
+        return null;
+    const match = url.match(/loom\.com\/share\/([A-Za-z0-9]+)/);
+    return match ? match[1] : null;
+}
+export function extractCaptionsUrl(html) {
+    const match = html.match(CAPTIONS_RE);
+    if (!match)
+        return null;
+    return match[0].replace(/\\u0026/g, '&').replace(/&/g, '&');
+}
+export function extractTranscriptionStatus(html) {
+    const match = html.match(STATUS_RE);
+    return match ? match[1] : null;
+}
+export async function loomTranscript(videoUrl, fetcher) {
+    const id = loomIdFromUrl(videoUrl);
+    if (!id)
+        return { status: 'unavailable', reason: `not a loom share url: ${videoUrl}`, provider: 'loom' };
+    const shareUrl = `https://www.loom.com/share/${id}`;
+    let html;
+    try {
+        html = await fetcher.getPage(shareUrl);
+    }
+    catch (error) {
+        return { status: 'failed', reason: `share page: ${error.message}`, provider: 'loom' };
+    }
+    const captionsUrl = extractCaptionsUrl(html);
+    if (!captionsUrl) {
+        const status = extractTranscriptionStatus(html) ?? 'unknown';
+        return { status: 'unavailable', reason: `no captions (transcription_status=${status})`, provider: 'loom' };
+    }
+    let vtt;
+    try {
+        vtt = await fetcher.getPage(captionsUrl);
+    }
+    catch (error) {
+        return { status: 'failed', reason: `captions: ${error.message}`, provider: 'loom' };
+    }
+    const parsed = parseVtt(vtt);
+    if (parsed.wordCount < MIN_WORDS) {
+        return {
+            status: 'unavailable',
+            reason: `transcript too short to trust (${parsed.wordCount} words)`,
+            provider: 'loom',
+        };
+    }
+    return {
+        status: 'ok',
+        text: parsed.text,
+        wordCount: parsed.wordCount,
+        provider: 'loom',
+        sourceUrl: shareUrl,
+    };
+}

package/dist/normalize/vtt.d.ts

@@ -0,0 +1,12 @@
+export interface VttCue {
+    start: string | null;
+    text: string;
+}
+export interface ParsedTranscript {
+    text: string;
+    wordCount: number;
+    cueCount: number;
+}
+/** Split a VTT body into cues, dropping structural lines and consecutive duplicates. */
+export declare function parseCues(vtt: string): VttCue[];
+export declare function parseVtt(vtt: string): ParsedTranscript;

package/dist/normalize/vtt.js

@@ -0,0 +1,109 @@
+const PARAGRAPH_MIN_CHARS = 450;
+/** Split a VTT body into cues, dropping structural lines and consecutive duplicates. */
+export function parseCues(vtt) {
+    const cues = [];
+    let currentStart = null;
+    let inNoteBlock = false;
+    let currentCueLines = [];
+    const flushCue = () => {
+        if (currentCueLines.length === 0)
+            return;
+        const text = currentCueLines.join(' ');
+        // Rolling captions repeat the previous cue verbatim. Only consecutive
+        // repeats are dropped — a speaker genuinely repeating a line later is kept.
+        if (cues.length > 0 && cues[cues.length - 1].text === text) {
+            currentCueLines = [];
+            currentStart = null;
+            return;
+        }
+        cues.push({ start: currentStart, text });
+        currentCueLines = [];
+        currentStart = null;
+    };
+    for (const raw of vtt.split(/\r?\n/)) {
+        const line = raw.trim();
+        // Blank lines end NOTE blocks and cue blocks
+        if (!line) {
+            inNoteBlock = false;
+            flushCue();
+            continue;
+        }
+        // Skip WEBVTT header
+        if (line === 'WEBVTT')
+            continue;
+        // Skip cue numbers (pure digits)
+        if (/^\d+$/.test(line))
+            continue;
+        // Enter NOTE block (lines starting with NOTE are skipped until blank line)
+        if (line.startsWith('NOTE')) {
+            inNoteBlock = true;
+            continue;
+        }
+        // Skip lines inside NOTE blocks
+        if (inNoteBlock)
+            continue;
+        // Handle timestamp lines
+        if (line.includes('-->')) {
+            flushCue();
+            currentStart = line.split('-->')[0].trim();
+            continue;
+        }
+        // Strip speaker tags and collect cue text (accumulate multi-line cues)
+        const text = line.replace(/<[^>]+>/g, '').trim();
+        if (!text)
+            continue;
+        currentCueLines.push(text);
+    }
+    // Flush any remaining cue lines
+    flushCue();
+    return cues;
+}
+/** "01:02:03.456" -> "[62:03]". Hours fold into minutes so anchors never wrap. */
+function anchor(ts) {
+    if (!ts)
+        return '';
+    const parts = ts.split(':');
+    if (parts.length === 2) {
+        // MM:SS.mmm format (no hours)
+        const minutes = Number(parts[0]);
+        const seconds = parts[1].split('.')[0];
+        if (!Number.isFinite(minutes))
+            return '';
+        return `[${minutes}:${seconds}] `;
+    }
+    if (parts.length === 3) {
+        // HH:MM:SS.mmm format
+        const hours = Number(parts[0]);
+        const minutes = Number(parts[1]);
+        const seconds = parts[2].split('.')[0];
+        if (!Number.isFinite(hours) || !Number.isFinite(minutes))
+            return '';
+        return `[${hours * 60 + minutes}:${seconds}] `;
+    }
+    return '';
+}
+export function parseVtt(vtt) {
+    const cues = parseCues(vtt);
+    const paragraphs = [];
+    let buffer = [];
+    let bufferStart = null;
+    let wordCount = 0;
+    const flush = () => {
+        if (buffer.length === 0)
+            return;
+        paragraphs.push(anchor(bufferStart) + buffer.join(' '));
+        buffer = [];
+        bufferStart = null;
+    };
+    for (const cue of cues) {
+        if (buffer.length === 0)
+            bufferStart = cue.start;
+        buffer.push(cue.text);
+        wordCount += cue.text.split(/\s+/).filter(Boolean).length;
+        const joined = buffer.join(' ');
+        if (joined.length > PARAGRAPH_MIN_CHARS && /[.!?]$/.test(cue.text))
+            flush();
+    }
+    flush();
+    return { text: paragraphs.join('\n\n'), wordCount, cueCount: cues.length };
+}

package/dist/store/db.d.ts

@@ -0,0 +1,15 @@
+import type { ContentItem, TranscriptResult } from '../types.js';
+export declare function contentHash(item: ContentItem): string;
+export interface Db {
+    upsertCommunity(slug: string, name: string): void;
+    upsertItem(community: string, item: ContentItem): {
+        id: string;
+        changed: boolean;
+    };
+    saveTranscript(itemId: string, result: TranscriptResult): void;
+    getTranscriptStatus(itemId: string): string | null;
+    getTranscriptReason(itemId: string): string | null;
+    markSynced(slug: string): void;
+    close(): void;
+}
+export declare function openDb(path: string): Db;