@mogulmoretti/skrape 0.1.0

package/dist/tui/flow.js

@@ -0,0 +1,30 @@
+export const MENU_ITEMS = [
+    { label: 'Sync another community', value: 'sync-another' },
+    { label: 'Sync this community again', value: 'sync-again' },
+    { label: 'Open output folder', value: 'open-folder' },
+    { label: 'Quit', value: 'quit' },
+];
+/**
+ * Given the state of the sync that just completed and the menu choice the
+ * user made, returns the next step transition — or `null` when the choice
+ * doesn't move to a new screen at all: `'open-folder'` is a side effect
+ * performed in place (the menu stays up), and `'quit'` exits instead of
+ * transitioning to another step.
+ */
+export function nextStepForMenuChoice(done, choice) {
+    switch (choice) {
+        case 'sync-another':
+            return { kind: 'discovering' };
+        case 'sync-again':
+            return {
+                kind: 'confirming',
+                slug: done.slug,
+                name: done.name,
+                courseCount: done.courseCount,
+                outDir: done.outDir,
+            };
+        case 'open-folder':
+        case 'quit':
+            return null;
+    }
+}

package/dist/tui/progress.d.ts

@@ -0,0 +1,21 @@
+import type { Outcome, ProgressEvent } from '../sync.js';
+export interface ProgressState {
+    total: number;
+    done: number;
+    counts: Record<Outcome, number>;
+    current: {
+        course: string;
+        title: string;
+    } | null;
+}
+export declare function createProgressState(total: number): ProgressState;
+/**
+ * Pure reducer folding one onProgress event into state. Events arrive out of
+ * order from concurrent workers, so `done`/`total` are taken straight from the
+ * event (sync.ts already owns and increments that counter atomically) rather
+ * than re-derived here, and `current` is simply whichever event was folded in
+ * most recently — a stable "last seen" snapshot, not a claim about true
+ * completion order.
+ */
+export declare function applyProgress(state: ProgressState, event: ProgressEvent): ProgressState;
+export declare function formatProgressBar(done: number, total: number, width?: number): string;

package/dist/tui/progress.js

@@ -0,0 +1,31 @@
+export function createProgressState(total) {
+    return {
+        total,
+        done: 0,
+        counts: { ok: 0, skipped: 0, 'no-video': 0, 'no-access': 0, unavailable: 0, failed: 0 },
+        current: null,
+    };
+}
+/**
+ * Pure reducer folding one onProgress event into state. Events arrive out of
+ * order from concurrent workers, so `done`/`total` are taken straight from the
+ * event (sync.ts already owns and increments that counter atomically) rather
+ * than re-derived here, and `current` is simply whichever event was folded in
+ * most recently — a stable "last seen" snapshot, not a claim about true
+ * completion order.
+ */
+export function applyProgress(state, event) {
+    return {
+        total: event.total,
+        done: event.done,
+        counts: { ...state.counts, [event.outcome]: state.counts[event.outcome] + 1 },
+        current: { course: event.course, title: event.title },
+    };
+}
+export function formatProgressBar(done, total, width = 24) {
+    const safeTotal = total > 0 ? total : 1;
+    const ratio = Math.min(1, Math.max(0, done / safeTotal));
+    const filled = Math.round(ratio * width);
+    const bar = '#'.repeat(filled) + '-'.repeat(width - filled);
+    return `[${bar}] ${done}/${total}`;
+}

package/dist/tui/reveal.d.ts

@@ -0,0 +1,17 @@
+export interface RevealDeps {
+    /** Injectable so tests never spawn a real process. Defaults to
+     *  node:child_process's execFile, promisified. */
+    run: (command: string, args: string[]) => Promise<void>;
+    platform: NodeJS.Platform;
+    /** Where to print the path when reveal isn't possible/fails. Defaults to
+     *  console.log. */
+    log: (message: string) => void;
+}
+export declare function defaultRevealDeps(): RevealDeps;
+/**
+ * Reveals `outDir` in the OS file manager. Tries the platform-appropriate
+ * command; if there isn't one for this platform, or the command fails (not
+ * installed, headless environment, etc.), this never throws — it just prints
+ * the path instead, so a failed reveal can never crash the TUI.
+ */
+export declare function revealOutputFolder(outDir: string, deps?: RevealDeps): Promise<void>;

package/dist/tui/reveal.js

@@ -0,0 +1,47 @@
+import { execFile } from 'node:child_process';
+function defaultRun(command, args) {
+    return new Promise((resolve, reject) => {
+        execFile(command, args, (error) => {
+            if (error)
+                reject(error);
+            else
+                resolve();
+        });
+    });
+}
+export function defaultRevealDeps() {
+    return { run: defaultRun, platform: process.platform, log: (message) => console.log(message) };
+}
+/** Maps a platform to the OS command that reveals a folder in the file
+ *  manager, or null when there's no known one to try. */
+function revealCommandFor(platform) {
+    switch (platform) {
+        case 'darwin':
+            return { command: 'open', args: (path) => [path] };
+        case 'win32':
+            return { command: 'explorer', args: (path) => [path] };
+        case 'linux':
+            return { command: 'xdg-open', args: (path) => [path] };
+        default:
+            return null;
+    }
+}
+/**
+ * Reveals `outDir` in the OS file manager. Tries the platform-appropriate
+ * command; if there isn't one for this platform, or the command fails (not
+ * installed, headless environment, etc.), this never throws — it just prints
+ * the path instead, so a failed reveal can never crash the TUI.
+ */
+export async function revealOutputFolder(outDir, deps = defaultRevealDeps()) {
+    const reveal = revealCommandFor(deps.platform);
+    if (!reveal) {
+        deps.log(outDir);
+        return;
+    }
+    try {
+        await deps.run(reveal.command, reveal.args(outDir));
+    }
+    catch {
+        deps.log(outDir);
+    }
+}

package/dist/tui/run.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Runs the guided, no-arguments flow: session check, community picker,
+ * confirmation, live sync progress, and a result summary. Owns the db and
+ * the browser lifecycle for the whole flow and guarantees both are closed
+ * on every exit path — normal completion, an in-flow error, or Ctrl+C.
+ *
+ * Browser lifetime is delegated to `createBrowserLifecycle` — see the
+ * invariant documented there: only one Chrome process may ever hold the
+ * profile directory at a time. That's what let the login step below run
+ * "for free" here without any direct knowledge of when browsers open or close.
+ */
+export declare function runGuidedFlow(): Promise<void>;

package/dist/tui/run.js

@@ -0,0 +1,96 @@
+import React from 'react';
+import { render } from 'ink';
+import { HttpFetcher } from '../fetch/http.js';
+import { BrowserFetcher } from '../fetch/browser.js';
+import { isChromeMarkedInstalled, installChromeViaCli, markChromeInstalled, probeChromeLaunchable } from '../fetch/chromeSetup.js';
+import { openDb } from '../store/db.js';
+import { syncClassroom } from '../sync.js';
+import { chromeMarkerPath, dbPath, ensureRoot, isLoggedIn, login, profileDir } from '../auth/session.js';
+import { listCourses } from '../discover/skool.js';
+import { listUserCommunities } from '../discover/communities.js';
+import { App } from './App.js';
+import { createBrowserLifecycle } from './browserLifecycle.js';
+import { ensureChromeReady } from './chromeSetup.js';
+const OUT_ROOT = './out';
+/**
+ * Runs the guided, no-arguments flow: session check, community picker,
+ * confirmation, live sync progress, and a result summary. Owns the db and
+ * the browser lifecycle for the whole flow and guarantees both are closed
+ * on every exit path — normal completion, an in-flow error, or Ctrl+C.
+ *
+ * Browser lifetime is delegated to `createBrowserLifecycle` — see the
+ * invariant documented there: only one Chrome process may ever hold the
+ * profile directory at a time. That's what let the login step below run
+ * "for free" here without any direct knowledge of when browsers open or close.
+ */
+export async function runGuidedFlow() {
+    await ensureRoot();
+    // First-run browser setup: must complete before the session check below,
+    // and before the db/browser lifecycle are created — a failure here leaves
+    // nothing to clean up and is safe to retry on the next invocation.
+    await ensureChromeReady({
+        isConfirmedInstalled: () => isChromeMarkedInstalled(chromeMarkerPath()),
+        probeLaunchable: probeChromeLaunchable,
+        markInstalled: (executablePath) => markChromeInstalled(chromeMarkerPath(), executablePath),
+        installChrome: async (onProgress) => {
+            console.log('\nSetting up the browser skrape needs — this happens once, ~150MB.\n');
+            await installChromeViaCli(onProgress);
+            console.log('\nBrowser setup complete.\n');
+            const executablePath = await probeChromeLaunchable();
+            if (!executablePath) {
+                throw new Error('Chrome installed, but still could not be launched. This usually means a platform-specific ' +
+                    'dependency is missing — see the Playwright install output above for details.');
+            }
+            return executablePath;
+        },
+    }, (elapsedSeconds) => {
+        process.stdout.write(`\r  installing… ${elapsedSeconds}s elapsed`);
+    });
+    const db = openDb(dbPath());
+    const lifecycle = createBrowserLifecycle({
+        http: new HttpFetcher(),
+        isLoggedIn,
+        makeBrowserFetcher: () => new BrowserFetcher(profileDir()),
+        performLogin: (onContext) => login(onContext),
+    });
+    let closed = false;
+    const cleanup = async () => {
+        if (closed)
+            return;
+        closed = true;
+        try {
+            await lifecycle.cleanup();
+        }
+        catch (error) {
+            console.error(`\nWarning: failed to close browser cleanly: ${error.message}`);
+        }
+        finally {
+            db.close();
+        }
+    };
+    const onSigint = () => {
+        void cleanup().finally(() => process.exit(130));
+    };
+    process.on('SIGINT', onSigint);
+    const controllers = {
+        outRoot: OUT_ROOT,
+        checkLoggedIn: async () => lifecycle.checkLoggedIn(),
+        login: async () => {
+            await lifecycle.login();
+        },
+        discoverCommunities: async () => listUserCommunities(lifecycle.getFetcher()),
+        countAccessibleCourses: async (slug) => {
+            const courses = await listCourses(slug, lifecycle.getFetcher());
+            return courses.filter((course) => course.hasAccess).length;
+        },
+        runSync: async (slug, outDir, onProgress) => syncClassroom({ slug, outDir, db, fetcher: lifecycle.getFetcher(), concurrency: 4, onProgress }),
+    };
+    const { waitUntilExit } = render(React.createElement(App, { controllers }));
+    try {
+        await waitUntilExit();
+    }
+    finally {
+        process.off('SIGINT', onSigint);
+        await cleanup();
+    }
+}

package/dist/tui/slug.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Strips a pasted skool.com URL (or a path within one) down to the bare slug,
+ * and trims whitespace. Users are told to enter "the part after skool.com/",
+ * but pasting the whole URL is common enough to normalize rather than reject.
+ */
+export declare function normalizeSlug(input: string): string;
+export declare function isValidSlug(input: string): boolean;

package/dist/tui/slug.js

@@ -0,0 +1,18 @@
+const SLUG_RE = /^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,98}[a-zA-Z0-9])?$/;
+/**
+ * Strips a pasted skool.com URL (or a path within one) down to the bare slug,
+ * and trims whitespace. Users are told to enter "the part after skool.com/",
+ * but pasting the whole URL is common enough to normalize rather than reject.
+ */
+export function normalizeSlug(input) {
+    let value = input.trim();
+    value = value.replace(/^https?:\/\//i, '');
+    value = value.replace(/^(www\.)?skool\.com\//i, '');
+    const slashIndex = value.indexOf('/');
+    if (slashIndex !== -1)
+        value = value.slice(0, slashIndex);
+    return value;
+}
+export function isValidSlug(input) {
+    return SLUG_RE.test(input);
+}

package/dist/tui/summary.d.ts

@@ -0,0 +1,7 @@
+import type { SyncSummary } from '../sync.js';
+export declare function formatCountLines(summary: SyncSummary): string[];
+export declare function formatProblemLines(summary: SyncSummary): string[];
+/** True when nothing was transcribed at all — the case that must never be
+ *  rendered as a cheerful, empty-looking summary. */
+export declare function nothingTranscribed(summary: SyncSummary): boolean;
+export declare function formatSummary(summary: SyncSummary, outDir: string): string[];

package/dist/tui/summary.js

@@ -0,0 +1,29 @@
+export function formatCountLines(summary) {
+    const lines = [];
+    for (const [outcome, count] of Object.entries(summary.counts)) {
+        if (count > 0)
+            lines.push(`  ${outcome.padEnd(14)} ${count}`);
+    }
+    return lines;
+}
+export function formatProblemLines(summary) {
+    return summary.problems.map((problem) => `  [${problem.outcome}] ${problem.course} / ${problem.title}: ${problem.reason}`);
+}
+/** True when nothing was transcribed at all — the case that must never be
+ *  rendered as a cheerful, empty-looking summary. */
+export function nothingTranscribed(summary) {
+    return summary.counts.ok === 0 && summary.totalWords === 0;
+}
+export function formatSummary(summary, outDir) {
+    const lines = [];
+    if (nothingTranscribed(summary)) {
+        lines.push('Nothing was transcribed.', '');
+    }
+    lines.push('--- summary ---', ...formatCountLines(summary));
+    lines.push(`  ${'words'.padEnd(14)} ${summary.totalWords.toLocaleString('en-US')}`);
+    if (summary.problems.length > 0) {
+        lines.push('', '--- not transcribed ---', ...formatProblemLines(summary));
+    }
+    lines.push('', `Output: ${outDir}`);
+    return lines;
+}

package/dist/types.d.ts

@@ -0,0 +1,33 @@
+export type ItemType = 'lesson' | 'call' | 'post';
+export interface ContentItem {
+    nativeId: string;
+    type: ItemType;
+    title: string;
+    index: number;
+    course: string | null;
+    section: string | null;
+    url: string | null;
+    videoUrl: string | null;
+    durationMs: number;
+    hasAccess: boolean;
+    publishedAt: string | null;
+    bodyText: string | null;
+}
+export type TranscriptResult = {
+    status: 'ok';
+    text: string;
+    wordCount: number;
+    provider: string;
+    sourceUrl: string;
+} | {
+    status: 'unavailable';
+    reason: string;
+    provider: string;
+} | {
+    status: 'failed';
+    reason: string;
+    provider: string;
+};
+export interface Fetcher {
+    getPage(url: string): Promise<string>;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@mogulmoretti/skrape",
+  "version": "0.1.0",
+  "description": "Read Skool course content instead of watching it — pulls the classroom of a community you belong to into clean, readable transcripts.",
+  "keywords": [
+    "skool",
+    "cli",
+    "transcript",
+    "video-to-text",
+    "loom",
+    "course"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/slyngg/skrape.git"
+  },
+  "bugs": {
+    "url": "https://github.com/slyngg/skrape/issues"
+  },
+  "homepage": "https://github.com/slyngg/skrape#readme",
+  "type": "module",
+  "bin": {
+    "skrape": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc && chmod +x dist/cli.js",
+    "prepublishOnly": "npm run build && npm test",
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "dependencies": {
+    "better-sqlite3": "^11.5.0",
+    "commander": "^12.1.0",
+    "ink": "^7.1.1",
+    "react": "^19.2.8",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.11",
+    "@types/node": "^22.9.0",
+    "@types/react": "^19.2.18",
+    "typescript": "^5.6.3",
+    "vitest": "^2.1.4"
+  },
+  "optionalDependencies": {
+    "playwright": "^1.48.2"
+  }
+}