@mogulmoretti/skrape 0.2.0 → 0.2.1

package/README.md

@@ -57,8 +57,9 @@ From there you can sync another community, re-sync the same one (it's
 incremental: already-fetched lessons are skipped), open the output folder,
 or quit.
 
-Output lands in `./out/<slug>/transcripts/<course>/NN-lesson.md`, and videos
-(if you asked for them) in `./out/<slug>/videos/<course>/NN-lesson.mp4`.
+Output lands in `~/skrape/<slug>/transcripts/<course>/NN-lesson.md`, and videos
+(if you asked for them) in `~/skrape/<slug>/videos/<course>/NN-lesson.mp4`,
+wherever you run skrape from.
 
 ### Downloading videos
 
@@ -79,7 +80,7 @@ The underlying subcommands work directly too, without the guided flow:
 
     skrape login                     # once: sign in by hand, session persists
     skrape sync <slug>                # slug is the part after skool.com/
-    skrape sync <slug> -o ./out -c 4  # custom output dir / concurrency
+    skrape sync <slug> -o ~/notes -c 4  # custom output dir / concurrency
     skrape sync <slug> --videos       # transcripts + every lesson video
 
 ### Building from source

package/dist/auth/session.d.ts

@@ -1,4 +1,6 @@
 export declare function profileDir(): string;
+/** Where synced communities land unless -o says otherwise: one fixed place, whatever the cwd. */
+export declare function defaultOutRoot(): string;
 export declare function dbPath(): string;
 /**
  * Marker file written once the one-time Chrome/Playwright browser install
@@ -6,6 +8,12 @@ export declare function dbPath(): string;
  * Its presence lets later runs skip re-probing with a real browser launch.
  */
 export declare function chromeMarkerPath(): string;
+/** Launch the browser first-run setup confirmed; before any setup ran, assume the user's Chrome. */
+export declare function browserLaunchOptions(): {
+    executablePath: string;
+} | {
+    channel: string;
+};
 export declare function ensureRoot(): Promise<void>;
 export declare function isLoggedIn(html: string): boolean;
 /** A handle callers can use to force-close a launched login context, e.g. on Ctrl+C. */

package/dist/auth/session.js

@@ -2,10 +2,15 @@ import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { mkdir } from 'node:fs/promises';
 import { extractNextData, PayloadParseError } from '../fetch/nextdata.js';
+import { markedExecutablePath } from '../fetch/chromeSetup.js';
 const ROOT = join(homedir(), '.skool-skrape');
 export function profileDir() {
     return join(ROOT, 'chrome-profile');
 }
+/** Where synced communities land unless -o says otherwise: one fixed place, whatever the cwd. */
+export function defaultOutRoot() {
+    return join(homedir(), 'skrape');
+}
 export function dbPath() {
     return join(ROOT, 'skool.db');
 }
@@ -17,6 +22,11 @@ export function dbPath() {
 export function chromeMarkerPath() {
     return join(ROOT, 'chrome-installed');
 }
+/** Launch the browser first-run setup confirmed; before any setup ran, assume the user's Chrome. */
+export function browserLaunchOptions() {
+    const executablePath = markedExecutablePath(chromeMarkerPath());
+    return executablePath ? { executablePath } : { channel: 'chrome' };
+}
 export async function ensureRoot() {
     await mkdir(ROOT, { recursive: true });
 }
@@ -46,14 +56,12 @@ export async function login(onContext) {
     await ensureRoot();
     const { chromium } = await import('playwright');
     const context = await chromium.launchPersistentContext(profileDir(), {
-        channel: 'chrome',
+        ...browserLaunchOptions(),
         headless: false,
     });
     onContext?.(context);
     const page = context.pages()[0] ?? (await context.newPage());
     await page.goto('https://www.skool.com/login', { waitUntil: 'domcontentloaded' });
-    console.log('\nA browser window is open. Sign in to Skool there.');
-    console.log('Waiting for you to reach a logged-in page (Ctrl+C to cancel)...\n');
     await page.waitForFunction(() => {
         const el = document.getElementById('__NEXT_DATA__');
         if (!el?.textContent)
@@ -65,6 +73,5 @@ export async function login(onContext) {
             return false;
         }
     }, undefined, { timeout: 0 });
-    console.log('Signed in. Session saved, future runs will not need this.');
     await context.close();
 }

package/dist/cli.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
+import { join, resolve } from 'node:path';
 import { styleText } from 'node:util';
 import { Command } from 'commander';
 import { HttpFetcher } from './fetch/http.js';
@@ -8,8 +9,8 @@ import { ResilientFetcher } from './fetch/resilient.js';
 import { openDb } from './store/db.js';
 import { syncClassroom } from './sync.js';
 import { formatSummary } from './tui/summary.js';
-import { OUTCOME_STYLE, formatDuration } from './tui/theme.js';
-import { login, profileDir, dbPath, ensureRoot, isLoggedIn } from './auth/session.js';
+import { OUTCOME_STYLE, displayPath, formatDuration } from './tui/theme.js';
+import { login, profileDir, dbPath, defaultOutRoot, ensureRoot, isLoggedIn } from './auth/session.js';
 // Color only for a human at a terminal; pipes and NO_COLOR get plain text.
 const useColor = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR;
 const paint = (color, text) => (useColor ? styleText(color, text) : text);
@@ -32,12 +33,14 @@ program
     .command('login')
     .description('Sign in to Skool once; the session is reused by later runs')
     .action(async () => {
+    console.log('A browser window is opening. Sign in to Skool there (Ctrl+C to cancel)…');
     await login();
+    console.log(`${paint('green', '✓')} Signed in. Future runs reuse this session.`);
 });
 program
     .command('sync')
     .argument('<slug>', 'community slug, e.g. demo from skool.com/demo')
-    .option('-o, --out <dir>', 'output directory', './out')
+    .option('-o, --out <dir>', 'output directory', defaultOutRoot())
     .option('-c, --concurrency <n>', 'parallel requests', '4')
     .option('--videos', 'also download every lesson video (needs yt-dlp)')
     .description('Pull a community classroom to disk as transcripts (and optionally videos)')
@@ -50,7 +53,7 @@ program
         return;
     }
     await ensureRoot();
-    const outDir = `${options.out}/${slug}`;
+    const outDir = join(resolve(options.out), slug);
     const db = openDb(dbPath());
     const browser = new BrowserFetcher(profileDir());
     const fetcher = new ResilientFetcher(new HttpFetcher(), async () => browser, isLoggedIn);
@@ -71,7 +74,7 @@ program
             },
         });
         console.log('');
-        for (const line of formatSummary(summary, outDir, paint))
+        for (const line of formatSummary(summary, displayPath(outDir), paint))
             console.log(line);
         console.log(paint('dim', `Finished in ${formatDuration(Date.now() - startedAt)}`));
         if (fetcher.escalatedRoutes.size > 0) {

package/dist/fetch/browser.js

@@ -1,3 +1,4 @@
+import { browserLaunchOptions } from '../auth/session.js';
 /**
  * Playwright-backed fallback. Playwright is imported dynamically so that users
  * who never hit the fallback never pay its startup cost — and so the package
@@ -20,7 +21,7 @@ export class BrowserFetcher {
                 this.contextPromise ??= (async () => {
                     const { chromium } = await import('playwright');
                     return await chromium.launchPersistentContext(this.profileDir, {
-                        channel: 'chrome',
+                        ...browserLaunchOptions(),
                         headless: true,
                     });
                 })();

package/dist/fetch/chromeSetup.d.ts

@@ -15,6 +15,8 @@
  * A missing/unreadable/empty marker is just treated as "not installed".
  */
 export declare function isChromeMarkedInstalled(markerPath: string): boolean;
+/** The browser executable a past run confirmed, if the marker exists and that path still does. */
+export declare function markedExecutablePath(markerPath: string): string | null;
 /** Records the confirmed Chrome executable's path, so later runs can skip
  *  the probe by cheaply checking that path still exists on disk. */
 export declare function markChromeInstalled(markerPath: string, executablePath: string): Promise<void>;
@@ -37,6 +39,8 @@ export declare function markChromeInstalled(markerPath: string, executablePath:
  * run) never pays Playwright's module-load cost at all.
  */
 export declare function probeChromeLaunchable(): Promise<string | undefined>;
+/** Which browser to install when none is launchable: Chrome where that needs no root, Chromium on Linux. */
+export declare const INSTALL_TARGET: string;
 /**
  * Runs the equivalent of `npx playwright install chrome` in-process, by
  * spawning Playwright's own bundled CLI script directly. Preferred over

package/dist/fetch/chromeSetup.js

@@ -20,16 +20,20 @@ import { fileURLToPath } from 'node:url';
  * A missing/unreadable/empty marker is just treated as "not installed".
  */
 export function isChromeMarkedInstalled(markerPath) {
+    return markedExecutablePath(markerPath) !== null;
+}
+/** The browser executable a past run confirmed, if the marker exists and that path still does. */
+export function markedExecutablePath(markerPath) {
     if (!existsSync(markerPath))
-        return false;
+        return null;
     let storedPath;
     try {
         storedPath = readFileSync(markerPath, 'utf8').trim();
     }
     catch {
-        return false;
+        return null;
     }
-    return storedPath.length > 0 && existsSync(storedPath);
+    return storedPath.length > 0 && existsSync(storedPath) ? storedPath : null;
 }
 /** Records the confirmed Chrome executable's path, so later runs can skip
  *  the probe by cheaply checking that path still exists on disk. */
@@ -55,20 +59,36 @@ export async function markChromeInstalled(markerPath, executablePath) {
  * run) never pays Playwright's module-load cost at all.
  */
 export async function probeChromeLaunchable() {
+    let chromium;
     try {
-        const { chromium } = await import('playwright');
-        const server = await chromium.launchServer({ channel: 'chrome', headless: true });
-        try {
-            return server.process().spawnfile;
-        }
-        finally {
-            await server.close();
-        }
+        ({ chromium } = await import('playwright'));
     }
     catch {
         return undefined;
     }
+    // Prefer the user's real Chrome; fall back to Playwright's own Chromium (what gets
+    // installed on Linux, where installing Chrome itself needs root).
+    const candidates = [{ channel: 'chrome' }];
+    if (existsSync(chromium.executablePath()))
+        candidates.push({ executablePath: chromium.executablePath() });
+    for (const options of candidates) {
+        try {
+            const server = await chromium.launchServer({ ...options, headless: true });
+            try {
+                return server.process().spawnfile;
+            }
+            finally {
+                await server.close();
+            }
+        }
+        catch {
+            // try the next candidate
+        }
+    }
+    return undefined;
 }
+/** Which browser to install when none is launchable: Chrome where that needs no root, Chromium on Linux. */
+export const INSTALL_TARGET = process.platform === 'linux' ? 'chromium' : 'chrome';
 /** Resolves the on-disk path to Playwright's own bundled CLI script, via the
  *  package's exported `./package.json` subpath (not `./cli.js`, which isn't
  *  in Playwright's `exports` map and so can't be resolved directly) — this
@@ -94,7 +114,7 @@ async function resolvePlaywrightCliPath() {
 export async function installChromeViaCli(onProgress) {
     const cliPath = await resolvePlaywrightCliPath();
     await new Promise((resolve, reject) => {
-        const child = spawn(process.execPath, [cliPath, 'install', 'chrome'], {
+        const child = spawn(process.execPath, [cliPath, 'install', INSTALL_TARGET], {
             stdio: ['ignore', 'pipe', 'pipe'],
         });
         let stdout = '';

package/dist/sync.js

@@ -1,7 +1,8 @@
 import { listCourses, listLessons } from './discover/skool.js';
 import { getTranscript } from './media/index.js';
 import { downloadVideo, hasYtDlp } from './media/download.js';
-import { videoPath, writeTranscript } from './store/markdown.js';
+import { existsSync } from 'node:fs';
+import { transcriptPath, videoPath, writeTranscript } from './store/markdown.js';
 /** Run tasks with a bounded number in flight. */
 async function pooled(tasks, limit) {
     const results = new Array(tasks.length);
@@ -156,7 +157,8 @@ export async function syncClassroom(options) {
         catch (error) {
             return record('failed', item, `could not read transcript status: ${errorMessage(error)}`);
         }
-        if (status === 'ok')
+        // 'ok' in the db only means it was written *somewhere*; skip only when the file is in this outDir.
+        if (status === 'ok' && existsSync(transcriptPath(outDir, item, padWidth)))
             return record('skipped', item);
         let result;
         try {

package/dist/tui/App.d.ts

@@ -2,12 +2,14 @@ import React from 'react';
 import type { CommunityRef } from '../discover/communities.js';
 import type { ProgressEvent, SyncSummary } from '../sync.js';
 export interface AppControllers {
-    /** Output root directory, e.g. './out' — the community slug is appended. */
+    /** Absolute output root, e.g. ~/skrape; the community slug is appended. */
     outRoot: string;
     checkLoggedIn: () => Promise<boolean>;
     login: () => Promise<void>;
     discoverCommunities: () => Promise<CommunityRef[] | null>;
     countAccessibleCourses: (slug: string) => Promise<number>;
+    /** Checked when the user picks videos, so installing it in another terminal works without a restart. */
+    hasYtDlp: () => Promise<boolean>;
     runSync: (slug: string, outDir: string, onProgress: (event: ProgressEvent) => void, videos: boolean) => Promise<SyncSummary>;
 }
 /**

package/dist/tui/App.js

@@ -4,7 +4,7 @@ import { Box, Text, useApp, useInput } from 'ink';
 import { SelectList } from './SelectList.js';
 import { applyProgress, createProgressState, formatProgressBar } from './progress.js';
 import { formatVideoLine, nothingTranscribed, sortedProblems } from './summary.js';
-import { ACCENT, OUTCOME_STYLE, estimateRemainingMs, formatDuration } from './theme.js';
+import { ACCENT, OUTCOME_STYLE, displayPath, estimateRemainingMs, formatDuration } from './theme.js';
 import { isValidSlug, normalizeSlug } from './slug.js';
 import { MENU_ITEMS, nextStepForMenuChoice } from './flow.js';
 import { revealOutputFolder } from './reveal.js';
@@ -70,6 +70,7 @@ function Timing({ startedAt, done, total }) {
     const remaining = estimateRemainingMs(elapsed, done, total);
     return (_jsxs(Text, { dimColor: true, children: [formatDuration(elapsed), " elapsed", remaining !== null && ` · ~${formatDuration(remaining)} left`] }));
 }
+const YT_DLP_MISSING = 'Video downloads need yt-dlp, which isn\'t installed. In another terminal run: brew install yt-dlp ffmpeg (or see github.com/yt-dlp/yt-dlp), then pick again.';
 const MODE_ITEMS = [
     { label: 'Transcripts', value: 'transcripts', hint: 'fast · text only' },
     { label: 'Transcripts + videos', value: 'videos', hint: 'needs yt-dlp · large download' },
@@ -300,19 +301,32 @@ export function App({ controllers }) {
                                 ['Community', _jsx(Text, { bold: true, children: name })],
                                 ['URL', `skool.com/${slug}`],
                                 ['Courses', `${courseCount} you can access`],
-                                ['Saving to', outDir],
+                                ['Saving to', displayPath(outDir)],
                             ] }), _jsx(Text, { children: " " }), _jsx(Text, { bold: true, children: "What should skrape pull?" }), _jsx(Text, { children: " " }), _jsx(SelectList, { items: MODE_ITEMS, onSelect: (mode) => {
                                 if (mode === 'quit') {
                                     exit();
                                     return;
                                 }
-                                setStep({
+                                const start = () => setStep({
                                     kind: 'syncing', slug, name, courseCount, outDir,
                                     videos: mode === 'videos',
                                     startedAt: Date.now(),
                                     progress: createProgressState(0),
                                 });
-                            } })] }));
+                                if (mode !== 'videos') {
+                                    start();
+                                    return;
+                                }
+                                controllers
+                                    .hasYtDlp()
+                                    .catch(() => false)
+                                    .then((ok) => {
+                                    if (ok)
+                                        start();
+                                    else
+                                        setStep({ ...step, notice: YT_DLP_MISSING });
+                                });
+                            } }), step.notice && (_jsx(Box, { marginTop: 1, children: _jsx(Text, { color: "yellow", children: step.notice }) }))] }));
             }
             case 'syncing': {
                 const { progress } = step;
@@ -335,7 +349,7 @@ export function App({ controllers }) {
                 ];
                 if (videoLine)
                     rows.push(['Videos', videoLine]);
-                rows.push(['Saved to', _jsx(Text, { color: ACCENT, children: step.outDir })]);
+                rows.push(['Saved to', _jsx(Text, { color: ACCENT, children: displayPath(step.outDir) })]);
                 return (_jsxs(Box, { flexDirection: "column", children: [empty && summary.counts.skipped === 0 ? (_jsx(Text, { color: "yellow", bold: true, children: "\u26A0 Nothing was transcribed." })) : (_jsxs(Text, { color: "green", bold: true, children: ["\u2713 ", empty ? `${step.name} is up to date` : `Synced ${step.name}`, _jsxs(Text, { dimColor: true, bold: false, children: ["  in ", formatDuration(step.elapsedMs)] })] })), _jsx(Text, { children: " " }), _jsx(Card, { rows: rows }), summary.problems.length > 0 && (_jsxs(Box, { flexDirection: "column", marginTop: 1, children: [_jsxs(Text, { bold: true, children: ["Needs attention (", summary.problems.length, ")"] }), sortedProblems(summary).map((problem, index) => (
                                 // eslint-disable-next-line react/no-array-index-key
                                 _jsxs(Text, { wrap: "truncate-end", children: [_jsx(Text, { color: OUTCOME_STYLE[problem.outcome].color, children: OUTCOME_STYLE[problem.outcome].icon }), ' ', problem.course, " \u203A ", problem.title, "  ", _jsx(Text, { dimColor: true, children: problem.reason })] }, index)))] })), _jsx(Text, { children: " " }), _jsx(Text, { bold: true, children: "What next?" }), _jsx(Text, { children: " " }), _jsx(SelectList, { items: MENU_ITEMS, onSelect: (choice) => handleMenuChoice(done, choice) })] }));

package/dist/tui/chromeSetup.js

@@ -1,11 +1,5 @@
-/**
- * Pure decision logic for the one-time "make sure Chrome is installed" step
- * that runs before the session check. Kept free of fs/child_process/Playwright
- * details (those live in `../fetch/chromeSetup.ts`) so the branching here is
- * unit-testable against plain fakes, the same split used by `flow.ts` and
- * `browserLifecycle.ts` elsewhere in this package.
- */
-const MANUAL_FALLBACK = 'npx playwright install chrome';
+import { INSTALL_TARGET } from '../fetch/chromeSetup.js';
+const MANUAL_FALLBACK = `npx playwright install ${INSTALL_TARGET}`;
 /** Thrown when the automatic install fails. Its message is already
  *  human-actionable (includes the manual fallback command) — callers can
  *  surface `.message` directly without needing to know install internals. */

package/dist/tui/run.js

@@ -5,13 +5,13 @@ import { BrowserFetcher } from '../fetch/browser.js';
 import { isChromeMarkedInstalled, installChromeViaCli, markChromeInstalled, probeChromeLaunchable } from '../fetch/chromeSetup.js';
 import { openDb } from '../store/db.js';
 import { syncClassroom } from '../sync.js';
-import { chromeMarkerPath, dbPath, ensureRoot, isLoggedIn, login, profileDir } from '../auth/session.js';
+import { hasYtDlp } from '../media/download.js';
+import { chromeMarkerPath, dbPath, defaultOutRoot, ensureRoot, isLoggedIn, login, profileDir } from '../auth/session.js';
 import { listCourses } from '../discover/skool.js';
 import { listUserCommunities } from '../discover/communities.js';
 import { App } from './App.js';
 import { createBrowserLifecycle } from './browserLifecycle.js';
 import { ensureChromeReady } from './chromeSetup.js';
-const OUT_ROOT = './out';
 /**
  * Runs the guided, no-arguments flow: session check, community picker,
  * confirmation, live sync progress, and a result summary. Owns the db and
@@ -38,8 +38,8 @@ export async function runGuidedFlow() {
             console.log('\nBrowser setup complete.\n');
             const executablePath = await probeChromeLaunchable();
             if (!executablePath) {
-                throw new Error('Chrome installed, but still could not be launched. This usually means a platform-specific ' +
-                    'dependency is missing. See the Playwright install output above for details.');
+                throw new Error('The browser installed, but still could not be launched. This usually means a system library is missing' +
+                    (process.platform === 'linux' ? '. Fix it with: sudo npx playwright install-deps chromium' : '. See the install output above.'));
             }
             return executablePath;
         },
@@ -73,7 +73,7 @@ export async function runGuidedFlow() {
     };
     process.on('SIGINT', onSigint);
     const controllers = {
-        outRoot: OUT_ROOT,
+        outRoot: defaultOutRoot(),
         checkLoggedIn: async () => lifecycle.checkLoggedIn(),
         login: async () => {
             await lifecycle.login();
@@ -83,6 +83,7 @@ export async function runGuidedFlow() {
             const courses = await listCourses(slug, lifecycle.getFetcher());
             return courses.filter((course) => course.hasAccess).length;
         },
+        hasYtDlp,
         runSync: async (slug, outDir, onProgress, videos) => syncClassroom({ slug, outDir, db, fetcher: lifecycle.getFetcher(), concurrency: 4, videos, onProgress }),
     };
     const { waitUntilExit } = render(React.createElement(App, { controllers }));

package/dist/tui/theme.d.ts

@@ -7,6 +7,8 @@ export declare const OUTCOME_STYLE: Record<Outcome, {
     label: string;
     color: OutcomeColor;
 }>;
+/** A path as a person reads it: absolute, with the home folder shortened to ~. */
+export declare function displayPath(path: string): string;
 export declare function formatDuration(ms: number): string;
 /** Remaining time from a linear rate; null until there's at least one data point. */
 export declare function estimateRemainingMs(elapsedMs: number, done: number, total: number): number | null;

package/dist/tui/theme.js

@@ -1,3 +1,4 @@
+import { homedir } from 'node:os';
 /** Brand accent: peach. Ink takes hex; the plain CLI falls back to a named color. */
 export const ACCENT = '#FFB38A';
 export const OUTCOME_STYLE = {
@@ -8,6 +9,11 @@ export const OUTCOME_STYLE = {
     unavailable: { icon: '!', label: 'unavailable', color: 'yellow' },
     failed: { icon: '✗', label: 'failed', color: 'red' },
 };
+/** A path as a person reads it: absolute, with the home folder shortened to ~. */
+export function displayPath(path) {
+    const home = homedir();
+    return path === home || path.startsWith(`${home}/`) ? `~${path.slice(home.length)}` : path;
+}
 export function formatDuration(ms) {
     const seconds = Math.max(0, Math.round(ms / 1000));
     if (seconds < 60)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mogulmoretti/skrape",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Read Skool course content instead of watching it — pulls the classroom of a community you belong to into clean, readable transcripts.",
   "keywords": [
     "skool",