@mogulmoretti/skrape 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/auth/session.d.ts +8 -0
- package/dist/auth/session.js +11 -4
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +9 -5
- package/dist/fetch/browser.js +2 -1
- package/dist/fetch/chromeSetup.d.ts +4 -0
- package/dist/fetch/chromeSetup.js +32 -12
- package/dist/quiet.d.ts +1 -0
- package/dist/quiet.js +9 -0
- package/dist/store/db.js +5 -4
- package/dist/sync.js +5 -3
- package/dist/tui/App.d.ts +3 -1
- package/dist/tui/App.js +19 -5
- package/dist/tui/chromeSetup.js +2 -8
- package/dist/tui/run.js +6 -5
- package/dist/tui/theme.d.ts +2 -0
- package/dist/tui/theme.js +6 -0
- package/package.json +3 -5
package/README.md
CHANGED
|
@@ -57,8 +57,9 @@ From there you can sync another community, re-sync the same one (it's
|
|
|
57
57
|
incremental: already-fetched lessons are skipped), open the output folder,
|
|
58
58
|
or quit.
|
|
59
59
|
|
|
60
|
-
Output lands in
|
|
61
|
-
(if you asked for them) in
|
|
60
|
+
Output lands in `~/skrape/<slug>/transcripts/<course>/NN-lesson.md`, and videos
|
|
61
|
+
(if you asked for them) in `~/skrape/<slug>/videos/<course>/NN-lesson.mp4`,
|
|
62
|
+
wherever you run skrape from.
|
|
62
63
|
|
|
63
64
|
### Downloading videos
|
|
64
65
|
|
|
@@ -79,7 +80,7 @@ The underlying subcommands work directly too, without the guided flow:
|
|
|
79
80
|
|
|
80
81
|
skrape login # once: sign in by hand, session persists
|
|
81
82
|
skrape sync <slug> # slug is the part after skool.com/
|
|
82
|
-
skrape sync <slug> -o
|
|
83
|
+
skrape sync <slug> -o ~/notes -c 4 # custom output dir / concurrency
|
|
83
84
|
skrape sync <slug> --videos # transcripts + every lesson video
|
|
84
85
|
|
|
85
86
|
### Building from source
|
package/dist/auth/session.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export declare function profileDir(): string;
|
|
2
|
+
/** Where synced communities land unless -o says otherwise: one fixed place, whatever the cwd. */
|
|
3
|
+
export declare function defaultOutRoot(): string;
|
|
2
4
|
export declare function dbPath(): string;
|
|
3
5
|
/**
|
|
4
6
|
* Marker file written once the one-time Chrome/Playwright browser install
|
|
@@ -6,6 +8,12 @@ export declare function dbPath(): string;
|
|
|
6
8
|
* Its presence lets later runs skip re-probing with a real browser launch.
|
|
7
9
|
*/
|
|
8
10
|
export declare function chromeMarkerPath(): string;
|
|
11
|
+
/** Launch the browser first-run setup confirmed; before any setup ran, assume the user's Chrome. */
|
|
12
|
+
export declare function browserLaunchOptions(): {
|
|
13
|
+
executablePath: string;
|
|
14
|
+
} | {
|
|
15
|
+
channel: string;
|
|
16
|
+
};
|
|
9
17
|
export declare function ensureRoot(): Promise<void>;
|
|
10
18
|
export declare function isLoggedIn(html: string): boolean;
|
|
11
19
|
/** A handle callers can use to force-close a launched login context, e.g. on Ctrl+C. */
|
package/dist/auth/session.js
CHANGED
|
@@ -2,10 +2,15 @@ import { homedir } from 'node:os';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { mkdir } from 'node:fs/promises';
|
|
4
4
|
import { extractNextData, PayloadParseError } from '../fetch/nextdata.js';
|
|
5
|
+
import { markedExecutablePath } from '../fetch/chromeSetup.js';
|
|
5
6
|
const ROOT = join(homedir(), '.skool-skrape');
|
|
6
7
|
export function profileDir() {
|
|
7
8
|
return join(ROOT, 'chrome-profile');
|
|
8
9
|
}
|
|
10
|
+
/** Where synced communities land unless -o says otherwise: one fixed place, whatever the cwd. */
|
|
11
|
+
export function defaultOutRoot() {
|
|
12
|
+
return join(homedir(), 'skrape');
|
|
13
|
+
}
|
|
9
14
|
export function dbPath() {
|
|
10
15
|
return join(ROOT, 'skool.db');
|
|
11
16
|
}
|
|
@@ -17,6 +22,11 @@ export function dbPath() {
|
|
|
17
22
|
export function chromeMarkerPath() {
|
|
18
23
|
return join(ROOT, 'chrome-installed');
|
|
19
24
|
}
|
|
25
|
+
/** Launch the browser first-run setup confirmed; before any setup ran, assume the user's Chrome. */
|
|
26
|
+
export function browserLaunchOptions() {
|
|
27
|
+
const executablePath = markedExecutablePath(chromeMarkerPath());
|
|
28
|
+
return executablePath ? { executablePath } : { channel: 'chrome' };
|
|
29
|
+
}
|
|
20
30
|
export async function ensureRoot() {
|
|
21
31
|
await mkdir(ROOT, { recursive: true });
|
|
22
32
|
}
|
|
@@ -46,14 +56,12 @@ export async function login(onContext) {
|
|
|
46
56
|
await ensureRoot();
|
|
47
57
|
const { chromium } = await import('playwright');
|
|
48
58
|
const context = await chromium.launchPersistentContext(profileDir(), {
|
|
49
|
-
|
|
59
|
+
...browserLaunchOptions(),
|
|
50
60
|
headless: false,
|
|
51
61
|
});
|
|
52
62
|
onContext?.(context);
|
|
53
63
|
const page = context.pages()[0] ?? (await context.newPage());
|
|
54
64
|
await page.goto('https://www.skool.com/login', { waitUntil: 'domcontentloaded' });
|
|
55
|
-
console.log('\nA browser window is open. Sign in to Skool there.');
|
|
56
|
-
console.log('Waiting for you to reach a logged-in page (Ctrl+C to cancel)...\n');
|
|
57
65
|
await page.waitForFunction(() => {
|
|
58
66
|
const el = document.getElementById('__NEXT_DATA__');
|
|
59
67
|
if (!el?.textContent)
|
|
@@ -65,6 +73,5 @@ export async function login(onContext) {
|
|
|
65
73
|
return false;
|
|
66
74
|
}
|
|
67
75
|
}, undefined, { timeout: 0 });
|
|
68
|
-
console.log('Signed in. Session saved, future runs will not need this.');
|
|
69
76
|
await context.close();
|
|
70
77
|
}
|
package/dist/cli.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
2
|
+
import './quiet.js';
|
package/dist/cli.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import './quiet.js';
|
|
2
3
|
import { createRequire } from 'node:module';
|
|
4
|
+
import { join, resolve } from 'node:path';
|
|
3
5
|
import { styleText } from 'node:util';
|
|
4
6
|
import { Command } from 'commander';
|
|
5
7
|
import { HttpFetcher } from './fetch/http.js';
|
|
@@ -8,8 +10,8 @@ import { ResilientFetcher } from './fetch/resilient.js';
|
|
|
8
10
|
import { openDb } from './store/db.js';
|
|
9
11
|
import { syncClassroom } from './sync.js';
|
|
10
12
|
import { formatSummary } from './tui/summary.js';
|
|
11
|
-
import { OUTCOME_STYLE, formatDuration } from './tui/theme.js';
|
|
12
|
-
import { login, profileDir, dbPath, ensureRoot, isLoggedIn } from './auth/session.js';
|
|
13
|
+
import { OUTCOME_STYLE, displayPath, formatDuration } from './tui/theme.js';
|
|
14
|
+
import { login, profileDir, dbPath, defaultOutRoot, ensureRoot, isLoggedIn } from './auth/session.js';
|
|
13
15
|
// Color only for a human at a terminal; pipes and NO_COLOR get plain text.
|
|
14
16
|
const useColor = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR;
|
|
15
17
|
const paint = (color, text) => (useColor ? styleText(color, text) : text);
|
|
@@ -32,12 +34,14 @@ program
|
|
|
32
34
|
.command('login')
|
|
33
35
|
.description('Sign in to Skool once; the session is reused by later runs')
|
|
34
36
|
.action(async () => {
|
|
37
|
+
console.log('A browser window is opening. Sign in to Skool there (Ctrl+C to cancel)…');
|
|
35
38
|
await login();
|
|
39
|
+
console.log(`${paint('green', '✓')} Signed in. Future runs reuse this session.`);
|
|
36
40
|
});
|
|
37
41
|
program
|
|
38
42
|
.command('sync')
|
|
39
43
|
.argument('<slug>', 'community slug, e.g. demo from skool.com/demo')
|
|
40
|
-
.option('-o, --out <dir>', 'output directory',
|
|
44
|
+
.option('-o, --out <dir>', 'output directory', defaultOutRoot())
|
|
41
45
|
.option('-c, --concurrency <n>', 'parallel requests', '4')
|
|
42
46
|
.option('--videos', 'also download every lesson video (needs yt-dlp)')
|
|
43
47
|
.description('Pull a community classroom to disk as transcripts (and optionally videos)')
|
|
@@ -50,7 +54,7 @@ program
|
|
|
50
54
|
return;
|
|
51
55
|
}
|
|
52
56
|
await ensureRoot();
|
|
53
|
-
const outDir =
|
|
57
|
+
const outDir = join(resolve(options.out), slug);
|
|
54
58
|
const db = openDb(dbPath());
|
|
55
59
|
const browser = new BrowserFetcher(profileDir());
|
|
56
60
|
const fetcher = new ResilientFetcher(new HttpFetcher(), async () => browser, isLoggedIn);
|
|
@@ -71,7 +75,7 @@ program
|
|
|
71
75
|
},
|
|
72
76
|
});
|
|
73
77
|
console.log('');
|
|
74
|
-
for (const line of formatSummary(summary, outDir, paint))
|
|
78
|
+
for (const line of formatSummary(summary, displayPath(outDir), paint))
|
|
75
79
|
console.log(line);
|
|
76
80
|
console.log(paint('dim', `Finished in ${formatDuration(Date.now() - startedAt)}`));
|
|
77
81
|
if (fetcher.escalatedRoutes.size > 0) {
|
package/dist/fetch/browser.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { browserLaunchOptions } from '../auth/session.js';
|
|
1
2
|
/**
|
|
2
3
|
* Playwright-backed fallback. Playwright is imported dynamically so that users
|
|
3
4
|
* who never hit the fallback never pay its startup cost — and so the package
|
|
@@ -20,7 +21,7 @@ export class BrowserFetcher {
|
|
|
20
21
|
this.contextPromise ??= (async () => {
|
|
21
22
|
const { chromium } = await import('playwright');
|
|
22
23
|
return await chromium.launchPersistentContext(this.profileDir, {
|
|
23
|
-
|
|
24
|
+
...browserLaunchOptions(),
|
|
24
25
|
headless: true,
|
|
25
26
|
});
|
|
26
27
|
})();
|
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
* A missing/unreadable/empty marker is just treated as "not installed".
|
|
16
16
|
*/
|
|
17
17
|
export declare function isChromeMarkedInstalled(markerPath: string): boolean;
|
|
18
|
+
/** The browser executable a past run confirmed, if the marker exists and that path still does. */
|
|
19
|
+
export declare function markedExecutablePath(markerPath: string): string | null;
|
|
18
20
|
/** Records the confirmed Chrome executable's path, so later runs can skip
|
|
19
21
|
* the probe by cheaply checking that path still exists on disk. */
|
|
20
22
|
export declare function markChromeInstalled(markerPath: string, executablePath: string): Promise<void>;
|
|
@@ -37,6 +39,8 @@ export declare function markChromeInstalled(markerPath: string, executablePath:
|
|
|
37
39
|
* run) never pays Playwright's module-load cost at all.
|
|
38
40
|
*/
|
|
39
41
|
export declare function probeChromeLaunchable(): Promise<string | undefined>;
|
|
42
|
+
/** Which browser to install when none is launchable: Chrome where that needs no root, Chromium on Linux. */
|
|
43
|
+
export declare const INSTALL_TARGET: string;
|
|
40
44
|
/**
|
|
41
45
|
* Runs the equivalent of `npx playwright install chrome` in-process, by
|
|
42
46
|
* spawning Playwright's own bundled CLI script directly. Preferred over
|
|
@@ -20,16 +20,20 @@ import { fileURLToPath } from 'node:url';
|
|
|
20
20
|
* A missing/unreadable/empty marker is just treated as "not installed".
|
|
21
21
|
*/
|
|
22
22
|
export function isChromeMarkedInstalled(markerPath) {
|
|
23
|
+
return markedExecutablePath(markerPath) !== null;
|
|
24
|
+
}
|
|
25
|
+
/** The browser executable a past run confirmed, if the marker exists and that path still does. */
|
|
26
|
+
export function markedExecutablePath(markerPath) {
|
|
23
27
|
if (!existsSync(markerPath))
|
|
24
|
-
return
|
|
28
|
+
return null;
|
|
25
29
|
let storedPath;
|
|
26
30
|
try {
|
|
27
31
|
storedPath = readFileSync(markerPath, 'utf8').trim();
|
|
28
32
|
}
|
|
29
33
|
catch {
|
|
30
|
-
return
|
|
34
|
+
return null;
|
|
31
35
|
}
|
|
32
|
-
return storedPath.length > 0 && existsSync(storedPath);
|
|
36
|
+
return storedPath.length > 0 && existsSync(storedPath) ? storedPath : null;
|
|
33
37
|
}
|
|
34
38
|
/** Records the confirmed Chrome executable's path, so later runs can skip
|
|
35
39
|
* the probe by cheaply checking that path still exists on disk. */
|
|
@@ -55,20 +59,36 @@ export async function markChromeInstalled(markerPath, executablePath) {
|
|
|
55
59
|
* run) never pays Playwright's module-load cost at all.
|
|
56
60
|
*/
|
|
57
61
|
export async function probeChromeLaunchable() {
|
|
62
|
+
let chromium;
|
|
58
63
|
try {
|
|
59
|
-
|
|
60
|
-
const server = await chromium.launchServer({ channel: 'chrome', headless: true });
|
|
61
|
-
try {
|
|
62
|
-
return server.process().spawnfile;
|
|
63
|
-
}
|
|
64
|
-
finally {
|
|
65
|
-
await server.close();
|
|
66
|
-
}
|
|
64
|
+
({ chromium } = await import('playwright'));
|
|
67
65
|
}
|
|
68
66
|
catch {
|
|
69
67
|
return undefined;
|
|
70
68
|
}
|
|
69
|
+
// Prefer the user's real Chrome; fall back to Playwright's own Chromium (what gets
|
|
70
|
+
// installed on Linux, where installing Chrome itself needs root).
|
|
71
|
+
const candidates = [{ channel: 'chrome' }];
|
|
72
|
+
if (existsSync(chromium.executablePath()))
|
|
73
|
+
candidates.push({ executablePath: chromium.executablePath() });
|
|
74
|
+
for (const options of candidates) {
|
|
75
|
+
try {
|
|
76
|
+
const server = await chromium.launchServer({ ...options, headless: true });
|
|
77
|
+
try {
|
|
78
|
+
return server.process().spawnfile;
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
await server.close();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
// try the next candidate
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return undefined;
|
|
71
89
|
}
|
|
90
|
+
/** Which browser to install when none is launchable: Chrome where that needs no root, Chromium on Linux. */
|
|
91
|
+
export const INSTALL_TARGET = process.platform === 'linux' ? 'chromium' : 'chrome';
|
|
72
92
|
/** Resolves the on-disk path to Playwright's own bundled CLI script, via the
|
|
73
93
|
* package's exported `./package.json` subpath (not `./cli.js`, which isn't
|
|
74
94
|
* in Playwright's `exports` map and so can't be resolved directly) — this
|
|
@@ -94,7 +114,7 @@ async function resolvePlaywrightCliPath() {
|
|
|
94
114
|
export async function installChromeViaCli(onProgress) {
|
|
95
115
|
const cliPath = await resolvePlaywrightCliPath();
|
|
96
116
|
await new Promise((resolve, reject) => {
|
|
97
|
-
const child = spawn(process.execPath, [cliPath, 'install',
|
|
117
|
+
const child = spawn(process.execPath, [cliPath, 'install', INSTALL_TARGET], {
|
|
98
118
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
99
119
|
});
|
|
100
120
|
let stdout = '';
|
package/dist/quiet.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/quiet.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Node 22 flags its built-in SQLite as experimental on first use. That notice means nothing to a
|
|
2
|
+
// skrape user and would print over the UI, so drop exactly that one warning. Imported first in cli.ts.
|
|
3
|
+
const emitWarning = process.emitWarning.bind(process);
|
|
4
|
+
process.emitWarning = ((warning, ...rest) => {
|
|
5
|
+
if (String(warning).includes('SQLite is an experimental feature'))
|
|
6
|
+
return;
|
|
7
|
+
emitWarning(warning, ...rest);
|
|
8
|
+
});
|
|
9
|
+
export {};
|
package/dist/store/db.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
// Node's built-in SQLite: no native module to compile, so installs never depend on a C toolchain.
|
|
2
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
2
3
|
import { createHash } from 'node:crypto';
|
|
3
4
|
import { dirname } from 'node:path';
|
|
4
5
|
import { mkdirSync } from 'node:fs';
|
|
@@ -57,9 +58,9 @@ export function openDb(path) {
|
|
|
57
58
|
mkdirSync(dir, { recursive: true });
|
|
58
59
|
}
|
|
59
60
|
}
|
|
60
|
-
const db = new
|
|
61
|
-
db.
|
|
62
|
-
db.
|
|
61
|
+
const db = new DatabaseSync(path);
|
|
62
|
+
db.exec('PRAGMA journal_mode = WAL');
|
|
63
|
+
db.exec('PRAGMA foreign_keys = ON');
|
|
63
64
|
db.exec(SCHEMA);
|
|
64
65
|
return {
|
|
65
66
|
upsertCommunity(slug, name) {
|
package/dist/sync.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { listCourses, listLessons } from './discover/skool.js';
|
|
2
2
|
import { getTranscript } from './media/index.js';
|
|
3
3
|
import { downloadVideo, hasYtDlp } from './media/download.js';
|
|
4
|
-
import {
|
|
4
|
+
import { existsSync } from 'node:fs';
|
|
5
|
+
import { transcriptPath, videoPath, writeTranscript } from './store/markdown.js';
|
|
5
6
|
/** Run tasks with a bounded number in flight. */
|
|
6
7
|
async function pooled(tasks, limit) {
|
|
7
8
|
const results = new Array(tasks.length);
|
|
@@ -107,7 +108,7 @@ export async function syncClassroom(options) {
|
|
|
107
108
|
// swallow — the caller's callback is not our concern
|
|
108
109
|
}
|
|
109
110
|
};
|
|
110
|
-
// A
|
|
111
|
+
// A SQLite error from any of the three store calls below (disk full, locked db, FK
|
|
111
112
|
// violation) must become a per-lesson 'failed' outcome, not a run-ending rejection: a Promise.all
|
|
112
113
|
// over the worker pool would reject on the first such error while sibling workers are still in
|
|
113
114
|
// flight, and the CLI's finally-block browser.close() would then race a live worker into
|
|
@@ -156,7 +157,8 @@ export async function syncClassroom(options) {
|
|
|
156
157
|
catch (error) {
|
|
157
158
|
return record('failed', item, `could not read transcript status: ${errorMessage(error)}`);
|
|
158
159
|
}
|
|
159
|
-
|
|
160
|
+
// 'ok' in the db only means it was written *somewhere*; skip only when the file is in this outDir.
|
|
161
|
+
if (status === 'ok' && existsSync(transcriptPath(outDir, item, padWidth)))
|
|
160
162
|
return record('skipped', item);
|
|
161
163
|
let result;
|
|
162
164
|
try {
|
package/dist/tui/App.d.ts
CHANGED
|
@@ -2,12 +2,14 @@ import React from 'react';
|
|
|
2
2
|
import type { CommunityRef } from '../discover/communities.js';
|
|
3
3
|
import type { ProgressEvent, SyncSummary } from '../sync.js';
|
|
4
4
|
export interface AppControllers {
|
|
5
|
-
/**
|
|
5
|
+
/** Absolute output root, e.g. ~/skrape; the community slug is appended. */
|
|
6
6
|
outRoot: string;
|
|
7
7
|
checkLoggedIn: () => Promise<boolean>;
|
|
8
8
|
login: () => Promise<void>;
|
|
9
9
|
discoverCommunities: () => Promise<CommunityRef[] | null>;
|
|
10
10
|
countAccessibleCourses: (slug: string) => Promise<number>;
|
|
11
|
+
/** Checked when the user picks videos, so installing it in another terminal works without a restart. */
|
|
12
|
+
hasYtDlp: () => Promise<boolean>;
|
|
11
13
|
runSync: (slug: string, outDir: string, onProgress: (event: ProgressEvent) => void, videos: boolean) => Promise<SyncSummary>;
|
|
12
14
|
}
|
|
13
15
|
/**
|
package/dist/tui/App.js
CHANGED
|
@@ -4,7 +4,7 @@ import { Box, Text, useApp, useInput } from 'ink';
|
|
|
4
4
|
import { SelectList } from './SelectList.js';
|
|
5
5
|
import { applyProgress, createProgressState, formatProgressBar } from './progress.js';
|
|
6
6
|
import { formatVideoLine, nothingTranscribed, sortedProblems } from './summary.js';
|
|
7
|
-
import { ACCENT, OUTCOME_STYLE, estimateRemainingMs, formatDuration } from './theme.js';
|
|
7
|
+
import { ACCENT, OUTCOME_STYLE, displayPath, estimateRemainingMs, formatDuration } from './theme.js';
|
|
8
8
|
import { isValidSlug, normalizeSlug } from './slug.js';
|
|
9
9
|
import { MENU_ITEMS, nextStepForMenuChoice } from './flow.js';
|
|
10
10
|
import { revealOutputFolder } from './reveal.js';
|
|
@@ -70,6 +70,7 @@ function Timing({ startedAt, done, total }) {
|
|
|
70
70
|
const remaining = estimateRemainingMs(elapsed, done, total);
|
|
71
71
|
return (_jsxs(Text, { dimColor: true, children: [formatDuration(elapsed), " elapsed", remaining !== null && ` · ~${formatDuration(remaining)} left`] }));
|
|
72
72
|
}
|
|
73
|
+
const YT_DLP_MISSING = 'Video downloads need yt-dlp, which isn\'t installed. In another terminal run: brew install yt-dlp ffmpeg (or see github.com/yt-dlp/yt-dlp), then pick again.';
|
|
73
74
|
const MODE_ITEMS = [
|
|
74
75
|
{ label: 'Transcripts', value: 'transcripts', hint: 'fast · text only' },
|
|
75
76
|
{ label: 'Transcripts + videos', value: 'videos', hint: 'needs yt-dlp · large download' },
|
|
@@ -300,19 +301,32 @@ export function App({ controllers }) {
|
|
|
300
301
|
['Community', _jsx(Text, { bold: true, children: name })],
|
|
301
302
|
['URL', `skool.com/${slug}`],
|
|
302
303
|
['Courses', `${courseCount} you can access`],
|
|
303
|
-
['Saving to', outDir],
|
|
304
|
+
['Saving to', displayPath(outDir)],
|
|
304
305
|
] }), _jsx(Text, { children: " " }), _jsx(Text, { bold: true, children: "What should skrape pull?" }), _jsx(Text, { children: " " }), _jsx(SelectList, { items: MODE_ITEMS, onSelect: (mode) => {
|
|
305
306
|
if (mode === 'quit') {
|
|
306
307
|
exit();
|
|
307
308
|
return;
|
|
308
309
|
}
|
|
309
|
-
setStep({
|
|
310
|
+
const start = () => setStep({
|
|
310
311
|
kind: 'syncing', slug, name, courseCount, outDir,
|
|
311
312
|
videos: mode === 'videos',
|
|
312
313
|
startedAt: Date.now(),
|
|
313
314
|
progress: createProgressState(0),
|
|
314
315
|
});
|
|
315
|
-
|
|
316
|
+
if (mode !== 'videos') {
|
|
317
|
+
start();
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
controllers
|
|
321
|
+
.hasYtDlp()
|
|
322
|
+
.catch(() => false)
|
|
323
|
+
.then((ok) => {
|
|
324
|
+
if (ok)
|
|
325
|
+
start();
|
|
326
|
+
else
|
|
327
|
+
setStep({ ...step, notice: YT_DLP_MISSING });
|
|
328
|
+
});
|
|
329
|
+
} }), step.notice && (_jsx(Box, { marginTop: 1, children: _jsx(Text, { color: "yellow", children: step.notice }) }))] }));
|
|
316
330
|
}
|
|
317
331
|
case 'syncing': {
|
|
318
332
|
const { progress } = step;
|
|
@@ -335,7 +349,7 @@ export function App({ controllers }) {
|
|
|
335
349
|
];
|
|
336
350
|
if (videoLine)
|
|
337
351
|
rows.push(['Videos', videoLine]);
|
|
338
|
-
rows.push(['Saved to', _jsx(Text, { color: ACCENT, children: step.outDir })]);
|
|
352
|
+
rows.push(['Saved to', _jsx(Text, { color: ACCENT, children: displayPath(step.outDir) })]);
|
|
339
353
|
return (_jsxs(Box, { flexDirection: "column", children: [empty && summary.counts.skipped === 0 ? (_jsx(Text, { color: "yellow", bold: true, children: "\u26A0 Nothing was transcribed." })) : (_jsxs(Text, { color: "green", bold: true, children: ["\u2713 ", empty ? `${step.name} is up to date` : `Synced ${step.name}`, _jsxs(Text, { dimColor: true, bold: false, children: [" in ", formatDuration(step.elapsedMs)] })] })), _jsx(Text, { children: " " }), _jsx(Card, { rows: rows }), summary.problems.length > 0 && (_jsxs(Box, { flexDirection: "column", marginTop: 1, children: [_jsxs(Text, { bold: true, children: ["Needs attention (", summary.problems.length, ")"] }), sortedProblems(summary).map((problem, index) => (
|
|
340
354
|
// eslint-disable-next-line react/no-array-index-key
|
|
341
355
|
_jsxs(Text, { wrap: "truncate-end", children: [_jsx(Text, { color: OUTCOME_STYLE[problem.outcome].color, children: OUTCOME_STYLE[problem.outcome].icon }), ' ', problem.course, " \u203A ", problem.title, " ", _jsx(Text, { dimColor: true, children: problem.reason })] }, index)))] })), _jsx(Text, { children: " " }), _jsx(Text, { bold: true, children: "What next?" }), _jsx(Text, { children: " " }), _jsx(SelectList, { items: MENU_ITEMS, onSelect: (choice) => handleMenuChoice(done, choice) })] }));
|
package/dist/tui/chromeSetup.js
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
* that runs before the session check. Kept free of fs/child_process/Playwright
|
|
4
|
-
* details (those live in `../fetch/chromeSetup.ts`) so the branching here is
|
|
5
|
-
* unit-testable against plain fakes, the same split used by `flow.ts` and
|
|
6
|
-
* `browserLifecycle.ts` elsewhere in this package.
|
|
7
|
-
*/
|
|
8
|
-
const MANUAL_FALLBACK = 'npx playwright install chrome';
|
|
1
|
+
import { INSTALL_TARGET } from '../fetch/chromeSetup.js';
|
|
2
|
+
const MANUAL_FALLBACK = `npx playwright install ${INSTALL_TARGET}`;
|
|
9
3
|
/** Thrown when the automatic install fails. Its message is already
|
|
10
4
|
* human-actionable (includes the manual fallback command) — callers can
|
|
11
5
|
* surface `.message` directly without needing to know install internals. */
|
package/dist/tui/run.js
CHANGED
|
@@ -5,13 +5,13 @@ import { BrowserFetcher } from '../fetch/browser.js';
|
|
|
5
5
|
import { isChromeMarkedInstalled, installChromeViaCli, markChromeInstalled, probeChromeLaunchable } from '../fetch/chromeSetup.js';
|
|
6
6
|
import { openDb } from '../store/db.js';
|
|
7
7
|
import { syncClassroom } from '../sync.js';
|
|
8
|
-
import {
|
|
8
|
+
import { hasYtDlp } from '../media/download.js';
|
|
9
|
+
import { chromeMarkerPath, dbPath, defaultOutRoot, ensureRoot, isLoggedIn, login, profileDir } from '../auth/session.js';
|
|
9
10
|
import { listCourses } from '../discover/skool.js';
|
|
10
11
|
import { listUserCommunities } from '../discover/communities.js';
|
|
11
12
|
import { App } from './App.js';
|
|
12
13
|
import { createBrowserLifecycle } from './browserLifecycle.js';
|
|
13
14
|
import { ensureChromeReady } from './chromeSetup.js';
|
|
14
|
-
const OUT_ROOT = './out';
|
|
15
15
|
/**
|
|
16
16
|
* Runs the guided, no-arguments flow: session check, community picker,
|
|
17
17
|
* confirmation, live sync progress, and a result summary. Owns the db and
|
|
@@ -38,8 +38,8 @@ export async function runGuidedFlow() {
|
|
|
38
38
|
console.log('\nBrowser setup complete.\n');
|
|
39
39
|
const executablePath = await probeChromeLaunchable();
|
|
40
40
|
if (!executablePath) {
|
|
41
|
-
throw new Error('
|
|
42
|
-
'
|
|
41
|
+
throw new Error('The browser installed, but still could not be launched. This usually means a system library is missing' +
|
|
42
|
+
(process.platform === 'linux' ? '. Fix it with: sudo npx playwright install-deps chromium' : '. See the install output above.'));
|
|
43
43
|
}
|
|
44
44
|
return executablePath;
|
|
45
45
|
},
|
|
@@ -73,7 +73,7 @@ export async function runGuidedFlow() {
|
|
|
73
73
|
};
|
|
74
74
|
process.on('SIGINT', onSigint);
|
|
75
75
|
const controllers = {
|
|
76
|
-
outRoot:
|
|
76
|
+
outRoot: defaultOutRoot(),
|
|
77
77
|
checkLoggedIn: async () => lifecycle.checkLoggedIn(),
|
|
78
78
|
login: async () => {
|
|
79
79
|
await lifecycle.login();
|
|
@@ -83,6 +83,7 @@ export async function runGuidedFlow() {
|
|
|
83
83
|
const courses = await listCourses(slug, lifecycle.getFetcher());
|
|
84
84
|
return courses.filter((course) => course.hasAccess).length;
|
|
85
85
|
},
|
|
86
|
+
hasYtDlp,
|
|
86
87
|
runSync: async (slug, outDir, onProgress, videos) => syncClassroom({ slug, outDir, db, fetcher: lifecycle.getFetcher(), concurrency: 4, videos, onProgress }),
|
|
87
88
|
};
|
|
88
89
|
const { waitUntilExit } = render(React.createElement(App, { controllers }));
|
package/dist/tui/theme.d.ts
CHANGED
|
@@ -7,6 +7,8 @@ export declare const OUTCOME_STYLE: Record<Outcome, {
|
|
|
7
7
|
label: string;
|
|
8
8
|
color: OutcomeColor;
|
|
9
9
|
}>;
|
|
10
|
+
/** A path as a person reads it: absolute, with the home folder shortened to ~. */
|
|
11
|
+
export declare function displayPath(path: string): string;
|
|
10
12
|
export declare function formatDuration(ms: number): string;
|
|
11
13
|
/** Remaining time from a linear rate; null until there's at least one data point. */
|
|
12
14
|
export declare function estimateRemainingMs(elapsedMs: number, done: number, total: number): number | null;
|
package/dist/tui/theme.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { homedir } from 'node:os';
|
|
1
2
|
/** Brand accent: peach. Ink takes hex; the plain CLI falls back to a named color. */
|
|
2
3
|
export const ACCENT = '#FFB38A';
|
|
3
4
|
export const OUTCOME_STYLE = {
|
|
@@ -8,6 +9,11 @@ export const OUTCOME_STYLE = {
|
|
|
8
9
|
unavailable: { icon: '!', label: 'unavailable', color: 'yellow' },
|
|
9
10
|
failed: { icon: '✗', label: 'failed', color: 'red' },
|
|
10
11
|
};
|
|
12
|
+
/** A path as a person reads it: absolute, with the home folder shortened to ~. */
|
|
13
|
+
export function displayPath(path) {
|
|
14
|
+
const home = homedir();
|
|
15
|
+
return path === home || path.startsWith(`${home}/`) ? `~${path.slice(home.length)}` : path;
|
|
16
|
+
}
|
|
11
17
|
export function formatDuration(ms) {
|
|
12
18
|
const seconds = Math.max(0, Math.round(ms / 1000));
|
|
13
19
|
if (seconds < 60)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mogulmoretti/skrape",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "Read Skool course content instead of watching it — pulls the classroom of a community you belong to into clean, readable transcripts.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"skool",
|
|
@@ -35,21 +35,19 @@
|
|
|
35
35
|
"typecheck": "tsc --noEmit"
|
|
36
36
|
},
|
|
37
37
|
"engines": {
|
|
38
|
-
"node": ">=22"
|
|
38
|
+
"node": ">=22.13"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"better-sqlite3": "^11.5.0",
|
|
42
41
|
"commander": "^12.1.0",
|
|
43
42
|
"ink": "^7.1.1",
|
|
44
43
|
"react": "^19.2.8",
|
|
45
44
|
"zod": "^3.23.8"
|
|
46
45
|
},
|
|
47
46
|
"devDependencies": {
|
|
48
|
-
"@types/better-sqlite3": "^7.6.11",
|
|
49
47
|
"@types/node": "^22.9.0",
|
|
50
48
|
"@types/react": "^19.2.18",
|
|
51
49
|
"typescript": "^5.6.3",
|
|
52
|
-
"vitest": "^2.
|
|
50
|
+
"vitest": "^3.2.7"
|
|
53
51
|
},
|
|
54
52
|
"optionalDependencies": {
|
|
55
53
|
"playwright": "^1.48.2"
|