@mogulmoretti/skrape 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -16
- package/dist/auth/session.js +1 -1
- package/dist/cli.js +33 -25
- package/dist/fetch/browser.js +1 -1
- package/dist/media/download.d.ts +12 -0
- package/dist/media/download.js +54 -0
- package/dist/store/markdown.d.ts +1 -0
- package/dist/store/markdown.js +10 -4
- package/dist/sync.d.ts +9 -0
- package/dist/sync.js +26 -6
- package/dist/tui/App.d.ts +1 -1
- package/dist/tui/App.js +129 -67
- package/dist/tui/SelectList.d.ts +2 -0
- package/dist/tui/SelectList.js +7 -2
- package/dist/tui/browserLifecycle.js +1 -1
- package/dist/tui/progress.js +1 -2
- package/dist/tui/run.js +3 -3
- package/dist/tui/summary.d.ts +7 -3
- package/dist/tui/summary.js +36 -16
- package/dist/tui/theme.d.ts +12 -0
- package/dist/tui/theme.js +25 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Read Skool course content instead of watching it.
|
|
4
4
|
|
|
5
5
|
`skrape` pulls the classroom of a Skool community **you already pay for and
|
|
6
|
-
belong to**, and writes clean, readable, timestamped transcripts to disk
|
|
6
|
+
belong to**, and writes clean, readable, timestamped transcripts to disk,
|
|
7
7
|
so you can read a lesson in two minutes instead of watching a 20-minute video.
|
|
8
8
|
|
|
9
9
|
It works because Skool ships the whole course tree as structured data inside
|
|
@@ -15,13 +15,13 @@ that isn't already sitting on the page you're allowed to see.
|
|
|
15
15
|
|
|
16
16
|
**This is a personal tool for your own communities.** It signs in as *you*,
|
|
17
17
|
using *your* real browser session, and only ever touches courses your account
|
|
18
|
-
already has access to
|
|
18
|
+
already has access to. It can't and won't bypass Skool's own paywall or
|
|
19
19
|
membership checks.
|
|
20
20
|
|
|
21
21
|
Automating access to a platform like this generally sits outside most
|
|
22
22
|
platforms' terms of service, even when you're a legitimate paying member and
|
|
23
23
|
even when nothing is being redistributed. Running this is a decision about
|
|
24
|
-
your own account, and it's yours to make
|
|
24
|
+
your own account, and it's yours to make. This tool doesn't make it for you,
|
|
25
25
|
and it doesn't hide what it's doing (see [How it works](#how-it-works)).
|
|
26
26
|
Don't use it on communities you don't belong to, don't share or republish
|
|
27
27
|
what it produces, and don't run it in a way designed to look like normal
|
|
@@ -39,33 +39,48 @@ one-time, ~150MB download) if it isn't already on your machine, then walks
|
|
|
39
39
|
you through everything:
|
|
40
40
|
|
|
41
41
|
1. Checks whether you're signed in, and if not, opens a real Chrome window for
|
|
42
|
-
you to sign in yourself
|
|
42
|
+
you to sign in yourself. Nothing is typed on your behalf, and your
|
|
43
43
|
password never passes through this tool.
|
|
44
44
|
2. Looks up the communities your account belongs to and lets you pick one
|
|
45
45
|
with the arrow keys (an "enter a slug manually" option is always there
|
|
46
46
|
too, in case discovery doesn't find one).
|
|
47
|
-
3. Shows you what's about to happen
|
|
48
|
-
found, where files will land
|
|
49
|
-
work starts.
|
|
47
|
+
3. Shows you what's about to happen (the community, how many courses were
|
|
48
|
+
found, where files will land) and waits for you to confirm before any
|
|
49
|
+
work starts. Pick **Transcripts** or **Transcripts + videos**.
|
|
50
50
|
4. Runs the sync with a live progress bar. You can Ctrl+C at any point; it
|
|
51
51
|
cleans up after itself.
|
|
52
52
|
5. Prints a summary: outcome counts, total words, the output path, and a
|
|
53
|
-
readable list of anything that wasn't transcribed and why
|
|
53
|
+
readable list of anything that wasn't transcribed and why. Nothing is
|
|
54
54
|
silently dropped from the count.
|
|
55
55
|
|
|
56
56
|
From there you can sync another community, re-sync the same one (it's
|
|
57
|
-
incremental
|
|
57
|
+
incremental: already-fetched lessons are skipped), open the output folder,
|
|
58
58
|
or quit.
|
|
59
59
|
|
|
60
|
-
Output lands in `./out/<slug>/transcripts/<course>/NN-lesson.md
|
|
60
|
+
Output lands in `./out/<slug>/transcripts/<course>/NN-lesson.md`, and videos
|
|
61
|
+
(if you asked for them) in `./out/<slug>/videos/<course>/NN-lesson.mp4`.
|
|
62
|
+
|
|
63
|
+
### Downloading videos
|
|
64
|
+
|
|
65
|
+
Video downloads are opt-in and use [yt-dlp](https://github.com/yt-dlp/yt-dlp),
|
|
66
|
+
which you install once yourself:
|
|
67
|
+
|
|
68
|
+
brew install yt-dlp ffmpeg # macOS; see yt-dlp's docs for other platforms
|
|
69
|
+
|
|
70
|
+
skrape hands each lesson's video link to yt-dlp, which covers Loom, YouTube,
|
|
71
|
+
Vimeo, Wistia and most other hosts. A video that's already on disk is skipped,
|
|
72
|
+
so re-running a sync only fetches what's missing. Videos are big (a 20-minute
|
|
73
|
+
Loom lesson is ~150MB), so check your free disk space before pulling a whole
|
|
74
|
+
community.
|
|
61
75
|
|
|
62
76
|
### Scripting / advanced use
|
|
63
77
|
|
|
64
78
|
The underlying subcommands work directly too, without the guided flow:
|
|
65
79
|
|
|
66
|
-
skrape login # once
|
|
80
|
+
skrape login # once: sign in by hand, session persists
|
|
67
81
|
skrape sync <slug> # slug is the part after skool.com/
|
|
68
82
|
skrape sync <slug> -o ./out -c 4 # custom output dir / concurrency
|
|
83
|
+
skrape sync <slug> --videos # transcripts + every lesson video
|
|
69
84
|
|
|
70
85
|
### Building from source
|
|
71
86
|
|
|
@@ -79,7 +94,7 @@ The underlying subcommands work directly too, without the guided flow:
|
|
|
79
94
|
|
|
80
95
|
## How it works
|
|
81
96
|
|
|
82
|
-
Skool ships the full course tree as JSON inside each classroom page
|
|
97
|
+
Skool ships the full course tree as JSON inside each classroom page, with no
|
|
83
98
|
clicking through lesson by lesson. `skrape` reads that structure the way any
|
|
84
99
|
browser or crawler would: an authenticated request to a page your account
|
|
85
100
|
can already open.
|
|
@@ -89,8 +104,8 @@ moment). Where a caption track exists, it's downloaded and cleaned into
|
|
|
89
104
|
readable prose with timestamp anchors. Where one doesn't, the lesson is
|
|
90
105
|
reported as not transcribed rather than silently skipped.
|
|
91
106
|
|
|
92
|
-
When a page doesn't come back with the data the tool expects
|
|
93
|
-
session, or Skool changing how a page is built
|
|
107
|
+
When a page doesn't come back with the data the tool expects (an expired
|
|
108
|
+
session, or Skool changing how a page is built), it falls back to driving an
|
|
94
109
|
actual browser window using your already-authenticated profile, rather than
|
|
95
110
|
guessing or failing silently.
|
|
96
111
|
|
|
@@ -100,18 +115,22 @@ run the command.
|
|
|
100
115
|
|
|
101
116
|
## What this doesn't do
|
|
102
117
|
|
|
103
|
-
- Doesn't work on communities you're not a member of
|
|
118
|
+
- Doesn't work on communities you're not a member of. There's no bypass for
|
|
104
119
|
Skool's own access checks, because none of the data is reachable without
|
|
105
120
|
them.
|
|
106
121
|
- Doesn't redistribute or publish anything. Output is written to your local
|
|
107
122
|
disk for you to read.
|
|
108
123
|
- Doesn't store your password. Sign-in happens in a real Chrome window that
|
|
109
124
|
only you interact with.
|
|
125
|
+
- Doesn't download videos unless you ask it to (`--videos`, or **Transcripts + videos**
|
|
126
|
+
in the guided flow).
|
|
127
|
+
- Doesn't download lesson `resources`: those are links to outside docs
|
|
128
|
+
(Google Docs, etc.), not files hosted on Skool.
|
|
110
129
|
- Doesn't transcribe video hosts it doesn't recognize. Those lessons show up
|
|
111
130
|
in the summary as not transcribed, with a reason.
|
|
112
131
|
|
|
113
132
|
## License
|
|
114
133
|
|
|
115
|
-
MIT
|
|
134
|
+
MIT. See [LICENSE](LICENSE). In short: do what you like with the code, no
|
|
116
135
|
warranty, use it at your own risk and your own judgment about the platforms
|
|
117
136
|
you point it at.
|
package/dist/auth/session.js
CHANGED
|
@@ -65,6 +65,6 @@ export async function login(onContext) {
|
|
|
65
65
|
return false;
|
|
66
66
|
}
|
|
67
67
|
}, undefined, { timeout: 0 });
|
|
68
|
-
console.log('Signed in. Session saved
|
|
68
|
+
console.log('Signed in. Session saved, future runs will not need this.');
|
|
69
69
|
await context.close();
|
|
70
70
|
}
|
package/dist/cli.js
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { createRequire } from 'node:module';
|
|
3
|
+
import { styleText } from 'node:util';
|
|
2
4
|
import { Command } from 'commander';
|
|
3
5
|
import { HttpFetcher } from './fetch/http.js';
|
|
4
6
|
import { BrowserFetcher } from './fetch/browser.js';
|
|
5
7
|
import { ResilientFetcher } from './fetch/resilient.js';
|
|
6
8
|
import { openDb } from './store/db.js';
|
|
7
9
|
import { syncClassroom } from './sync.js';
|
|
10
|
+
import { formatSummary } from './tui/summary.js';
|
|
11
|
+
import { OUTCOME_STYLE, formatDuration } from './tui/theme.js';
|
|
8
12
|
import { login, profileDir, dbPath, ensureRoot, isLoggedIn } from './auth/session.js';
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
13
|
+
// Color only for a human at a terminal; pipes and NO_COLOR get plain text.
|
|
14
|
+
const useColor = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR;
|
|
15
|
+
const paint = (color, text) => (useColor ? styleText(color, text) : text);
|
|
12
16
|
const MAX_CONCURRENCY = 16;
|
|
13
17
|
/** Parse the --concurrency flag to a positive integer, or return null if invalid. */
|
|
14
18
|
function parseConcurrency(raw) {
|
|
@@ -18,7 +22,12 @@ function parseConcurrency(raw) {
|
|
|
18
22
|
return Math.min(n, MAX_CONCURRENCY);
|
|
19
23
|
}
|
|
20
24
|
const program = new Command();
|
|
21
|
-
|
|
25
|
+
const { version } = createRequire(import.meta.url)('../package.json');
|
|
26
|
+
program
|
|
27
|
+
.name('skrape')
|
|
28
|
+
.description('Read Skool course content instead of watching it')
|
|
29
|
+
.version(version, '-v, --version')
|
|
30
|
+
.addHelpText('after', '\nRun `skrape` with no arguments for the guided, interactive flow.');
|
|
22
31
|
program
|
|
23
32
|
.command('login')
|
|
24
33
|
.description('Sign in to Skool once; the session is reused by later runs')
|
|
@@ -30,7 +39,8 @@ program
|
|
|
30
39
|
.argument('<slug>', 'community slug, e.g. demo from skool.com/demo')
|
|
31
40
|
.option('-o, --out <dir>', 'output directory', './out')
|
|
32
41
|
.option('-c, --concurrency <n>', 'parallel requests', '4')
|
|
33
|
-
.
|
|
42
|
+
.option('--videos', 'also download every lesson video (needs yt-dlp)')
|
|
43
|
+
.description('Pull a community classroom to disk as transcripts (and optionally videos)')
|
|
34
44
|
.action(async (slug, options) => {
|
|
35
45
|
const concurrency = parseConcurrency(options.concurrency);
|
|
36
46
|
if (concurrency === null) {
|
|
@@ -44,6 +54,8 @@ program
|
|
|
44
54
|
const db = openDb(dbPath());
|
|
45
55
|
const browser = new BrowserFetcher(profileDir());
|
|
46
56
|
const fetcher = new ResilientFetcher(new HttpFetcher(), async () => browser, isLoggedIn);
|
|
57
|
+
const startedAt = Date.now();
|
|
58
|
+
console.log(`${paint('bold', '◆ skrape')} ${paint('dim', `syncing skool.com/${slug}${options.videos ? ' · with videos' : ''}`)}\n`);
|
|
47
59
|
try {
|
|
48
60
|
const summary = await syncClassroom({
|
|
49
61
|
slug,
|
|
@@ -51,39 +63,35 @@ program
|
|
|
51
63
|
db,
|
|
52
64
|
fetcher,
|
|
53
65
|
concurrency,
|
|
66
|
+
videos: options.videos,
|
|
54
67
|
onProgress: (event) => {
|
|
55
|
-
const
|
|
56
|
-
|
|
68
|
+
const style = OUTCOME_STYLE[event.outcome];
|
|
69
|
+
const counter = `${event.done}/${event.total}`.padStart(String(event.total).length * 2 + 1);
|
|
70
|
+
console.log(`${paint(style.color, style.icon)} ${paint('dim', counter)} ${paint('dim', `${event.course.slice(0, 28)} ›`)} ${event.title.slice(0, 60)}`);
|
|
57
71
|
},
|
|
58
72
|
});
|
|
59
|
-
console.log('
|
|
60
|
-
for (const
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
}
|
|
64
|
-
console.log(` ${'words'.padEnd(14)} ${summary.totalWords.toLocaleString('en-US')}`);
|
|
73
|
+
console.log('');
|
|
74
|
+
for (const line of formatSummary(summary, outDir, paint))
|
|
75
|
+
console.log(line);
|
|
76
|
+
console.log(paint('dim', `Finished in ${formatDuration(Date.now() - startedAt)}`));
|
|
65
77
|
if (fetcher.escalatedRoutes.size > 0) {
|
|
66
|
-
console.log(
|
|
67
|
-
}
|
|
68
|
-
if (summary.problems.length > 0) {
|
|
69
|
-
console.log('\n--- not transcribed ---');
|
|
70
|
-
for (const problem of summary.problems) {
|
|
71
|
-
console.log(` [${problem.outcome}] ${problem.course} / ${problem.title}: ${problem.reason}`);
|
|
72
|
-
}
|
|
78
|
+
console.log(paint('dim', `Escalated to browser: ${[...fetcher.escalatedRoutes].join(', ')}`));
|
|
73
79
|
}
|
|
74
|
-
console.log(`\nOutput: ${outDir}`);
|
|
75
80
|
}
|
|
76
81
|
catch (error) {
|
|
77
|
-
console.error(`\
|
|
78
|
-
if (
|
|
82
|
+
console.error(`\n${paint('red', '✗ Sync failed:')} ${error.message}`);
|
|
83
|
+
if (error.message.includes('HTTP 404')) {
|
|
84
|
+
console.error(`No classroom at skool.com/${slug}. Check the slug (the part after skool.com/).`);
|
|
85
|
+
}
|
|
86
|
+
else if (fetcher.escalatedRoutes.size > 0) {
|
|
79
87
|
console.error(`\n escalated to browser: ${[...fetcher.escalatedRoutes].join(', ')}`);
|
|
80
88
|
console.error('If this mentions an unexpected payload, the authenticated browser path was already tried ' +
|
|
81
89
|
"for the route(s) above, so Skool's page structure most likely changed rather than the " +
|
|
82
|
-
'session being expired. A fresh `
|
|
90
|
+
'session being expired. A fresh `skrape login` is still worth trying, but treat it as a ' +
|
|
83
91
|
'secondary guess.');
|
|
84
92
|
}
|
|
85
93
|
else {
|
|
86
|
-
console.error('If this mentions an unexpected payload, your session may have expired
|
|
94
|
+
console.error('If this mentions an unexpected payload, your session may have expired. Run: skrape login. ' +
|
|
87
95
|
"It is also possible Skool's page structure changed; if a retry after login fails the same " +
|
|
88
96
|
'way, that is more likely.');
|
|
89
97
|
}
|
package/dist/fetch/browser.js
CHANGED
|
@@ -13,7 +13,7 @@ export class BrowserFetcher {
|
|
|
13
13
|
}
|
|
14
14
|
async ensureContext() {
|
|
15
15
|
if (this.closed) {
|
|
16
|
-
throw new Error('BrowserFetcher is closed
|
|
16
|
+
throw new Error('BrowserFetcher is closed, cannot open a new context after close()');
|
|
17
17
|
}
|
|
18
18
|
if (!this.context) {
|
|
19
19
|
try {
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare function hasYtDlp(): Promise<boolean>;
|
|
2
|
+
export type DownloadResult = {
|
|
3
|
+
status: 'downloaded' | 'exists';
|
|
4
|
+
} | {
|
|
5
|
+
status: 'failed';
|
|
6
|
+
reason: string;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Download one lesson video to `destPath` (an .mp4). An existing file is treated as complete:
|
|
10
|
+
* yt-dlp writes to a .part file and only renames on success, so a partial never lands at destPath.
|
|
11
|
+
*/
|
|
12
|
+
export declare function downloadVideo(videoUrl: string, destPath: string): Promise<DownloadResult>;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { access, mkdir } from 'node:fs/promises';
|
|
3
|
+
import { dirname } from 'node:path';
|
|
4
|
+
/** Run yt-dlp, resolving with its exit code and the tail of stderr. Never rejects on a bad exit. */
|
|
5
|
+
function run(args) {
|
|
6
|
+
return new Promise((resolve, reject) => {
|
|
7
|
+
const child = spawn(process.env.SKRAPE_YT_DLP ?? 'yt-dlp', args, { stdio: ['ignore', 'ignore', 'pipe'] });
|
|
8
|
+
let stderr = '';
|
|
9
|
+
child.stderr.on('data', (chunk) => {
|
|
10
|
+
stderr = (stderr + chunk.toString()).slice(-2000);
|
|
11
|
+
});
|
|
12
|
+
child.on('error', reject);
|
|
13
|
+
child.on('close', (code) => resolve({ code: code ?? 1, stderr }));
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
export async function hasYtDlp() {
|
|
17
|
+
try {
|
|
18
|
+
return (await run(['--version'])).code === 0;
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Download one lesson video to `destPath` (an .mp4). An existing file is treated as complete:
|
|
26
|
+
* yt-dlp writes to a .part file and only renames on success, so a partial never lands at destPath.
|
|
27
|
+
*/
|
|
28
|
+
export async function downloadVideo(videoUrl, destPath) {
|
|
29
|
+
try {
|
|
30
|
+
await access(destPath);
|
|
31
|
+
return { status: 'exists' };
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
// not there yet — download it
|
|
35
|
+
}
|
|
36
|
+
await mkdir(dirname(destPath), { recursive: true });
|
|
37
|
+
try {
|
|
38
|
+
const { code, stderr } = await run([
|
|
39
|
+
'--quiet', '--no-progress', '--no-playlist',
|
|
40
|
+
'--referer', 'https://www.skool.com/',
|
|
41
|
+
'--merge-output-format', 'mp4', '--remux-video', 'mp4',
|
|
42
|
+
// Let yt-dlp pick the source extension; --remux-video lands the final file at destPath.
|
|
43
|
+
'-o', destPath.replace(/\.mp4$/, '.%(ext)s'),
|
|
44
|
+
videoUrl,
|
|
45
|
+
]);
|
|
46
|
+
if (code === 0)
|
|
47
|
+
return { status: 'downloaded' };
|
|
48
|
+
const lastError = stderr.split('\n').filter((line) => line.includes('ERROR')).pop();
|
|
49
|
+
return { status: 'failed', reason: (lastError ?? stderr.trim().split('\n').pop() ?? `yt-dlp exit ${code}`).trim() };
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
return { status: 'failed', reason: error.message };
|
|
53
|
+
}
|
|
54
|
+
}
|
package/dist/store/markdown.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ContentItem } from '../types.js';
|
|
2
2
|
export declare function slugify(text: string, maxLength?: number): string;
|
|
3
3
|
export declare function transcriptPath(outDir: string, item: ContentItem, padWidth?: number): string;
|
|
4
|
+
export declare function videoPath(outDir: string, item: ContentItem, padWidth?: number): string;
|
|
4
5
|
export declare function writeTranscript(outDir: string, item: ContentItem, body: string, meta: {
|
|
5
6
|
wordCount: number;
|
|
6
7
|
sourceUrl: string;
|
package/dist/store/markdown.js
CHANGED
|
@@ -12,18 +12,24 @@ export function slugify(text, maxLength = 60) {
|
|
|
12
12
|
.replace(/-+$/g, '');
|
|
13
13
|
return cleaned || 'untitled';
|
|
14
14
|
}
|
|
15
|
-
|
|
15
|
+
function lessonPath(outDir, kind, ext, item, padWidth) {
|
|
16
16
|
const course = slugify(item.course ?? 'uncategorized');
|
|
17
17
|
const width = Math.max(2, padWidth);
|
|
18
|
-
const name = `${String(item.index).padStart(width, '0')}-${slugify(item.title)}
|
|
19
|
-
return join(outDir,
|
|
18
|
+
const name = `${String(item.index).padStart(width, '0')}-${slugify(item.title)}.${ext}`;
|
|
19
|
+
return join(outDir, kind, course, name);
|
|
20
|
+
}
|
|
21
|
+
export function transcriptPath(outDir, item, padWidth = 2) {
|
|
22
|
+
return lessonPath(outDir, 'transcripts', 'md', item, padWidth);
|
|
23
|
+
}
|
|
24
|
+
export function videoPath(outDir, item, padWidth = 2) {
|
|
25
|
+
return lessonPath(outDir, 'videos', 'mp4', item, padWidth);
|
|
20
26
|
}
|
|
21
27
|
export async function writeTranscript(outDir, item, body, meta, padWidth = 2) {
|
|
22
28
|
const path = transcriptPath(outDir, item, padWidth);
|
|
23
29
|
await mkdir(dirname(path), { recursive: true });
|
|
24
30
|
const minutes = Math.round(item.durationMs / 60_000);
|
|
25
31
|
const cleanTitle = item.title.replace(/\s+/g, ' ');
|
|
26
|
-
const cleanCourse = (item.course ?? '
|
|
32
|
+
const cleanCourse = (item.course ?? '-').replace(/\s+/g, ' ');
|
|
27
33
|
const cleanSection = item.section?.replace(/\s+/g, ' ');
|
|
28
34
|
const header = [
|
|
29
35
|
`# ${cleanTitle}`,
|
package/dist/sync.d.ts
CHANGED
|
@@ -18,6 +18,13 @@ export interface SyncSummary {
|
|
|
18
18
|
counts: Record<Outcome, number>;
|
|
19
19
|
problems: Problem[];
|
|
20
20
|
totalWords: number;
|
|
21
|
+
/** Present only when the sync was asked to download videos. */
|
|
22
|
+
videos?: VideoCounts;
|
|
23
|
+
}
|
|
24
|
+
export interface VideoCounts {
|
|
25
|
+
downloaded: number;
|
|
26
|
+
existing: number;
|
|
27
|
+
failed: number;
|
|
21
28
|
}
|
|
22
29
|
export interface SyncOptions {
|
|
23
30
|
slug: string;
|
|
@@ -25,6 +32,8 @@ export interface SyncOptions {
|
|
|
25
32
|
db: Db;
|
|
26
33
|
fetcher: Fetcher;
|
|
27
34
|
concurrency?: number;
|
|
35
|
+
/** Also download each lesson's video (needs yt-dlp on PATH). */
|
|
36
|
+
videos?: boolean;
|
|
28
37
|
onProgress?: (event: ProgressEvent) => void;
|
|
29
38
|
}
|
|
30
39
|
export declare function syncClassroom(options: SyncOptions): Promise<SyncSummary>;
|
package/dist/sync.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { listCourses, listLessons } from './discover/skool.js';
|
|
2
2
|
import { getTranscript } from './media/index.js';
|
|
3
|
-
import {
|
|
3
|
+
import { downloadVideo, hasYtDlp } from './media/download.js';
|
|
4
|
+
import { videoPath, writeTranscript } from './store/markdown.js';
|
|
4
5
|
/** Run tasks with a bounded number in flight. */
|
|
5
6
|
async function pooled(tasks, limit) {
|
|
6
7
|
const results = new Array(tasks.length);
|
|
@@ -31,7 +32,12 @@ function errorMessage(error) {
|
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
export async function syncClassroom(options) {
|
|
34
|
-
const { slug, outDir, db, fetcher, concurrency = 4, onProgress } = options;
|
|
35
|
+
const { slug, outDir, db, fetcher, concurrency = 4, videos = false, onProgress } = options;
|
|
36
|
+
if (videos && !(await hasYtDlp())) {
|
|
37
|
+
throw new Error('Video downloads need yt-dlp, which was not found on your PATH. ' +
|
|
38
|
+
'Install it (macOS: brew install yt-dlp ffmpeg; others: https://github.com/yt-dlp/yt-dlp#installation) and try again.');
|
|
39
|
+
}
|
|
40
|
+
const videoCounts = videos ? { downloaded: 0, existing: 0, failed: 0 } : undefined;
|
|
35
41
|
const counts = {
|
|
36
42
|
ok: 0, skipped: 0, 'no-video': 0, 'no-access': 0, unavailable: 0, failed: 0,
|
|
37
43
|
};
|
|
@@ -44,7 +50,7 @@ export async function syncClassroom(options) {
|
|
|
44
50
|
counts['no-access']++;
|
|
45
51
|
problems.push({
|
|
46
52
|
outcome: 'no-access', course: locked.title, title: '(entire course)',
|
|
47
|
-
reason: 'hasAccess is 0
|
|
53
|
+
reason: 'hasAccess is 0, not entitled, skipped without probing',
|
|
48
54
|
});
|
|
49
55
|
}
|
|
50
56
|
// One course's listing failing (network blip, malformed payload) must not take down the whole
|
|
@@ -90,12 +96,12 @@ export async function syncClassroom(options) {
|
|
|
90
96
|
const record = (outcome, item, reason) => {
|
|
91
97
|
counts[outcome]++;
|
|
92
98
|
if (reason) {
|
|
93
|
-
problems.push({ outcome, course: item.course ?? '
|
|
99
|
+
problems.push({ outcome, course: item.course ?? '-', title: item.title, reason });
|
|
94
100
|
}
|
|
95
101
|
done++;
|
|
96
102
|
// Progress reporting is cosmetic. A caller's callback throwing must never fail the sync.
|
|
97
103
|
try {
|
|
98
|
-
onProgress?.({ outcome, course: item.course ?? '
|
|
104
|
+
onProgress?.({ outcome, course: item.course ?? '-', title: item.title, done, total });
|
|
99
105
|
}
|
|
100
106
|
catch {
|
|
101
107
|
// swallow — the caller's callback is not our concern
|
|
@@ -129,6 +135,20 @@ export async function syncClassroom(options) {
|
|
|
129
135
|
return record('no-access', item, 'lesson locked');
|
|
130
136
|
if (!item.videoUrl)
|
|
131
137
|
return record('no-video', item, 'lesson has no video attached');
|
|
138
|
+
// Download before the transcript step so an already-transcribed lesson still gets its video.
|
|
139
|
+
if (videoCounts) {
|
|
140
|
+
const download = await downloadVideo(item.videoUrl, videoPath(outDir, item, padWidth));
|
|
141
|
+
if (download.status === 'failed') {
|
|
142
|
+
videoCounts.failed++;
|
|
143
|
+
problems.push({
|
|
144
|
+
outcome: 'failed', course: item.course ?? '-', title: item.title,
|
|
145
|
+
reason: `video download: ${download.reason}`,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
else {
|
|
149
|
+
videoCounts[download.status === 'downloaded' ? 'downloaded' : 'existing']++;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
132
152
|
let status;
|
|
133
153
|
try {
|
|
134
154
|
status = db.getTranscriptStatus(id);
|
|
@@ -174,5 +194,5 @@ export async function syncClassroom(options) {
|
|
|
174
194
|
return record(result.status, item, result.reason);
|
|
175
195
|
}), concurrency);
|
|
176
196
|
db.markSynced(slug);
|
|
177
|
-
return { counts, problems, totalWords };
|
|
197
|
+
return { counts, problems, totalWords, videos: videoCounts };
|
|
178
198
|
}
|
package/dist/tui/App.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export interface AppControllers {
|
|
|
8
8
|
login: () => Promise<void>;
|
|
9
9
|
discoverCommunities: () => Promise<CommunityRef[] | null>;
|
|
10
10
|
countAccessibleCourses: (slug: string) => Promise<number>;
|
|
11
|
-
runSync: (slug: string, outDir: string, onProgress: (event: ProgressEvent) => void) => Promise<SyncSummary>;
|
|
11
|
+
runSync: (slug: string, outDir: string, onProgress: (event: ProgressEvent) => void, videos: boolean) => Promise<SyncSummary>;
|
|
12
12
|
}
|
|
13
13
|
/**
|
|
14
14
|
* Caps error text shown in the TUI. Some failures (e.g. a raw Playwright
|
package/dist/tui/App.js
CHANGED
|
@@ -3,7 +3,8 @@ import React from 'react';
|
|
|
3
3
|
import { Box, Text, useApp, useInput } from 'ink';
|
|
4
4
|
import { SelectList } from './SelectList.js';
|
|
5
5
|
import { applyProgress, createProgressState, formatProgressBar } from './progress.js';
|
|
6
|
-
import {
|
|
6
|
+
import { formatVideoLine, nothingTranscribed, sortedProblems } from './summary.js';
|
|
7
|
+
import { ACCENT, OUTCOME_STYLE, estimateRemainingMs, formatDuration } from './theme.js';
|
|
7
8
|
import { isValidSlug, normalizeSlug } from './slug.js';
|
|
8
9
|
import { MENU_ITEMS, nextStepForMenuChoice } from './flow.js';
|
|
9
10
|
import { revealOutputFolder } from './reveal.js';
|
|
@@ -35,8 +36,45 @@ function ConfirmPrompt({ onConfirm }) {
|
|
|
35
36
|
else if (input === 'q' || key.escape)
|
|
36
37
|
exit();
|
|
37
38
|
});
|
|
38
|
-
return _jsx(Text, { dimColor: true, children: "
|
|
39
|
+
return _jsx(Text, { dimColor: true, children: "enter continue \u00B7 q quit" });
|
|
39
40
|
}
|
|
41
|
+
function Header() {
|
|
42
|
+
return (_jsxs(Box, { marginBottom: 1, children: [_jsx(Text, { color: ACCENT, bold: true, children: "\u25C6 skrape" }), _jsx(Text, { dimColor: true, children: " read the course, skip the video" })] }));
|
|
43
|
+
}
|
|
44
|
+
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
45
|
+
function Spinner({ label }) {
|
|
46
|
+
const [frame, setFrame] = React.useState(0);
|
|
47
|
+
React.useEffect(() => {
|
|
48
|
+
const timer = setInterval(() => setFrame((current) => (current + 1) % SPINNER_FRAMES.length), 80);
|
|
49
|
+
return () => clearInterval(timer);
|
|
50
|
+
}, []);
|
|
51
|
+
return (_jsxs(Text, { children: [_jsx(Text, { color: ACCENT, children: SPINNER_FRAMES[frame] }), " ", label] }));
|
|
52
|
+
}
|
|
53
|
+
/** Label/value rows inside a rounded card, labels aligned. */
|
|
54
|
+
function Card({ rows }) {
|
|
55
|
+
const width = Math.max(...rows.map(([label]) => label.length));
|
|
56
|
+
return (_jsx(Box, { flexDirection: "column", borderStyle: "round", borderColor: "gray", paddingX: 1, alignSelf: "flex-start", children: rows.map(([label, value]) => (_jsxs(Box, { children: [_jsx(Text, { dimColor: true, children: label.padEnd(width + 3) }), _jsx(Text, { children: value })] }, label))) }));
|
|
57
|
+
}
|
|
58
|
+
function OutcomeChips({ counts, always = [] }) {
|
|
59
|
+
const shown = Object.keys(OUTCOME_STYLE).filter((o) => counts[o] > 0 || always.includes(o));
|
|
60
|
+
return (_jsx(Text, { children: shown.map((outcome, index) => (_jsxs(Text, { children: [index > 0 && ' ', _jsx(Text, { color: OUTCOME_STYLE[outcome].color, children: OUTCOME_STYLE[outcome].icon }), " ", counts[outcome], ' ', _jsx(Text, { dimColor: true, children: OUTCOME_STYLE[outcome].label })] }, outcome))) }));
|
|
61
|
+
}
|
|
62
|
+
/** Ticks once a second on its own so the rest of the screen only re-renders on progress. */
|
|
63
|
+
function Timing({ startedAt, done, total }) {
|
|
64
|
+
const [now, setNow] = React.useState(Date.now());
|
|
65
|
+
React.useEffect(() => {
|
|
66
|
+
const timer = setInterval(() => setNow(Date.now()), 1000);
|
|
67
|
+
return () => clearInterval(timer);
|
|
68
|
+
}, []);
|
|
69
|
+
const elapsed = now - startedAt;
|
|
70
|
+
const remaining = estimateRemainingMs(elapsed, done, total);
|
|
71
|
+
return (_jsxs(Text, { dimColor: true, children: [formatDuration(elapsed), " elapsed", remaining !== null && ` · ~${formatDuration(remaining)} left`] }));
|
|
72
|
+
}
|
|
73
|
+
const MODE_ITEMS = [
|
|
74
|
+
{ label: 'Transcripts', value: 'transcripts', hint: 'fast · text only' },
|
|
75
|
+
{ label: 'Transcripts + videos', value: 'videos', hint: 'needs yt-dlp · large download' },
|
|
76
|
+
{ label: 'Quit', value: 'quit' },
|
|
77
|
+
];
|
|
40
78
|
function ExitPrompt({ label }) {
|
|
41
79
|
const { exit } = useApp();
|
|
42
80
|
useInput(() => exit());
|
|
@@ -166,16 +204,16 @@ export function App({ controllers }) {
|
|
|
166
204
|
if (step.kind !== 'syncing')
|
|
167
205
|
return;
|
|
168
206
|
let cancelled = false;
|
|
169
|
-
const { slug, name, courseCount, outDir } = step;
|
|
207
|
+
const { slug, name, courseCount, outDir, videos, startedAt } = step;
|
|
170
208
|
controllers
|
|
171
209
|
.runSync(slug, outDir, (event) => {
|
|
172
210
|
if (cancelled)
|
|
173
211
|
return;
|
|
174
212
|
setStep((previous) => previous.kind === 'syncing' ? { ...previous, progress: applyProgress(previous.progress, event) } : previous);
|
|
175
|
-
})
|
|
213
|
+
}, videos)
|
|
176
214
|
.then((summary) => {
|
|
177
215
|
if (!cancelled)
|
|
178
|
-
setStep({ kind: 'done', slug, name, courseCount, outDir, summary });
|
|
216
|
+
setStep({ kind: 'done', slug, name, courseCount, outDir, elapsedMs: Date.now() - startedAt, summary });
|
|
179
217
|
})
|
|
180
218
|
.catch((error) => {
|
|
181
219
|
if (!cancelled)
|
|
@@ -218,68 +256,92 @@ export function App({ controllers }) {
|
|
|
218
256
|
});
|
|
219
257
|
}
|
|
220
258
|
}, [exit]);
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
259
|
+
return (_jsxs(Box, { flexDirection: "column", paddingX: 1, paddingY: 1, children: [_jsx(Header, {}), renderStep()] }));
|
|
260
|
+
function renderStep() {
|
|
261
|
+
switch (step.kind) {
|
|
262
|
+
case 'checking-session':
|
|
263
|
+
return _jsx(Spinner, { label: "Checking your Skool session\u2026" });
|
|
264
|
+
case 'login-needed':
|
|
265
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Text, { color: "yellow", children: "You're not signed in to Skool yet." }), _jsx(Text, { dimColor: true, children: "A browser window will open and you sign in yourself. skrape never sees your password." }), _jsx(Text, { children: " " }), _jsx(ConfirmPrompt, { onConfirm: () => setStep({ kind: 'logging-in' }) })] }));
|
|
266
|
+
case 'logging-in':
|
|
267
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Spinner, { label: "Waiting for you to sign in in the browser window\u2026" }), _jsx(Text, { dimColor: true, children: "Ctrl+C to cancel" })] }));
|
|
268
|
+
case 'discovering':
|
|
269
|
+
return _jsx(Spinner, { label: "Finding your communities\u2026" });
|
|
270
|
+
case 'picking': {
|
|
271
|
+
const items = [
|
|
272
|
+
...step.communities.map((community) => ({
|
|
273
|
+
label: community.name,
|
|
274
|
+
hint: `skool.com/${community.slug}`,
|
|
275
|
+
value: community,
|
|
276
|
+
})),
|
|
277
|
+
{ label: 'Enter a slug manually…', value: null },
|
|
278
|
+
];
|
|
279
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Text, { bold: true, children: "Which community?" }), _jsx(Text, { children: " " }), _jsx(SelectList, { items: items, onSelect: (value) => {
|
|
280
|
+
if (value === null)
|
|
281
|
+
setStep({ kind: 'manual-slug', value: '', error: null });
|
|
282
|
+
else
|
|
283
|
+
chooseCommunity(value.slug, value.name);
|
|
284
|
+
} })] }));
|
|
285
|
+
}
|
|
286
|
+
case 'manual-slug':
|
|
287
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Text, { bold: true, children: "Community slug" }), _jsxs(Text, { dimColor: true, children: ["The part after skool.com/, e.g. skool.com/", _jsx(Text, { color: ACCENT, children: "my-community" })] }), _jsx(Text, { children: " " }), _jsxs(Text, { children: [_jsx(Text, { color: ACCENT, children: "\u276F " }), step.value, _jsx(Text, { color: ACCENT, children: "\u258C" })] }), step.error && _jsx(Text, { color: "red", children: step.error }), _jsx(ManualSlugInput, { value: step.value, onChange: (value) => setStep({ kind: 'manual-slug', value, error: null }), onSubmit: (value) => {
|
|
288
|
+
const slug = normalizeSlug(value);
|
|
289
|
+
if (!isValidSlug(slug)) {
|
|
290
|
+
setStep({ kind: 'manual-slug', value, error: `"${value}" doesn't look like a valid slug.` });
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
chooseCommunity(slug, slug);
|
|
294
|
+
} })] }));
|
|
295
|
+
case 'counting':
|
|
296
|
+
return _jsx(Spinner, { label: `Looking up courses in ${step.name}…` });
|
|
297
|
+
case 'confirming': {
|
|
298
|
+
const { slug, name, courseCount, outDir } = step;
|
|
299
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Card, { rows: [
|
|
300
|
+
['Community', _jsx(Text, { bold: true, children: name })],
|
|
301
|
+
['URL', `skool.com/${slug}`],
|
|
302
|
+
['Courses', `${courseCount} you can access`],
|
|
303
|
+
['Saving to', outDir],
|
|
304
|
+
] }), _jsx(Text, { children: " " }), _jsx(Text, { bold: true, children: "What should skrape pull?" }), _jsx(Text, { children: " " }), _jsx(SelectList, { items: MODE_ITEMS, onSelect: (mode) => {
|
|
305
|
+
if (mode === 'quit') {
|
|
306
|
+
exit();
|
|
307
|
+
return;
|
|
308
|
+
}
|
|
309
|
+
setStep({
|
|
310
|
+
kind: 'syncing', slug, name, courseCount, outDir,
|
|
311
|
+
videos: mode === 'videos',
|
|
312
|
+
startedAt: Date.now(),
|
|
313
|
+
progress: createProgressState(0),
|
|
314
|
+
});
|
|
315
|
+
} })] }));
|
|
316
|
+
}
|
|
317
|
+
case 'syncing': {
|
|
318
|
+
const { progress } = step;
|
|
319
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsxs(Text, { children: ["Syncing ", _jsx(Text, { bold: true, children: step.name }), _jsx(Text, { dimColor: true, children: step.videos ? ' · transcripts + videos' : ' · transcripts' })] }), _jsx(Text, { children: " " }), _jsx(Text, { color: ACCENT, children: formatProgressBar(progress.done, progress.total, 32) }), progress.current ? (_jsx(Spinner, { label: `${progress.current.course} › ${progress.current.title}` })) : (_jsx(Spinner, { label: "Reading the course tree\u2026" })), _jsx(Text, { children: " " }), _jsx(OutcomeChips, { counts: progress.counts, always: ['ok'] }), _jsx(Timing, { startedAt: step.startedAt, done: progress.done, total: progress.total })] }));
|
|
320
|
+
}
|
|
321
|
+
case 'done': {
|
|
322
|
+
const { summary } = step;
|
|
323
|
+
const empty = nothingTranscribed(summary);
|
|
324
|
+
const videoLine = formatVideoLine(summary);
|
|
325
|
+
const done = {
|
|
326
|
+
slug: step.slug,
|
|
327
|
+
name: step.name,
|
|
328
|
+
courseCount: step.courseCount,
|
|
329
|
+
outDir: step.outDir,
|
|
330
|
+
summary,
|
|
331
|
+
};
|
|
332
|
+
const rows = [
|
|
333
|
+
['Words', summary.totalWords.toLocaleString('en-US')],
|
|
334
|
+
['Lessons', _jsx(OutcomeChips, { counts: summary.counts })],
|
|
335
|
+
];
|
|
336
|
+
if (videoLine)
|
|
337
|
+
rows.push(['Videos', videoLine]);
|
|
338
|
+
rows.push(['Saved to', _jsx(Text, { color: ACCENT, children: step.outDir })]);
|
|
339
|
+
return (_jsxs(Box, { flexDirection: "column", children: [empty && summary.counts.skipped === 0 ? (_jsx(Text, { color: "yellow", bold: true, children: "\u26A0 Nothing was transcribed." })) : (_jsxs(Text, { color: "green", bold: true, children: ["\u2713 ", empty ? `${step.name} is up to date` : `Synced ${step.name}`, _jsxs(Text, { dimColor: true, bold: false, children: [" in ", formatDuration(step.elapsedMs)] })] })), _jsx(Text, { children: " " }), _jsx(Card, { rows: rows }), summary.problems.length > 0 && (_jsxs(Box, { flexDirection: "column", marginTop: 1, children: [_jsxs(Text, { bold: true, children: ["Needs attention (", summary.problems.length, ")"] }), sortedProblems(summary).map((problem, index) => (
|
|
340
|
+
// eslint-disable-next-line react/no-array-index-key
|
|
341
|
+
_jsxs(Text, { wrap: "truncate-end", children: [_jsx(Text, { color: OUTCOME_STYLE[problem.outcome].color, children: OUTCOME_STYLE[problem.outcome].icon }), ' ', problem.course, " \u203A ", problem.title, " ", _jsx(Text, { dimColor: true, children: problem.reason })] }, index)))] })), _jsx(Text, { children: " " }), _jsx(Text, { bold: true, children: "What next?" }), _jsx(Text, { children: " " }), _jsx(SelectList, { items: MENU_ITEMS, onSelect: (choice) => handleMenuChoice(done, choice) })] }));
|
|
342
|
+
}
|
|
343
|
+
case 'error':
|
|
344
|
+
return (_jsxs(Box, { flexDirection: "column", borderStyle: "round", borderColor: "red", paddingX: 1, alignSelf: "flex-start", children: [_jsx(Text, { color: "red", bold: true, children: "\u2717 Something went wrong" }), _jsx(Text, { children: step.message }), _jsx(Text, { children: " " }), _jsx(ExitPrompt, { label: "Press any key to exit." })] }));
|
|
281
345
|
}
|
|
282
|
-
case 'error':
|
|
283
|
-
return (_jsxs(Box, { flexDirection: "column", children: [_jsxs(Text, { color: "red", children: ["Something went wrong: ", step.message] }), _jsx(ExitPrompt, { label: "Press any key to exit." })] }));
|
|
284
346
|
}
|
|
285
347
|
}
|
package/dist/tui/SelectList.d.ts
CHANGED
package/dist/tui/SelectList.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
2
|
import React from 'react';
|
|
3
3
|
import { Box, Text, useInput } from 'ink';
|
|
4
|
+
import { ACCENT } from './theme.js';
|
|
4
5
|
/** Minimal arrow-key list, built directly on Ink's useInput rather than pulling
|
|
5
6
|
* in a select-input dependency. Cheap to render: a handful of Text lines,
|
|
6
7
|
* re-rendered only on cursor movement or selection. */
|
|
@@ -21,5 +22,9 @@ export function SelectList({ items, onSelect }) {
|
|
|
21
22
|
onSelect(item.value);
|
|
22
23
|
}
|
|
23
24
|
});
|
|
24
|
-
|
|
25
|
+
const labelWidth = Math.max(...items.map((item) => item.label.length));
|
|
26
|
+
return (_jsxs(Box, { flexDirection: "column", children: [items.map((item, index) => {
|
|
27
|
+
const active = index === cursor;
|
|
28
|
+
return (_jsxs(Text, { children: [_jsx(Text, { color: ACCENT, children: active ? '❯ ' : ' ' }), _jsx(Text, { color: active ? ACCENT : undefined, bold: active, children: item.hint ? item.label.padEnd(labelWidth) : item.label }), item.hint && _jsxs(Text, { dimColor: true, children: [" ", item.hint] })] }, item.label));
|
|
29
|
+
}), _jsxs(Text, { dimColor: true, children: ['\n', "\u2191\u2193 move \u00B7 enter select"] })] }));
|
|
25
30
|
}
|
|
@@ -13,7 +13,7 @@ export function isProfileLockError(error) {
|
|
|
13
13
|
*/
|
|
14
14
|
export function translateLaunchError(error) {
|
|
15
15
|
if (isProfileLockError(error)) {
|
|
16
|
-
const friendly = new Error('The Chrome profile is already in use. Another skrape process may be running
|
|
16
|
+
const friendly = new Error('The Chrome profile is already in use. Another skrape process may be running. ' +
|
|
17
17
|
'if not, delete ~/.skool-skrape/chrome-profile/SingletonLock and try again.');
|
|
18
18
|
friendly.cause = error;
|
|
19
19
|
return friendly;
|
package/dist/tui/progress.js
CHANGED
|
@@ -26,6 +26,5 @@ export function formatProgressBar(done, total, width = 24) {
|
|
|
26
26
|
const safeTotal = total > 0 ? total : 1;
|
|
27
27
|
const ratio = Math.min(1, Math.max(0, done / safeTotal));
|
|
28
28
|
const filled = Math.round(ratio * width);
|
|
29
|
-
|
|
30
|
-
return `[${bar}] ${done}/${total}`;
|
|
29
|
+
return `${'█'.repeat(filled)}${'░'.repeat(width - filled)} ${done}/${total} ${Math.round(ratio * 100)}%`;
|
|
31
30
|
}
|
package/dist/tui/run.js
CHANGED
|
@@ -33,13 +33,13 @@ export async function runGuidedFlow() {
|
|
|
33
33
|
probeLaunchable: probeChromeLaunchable,
|
|
34
34
|
markInstalled: (executablePath) => markChromeInstalled(chromeMarkerPath(), executablePath),
|
|
35
35
|
installChrome: async (onProgress) => {
|
|
36
|
-
console.log('\nSetting up the browser skrape needs
|
|
36
|
+
console.log('\nSetting up the browser skrape needs. This happens once, ~150MB.\n');
|
|
37
37
|
await installChromeViaCli(onProgress);
|
|
38
38
|
console.log('\nBrowser setup complete.\n');
|
|
39
39
|
const executablePath = await probeChromeLaunchable();
|
|
40
40
|
if (!executablePath) {
|
|
41
41
|
throw new Error('Chrome installed, but still could not be launched. This usually means a platform-specific ' +
|
|
42
|
-
'dependency is missing
|
|
42
|
+
'dependency is missing. See the Playwright install output above for details.');
|
|
43
43
|
}
|
|
44
44
|
return executablePath;
|
|
45
45
|
},
|
|
@@ -83,7 +83,7 @@ export async function runGuidedFlow() {
|
|
|
83
83
|
const courses = await listCourses(slug, lifecycle.getFetcher());
|
|
84
84
|
return courses.filter((course) => course.hasAccess).length;
|
|
85
85
|
},
|
|
86
|
-
runSync: async (slug, outDir, onProgress) => syncClassroom({ slug, outDir, db, fetcher: lifecycle.getFetcher(), concurrency: 4, onProgress }),
|
|
86
|
+
runSync: async (slug, outDir, onProgress, videos) => syncClassroom({ slug, outDir, db, fetcher: lifecycle.getFetcher(), concurrency: 4, videos, onProgress }),
|
|
87
87
|
};
|
|
88
88
|
const { waitUntilExit } = render(React.createElement(App, { controllers }));
|
|
89
89
|
try {
|
package/dist/tui/summary.d.ts
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import type { SyncSummary } from '../sync.js';
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
import { type OutcomeColor } from './theme.js';
|
|
3
|
+
/** Colors a piece of text; identity by default so output stays plain for tests and pipes. */
|
|
4
|
+
export type Paint = (color: OutcomeColor | 'bold' | 'dim', text: string) => string;
|
|
5
|
+
export declare function formatVideoLine(summary: SyncSummary): string | null;
|
|
6
|
+
/** Problems worst-first, so real failures aren't buried under routine "no video" lessons. */
|
|
7
|
+
export declare function sortedProblems(summary: SyncSummary): SyncSummary['problems'];
|
|
4
8
|
/** True when nothing was transcribed at all — the case that must never be
|
|
5
9
|
* rendered as a cheerful, empty-looking summary. */
|
|
6
10
|
export declare function nothingTranscribed(summary: SyncSummary): boolean;
|
|
7
|
-
export declare function formatSummary(summary: SyncSummary, outDir: string): string[];
|
|
11
|
+
export declare function formatSummary(summary: SyncSummary, outDir: string, paint?: Paint): string[];
|
package/dist/tui/summary.js
CHANGED
|
@@ -1,29 +1,49 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
return
|
|
1
|
+
import { OUTCOME_STYLE } from './theme.js';
|
|
2
|
+
const plain = (_color, text) => text;
|
|
3
|
+
export function formatVideoLine(summary) {
|
|
4
|
+
const v = summary.videos;
|
|
5
|
+
if (!v)
|
|
6
|
+
return null;
|
|
7
|
+
return `${v.downloaded} downloaded · ${v.existing} already on disk · ${v.failed} failed`;
|
|
8
8
|
}
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
const SEVERITY = ['failed', 'unavailable', 'no-access', 'no-video', 'skipped', 'ok'];
|
|
10
|
+
/** Problems worst-first, so real failures aren't buried under routine "no video" lessons. */
|
|
11
|
+
export function sortedProblems(summary) {
|
|
12
|
+
return [...summary.problems].sort((a, b) => SEVERITY.indexOf(a.outcome) - SEVERITY.indexOf(b.outcome));
|
|
11
13
|
}
|
|
12
14
|
/** True when nothing was transcribed at all — the case that must never be
|
|
13
15
|
* rendered as a cheerful, empty-looking summary. */
|
|
14
16
|
export function nothingTranscribed(summary) {
|
|
15
17
|
return summary.counts.ok === 0 && summary.totalWords === 0;
|
|
16
18
|
}
|
|
17
|
-
|
|
19
|
+
function headline(summary, paint) {
|
|
20
|
+
if (!nothingTranscribed(summary)) {
|
|
21
|
+
return paint('green', `✓ Transcribed ${summary.totalWords.toLocaleString('en-US')} words`);
|
|
22
|
+
}
|
|
23
|
+
// A re-sync where every lesson was already on disk is a success, not an empty run.
|
|
24
|
+
if (summary.counts.skipped > 0)
|
|
25
|
+
return paint('green', '✓ Up to date, nothing new to transcribe.');
|
|
26
|
+
return paint('yellow', '⚠ Nothing was transcribed.');
|
|
27
|
+
}
|
|
28
|
+
export function formatSummary(summary, outDir, paint = plain) {
|
|
18
29
|
const lines = [];
|
|
19
|
-
|
|
20
|
-
|
|
30
|
+
lines.push(headline(summary, paint), '');
|
|
31
|
+
for (const [outcome, count] of Object.entries(summary.counts)) {
|
|
32
|
+
if (count === 0)
|
|
33
|
+
continue;
|
|
34
|
+
const style = OUTCOME_STYLE[outcome];
|
|
35
|
+
lines.push(` ${paint(style.color, style.icon)} ${style.label.padEnd(14)} ${count}`);
|
|
21
36
|
}
|
|
22
|
-
|
|
23
|
-
|
|
37
|
+
const videoLine = formatVideoLine(summary);
|
|
38
|
+
if (videoLine)
|
|
39
|
+
lines.push(` ${paint('dim', '▸')} ${'videos'.padEnd(14)} ${videoLine}`);
|
|
24
40
|
if (summary.problems.length > 0) {
|
|
25
|
-
lines.push('', '
|
|
41
|
+
lines.push('', paint('bold', `Needs attention (${summary.problems.length})`));
|
|
42
|
+
for (const problem of sortedProblems(summary)) {
|
|
43
|
+
const style = OUTCOME_STYLE[problem.outcome];
|
|
44
|
+
lines.push(` ${paint(style.color, style.icon)} ${problem.course} › ${problem.title} ${paint('dim', problem.reason)}`);
|
|
45
|
+
}
|
|
26
46
|
}
|
|
27
|
-
lines.push('',
|
|
47
|
+
lines.push('', `${paint('dim', 'Saved to')} ${outDir}`);
|
|
28
48
|
return lines;
|
|
29
49
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Outcome } from '../sync.js';
|
|
2
|
+
/** Brand accent: peach. Ink takes hex; the plain CLI falls back to a named color. */
|
|
3
|
+
export declare const ACCENT = "#FFB38A";
|
|
4
|
+
export type OutcomeColor = 'green' | 'gray' | 'yellow' | 'red';
|
|
5
|
+
export declare const OUTCOME_STYLE: Record<Outcome, {
|
|
6
|
+
icon: string;
|
|
7
|
+
label: string;
|
|
8
|
+
color: OutcomeColor;
|
|
9
|
+
}>;
|
|
10
|
+
export declare function formatDuration(ms: number): string;
|
|
11
|
+
/** Remaining time from a linear rate; null until there's at least one data point. */
|
|
12
|
+
export declare function estimateRemainingMs(elapsedMs: number, done: number, total: number): number | null;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** Brand accent: peach. Ink takes hex; the plain CLI falls back to a named color. */
|
|
2
|
+
export const ACCENT = '#FFB38A';
|
|
3
|
+
export const OUTCOME_STYLE = {
|
|
4
|
+
ok: { icon: '✓', label: 'transcribed', color: 'green' },
|
|
5
|
+
skipped: { icon: '↷', label: 'already done', color: 'gray' },
|
|
6
|
+
'no-video': { icon: '·', label: 'no video', color: 'gray' },
|
|
7
|
+
'no-access': { icon: '⊘', label: 'locked', color: 'yellow' },
|
|
8
|
+
unavailable: { icon: '!', label: 'unavailable', color: 'yellow' },
|
|
9
|
+
failed: { icon: '✗', label: 'failed', color: 'red' },
|
|
10
|
+
};
|
|
11
|
+
export function formatDuration(ms) {
|
|
12
|
+
const seconds = Math.max(0, Math.round(ms / 1000));
|
|
13
|
+
if (seconds < 60)
|
|
14
|
+
return `${seconds}s`;
|
|
15
|
+
const minutes = Math.floor(seconds / 60);
|
|
16
|
+
if (minutes < 60)
|
|
17
|
+
return `${minutes}m ${seconds % 60}s`;
|
|
18
|
+
return `${Math.floor(minutes / 60)}h ${minutes % 60}m`;
|
|
19
|
+
}
|
|
20
|
+
/** Remaining time from a linear rate; null until there's at least one data point. */
|
|
21
|
+
export function estimateRemainingMs(elapsedMs, done, total) {
|
|
22
|
+
if (done <= 0 || done >= total)
|
|
23
|
+
return null;
|
|
24
|
+
return (elapsedMs / done) * (total - done);
|
|
25
|
+
}
|
package/package.json
CHANGED