@mogulmoretti/skrape 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +32 -0
- package/README.md +117 -0
- package/dist/auth/session.d.ts +26 -0
- package/dist/auth/session.js +70 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +123 -0
- package/dist/discover/communities.d.ts +22 -0
- package/dist/discover/communities.js +92 -0
- package/dist/discover/skool.d.ts +17 -0
- package/dist/discover/skool.js +91 -0
- package/dist/fetch/browser.d.ts +16 -0
- package/dist/fetch/browser.js +58 -0
- package/dist/fetch/chromeSetup.d.ts +54 -0
- package/dist/fetch/chromeSetup.js +126 -0
- package/dist/fetch/http.d.ts +17 -0
- package/dist/fetch/http.js +64 -0
- package/dist/fetch/nextdata.d.ts +9 -0
- package/dist/fetch/nextdata.js +23 -0
- package/dist/fetch/resilient.d.ts +12 -0
- package/dist/fetch/resilient.js +68 -0
- package/dist/media/index.d.ts +4 -0
- package/dist/media/index.js +11 -0
- package/dist/media/loom.d.ts +5 -0
- package/dist/media/loom.js +60 -0
- package/dist/normalize/vtt.d.ts +12 -0
- package/dist/normalize/vtt.js +109 -0
- package/dist/store/db.d.ts +15 -0
- package/dist/store/db.js +125 -0
- package/dist/store/markdown.d.ts +7 -0
- package/dist/store/markdown.js +41 -0
- package/dist/sync.d.ts +30 -0
- package/dist/sync.js +178 -0
- package/dist/tui/App.d.ts +22 -0
- package/dist/tui/App.js +285 -0
- package/dist/tui/SelectList.d.ts +14 -0
- package/dist/tui/SelectList.js +25 -0
- package/dist/tui/browserLifecycle.d.ts +71 -0
- package/dist/tui/browserLifecycle.js +114 -0
- package/dist/tui/chromeSetup.d.ts +50 -0
- package/dist/tui/chromeSetup.js +52 -0
- package/dist/tui/flow.d.ts +53 -0
- package/dist/tui/flow.js +30 -0
- package/dist/tui/progress.d.ts +21 -0
- package/dist/tui/progress.js +31 -0
- package/dist/tui/reveal.d.ts +17 -0
- package/dist/tui/reveal.js +47 -0
- package/dist/tui/run.d.ts +12 -0
- package/dist/tui/run.js +96 -0
- package/dist/tui/slug.d.ts +7 -0
- package/dist/tui/slug.js +18 -0
- package/dist/tui/summary.d.ts +7 -0
- package/dist/tui/summary.js +29 -0
- package/dist/types.d.ts +33 -0
- package/dist/types.js +1 -0
- package/package.json +57 -0
package/dist/tui/App.js
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
|
+
import React from 'react';
|
|
3
|
+
import { Box, Text, useApp, useInput } from 'ink';
|
|
4
|
+
import { SelectList } from './SelectList.js';
|
|
5
|
+
import { applyProgress, createProgressState, formatProgressBar } from './progress.js';
|
|
6
|
+
import { formatSummary } from './summary.js';
|
|
7
|
+
import { isValidSlug, normalizeSlug } from './slug.js';
|
|
8
|
+
import { MENU_ITEMS, nextStepForMenuChoice } from './flow.js';
|
|
9
|
+
import { revealOutputFolder } from './reveal.js';
|
|
10
|
+
const MAX_ERROR_LINES = 6;
|
|
11
|
+
const MAX_ERROR_CHARS = 800;
|
|
12
|
+
/**
|
|
13
|
+
* Caps error text shown in the TUI. Some failures (e.g. a raw Playwright
|
|
14
|
+
* browser-launch error) can be a multi-hundred-line dump — that's useless in
|
|
15
|
+
* a terminal UI and pushes everything else off screen, so keep only the
|
|
16
|
+
* first few lines and a hard character cap.
|
|
17
|
+
*/
|
|
18
|
+
export function truncateErrorText(message) {
|
|
19
|
+
const lines = message.split('\n');
|
|
20
|
+
const truncatedByLines = lines.length > MAX_ERROR_LINES;
|
|
21
|
+
const limitedLines = lines.slice(0, MAX_ERROR_LINES).join('\n');
|
|
22
|
+
const truncatedByChars = limitedLines.length > MAX_ERROR_CHARS;
|
|
23
|
+
const limited = limitedLines.slice(0, MAX_ERROR_CHARS);
|
|
24
|
+
return truncatedByLines || truncatedByChars ? `${limited}\n… (truncated)` : limited;
|
|
25
|
+
}
|
|
26
|
+
function errorMessage(error) {
|
|
27
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
28
|
+
return truncateErrorText(message);
|
|
29
|
+
}
|
|
30
|
+
function ConfirmPrompt({ onConfirm }) {
|
|
31
|
+
const { exit } = useApp();
|
|
32
|
+
useInput((input, key) => {
|
|
33
|
+
if (key.return)
|
|
34
|
+
onConfirm();
|
|
35
|
+
else if (input === 'q' || key.escape)
|
|
36
|
+
exit();
|
|
37
|
+
});
|
|
38
|
+
return _jsx(Text, { dimColor: true, children: "[enter] continue [q] quit" });
|
|
39
|
+
}
|
|
40
|
+
function ExitPrompt({ label }) {
|
|
41
|
+
const { exit } = useApp();
|
|
42
|
+
useInput(() => exit());
|
|
43
|
+
return _jsx(Text, { dimColor: true, children: label });
|
|
44
|
+
}
|
|
45
|
+
function ManualSlugInput({ value, onChange, onSubmit, }) {
|
|
46
|
+
useInput((input, key) => {
|
|
47
|
+
if (key.return) {
|
|
48
|
+
onSubmit(value);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
if (key.backspace || key.delete) {
|
|
52
|
+
onChange(value.slice(0, -1));
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
if (key.ctrl || key.meta || key.upArrow || key.downArrow || key.leftArrow || key.rightArrow)
|
|
56
|
+
return;
|
|
57
|
+
if (input)
|
|
58
|
+
onChange(value + input);
|
|
59
|
+
});
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
export function App({ controllers }) {
|
|
63
|
+
const { exit } = useApp();
|
|
64
|
+
const [step, setStep] = React.useState({ kind: 'checking-session' });
|
|
65
|
+
useInput((input, key) => {
|
|
66
|
+
if (key.ctrl && input === 'c')
|
|
67
|
+
exit();
|
|
68
|
+
});
|
|
69
|
+
const fail = React.useCallback((error) => {
|
|
70
|
+
setStep({ kind: 'error', message: errorMessage(error) });
|
|
71
|
+
}, []);
|
|
72
|
+
// Step 1: is the saved Chrome profile signed in?
|
|
73
|
+
React.useEffect(() => {
|
|
74
|
+
if (step.kind !== 'checking-session')
|
|
75
|
+
return;
|
|
76
|
+
let cancelled = false;
|
|
77
|
+
controllers
|
|
78
|
+
.checkLoggedIn()
|
|
79
|
+
.then((loggedIn) => {
|
|
80
|
+
if (cancelled)
|
|
81
|
+
return;
|
|
82
|
+
setStep(loggedIn ? { kind: 'discovering' } : { kind: 'login-needed' });
|
|
83
|
+
})
|
|
84
|
+
.catch((error) => {
|
|
85
|
+
if (!cancelled)
|
|
86
|
+
fail(error);
|
|
87
|
+
});
|
|
88
|
+
return () => {
|
|
89
|
+
cancelled = true;
|
|
90
|
+
};
|
|
91
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
92
|
+
}, [step.kind]);
|
|
93
|
+
// Step 1b: the user asked us to open the browser so they can sign in.
|
|
94
|
+
React.useEffect(() => {
|
|
95
|
+
if (step.kind !== 'logging-in')
|
|
96
|
+
return;
|
|
97
|
+
let cancelled = false;
|
|
98
|
+
controllers
|
|
99
|
+
.login()
|
|
100
|
+
.then(() => {
|
|
101
|
+
if (!cancelled)
|
|
102
|
+
setStep({ kind: 'discovering' });
|
|
103
|
+
})
|
|
104
|
+
.catch((error) => {
|
|
105
|
+
if (!cancelled)
|
|
106
|
+
fail(error);
|
|
107
|
+
});
|
|
108
|
+
return () => {
|
|
109
|
+
cancelled = true;
|
|
110
|
+
};
|
|
111
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
112
|
+
}, [step.kind]);
|
|
113
|
+
// Step 2: fetch the communities the signed-in user belongs to.
|
|
114
|
+
React.useEffect(() => {
|
|
115
|
+
if (step.kind !== 'discovering')
|
|
116
|
+
return;
|
|
117
|
+
let cancelled = false;
|
|
118
|
+
controllers
|
|
119
|
+
.discoverCommunities()
|
|
120
|
+
.then((communities) => {
|
|
121
|
+
if (cancelled)
|
|
122
|
+
return;
|
|
123
|
+
if (communities && communities.length > 0) {
|
|
124
|
+
setStep({ kind: 'picking', communities });
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
setStep({ kind: 'manual-slug', value: '', error: null });
|
|
128
|
+
}
|
|
129
|
+
})
|
|
130
|
+
.catch((error) => {
|
|
131
|
+
if (!cancelled)
|
|
132
|
+
fail(error);
|
|
133
|
+
});
|
|
134
|
+
return () => {
|
|
135
|
+
cancelled = true;
|
|
136
|
+
};
|
|
137
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
138
|
+
}, [step.kind]);
|
|
139
|
+
const chooseCommunity = React.useCallback((slug, name) => {
|
|
140
|
+
setStep({ kind: 'counting', slug, name });
|
|
141
|
+
}, []);
|
|
142
|
+
// Step 2b/3: count accessible courses for the confirmation screen.
|
|
143
|
+
React.useEffect(() => {
|
|
144
|
+
if (step.kind !== 'counting')
|
|
145
|
+
return;
|
|
146
|
+
let cancelled = false;
|
|
147
|
+
const { slug, name } = step;
|
|
148
|
+
controllers
|
|
149
|
+
.countAccessibleCourses(slug)
|
|
150
|
+
.then((courseCount) => {
|
|
151
|
+
if (cancelled)
|
|
152
|
+
return;
|
|
153
|
+
setStep({ kind: 'confirming', slug, name, courseCount, outDir: `${controllers.outRoot}/${slug}` });
|
|
154
|
+
})
|
|
155
|
+
.catch((error) => {
|
|
156
|
+
if (!cancelled)
|
|
157
|
+
fail(error);
|
|
158
|
+
});
|
|
159
|
+
return () => {
|
|
160
|
+
cancelled = true;
|
|
161
|
+
};
|
|
162
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
163
|
+
}, [step]);
|
|
164
|
+
// Step 4: run the sync, folding progress events into render state.
|
|
165
|
+
React.useEffect(() => {
|
|
166
|
+
if (step.kind !== 'syncing')
|
|
167
|
+
return;
|
|
168
|
+
let cancelled = false;
|
|
169
|
+
const { slug, name, courseCount, outDir } = step;
|
|
170
|
+
controllers
|
|
171
|
+
.runSync(slug, outDir, (event) => {
|
|
172
|
+
if (cancelled)
|
|
173
|
+
return;
|
|
174
|
+
setStep((previous) => previous.kind === 'syncing' ? { ...previous, progress: applyProgress(previous.progress, event) } : previous);
|
|
175
|
+
})
|
|
176
|
+
.then((summary) => {
|
|
177
|
+
if (!cancelled)
|
|
178
|
+
setStep({ kind: 'done', slug, name, courseCount, outDir, summary });
|
|
179
|
+
})
|
|
180
|
+
.catch((error) => {
|
|
181
|
+
if (!cancelled)
|
|
182
|
+
fail(error);
|
|
183
|
+
});
|
|
184
|
+
return () => {
|
|
185
|
+
cancelled = true;
|
|
186
|
+
};
|
|
187
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
188
|
+
}, [step.kind]);
|
|
189
|
+
// Step 5: after a sync finishes, act on the post-summary menu choice —
|
|
190
|
+
// loops back into an earlier step (never nests/recurses; `step` is simply
|
|
191
|
+
// replaced again like every other transition) or performs an in-place
|
|
192
|
+
// side effect that leaves the menu on screen.
|
|
193
|
+
const handleMenuChoice = React.useCallback((done, choice) => {
|
|
194
|
+
if (choice === 'quit') {
|
|
195
|
+
exit();
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
if (choice === 'open-folder') {
|
|
199
|
+
// revealOutputFolder never throws (it falls back to printing the
|
|
200
|
+
// path on failure) — this catch is a last-resort guard so a future
|
|
201
|
+
// change there can never crash the TUI.
|
|
202
|
+
void revealOutputFolder(done.outDir).catch(() => { });
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
const transition = nextStepForMenuChoice(done, choice);
|
|
206
|
+
if (!transition)
|
|
207
|
+
return;
|
|
208
|
+
if (transition.kind === 'discovering') {
|
|
209
|
+
setStep({ kind: 'discovering' });
|
|
210
|
+
}
|
|
211
|
+
else {
|
|
212
|
+
setStep({
|
|
213
|
+
kind: 'confirming',
|
|
214
|
+
slug: transition.slug,
|
|
215
|
+
name: transition.name,
|
|
216
|
+
courseCount: transition.courseCount,
|
|
217
|
+
outDir: transition.outDir,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
}, [exit]);
|
|
221
|
+
switch (step.kind) {
|
|
222
|
+
case 'checking-session':
|
|
223
|
+
return _jsx(Text, { children: "Checking your Skool session\u2026" });
|
|
224
|
+
case 'login-needed':
|
|
225
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Text, { color: "yellow", children: "You are not signed in to Skool." }), _jsx(Text, { dimColor: true, children: "Nothing is typed for you \u2014 a browser window opens and you sign in yourself." }), _jsx(ConfirmPrompt, { onConfirm: () => setStep({ kind: 'logging-in' }) })] }));
|
|
226
|
+
case 'logging-in':
|
|
227
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Text, { children: "A browser window is open. Sign in there." }), _jsx(Text, { dimColor: true, children: "Waiting for you to finish (Ctrl+C to cancel)\u2026" })] }));
|
|
228
|
+
case 'discovering':
|
|
229
|
+
return _jsx(Text, { children: "Looking up the communities you belong to\u2026" });
|
|
230
|
+
case 'picking': {
|
|
231
|
+
const items = [
|
|
232
|
+
...step.communities.map((community) => ({
|
|
233
|
+
label: `${community.name} (skool.com/${community.slug})`,
|
|
234
|
+
value: community,
|
|
235
|
+
})),
|
|
236
|
+
{ label: 'Enter a slug manually…', value: null },
|
|
237
|
+
];
|
|
238
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Text, { children: "Which community?" }), _jsx(SelectList, { items: items, onSelect: (value) => {
|
|
239
|
+
if (value === null)
|
|
240
|
+
setStep({ kind: 'manual-slug', value: '', error: null });
|
|
241
|
+
else
|
|
242
|
+
chooseCommunity(value.slug, value.name);
|
|
243
|
+
} })] }));
|
|
244
|
+
}
|
|
245
|
+
case 'manual-slug':
|
|
246
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Text, { children: "Enter the community slug \u2014 the part after skool.com/" }), _jsxs(Text, { children: ["Slug: ", step.value, _jsx(Text, { color: "gray", children: "\u2588" })] }), step.error && _jsx(Text, { color: "red", children: step.error }), _jsx(ManualSlugInput, { value: step.value, onChange: (value) => setStep({ kind: 'manual-slug', value, error: null }), onSubmit: (value) => {
|
|
247
|
+
const slug = normalizeSlug(value);
|
|
248
|
+
if (!isValidSlug(slug)) {
|
|
249
|
+
setStep({ kind: 'manual-slug', value, error: `"${value}" doesn't look like a valid slug.` });
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
chooseCommunity(slug, slug);
|
|
253
|
+
} })] }));
|
|
254
|
+
case 'counting':
|
|
255
|
+
return _jsxs(Text, { children: ["Looking up courses in ", step.name, "\u2026"] });
|
|
256
|
+
case 'confirming':
|
|
257
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsxs(Text, { children: ["Community: ", step.name, " (skool.com/", step.slug, ")"] }), _jsxs(Text, { children: ["Courses found: ", step.courseCount] }), _jsxs(Text, { children: ["Output will be written to: ", step.outDir] }), _jsx(Text, { children: " " }), _jsx(ConfirmPrompt, { onConfirm: () => setStep({
|
|
258
|
+
kind: 'syncing',
|
|
259
|
+
slug: step.slug,
|
|
260
|
+
name: step.name,
|
|
261
|
+
courseCount: step.courseCount,
|
|
262
|
+
outDir: step.outDir,
|
|
263
|
+
progress: createProgressState(0),
|
|
264
|
+
}) })] }));
|
|
265
|
+
case 'syncing': {
|
|
266
|
+
const { progress } = step;
|
|
267
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsx(Text, { children: formatProgressBar(progress.done, progress.total) }), progress.current && (_jsxs(Text, { dimColor: true, children: ["current: ", progress.current.course, " / ", progress.current.title] })), _jsxs(Text, { dimColor: true, children: ["ok ", progress.counts.ok, " skipped ", progress.counts.skipped, " no-video ", progress.counts['no-video'], ' ', "no-access ", progress.counts['no-access'], " unavailable ", progress.counts.unavailable, " failed", ' ', progress.counts.failed] })] }));
|
|
268
|
+
}
|
|
269
|
+
case 'done': {
|
|
270
|
+
const lines = formatSummary(step.summary, step.outDir);
|
|
271
|
+
const done = {
|
|
272
|
+
slug: step.slug,
|
|
273
|
+
name: step.name,
|
|
274
|
+
courseCount: step.courseCount,
|
|
275
|
+
outDir: step.outDir,
|
|
276
|
+
summary: step.summary,
|
|
277
|
+
};
|
|
278
|
+
return (_jsxs(Box, { flexDirection: "column", children: [lines.map((line, index) => (
|
|
279
|
+
// eslint-disable-next-line react/no-array-index-key
|
|
280
|
+
_jsx(Text, { children: line || ' ' }, index))), _jsx(Text, { children: " " }), _jsx(Text, { children: "What next?" }), _jsx(SelectList, { items: MENU_ITEMS, onSelect: (choice) => handleMenuChoice(done, choice) })] }));
|
|
281
|
+
}
|
|
282
|
+
case 'error':
|
|
283
|
+
return (_jsxs(Box, { flexDirection: "column", children: [_jsxs(Text, { color: "red", children: ["Something went wrong: ", step.message] }), _jsx(ExitPrompt, { label: "Press any key to exit." })] }));
|
|
284
|
+
}
|
|
285
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
export interface SelectItem<T> {
|
|
3
|
+
label: string;
|
|
4
|
+
value: T;
|
|
5
|
+
}
|
|
6
|
+
interface SelectListProps<T> {
|
|
7
|
+
items: Array<SelectItem<T>>;
|
|
8
|
+
onSelect: (value: T) => void;
|
|
9
|
+
}
|
|
10
|
+
/** Minimal arrow-key list, built directly on Ink's useInput rather than pulling
|
|
11
|
+
* in a select-input dependency. Cheap to render: a handful of Text lines,
|
|
12
|
+
* re-rendered only on cursor movement or selection. */
|
|
13
|
+
export declare function SelectList<T>({ items, onSelect }: SelectListProps<T>): React.JSX.Element;
|
|
14
|
+
export {};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { jsxs as _jsxs, jsx as _jsx } from "react/jsx-runtime";
|
|
2
|
+
import React from 'react';
|
|
3
|
+
import { Box, Text, useInput } from 'ink';
|
|
4
|
+
/** Minimal arrow-key list, built directly on Ink's useInput rather than pulling
|
|
5
|
+
* in a select-input dependency. Cheap to render: a handful of Text lines,
|
|
6
|
+
* re-rendered only on cursor movement or selection. */
|
|
7
|
+
export function SelectList({ items, onSelect }) {
|
|
8
|
+
const [cursor, setCursor] = React.useState(0);
|
|
9
|
+
useInput((input, key) => {
|
|
10
|
+
if (items.length === 0)
|
|
11
|
+
return;
|
|
12
|
+
if (key.upArrow || input === 'k') {
|
|
13
|
+
setCursor((current) => (current - 1 + items.length) % items.length);
|
|
14
|
+
}
|
|
15
|
+
else if (key.downArrow || input === 'j') {
|
|
16
|
+
setCursor((current) => (current + 1) % items.length);
|
|
17
|
+
}
|
|
18
|
+
else if (key.return) {
|
|
19
|
+
const item = items[cursor];
|
|
20
|
+
if (item)
|
|
21
|
+
onSelect(item.value);
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
return (_jsx(Box, { flexDirection: "column", children: items.map((item, index) => (_jsxs(Text, { color: index === cursor ? 'cyan' : undefined, children: [index === cursor ? '> ' : ' ', item.label] }, item.label))) }));
|
|
25
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { Fetcher } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* A Fetcher that also owns a real Chrome process and must be closed to
|
|
4
|
+
* release it.
|
|
5
|
+
*/
|
|
6
|
+
export interface CloseableFetcher extends Fetcher {
|
|
7
|
+
close(): Promise<void>;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* ============================================================================
|
|
11
|
+
* INVARIANT: only one Chrome process may hold the persistent profile
|
|
12
|
+
* directory at a time. Chrome enforces this itself (SingletonLock) and
|
|
13
|
+
* aborts a second launch against the same profile. Every operation below
|
|
14
|
+
* that needs a browser always closes whatever browser is currently live
|
|
15
|
+
* (via closeCurrent()) before handing out a new one — so the profile is
|
|
16
|
+
* provably held by at most one Chrome at any instant.
|
|
17
|
+
* ============================================================================
|
|
18
|
+
*/
|
|
19
|
+
/** A handle callers can use to force-close a launched login context, e.g. on Ctrl+C. */
|
|
20
|
+
export interface LoginContextHandle {
|
|
21
|
+
close(): Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
export interface BrowserLifecycleDeps {
|
|
24
|
+
http: Fetcher;
|
|
25
|
+
isLoggedIn: (html: string) => boolean;
|
|
26
|
+
/** Creates a fresh, not-yet-launched BrowserFetcher over the profile dir. */
|
|
27
|
+
makeBrowserFetcher: () => CloseableFetcher;
|
|
28
|
+
/**
|
|
29
|
+
* The real interactive sign-in flow (headed Chrome, same profile dir).
|
|
30
|
+
* `onContext` is called as soon as the headed browser launches, so the
|
|
31
|
+
* lifecycle can track it and force-close it on SIGINT even while
|
|
32
|
+
* performLogin() is still awaiting the user.
|
|
33
|
+
*/
|
|
34
|
+
performLogin: (onContext: (context: LoginContextHandle) => void) => Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
export interface BrowserLifecycle {
|
|
37
|
+
/**
|
|
38
|
+
* Checks whether the saved profile is already signed in. Owns a
|
|
39
|
+
* short-lived browser for the duration of the check only — the profile is
|
|
40
|
+
* guaranteed free again the instant this call resolves or rejects, so a
|
|
41
|
+
* following login() never collides with it.
|
|
42
|
+
*/
|
|
43
|
+
checkLoggedIn(): Promise<boolean>;
|
|
44
|
+
/**
|
|
45
|
+
* Runs the interactive login flow. Closes any browser this lifecycle is
|
|
46
|
+
* currently holding first, so the profile is always free before the real
|
|
47
|
+
* (headed) Chrome launches against it.
|
|
48
|
+
*/
|
|
49
|
+
login(): Promise<void>;
|
|
50
|
+
/**
|
|
51
|
+
* Returns the long-lived fetcher used for discovery and sync. Lazily
|
|
52
|
+
* launches one fresh browser behind it on first call and reuses it for
|
|
53
|
+
* every call after that.
|
|
54
|
+
*/
|
|
55
|
+
getFetcher(): Fetcher;
|
|
56
|
+
/**
|
|
57
|
+
* Closes whichever browser instance is currently live, if any. Safe to
|
|
58
|
+
* call when nothing was ever created and safe to call more than once.
|
|
59
|
+
*/
|
|
60
|
+
cleanup(): Promise<void>;
|
|
61
|
+
}
|
|
62
|
+
/** True when an error looks like Chrome's "profile already in use" failure. */
|
|
63
|
+
export declare function isProfileLockError(error: unknown): boolean;
|
|
64
|
+
/**
|
|
65
|
+
* Turns Playwright's multi-hundred-line ProcessSingleton/SingletonLock dump
|
|
66
|
+
* into one short, actionable line. The raw error is kept on `.cause` for
|
|
67
|
+
* non-TUI diagnostics (e.g. `--verbose` logging) rather than shown to the
|
|
68
|
+
* user. Anything else passes through unchanged.
|
|
69
|
+
*/
|
|
70
|
+
export declare function translateLaunchError(error: unknown): Error;
|
|
71
|
+
export declare function createBrowserLifecycle(deps: BrowserLifecycleDeps): BrowserLifecycle;
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { ResilientFetcher } from '../fetch/resilient.js';
|
|
2
|
+
const LOCK_ERROR_PATTERN = /ProcessSingleton|SingletonLock/i;
|
|
3
|
+
/** True when an error looks like Chrome's "profile already in use" failure. */
|
|
4
|
+
export function isProfileLockError(error) {
|
|
5
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6
|
+
return LOCK_ERROR_PATTERN.test(message);
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Turns Playwright's multi-hundred-line ProcessSingleton/SingletonLock dump
|
|
10
|
+
* into one short, actionable line. The raw error is kept on `.cause` for
|
|
11
|
+
* non-TUI diagnostics (e.g. `--verbose` logging) rather than shown to the
|
|
12
|
+
* user. Anything else passes through unchanged.
|
|
13
|
+
*/
|
|
14
|
+
export function translateLaunchError(error) {
|
|
15
|
+
if (isProfileLockError(error)) {
|
|
16
|
+
const friendly = new Error('The Chrome profile is already in use. Another skrape process may be running — ' +
|
|
17
|
+
'if not, delete ~/.skool-skrape/chrome-profile/SingletonLock and try again.');
|
|
18
|
+
friendly.cause = error;
|
|
19
|
+
return friendly;
|
|
20
|
+
}
|
|
21
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
22
|
+
}
|
|
23
|
+
function wrapLaunchErrors(browser) {
|
|
24
|
+
return {
|
|
25
|
+
async getPage(url) {
|
|
26
|
+
try {
|
|
27
|
+
return await browser.getPage(url);
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
throw translateLaunchError(error);
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
async close() {
|
|
34
|
+
try {
|
|
35
|
+
await browser.close();
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
throw translateLaunchError(error);
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
export function createBrowserLifecycle(deps) {
|
|
44
|
+
const { http, isLoggedIn, makeBrowserFetcher, performLogin } = deps;
|
|
45
|
+
// Tracks whichever browser (BrowserFetcher-backed, or the raw login
|
|
46
|
+
// context) currently owns the profile directory, if any. Never held by
|
|
47
|
+
// more than one at a time — see invariant above. At most one of these two
|
|
48
|
+
// is non-null at any instant.
|
|
49
|
+
let current = null;
|
|
50
|
+
let loginContext = null;
|
|
51
|
+
let sharedFetcher = null;
|
|
52
|
+
async function closeCurrent() {
|
|
53
|
+
const browser = current;
|
|
54
|
+
current = null;
|
|
55
|
+
const login = loginContext;
|
|
56
|
+
loginContext = null;
|
|
57
|
+
if (browser)
|
|
58
|
+
await browser.close();
|
|
59
|
+
if (login)
|
|
60
|
+
await login.close();
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
async checkLoggedIn() {
|
|
64
|
+
const browser = wrapLaunchErrors(makeBrowserFetcher());
|
|
65
|
+
current = browser;
|
|
66
|
+
const fetcher = new ResilientFetcher(http, async () => browser, isLoggedIn);
|
|
67
|
+
try {
|
|
68
|
+
const html = await fetcher.getPage('https://www.skool.com/');
|
|
69
|
+
return isLoggedIn(html);
|
|
70
|
+
}
|
|
71
|
+
finally {
|
|
72
|
+
// The check's browser is never reused — always release it here so
|
|
73
|
+
// the profile is free the instant the check is done, whatever the
|
|
74
|
+
// outcome.
|
|
75
|
+
await closeCurrent();
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
async login() {
|
|
79
|
+
// Defensive: guarantees the invariant even if a caller invokes login()
|
|
80
|
+
// out of the usual check -> login order.
|
|
81
|
+
await closeCurrent();
|
|
82
|
+
try {
|
|
83
|
+
await performLogin((context) => {
|
|
84
|
+
loginContext = context;
|
|
85
|
+
});
|
|
86
|
+
// performLogin() already closed its own context on success.
|
|
87
|
+
loginContext = null;
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
// If a browser did launch before the failure, close it ourselves —
|
|
91
|
+
// otherwise a failed/interrupted login leaves the profile locked
|
|
92
|
+
// for the next run. If SIGINT already closed it (see cleanup()),
|
|
93
|
+
// loginContext is already null here and there is nothing to do.
|
|
94
|
+
const context = loginContext;
|
|
95
|
+
loginContext = null;
|
|
96
|
+
if (context) {
|
|
97
|
+
await context.close().catch(() => {
|
|
98
|
+
// Best-effort: we're already reporting the original error.
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
throw translateLaunchError(error);
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
getFetcher() {
|
|
105
|
+
if (!sharedFetcher) {
|
|
106
|
+
const browser = wrapLaunchErrors(makeBrowserFetcher());
|
|
107
|
+
current = browser;
|
|
108
|
+
sharedFetcher = new ResilientFetcher(http, async () => browser, isLoggedIn);
|
|
109
|
+
}
|
|
110
|
+
return sharedFetcher;
|
|
111
|
+
},
|
|
112
|
+
cleanup: closeCurrent,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure decision logic for the one-time "make sure Chrome is installed" step
|
|
3
|
+
* that runs before the session check. Kept free of fs/child_process/Playwright
|
|
4
|
+
* details (those live in `../fetch/chromeSetup.ts`) so the branching here is
|
|
5
|
+
* unit-testable against plain fakes, the same split used by `flow.ts` and
|
|
6
|
+
* `browserLifecycle.ts` elsewhere in this package.
|
|
7
|
+
*/
|
|
8
|
+
export interface ChromeSetupDeps {
|
|
9
|
+
/** Cheap on-disk check: was this already confirmed on a past run, and does
|
|
10
|
+
* the executable path recorded then still exist now? */
|
|
11
|
+
isConfirmedInstalled: () => Promise<boolean> | boolean;
|
|
12
|
+
/** Real (but cheap: headless, throwaway, closes fast) launch-and-close
|
|
13
|
+
* check. Resolves the launched Chrome's executable path on success, or
|
|
14
|
+
* `undefined` if it couldn't be launched. */
|
|
15
|
+
probeLaunchable: () => Promise<string | undefined>;
|
|
16
|
+
/** Persists the confirmed executable path so future calls can skip
|
|
17
|
+
* straight to it (via a cheap existence check, not another launch). */
|
|
18
|
+
markInstalled: (executablePath: string) => Promise<void>;
|
|
19
|
+
/** Runs the actual installer and resolves the freshly-installed Chrome's
|
|
20
|
+
* executable path; rejects on failure (network/disk/permissions/platform,
|
|
21
|
+
* or an install that "succeeded" but still isn't launchable). */
|
|
22
|
+
installChrome: (onProgress: (elapsedSeconds: number) => void) => Promise<string>;
|
|
23
|
+
}
|
|
24
|
+
export interface ChromeReadyResult {
|
|
25
|
+
/** True only when this call actually ran the installer. */
|
|
26
|
+
installed: boolean;
|
|
27
|
+
}
|
|
28
|
+
/** Thrown when the automatic install fails. Its message is already
|
|
29
|
+
* human-actionable (includes the manual fallback command) — callers can
|
|
30
|
+
* surface `.message` directly without needing to know install internals. */
|
|
31
|
+
export declare class ChromeInstallFailedError extends Error {
|
|
32
|
+
constructor(message: string, options?: {
|
|
33
|
+
cause?: unknown;
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Ensures Chrome is installed and launchable, installing it if not.
|
|
38
|
+
*
|
|
39
|
+
* Order, cheapest-first:
|
|
40
|
+
* 1. Trust a previously-written "confirmed installed" marker — no launch at
|
|
41
|
+
* all, as long as the executable path it recorded still exists on disk.
|
|
42
|
+
* 2. Otherwise, do the real (but cheap) launch-and-close probe.
|
|
43
|
+
* 3. Only if that fails, run the installer — and mark the resolved path it
|
|
44
|
+
* hands back so steps 2/3 are never repeated on later runs.
|
|
45
|
+
*
|
|
46
|
+
* Never throws a raw/undecorated error: any installChrome() failure is
|
|
47
|
+
* wrapped in `ChromeInstallFailedError` with the manual fallback command
|
|
48
|
+
* included, per this project's no-silent-failures rule.
|
|
49
|
+
*/
|
|
50
|
+
export declare function ensureChromeReady(deps: ChromeSetupDeps, onProgress?: (elapsedSeconds: number) => void): Promise<ChromeReadyResult>;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure decision logic for the one-time "make sure Chrome is installed" step
|
|
3
|
+
* that runs before the session check. Kept free of fs/child_process/Playwright
|
|
4
|
+
* details (those live in `../fetch/chromeSetup.ts`) so the branching here is
|
|
5
|
+
* unit-testable against plain fakes, the same split used by `flow.ts` and
|
|
6
|
+
* `browserLifecycle.ts` elsewhere in this package.
|
|
7
|
+
*/
|
|
8
|
+
const MANUAL_FALLBACK = 'npx playwright install chrome';
|
|
9
|
+
/** Thrown when the automatic install fails. Its message is already
|
|
10
|
+
* human-actionable (includes the manual fallback command) — callers can
|
|
11
|
+
* surface `.message` directly without needing to know install internals. */
|
|
12
|
+
export class ChromeInstallFailedError extends Error {
|
|
13
|
+
constructor(message, options) {
|
|
14
|
+
super(message, options);
|
|
15
|
+
this.name = 'ChromeInstallFailedError';
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Ensures Chrome is installed and launchable, installing it if not.
|
|
20
|
+
*
|
|
21
|
+
* Order, cheapest-first:
|
|
22
|
+
* 1. Trust a previously-written "confirmed installed" marker — no launch at
|
|
23
|
+
* all, as long as the executable path it recorded still exists on disk.
|
|
24
|
+
* 2. Otherwise, do the real (but cheap) launch-and-close probe.
|
|
25
|
+
* 3. Only if that fails, run the installer — and mark the resolved path it
|
|
26
|
+
* hands back so steps 2/3 are never repeated on later runs.
|
|
27
|
+
*
|
|
28
|
+
* Never throws a raw/undecorated error: any installChrome() failure is
|
|
29
|
+
* wrapped in `ChromeInstallFailedError` with the manual fallback command
|
|
30
|
+
* included, per this project's no-silent-failures rule.
|
|
31
|
+
*/
|
|
32
|
+
export async function ensureChromeReady(deps, onProgress) {
|
|
33
|
+
if (await deps.isConfirmedInstalled()) {
|
|
34
|
+
return { installed: false };
|
|
35
|
+
}
|
|
36
|
+
const probedPath = await deps.probeLaunchable();
|
|
37
|
+
if (probedPath) {
|
|
38
|
+
await deps.markInstalled(probedPath);
|
|
39
|
+
return { installed: false };
|
|
40
|
+
}
|
|
41
|
+
let installedPath;
|
|
42
|
+
try {
|
|
43
|
+
installedPath = await deps.installChrome(onProgress ?? (() => { }));
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
47
|
+
throw new ChromeInstallFailedError(`Could not automatically install the browser skrape needs.\n${message}\n\n` +
|
|
48
|
+
`You can install it yourself, then run skrape again:\n ${MANUAL_FALLBACK}`, { cause: error });
|
|
49
|
+
}
|
|
50
|
+
await deps.markInstalled(installedPath);
|
|
51
|
+
return { installed: true };
|
|
52
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { SyncSummary } from '../sync.js';
|
|
2
|
+
/**
|
|
3
|
+
* Pure screen-sequencing logic for what happens after a sync finishes and the
|
|
4
|
+
* user picks an option from the post-summary menu. Kept free of Ink/React so
|
|
5
|
+
* the transitions themselves — independent of how they get rendered or which
|
|
6
|
+
* keys drive them — are unit-testable.
|
|
7
|
+
*
|
|
8
|
+
* This models a genuine loop: the caller (App.tsx) folds the returned Step
|
|
9
|
+
* back into its single `step` state slot exactly like every other
|
|
10
|
+
* transition already in the guided flow, so a long interactive session
|
|
11
|
+
* (sync A -> menu -> sync A again -> menu -> sync B -> menu -> quit -> ...)
|
|
12
|
+
* never nests components or accumulates state — each transition just
|
|
13
|
+
* replaces `step`.
|
|
14
|
+
*/
|
|
15
|
+
/** The subset of an App `Step` this module needs to know about, expressed
|
|
16
|
+
* structurally so this file doesn't have to import App's full Step union
|
|
17
|
+
* (which would create a cycle back into the Ink-aware module). */
|
|
18
|
+
export interface DoneState {
|
|
19
|
+
slug: string;
|
|
20
|
+
name: string;
|
|
21
|
+
courseCount: number;
|
|
22
|
+
outDir: string;
|
|
23
|
+
summary: SyncSummary;
|
|
24
|
+
}
|
|
25
|
+
export type MenuChoice = 'sync-another' | 'sync-again' | 'open-folder' | 'quit';
|
|
26
|
+
export interface MenuItem {
|
|
27
|
+
label: string;
|
|
28
|
+
value: MenuChoice;
|
|
29
|
+
}
|
|
30
|
+
export declare const MENU_ITEMS: MenuItem[];
|
|
31
|
+
/** The next screen-sequencing step to fold into state after a menu choice.
|
|
32
|
+
* `'discovering'` re-enters the same community-discovery step that ran
|
|
33
|
+
* after login, skipping the session check entirely since the user is
|
|
34
|
+
* already signed in. `'confirming'` re-uses the slug/name/courseCount/outDir
|
|
35
|
+
* from the sync that just finished, skipping both the picker and the course
|
|
36
|
+
* count so "sync again" goes straight to confirm -> progress -> summary. */
|
|
37
|
+
export type FlowTransition = {
|
|
38
|
+
kind: 'discovering';
|
|
39
|
+
} | {
|
|
40
|
+
kind: 'confirming';
|
|
41
|
+
slug: string;
|
|
42
|
+
name: string;
|
|
43
|
+
courseCount: number;
|
|
44
|
+
outDir: string;
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* Given the state of the sync that just completed and the menu choice the
|
|
48
|
+
* user made, returns the next step transition — or `null` when the choice
|
|
49
|
+
* doesn't move to a new screen at all: `'open-folder'` is a side effect
|
|
50
|
+
* performed in place (the menu stays up), and `'quit'` exits instead of
|
|
51
|
+
* transitioning to another step.
|
|
52
|
+
*/
|
|
53
|
+
export declare function nextStepForMenuChoice(done: DoneState, choice: MenuChoice): FlowTransition | null;
|