@mogulmoretti/skrape 0.1.0

package/dist/store/db.js

@@ -0,0 +1,125 @@
+import Database from 'better-sqlite3';
+import { createHash } from 'node:crypto';
+import { dirname } from 'node:path';
+import { mkdirSync } from 'node:fs';
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS communities (
+  slug TEXT PRIMARY KEY,
+  name TEXT,
+  last_synced_at TEXT
+);
+CREATE TABLE IF NOT EXISTS items (
+  id TEXT PRIMARY KEY,
+  community TEXT NOT NULL,
+  type TEXT NOT NULL CHECK (type IN ('lesson','call','post')),
+  native_id TEXT NOT NULL,
+  section TEXT,
+  course TEXT,
+  position INTEGER NOT NULL DEFAULT 0,
+  title TEXT NOT NULL,
+  url TEXT,
+  video_url TEXT,
+  duration_ms INTEGER NOT NULL DEFAULT 0,
+  has_access INTEGER NOT NULL DEFAULT 1,
+  published_at TEXT,
+  body_text TEXT,
+  content_hash TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS items_community_type ON items(community, type);
+CREATE TABLE IF NOT EXISTS transcripts (
+  item_id TEXT PRIMARY KEY,
+  provider TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('ok','unavailable','failed')),
+  reason TEXT,
+  text TEXT,
+  word_count INTEGER,
+  source_url TEXT,
+  fetched_at TEXT NOT NULL,
+  FOREIGN KEY (item_id) REFERENCES items(id)
+);
+`;
+export function contentHash(item) {
+    // Use \0 as separator to prevent field boundary collisions.
+    // Nullable fields are collapsed to '' to ensure stable hashing.
+    const salient = [
+        item.nativeId, item.title, item.section ?? '', item.course ?? '',
+        item.videoUrl ?? '', String(item.durationMs), item.bodyText ?? '',
+        String(item.hasAccess),
+    ].join('\0');
+    return createHash('sha256').update(salient).digest('hex');
+}
+export function openDb(path) {
+    // Create parent directories for file-based databases (not :memory:)
+    if (path !== ':memory:') {
+        const dir = dirname(path);
+        if (dir && dir !== '.') {
+            mkdirSync(dir, { recursive: true });
+        }
+    }
+    const db = new Database(path);
+    db.pragma('journal_mode = WAL');
+    db.pragma('foreign_keys = ON');
+    db.exec(SCHEMA);
+    return {
+        upsertCommunity(slug, name) {
+            db.prepare(`INSERT INTO communities (slug, name) VALUES (?, ?)
+         ON CONFLICT(slug) DO UPDATE SET name = excluded.name`).run(slug, name);
+        },
+        upsertItem(community, item) {
+            const id = `${community}:${item.type}:${item.nativeId}`;
+            const hash = contentHash(item);
+            const existing = db.prepare('SELECT content_hash FROM items WHERE id = ?').get(id);
+            db.prepare(`INSERT INTO items (id, community, type, native_id, section, course, position, title,
+                            url, video_url, duration_ms, has_access, published_at, body_text,
+                            content_hash, updated_at)
+         VALUES (@id, @community, @type, @nativeId, @section, @course, @position, @title,
+                 @url, @videoUrl, @durationMs, @hasAccess, @publishedAt, @bodyText,
+                 @hash, @updatedAt)
+         ON CONFLICT(id) DO UPDATE SET
+           section = excluded.section, course = excluded.course, position = excluded.position,
+           title = excluded.title, url = excluded.url, video_url = excluded.video_url,
+           duration_ms = excluded.duration_ms, has_access = excluded.has_access,
+           published_at = excluded.published_at, body_text = excluded.body_text,
+           content_hash = excluded.content_hash, updated_at = excluded.updated_at`).run({
+                id, community, type: item.type, nativeId: item.nativeId,
+                section: item.section, course: item.course, position: item.index,
+                title: item.title, url: item.url, videoUrl: item.videoUrl,
+                durationMs: item.durationMs, hasAccess: item.hasAccess ? 1 : 0,
+                publishedAt: item.publishedAt, bodyText: item.bodyText,
+                hash, updatedAt: new Date().toISOString(),
+            });
+            return { id, changed: existing?.content_hash !== hash };
+        },
+        saveTranscript(itemId, result) {
+            db.prepare(`INSERT INTO transcripts (item_id, provider, status, reason, text, word_count, source_url, fetched_at)
+         VALUES (@itemId, @provider, @status, @reason, @text, @wordCount, @sourceUrl, @fetchedAt)
+         ON CONFLICT(item_id) DO UPDATE SET
+           provider = excluded.provider, status = excluded.status, reason = excluded.reason,
+           text = excluded.text, word_count = excluded.word_count,
+           source_url = excluded.source_url, fetched_at = excluded.fetched_at`).run({
+                itemId,
+                provider: result.provider,
+                status: result.status,
+                reason: result.status === 'ok' ? null : result.reason,
+                text: result.status === 'ok' ? result.text : null,
+                wordCount: result.status === 'ok' ? result.wordCount : null,
+                sourceUrl: result.status === 'ok' ? result.sourceUrl : null,
+                fetchedAt: new Date().toISOString(),
+            });
+        },
+        getTranscriptStatus(itemId) {
+            const row = db.prepare('SELECT status FROM transcripts WHERE item_id = ?').get(itemId);
+            return row?.status ?? null;
+        },
+        getTranscriptReason(itemId) {
+            const row = db.prepare('SELECT reason FROM transcripts WHERE item_id = ?').get(itemId);
+            return row?.reason ?? null;
+        },
+        markSynced(slug) {
+            db.prepare('UPDATE communities SET last_synced_at = ? WHERE slug = ?')
+                .run(new Date().toISOString(), slug);
+        },
+        close() { db.close(); },
+    };
+}

package/dist/store/markdown.d.ts

@@ -0,0 +1,7 @@
+import type { ContentItem } from '../types.js';
+export declare function slugify(text: string, maxLength?: number): string;
+export declare function transcriptPath(outDir: string, item: ContentItem, padWidth?: number): string;
+export declare function writeTranscript(outDir: string, item: ContentItem, body: string, meta: {
+    wordCount: number;
+    sourceUrl: string;
+}, padWidth?: number): Promise<string>;

package/dist/store/markdown.js

@@ -0,0 +1,41 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+export function slugify(text, maxLength = 60) {
+    const cleaned = text
+        .normalize('NFKD')
+        .replace(/[^\p{L}\p{N}\s-]/gu, '')
+        .trim()
+        .toLowerCase()
+        .replace(/[\s_-]+/g, '-')
+        .replace(/^-+|-+$/g, '')
+        .slice(0, maxLength)
+        .replace(/-+$/g, '');
+    return cleaned || 'untitled';
+}
+export function transcriptPath(outDir, item, padWidth = 2) {
+    const course = slugify(item.course ?? 'uncategorized');
+    const width = Math.max(2, padWidth);
+    const name = `${String(item.index).padStart(width, '0')}-${slugify(item.title)}.md`;
+    return join(outDir, 'transcripts', course, name);
+}
+export async function writeTranscript(outDir, item, body, meta, padWidth = 2) {
+    const path = transcriptPath(outDir, item, padWidth);
+    await mkdir(dirname(path), { recursive: true });
+    const minutes = Math.round(item.durationMs / 60_000);
+    const cleanTitle = item.title.replace(/\s+/g, ' ');
+    const cleanCourse = (item.course ?? '—').replace(/\s+/g, ' ');
+    const cleanSection = item.section?.replace(/\s+/g, ' ');
+    const header = [
+        `# ${cleanTitle}`,
+        '',
+        `> Course: ${cleanCourse}` +
+            (cleanSection ? `  |  Section: ${cleanSection}` : '') +
+            `  |  ${minutes} min  |  ${meta.wordCount.toLocaleString('en-US')} words`,
+        `> Source: ${meta.sourceUrl}`,
+        '',
+        '---',
+        '',
+    ].join('\n');
+    await writeFile(path, `${header}${body}\n`, 'utf-8');
+    return path;
+}

package/dist/sync.d.ts

@@ -0,0 +1,30 @@
+import type { Db } from './store/db.js';
+import type { Fetcher } from './types.js';
+export type Outcome = 'ok' | 'skipped' | 'no-video' | 'no-access' | 'unavailable' | 'failed';
+export interface Problem {
+    outcome: Outcome;
+    course: string;
+    title: string;
+    reason: string;
+}
+export interface ProgressEvent {
+    outcome: Outcome;
+    course: string;
+    title: string;
+    done: number;
+    total: number;
+}
+export interface SyncSummary {
+    counts: Record<Outcome, number>;
+    problems: Problem[];
+    totalWords: number;
+}
+export interface SyncOptions {
+    slug: string;
+    outDir: string;
+    db: Db;
+    fetcher: Fetcher;
+    concurrency?: number;
+    onProgress?: (event: ProgressEvent) => void;
+}
+export declare function syncClassroom(options: SyncOptions): Promise<SyncSummary>;

package/dist/sync.js

@@ -0,0 +1,178 @@
+import { listCourses, listLessons } from './discover/skool.js';
+import { getTranscript } from './media/index.js';
+import { writeTranscript } from './store/markdown.js';
+/** Run tasks with a bounded number in flight. */
+async function pooled(tasks, limit) {
+    const results = new Array(tasks.length);
+    let cursor = 0;
+    // Clamp to at least 1 worker: a caller passing 0, a negative number, or NaN must never
+    // produce a zero-worker pool that silently processes nothing.
+    const safeLimit = Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1;
+    const workers = Array.from({ length: Math.min(safeLimit, tasks.length) }, async () => {
+        while (cursor < tasks.length) {
+            const index = cursor++;
+            results[index] = await tasks[index]();
+        }
+    });
+    await Promise.all(workers);
+    return results;
+}
+/** Extract a readable message from any thrown value, not just Error instances. */
+function errorMessage(error) {
+    if (error instanceof Error)
+        return error.message;
+    if (typeof error === 'string')
+        return error;
+    try {
+        return JSON.stringify(error);
+    }
+    catch {
+        return String(error);
+    }
+}
+export async function syncClassroom(options) {
+    const { slug, outDir, db, fetcher, concurrency = 4, onProgress } = options;
+    const counts = {
+        ok: 0, skipped: 0, 'no-video': 0, 'no-access': 0, unavailable: 0, failed: 0,
+    };
+    const problems = [];
+    let totalWords = 0;
+    db.upsertCommunity(slug, slug);
+    const courses = await listCourses(slug, fetcher);
+    const accessible = courses.filter((course) => course.hasAccess);
+    for (const locked of courses.filter((course) => !course.hasAccess)) {
+        counts['no-access']++;
+        problems.push({
+            outcome: 'no-access', course: locked.title, title: '(entire course)',
+            reason: 'hasAccess is 0 — not entitled, skipped without probing',
+        });
+    }
+    // One course's listing failing (network blip, malformed payload) must not take down the whole
+    // sync. Each course's fetch is isolated in its own try/catch so Promise.all can never reject —
+    // a failure here is recorded as a course-level 'failed' problem and the rest of the run
+    // continues.
+    const courseResults = await Promise.all(accessible.map(async (course) => {
+        try {
+            const { items, skipped } = await listLessons(slug, course, fetcher);
+            return { course, items, skipped, error: null };
+        }
+        catch (error) {
+            return { course, items: [], skipped: [], error: errorMessage(error) };
+        }
+    }));
+    const lessonEntries = [];
+    for (const { course, items, skipped, error } of courseResults) {
+        if (error !== null) {
+            counts.failed++;
+            problems.push({
+                outcome: 'failed', course: course.title, title: '(entire course)',
+                reason: `could not list lessons: ${error}`,
+            });
+            continue;
+        }
+        // Deviation from brief (see task-10-brief.md DEVIATION note 1): writeTranscript now takes an
+        // optional padWidth so filename index prefixes sort lexicographically past 99 items. Compute
+        // a width per course, based on that course's own lesson count, and carry it alongside each
+        // lesson so it's never looked up by a course-title key that two courses could share.
+        const padWidth = Math.max(2, String(items.length).length);
+        for (const item of items)
+            lessonEntries.push({ item, padWidth });
+        // A node that parsed but had no usable title (or was a null/non-object tombstone) must not
+        // vanish uncounted — the acceptance criterion is a lesson count. Fold it into 'failed' rather
+        // than adding a new Outcome member, which would ripple through the CLI and summary.
+        for (const skip of skipped) {
+            counts.failed++;
+            problems.push({ outcome: 'failed', course: course.title, title: '(skipped node)', reason: skip.reason });
+        }
+    }
+    let done = 0;
+    const total = lessonEntries.length;
+    const record = (outcome, item, reason) => {
+        counts[outcome]++;
+        if (reason) {
+            problems.push({ outcome, course: item.course ?? '—', title: item.title, reason });
+        }
+        done++;
+        // Progress reporting is cosmetic. A caller's callback throwing must never fail the sync.
+        try {
+            onProgress?.({ outcome, course: item.course ?? '—', title: item.title, done, total });
+        }
+        catch {
+            // swallow — the caller's callback is not our concern
+        }
+    };
+    // A better-sqlite3 error from any of the three store calls below (disk full, locked db, FK
+    // violation) must become a per-lesson 'failed' outcome, not a run-ending rejection: a Promise.all
+    // over the worker pool would reject on the first such error while sibling workers are still in
+    // flight, and the CLI's finally-block browser.close() would then race a live worker into
+    // launching a brand-new Chrome context that nothing ever closes. Wrap every db call here.
+    const safeSaveTranscript = (id, result) => {
+        try {
+            db.saveTranscript(id, result);
+            return null;
+        }
+        catch (error) {
+            return `could not save transcript status: ${errorMessage(error)}`;
+        }
+    };
+    await pooled(lessonEntries.map(({ item, padWidth }) => async () => {
+        let id;
+        try {
+            ({ id } = db.upsertItem(slug, item));
+        }
+        catch (error) {
+            // No item id was ever created, so there's nothing further to key a transcript row to —
+            // still record the lesson so it isn't silently dropped from the counts.
+            return record('failed', item, `could not save item: ${errorMessage(error)}`);
+        }
+        if (!item.hasAccess)
+            return record('no-access', item, 'lesson locked');
+        if (!item.videoUrl)
+            return record('no-video', item, 'lesson has no video attached');
+        let status;
+        try {
+            status = db.getTranscriptStatus(id);
+        }
+        catch (error) {
+            return record('failed', item, `could not read transcript status: ${errorMessage(error)}`);
+        }
+        if (status === 'ok')
+            return record('skipped', item);
+        let result;
+        try {
+            result = await getTranscript(item.videoUrl, fetcher);
+        }
+        catch (error) {
+            // A provider should never throw, but one bad lesson must not end the run.
+            result = { status: 'failed', reason: errorMessage(error), provider: 'unknown' };
+        }
+        if (result.status === 'ok') {
+            // Write the file BEFORE committing the DB row. If the write fails (disk full, permission
+            // denied) and the DB already said 'ok', resumability would skip this lesson forever on
+            // every future run — a permanent silent data loss. Writing first means the only failure
+            // mode is a DB save failing after a successful write, which is harmless: the next run
+            // just redoes the lesson and overwrites an identical file.
+            try {
+                await writeTranscript(outDir, item, result.text, {
+                    wordCount: result.wordCount, sourceUrl: result.sourceUrl,
+                }, padWidth);
+            }
+            catch (error) {
+                const reason = `could not write transcript file: ${errorMessage(error)}`;
+                safeSaveTranscript(id, { status: 'failed', reason, provider: result.provider });
+                return record('failed', item, reason);
+            }
+            const saveError = safeSaveTranscript(id, result);
+            if (saveError)
+                return record('failed', item, saveError);
+            totalWords += result.wordCount;
+            return record('ok', item);
+        }
+        const saveError = safeSaveTranscript(id, result);
+        if (saveError)
+            return record('failed', item, saveError);
+        return record(result.status, item, result.reason);
+    }), concurrency);
+    db.markSynced(slug);
+    return { counts, problems, totalWords };
+}

package/dist/tui/App.d.ts

@@ -0,0 +1,22 @@
+import React from 'react';
+import type { CommunityRef } from '../discover/communities.js';
+import type { ProgressEvent, SyncSummary } from '../sync.js';
+export interface AppControllers {
+    /** Output root directory, e.g. './out' — the community slug is appended. */
+    outRoot: string;
+    checkLoggedIn: () => Promise<boolean>;
+    login: () => Promise<void>;
+    discoverCommunities: () => Promise<CommunityRef[] | null>;
+    countAccessibleCourses: (slug: string) => Promise<number>;
+    runSync: (slug: string, outDir: string, onProgress: (event: ProgressEvent) => void) => Promise<SyncSummary>;
+}
+/**
+ * Caps error text shown in the TUI. Some failures (e.g. a raw Playwright
+ * browser-launch error) can be a multi-hundred-line dump — that's useless in
+ * a terminal UI and pushes everything else off screen, so keep only the
+ * first few lines and a hard character cap.
+ */
+export declare function truncateErrorText(message: string): string;
+export declare function App({ controllers }: {
+    controllers: AppControllers;
+}): React.JSX.Element;