imperium-crawl 1.5.3 → 2.1.0

package/dist/social/parsers.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Shared parsers for social media tools.
+ *
+ * parseCompactNumber: "1.2M" -> 1200000
+ * parseRelativeTime: "3 hours ago" -> ISO date string
+ * sanitizeText: strip HTML tags, collapse whitespace
+ */
+/**
+ * Parse compact number strings like "1.2M", "842K", "3.5B", "1,234".
+ * Returns NaN for unparseable input.
+ */
+export declare function parseCompactNumber(str: string): number;
+/**
+ * Parse relative time strings like "3 hours ago", "1 day ago", "2 months ago".
+ * Returns ISO date string, or null if unparseable.
+ */
+export declare function parseRelativeTime(str: string): string | null;
+/**
+ * Strip HTML tags and collapse whitespace.
+ */
+export declare function sanitizeText(html: string): string;
+/**
+ * Extract JSON from a script tag pattern like `var ytInitialData = {...};`
+ * Also handles `<script id="varName" type="application/json">{...}</script>`.
+ * Returns parsed object or null.
+ */
+export declare function extractScriptJson(html: string, varName: string): unknown | null;
+//# sourceMappingURL=parsers.d.ts.map

package/dist/social/parsers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parsers.d.ts","sourceRoot":"","sources":["../../src/social/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AASH;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAgBtD;AAYD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAY5D;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAYjD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAwE/E"}

package/dist/social/parsers.js

@@ -0,0 +1,146 @@
+/**
+ * Shared parsers for social media tools.
+ *
+ * parseCompactNumber: "1.2M" -> 1200000
+ * parseRelativeTime: "3 hours ago" -> ISO date string
+ * sanitizeText: strip HTML tags, collapse whitespace
+ */
+const MULTIPLIERS = {
+    K: 1_000,
+    M: 1_000_000,
+    B: 1_000_000_000,
+    T: 1_000_000_000_000,
+};
+/**
+ * Parse compact number strings like "1.2M", "842K", "3.5B", "1,234".
+ * Returns NaN for unparseable input.
+ */
+export function parseCompactNumber(str) {
+    if (!str)
+        return NaN;
+    // Clean: remove commas, spaces, plus signs
+    const cleaned = str.replace(/[,\s+]/g, "").trim();
+    // Try suffix match: "1.2M", "842K"
+    const match = cleaned.match(/^([\d.]+)\s*([KMBT])/i);
+    if (match) {
+        const num = parseFloat(match[1]);
+        const suffix = match[2].toUpperCase();
+        return Math.round(num * (MULTIPLIERS[suffix] || 1));
+    }
+    // Plain number
+    return parseFloat(cleaned) || NaN;
+}
+const TIME_UNITS = {
+    second: 1_000,
+    minute: 60_000,
+    hour: 3_600_000,
+    day: 86_400_000,
+    week: 604_800_000,
+    month: 2_592_000_000, // ~30 days
+    year: 31_536_000_000, // ~365 days
+};
+/**
+ * Parse relative time strings like "3 hours ago", "1 day ago", "2 months ago".
+ * Returns ISO date string, or null if unparseable.
+ */
+export function parseRelativeTime(str) {
+    if (!str)
+        return null;
+    const match = str.match(/(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago/i);
+    if (!match)
+        return null;
+    const amount = parseInt(match[1], 10);
+    const unit = match[2].toLowerCase();
+    const ms = TIME_UNITS[unit];
+    if (!ms)
+        return null;
+    return new Date(Date.now() - amount * ms).toISOString();
+}
+/**
+ * Strip HTML tags and collapse whitespace.
+ */
+export function sanitizeText(html) {
+    if (!html)
+        return "";
+    return html
+        .replace(/<[^>]*>/g, "")
+        .replace(/&amp;/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&nbsp;/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+/**
+ * Extract JSON from a script tag pattern like `var ytInitialData = {...};`
+ * Also handles `<script id="varName" type="application/json">{...}</script>`.
+ * Returns parsed object or null.
+ */
+export function extractScriptJson(html, varName) {
+    // Strategy 1: <script id="varName" type="application/json">{...}</script>
+    const scriptTagPattern = new RegExp(`<script[^>]+id=["']${varName}["'][^>]*>([\\s\\S]*?)</script>`, "i");
+    const scriptTagMatch = scriptTagPattern.exec(html);
+    if (scriptTagMatch) {
+        try {
+            return JSON.parse(scriptTagMatch[1]);
+        }
+        catch {
+            // Fall through to other strategies
+        }
+    }
+    // Strategy 2: var varName = {...}; or window['varName'] = {...};
+    const patterns = [
+        new RegExp(`var\\s+${varName}\\s*=\\s*`, "i"),
+        new RegExp(`window\\[['"]${varName}['"]\\]\\s*=\\s*`, "i"),
+    ];
+    for (const pattern of patterns) {
+        const match = pattern.exec(html);
+        if (!match)
+            continue;
+        const start = match.index + match[0].length;
+        // Find matching brace
+        let depth = 0;
+        let inString = false;
+        let escape = false;
+        let stringChar = "";
+        for (let i = start; i < html.length; i++) {
+            const ch = html[i];
+            if (escape) {
+                escape = false;
+                continue;
+            }
+            if (ch === "\\") {
+                escape = true;
+                continue;
+            }
+            if (inString) {
+                if (ch === stringChar)
+                    inString = false;
+                continue;
+            }
+            if (ch === '"' || ch === "'") {
+                inString = true;
+                stringChar = ch;
+                continue;
+            }
+            if (ch === "{" || ch === "[")
+                depth++;
+            if (ch === "}" || ch === "]") {
+                depth--;
+                if (depth === 0) {
+                    const jsonStr = html.substring(start, i + 1);
+                    try {
+                        return JSON.parse(jsonStr);
+                    }
+                    catch {
+                        return null;
+                    }
+                }
+            }
+        }
+    }
+    return null;
+}
+//# sourceMappingURL=parsers.js.map

package/dist/social/parsers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parsers.js","sourceRoot":"","sources":["../../src/social/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,GAA2B;IAC1C,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,SAAS;IACZ,CAAC,EAAE,aAAa;IAChB,CAAC,EAAE,iBAAiB;CACrB,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,IAAI,CAAC,GAAG;QAAE,OAAO,GAAG,CAAC;IAErB,2CAA2C;IAC3C,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAElD,mCAAmC;IACnC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACrD,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,eAAe;IACf,OAAO,UAAU,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,GAA2B;IACzC,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,MAAM;IACd,IAAI,EAAE,SAAS;IACf,GAAG,EAAE,UAAU;IACf,IAAI,EAAE,WAAW;IACjB,KAAK,EAAE,aAAa,EAAI,WAAW;IACnC,IAAI,EAAE,cAAc,EAAI,YAAY;CACrC,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAW;IAC3C,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;IACrF,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IAErB,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,OAAO,IAAI;SACR,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,OAAe;IAC7D,0EAA0E;IAC1E,MAAM,gBAAgB,GAAG,IAAI,MAAM,CACjC,sBAAsB,OAAO,iCAAiC,EAC9D,GAAG,CACJ,CAAC;IACF,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,mCAAmC;QACrC,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,MAAM,QAAQ,GAAG;QACf,IAAI,MAAM,CAAC,UAAU,OAAO,WAAW,EAAE,GAAG,CAAC;QAC7C,IAAI,MAAM,CAAC,gBAAgB,OAAO,kBAAkB,EAAE,GAAG,CAAC;KAC3D,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK;YAAE,SAAS;QAErB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5C,sBAAsB;QACtB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAEnB,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,GAAG,KAAK,CAAC;gBACf,SAAS;YACX,CAAC;YAED,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;gBAChB,MAAM,GAAG,IAAI,CAAC;gBACd,SAAS;YACX,CAAC;YAED,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,EAAE,KAAK,UAAU;oBAAE,QAAQ,GAAG,KAAK,CAAC;gBACxC,SAAS;YACX,CAAC;YAED,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBAC7B,QAAQ,GAAG,IAAI,CAAC;gBAChB,UAAU,GAAG,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG;gBAAE,KAAK,EAAE,CAAC;YACtC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBAC7B,KAAK,EAAE,CAAC;gBACR,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;oBAChB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;oBAC7C,IAAI,CAAC;wBACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;oBAAC,MAAM,CAAC;wBACP,OAAO,IAAI,CAAC;oBACd,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/social/types.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Shared types for social media tools (YouTube, Reddit).
+ */
+export interface SocialVideo {
+    id: string;
+    title: string;
+    url: string;
+    thumbnail?: string;
+    duration?: string;
+    views?: number;
+    likes?: number;
+    author: string;
+    author_url?: string;
+    published?: string;
+    description?: string;
+}
+export interface SocialPost {
+    id: string;
+    title: string;
+    url: string;
+    author: string;
+    score?: number;
+    comments_count?: number;
+    published?: string;
+    subreddit?: string;
+    text?: string;
+    thumbnail?: string;
+    is_video?: boolean;
+    flair?: string;
+}
+export interface SocialComment {
+    id: string;
+    author: string;
+    text: string;
+    score?: number;
+    published?: string;
+    replies_count?: number;
+}
+export interface SocialProfile {
+    name: string;
+    url: string;
+    description?: string;
+    subscribers?: number;
+    avatar?: string;
+    verified?: boolean;
+    video_count?: number;
+    created?: string;
+}
+export interface SocialSearchResult<T> {
+    query?: string;
+    platform: "youtube" | "reddit";
+    results: T[];
+    total?: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/social/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/social/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB,CAAC,CAAC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,SAAS,GAAG,QAAQ,CAAC;IAC/B,OAAO,EAAE,CAAC,EAAE,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}

package/dist/social/types.js

@@ -0,0 +1,5 @@
+/**
+ * Shared types for social media tools (YouTube, Reddit).
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/social/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/social/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/social/whisper.d.ts

@@ -0,0 +1,29 @@
+/**
+ * OpenAI Whisper integration for audio transcription.
+ * Used as fallback when YouTube captions are unavailable.
+ *
+ * Requires LLM_API_KEY with LLM_PROVIDER=openai (or any OpenAI-compatible key).
+ */
+export interface WhisperResult {
+    text: string;
+    segments?: Array<{
+        start: number;
+        end: number;
+        text: string;
+    }>;
+    language?: string;
+    duration?: number;
+}
+/**
+ * Check if Whisper transcription is available (OpenAI API key configured).
+ */
+export declare function hasWhisperConfigured(): boolean;
+/**
+ * Transcribe audio data using OpenAI Whisper API.
+ * Returns timestamped segments.
+ */
+export declare function transcribeAudio(audioData: Buffer | Uint8Array, options?: {
+    language?: string;
+    filename?: string;
+}): Promise<WhisperResult>;
+//# sourceMappingURL=whisper.d.ts.map

package/dist/social/whisper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"whisper.d.ts","sourceRoot":"","sources":["../../src/social/whisper.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,OAAO,CAM9C;AAgBD;;;GAGG;AACH,wBAAsB,eAAe,CACnC,SAAS,EAAE,MAAM,GAAG,UAAU,EAC9B,OAAO,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GACjD,OAAO,CAAC,aAAa,CAAC,CA6ExB"}

package/dist/social/whisper.js

@@ -0,0 +1,88 @@
+/**
+ * OpenAI Whisper integration for audio transcription.
+ * Used as fallback when YouTube captions are unavailable.
+ *
+ * Requires LLM_API_KEY with LLM_PROVIDER=openai (or any OpenAI-compatible key).
+ */
+import { getLLMConfig } from "../llm/index.js";
+const WHISPER_API_URL = "https://api.openai.com/v1/audio/transcriptions";
+const WHISPER_MODEL = "whisper-1";
+const MAX_AUDIO_SIZE = 25 * 1024 * 1024; // 25MB Whisper limit
+/**
+ * Check if Whisper transcription is available (OpenAI API key configured).
+ */
+export function hasWhisperConfigured() {
+    const config = getLLMConfig();
+    if (!config)
+        return false;
+    // Whisper only works with OpenAI API keys
+    // But users might have LLM_PROVIDER=anthropic with a separate OPENAI_API_KEY
+    return config.provider === "openai" || !!process.env.OPENAI_API_KEY?.trim();
+}
+/**
+ * Get the OpenAI API key for Whisper.
+ * Prefers OPENAI_API_KEY env var, falls back to LLM_API_KEY if provider is openai.
+ */
+function getWhisperApiKey() {
+    const explicit = process.env.OPENAI_API_KEY?.trim();
+    if (explicit)
+        return explicit;
+    const config = getLLMConfig();
+    if (config && config.provider === "openai")
+        return config.apiKey;
+    return null;
+}
+/**
+ * Transcribe audio data using OpenAI Whisper API.
+ * Returns timestamped segments.
+ */
+export async function transcribeAudio(audioData, options) {
+    const apiKey = getWhisperApiKey();
+    if (!apiKey) {
+        throw new Error("OpenAI API key required for Whisper transcription. Set OPENAI_API_KEY or LLM_API_KEY with LLM_PROVIDER=openai.");
+    }
+    if (audioData.length > MAX_AUDIO_SIZE) {
+        throw new Error(`Audio file too large (${Math.round(audioData.length / 1024 / 1024)}MB). Whisper limit is 25MB. Try a shorter video.`);
+    }
+    const filename = options?.filename || "audio.webm";
+    // Build multipart form data manually (no FormData in Node < 18 without polyfill)
+    const boundary = `----WhisperBoundary${Date.now()}`;
+    const parts = [];
+    // Add model field
+    parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="model"\r\n\r\n${WHISPER_MODEL}\r\n`));
+    // Add response_format for segments
+    parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="response_format"\r\n\r\nverbose_json\r\n`));
+    // Add language if specified
+    if (options?.language) {
+        parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="language"\r\n\r\n${options.language}\r\n`));
+    }
+    // Add audio file
+    parts.push(Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${filename}"\r\nContent-Type: audio/webm\r\n\r\n`));
+    parts.push(Buffer.from(audioData));
+    parts.push(Buffer.from(`\r\n--${boundary}--\r\n`));
+    const body = Buffer.concat(parts);
+    const res = await fetch(WHISPER_API_URL, {
+        method: "POST",
+        headers: {
+            Authorization: `Bearer ${apiKey}`,
+            "Content-Type": `multipart/form-data; boundary=${boundary}`,
+        },
+        body,
+    });
+    if (!res.ok) {
+        const errorText = await res.text().catch(() => res.statusText);
+        throw new Error(`Whisper API error ${res.status}: ${errorText}`);
+    }
+    const data = await res.json();
+    return {
+        text: data.text,
+        segments: data.segments?.map((s) => ({
+            start: Math.round(s.start * 100) / 100,
+            end: Math.round(s.end * 100) / 100,
+            text: s.text.trim(),
+        })),
+        language: data.language,
+        duration: data.duration,
+    };
+}
+//# sourceMappingURL=whisper.js.map

package/dist/social/whisper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"whisper.js","sourceRoot":"","sources":["../../src/social/whisper.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,MAAM,eAAe,GAAG,gDAAgD,CAAC;AACzE,MAAM,aAAa,GAAG,WAAW,CAAC;AAClC,MAAM,cAAc,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,qBAAqB;AAS9D;;GAEG;AACH,MAAM,UAAU,oBAAoB;IAClC,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,0CAA0C;IAC1C,6EAA6E;IAC7E,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC;AAC9E,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB;IACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC;IACpD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAE9B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC;IAEjE,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,SAA8B,EAC9B,OAAkD;IAElD,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,gHAAgH,CACjH,CAAC;IACJ,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,yBAAyB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC,kDAAkD,CACtH,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,YAAY,CAAC;IAEnD,iFAAiF;IACjF,MAAM,QAAQ,GAAG,sBAAsB,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IACpD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,kBAAkB;IAClB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CACpB,KAAK,QAAQ,2DAA2D,aAAa,MAAM,CAC5F,CAAC,CAAC;IAEH,mCAAmC;IACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CACpB,KAAK,QAAQ,oFAAoF,CAClG,CAAC,CAAC;IAEH,4BAA4B;IAC5B,IAAI,OAAO,EAAE,QAAQ,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CACpB,KAAK,QAAQ,8DAA8D,OAAO,CAAC,QAAQ,MAAM,CAClG,CAAC,CAAC;IACL,CAAC;IAED,iBAAiB;IACjB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CACpB,KAAK,QAAQ,8DAA8D,QAAQ,uCAAuC,CAC3H,CAAC,CAAC;IACH,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,QAAQ,QAAQ,CAAC,CAAC,CAAC;IAEnD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAElC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,eAAe,EAAE;QACvC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,aAAa,EAAE,UAAU,MAAM,EAAE;YACjC,cAAc,EAAE,iCAAiC,QAAQ,EAAE;SAC5D;QACD,IAAI;KACL,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC/D,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAK1B,CAAC;IAEF,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACnC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG;YACtC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG;YAClC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE;SACpB,CAAC,CAAC;QACH,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;KACxB,CAAC;AACJ,CAAC"}

package/dist/tools/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyBxB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC;IAElB,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC,CAAC;CACzH;AAED,eAAO,MAAM,QAAQ,EAAE,cAAc,EA8BpC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AA4BxB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC;IAElB,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC,CAAC;CACzH;AAED,eAAO,MAAM,QAAQ,EAAE,cAAc,EAkCpC,CAAC"}

package/dist/tools/index.js

@@ -16,10 +16,13 @@ import * as queryApi from "./query-api.js";
 import * as monitorWebsocket from "./monitor-websocket.js";
 import * as aiExtract from "./ai-extract.js";
 import * as interact from "./interact.js";
+import * as snapshot from "./snapshot.js";
 import * as batchScrape from "./batch-scrape.js";
 import * as listJobs from "./list-jobs.js";
 import * as jobStatus from "./job-status.js";
 import * as deleteJob from "./delete-job.js";
+import * as youtube from "./youtube.js";
+import * as reddit from "./reddit.js";
 export const allTools = [
     // Scraping tools (no API key needed)
     scrape,
@@ -45,10 +48,14 @@ export const allTools = [
     aiExtract,
     // Interaction & session tools (Playwright needed)
     interact,
+    snapshot,
     // Batch processing
     batchScrape,
     listJobs,
     jobStatus,
     deleteJob,
+    // Social media tools (no API key needed)
+    youtube,
+    reddit,
 ];
 //# sourceMappingURL=index.js.map

package/dist/tools/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAC;AACpC,OAAO,KAAK,GAAG,MAAM,UAAU,CAAC;AAChC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAC;AACxC,OAAO,KAAK,WAAW,MAAM,kBAAkB,CAAC;AAChD,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAC;AAC9C,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,UAAU,MAAM,kBAAkB,CAAC;AAC/C,OAAO,KAAK,WAAW,MAAM,mBAAmB,CAAC;AACjD,OAAO,KAAK,WAAW,MAAM,mBAAmB,CAAC;AACjD,OAAO,KAAK,WAAW,MAAM,mBAAmB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,UAAU,MAAM,kBAAkB,CAAC;AAC/C,OAAO,KAAK,YAAY,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,gBAAgB,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAC;AAC1C,OAAO,KAAK,WAAW,MAAM,mBAAmB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAU7C,MAAM,CAAC,MAAM,QAAQ,GAAqB;IACxC,qCAAqC;IACrC,MAAwB;IACxB,KAAuB;IACvB,GAAqB;IACrB,OAAyB;IACzB,WAA6B;IAC7B,UAA4B;IAC5B,sCAAsC;IACtC,MAAwB;IACxB,UAA4B;IAC5B,WAA6B;IAC7B,WAA6B;IAC7B,eAAe;IACf,WAA6B;IAC7B,QAA0B;IAC1B,UAA4B;IAC5B,sDAAsD;IACtD,YAA8B;IAC9B,QAA0B;IAC1B,gBAAkC;IAClC,oCAAoC;IACpC,SAA2B;IAC3B,kDAAkD;IAClD,QAA0B;IAC1B,mBAAmB;IACnB,WAA6B;IAC7B,QAA0B;IAC1B,SAA2B;IAC3B,SAA2B;CAC5B,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,KAAK,MAAM,YAAY,CAAC;AACpC,OAAO,KAAK,GAAG,MAAM,UAAU,CAAC;AAChC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAC;AACxC,OAAO,KAAK,WAAW,MAAM,kBAAkB,CAAC;AAChD,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAC;AAC9C,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,UAAU,MAAM,kBAAkB,CAAC;AAC/C,OAAO,KAAK,WAAW,MAAM,mBAAmB,CAAC;AACjD,OAAO,KAAK,WAAW,MAAM,mBAAmB,CAAC;AACjD,OAAO,KAAK,WAAW,MAAM,mBAAmB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,UAAU,MAAM,kBAAkB,CAAC;AAC/C,OAAO,KAAK,YAAY,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,gBAAgB,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAC;AAC1C,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAC;AAC1C,OAAO,KAAK,WAAW,MAAM,mBAAmB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,OAAO,MAAM,cAAc,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAUtC,MAAM,CAAC,MAAM,QAAQ,GAAqB;IACxC,qCAAqC;IACrC,MAAwB;IACxB,KAAuB;IACvB,GAAqB;IACrB,OAAyB;IACzB,WAA6B;IAC7B,UAA4B;IAC5B,sCAAsC;IACtC,MAAwB;IACxB,UAA4B;IAC5B,WAA6B;IAC7B,WAA6B;IAC7B,eAAe;IACf,WAA6B;IAC7B,QAA0B;IAC1B,UAA4B;IAC5B,sDAAsD;IACtD,YAA8B;IAC9B,QAA0B;IAC1B,gBAAkC;IAClC,oCAAoC;IACpC,SAA2B;IAC3B,kDAAkD;IAClD,QAA0B;IAC1B,QAA0B;IAC1B,mBAAmB;IACnB,WAA6B;IAC7B,QAA0B;IAC1B,SAA2B;IAC3B,SAA2B;IAC3B,yCAAyC;IACzC,OAAyB;IACzB,MAAwB;CACzB,CAAC"}