llm-search-tools 1.1.0

package/dist/modules/parser.js

@@ -0,0 +1,361 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parse = parse;
+// parser.ts - unified parser for various file types
+const fs_1 = require("fs");
+const path_1 = require("path");
+const os_1 = require("os");
+const crypto_1 = require("crypto");
+const pdf_parse_1 = __importDefault(require("pdf-parse"));
+const mammoth_1 = __importDefault(require("mammoth"));
+const sync_1 = require("csv-parse/sync");
+const tesseract_js_1 = require("tesseract.js");
+const fast_xml_parser_1 = require("fast-xml-parser");
+// detect file type from path or buffer
+function detectFileType(pathOrBuffer, filename) {
+    // if we got a string path, use that
+    if (typeof pathOrBuffer === "string") {
+        const ext = (0, path_1.extname)(pathOrBuffer).toLowerCase();
+        return getTypeFromExtension(ext);
+    }
+    // if we got a filename hint with the buffer, use that
+    if (filename) {
+        const ext = (0, path_1.extname)(filename).toLowerCase();
+        return getTypeFromExtension(ext);
+    }
+    // ok fine we'll try to detect from buffer magic numbers
+    const header = pathOrBuffer.slice(0, 4).toString("hex");
+    // check magic numbers
+    if (header.startsWith("89504e47"))
+        return "image"; // PNG
+    if (header.startsWith("ffd8"))
+        return "image"; // JPEG
+    if (header.startsWith("424d"))
+        return "image"; // BMP
+    if (header.startsWith("47494638"))
+        return "image"; // GIF
+    if (header.startsWith("25504446"))
+        return "pdf"; // PDF
+    if (header.startsWith("504b"))
+        return "docx"; // ZIP/DOCX
+    if (pathOrBuffer.slice(0, 5).toString() === "<?xml")
+        return "xml";
+    // attempt json detection
+    try {
+        JSON.parse(pathOrBuffer.toString());
+        return "json";
+    }
+    catch {
+        // not json, continue
+    }
+    // check if it looks like csv
+    const firstLine = pathOrBuffer.toString().split("\n")[0];
+    if (firstLine && firstLine.includes(","))
+        return "csv";
+    // probably just text if we got here
+    if (pathOrBuffer.toString().trim())
+        return "text";
+    return "unknown";
+}
+// helper to get type from file extension
+function getTypeFromExtension(ext) {
+    switch (ext) {
+        case ".pdf":
+            return "pdf";
+        case ".docx":
+            return "docx";
+        case ".csv":
+            return "csv";
+        case ".txt":
+            return "text";
+        case ".xml":
+            return "xml";
+        case ".json":
+            return "json";
+        case ".png":
+        case ".jpg":
+        case ".jpeg":
+        case ".bmp":
+        case ".gif":
+            return "image";
+        default:
+            return "unknown";
+    }
+}
+// parse JSON files
+function parseJSON(buffer) {
+    try {
+        const text = buffer.toString();
+        const data = JSON.parse(text);
+        return {
+            type: "json",
+            text: text,
+            data: data,
+            metadata: {
+                length: text.length,
+                lines: text.split("\n").length,
+            },
+        };
+    }
+    catch (error) {
+        throw {
+            message: "Failed to parse JSON file",
+            code: "JSON_PARSE_ERROR",
+            originalError: error,
+        };
+    }
+}
+// parse PDF files
+async function parsePDF(buffer) {
+    try {
+        const data = await (0, pdf_parse_1.default)(buffer);
+        return {
+            type: "pdf",
+            text: data.text,
+            metadata: {
+                pages: data.numpages,
+                info: data.info,
+                metadata: data.metadata,
+                version: data.version,
+            },
+        };
+    }
+    catch (error) {
+        throw {
+            message: "Failed to parse PDF file",
+            code: "PDF_PARSE_ERROR",
+            originalError: error,
+        };
+    }
+}
+// parse CSV files
+function parseCSV(buffer, options) {
+    try {
+        const text = buffer.toString();
+        const lines = text.trim().split("\n");
+        if (lines.length === 0) {
+            throw new Error("Empty CSV file");
+        }
+        // Count columns from the header row
+        const headerRow = lines[0];
+        const columnCount = headerRow.split(options?.csv?.delimiter || ",").length;
+        // Parse the CSV
+        const records = (0, sync_1.parse)(text, {
+            delimiter: options?.csv?.delimiter,
+            columns: options?.csv?.columns ?? true, // Default to true for column headers
+            skip_empty_lines: true,
+        });
+        const headers = options?.csv?.columns !== false ? Object.keys(records[0] || {}) : undefined;
+        return {
+            type: "csv",
+            text: text,
+            data: records,
+            metadata: {
+                rowCount: records.length,
+                columnCount: columnCount,
+                headers: headers,
+            },
+        };
+    }
+    catch (error) {
+        throw {
+            message: "Failed to parse CSV file",
+            code: "CSV_PARSE_ERROR",
+            originalError: error,
+        };
+    }
+}
+// Helper function to extract text from HTML
+function htmlToText(html) {
+    return html
+        .replace(/<br\s*\/?>/gi, "\n")
+        .replace(/<\/p>/gi, "\n")
+        .replace(/<\/div>/gi, "\n")
+        .replace(/<\/h[1-6]>/gi, "\n")
+        .replace(/<\/li>/gi, "\n")
+        .replace(/<[^>]+>/g, "")
+        .replace(/&nbsp;/g, " ")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+// Helper function to create temp file
+async function withTempFile(buffer, extension, callback) {
+    const tempFileName = `temp-${(0, crypto_1.randomBytes)(16).toString("hex")}${extension}`;
+    const tempPath = (0, path_1.join)((0, os_1.tmpdir)(), tempFileName);
+    try {
+        (0, fs_1.writeFileSync)(tempPath, buffer);
+        return await callback(tempPath);
+    }
+    finally {
+        try {
+            (0, fs_1.unlinkSync)(tempPath);
+        }
+        catch {
+            // ignore cleanup error
+        }
+    }
+}
+// parse DOCX files using mammoth
+async function parseDOCX(buffer) {
+    try {
+        return await withTempFile(buffer, ".docx", async (tempPath) => {
+            // Try HTML conversion first
+            const htmlResult = await mammoth_1.default.convertToHtml({ path: tempPath });
+            if (!htmlResult.value) {
+                // Fallback to raw text extraction
+                const textResult = await mammoth_1.default.extractRawText({ path: tempPath });
+                if (!textResult.value) {
+                    throw new Error("No content found in DOCX file");
+                }
+                const cleanText = textResult.value.trim();
+                return {
+                    type: "docx",
+                    text: cleanText,
+                    metadata: {
+                        type: "docx",
+                        paragraphs: cleanText.split("\n").filter(Boolean).length,
+                        warnings: textResult.messages,
+                        method: "raw",
+                    },
+                };
+            }
+            // Convert HTML to plain text
+            const text = htmlToText(htmlResult.value);
+            const paragraphs = text.split("\n").filter(Boolean);
+            return {
+                type: "docx",
+                text: text,
+                metadata: {
+                    type: "docx",
+                    paragraphs: paragraphs.length,
+                    warnings: htmlResult.messages,
+                    hasHtml: true,
+                    method: "html",
+                },
+            };
+        });
+    }
+    catch (error) {
+        throw {
+            message: "Failed to parse DOCX file",
+            code: "DOCX_PARSE_ERROR",
+            originalError: error,
+        };
+    }
+}
+// parse plain text files
+function parseText(buffer) {
+    try {
+        const text = buffer.toString();
+        return {
+            type: "text",
+            text: text,
+            metadata: {
+                length: text.length,
+                lines: text.split("\n").length,
+            },
+        };
+    }
+    catch (error) {
+        throw {
+            message: "Failed to parse text file",
+            code: "TEXT_PARSE_ERROR",
+            originalError: error,
+        };
+    }
+}
+// parse XML files
+function parseXML(buffer, options) {
+    try {
+        const text = buffer.toString();
+        const parser = new fast_xml_parser_1.XMLParser({
+            ignoreAttributes: options?.xml?.ignoreAttributes ?? false,
+            parseAttributeValue: options?.xml?.parseAttributeValue ?? true,
+        });
+        const data = parser.parse(text);
+        return {
+            type: "xml",
+            text: text,
+            data: data,
+            metadata: {
+                length: text.length,
+                lines: text.split("\n").length,
+            },
+        };
+    }
+    catch (error) {
+        throw {
+            message: "Failed to parse XML file",
+            code: "XML_PARSE_ERROR",
+            originalError: error,
+        };
+    }
+}
+// parse images using OCR
+async function parseImage(buffer, options) {
+    try {
+        const worker = await (0, tesseract_js_1.createWorker)();
+        const lang = options?.language || "eng";
+        // Initialize worker with language
+        await worker.reinitialize(lang);
+        const result = await worker.recognize(buffer);
+        await worker.terminate();
+        return {
+            type: "image",
+            text: result.data.text,
+            metadata: {
+                language: lang,
+                confidence: result.data.confidence,
+            },
+        };
+    }
+    catch (error) {
+        throw {
+            message: "Failed to parse image file",
+            code: "IMAGE_PARSE_ERROR",
+            originalError: error,
+        };
+    }
+}
+// main parse function
+async function parse(pathOrBuffer, options = {}, filename) {
+    try {
+        // Get file buffer
+        const buffer = typeof pathOrBuffer === "string" ? (0, fs_1.readFileSync)(pathOrBuffer) : pathOrBuffer;
+        // Detect file type (pass filename hint if we have it)
+        const fileType = detectFileType(pathOrBuffer, filename);
+        // Parse based on file type
+        switch (fileType) {
+            case "pdf":
+                return await parsePDF(buffer);
+            case "docx":
+                return await parseDOCX(buffer);
+            case "csv":
+                return parseCSV(buffer, options);
+            case "text":
+                return parseText(buffer);
+            case "xml":
+                return parseXML(buffer, options);
+            case "json":
+                return parseJSON(buffer);
+            case "image":
+                return await parseImage(buffer, options);
+            default:
+                throw new Error(`Unsupported file type: ${fileType}`);
+        }
+    }
+    catch (error) {
+        const searchError = error;
+        if (searchError.code) {
+            throw searchError;
+        }
+        throw {
+            message: `Failed to parse file: ${error instanceof Error ? error.message : String(error)}`,
+            code: "PARSE_ERROR",
+            originalError: error,
+        };
+    }
+}

package/dist/modules/parser.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/modules/parser.test.js

@@ -0,0 +1,151 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const parser_1 = require("./parser");
+const pdf_parse_1 = __importDefault(require("pdf-parse"));
+const mammoth_1 = __importDefault(require("mammoth"));
+const tesseract_js_1 = require("tesseract.js");
+const fs_1 = require("fs");
+// Mock dependencies
+vitest_1.vi.mock("pdf-parse", () => ({
+    default: vitest_1.vi.fn(),
+}));
+vitest_1.vi.mock("mammoth");
+vitest_1.vi.mock("tesseract.js");
+vitest_1.vi.mock("fs", async () => {
+    const actual = await vitest_1.vi.importActual("fs");
+    return {
+        ...actual,
+        readFileSync: vitest_1.vi.fn(),
+        // We don't mock writeFileSync/unlinkSync as they are used for temp files in docx parsing
+        // and we want that to work or we mock the whole flow.
+        // For now, let's just mock readFileSync for the input file reading.
+    };
+});
+(0, vitest_1.describe)("Parser Module", () => {
+    (0, vitest_1.beforeEach)(() => {
+        vitest_1.vi.resetAllMocks();
+    });
+    (0, vitest_1.describe)("Text Parsing", () => {
+        (0, vitest_1.it)("should parse text from buffer", async () => {
+            const buffer = Buffer.from("Hello world");
+            const result = await (0, parser_1.parse)(buffer);
+            (0, vitest_1.expect)(result).toEqual({
+                type: "text",
+                text: "Hello world",
+                metadata: {
+                    length: 11,
+                    lines: 1,
+                },
+            });
+        });
+        (0, vitest_1.it)("should parse text from file path", async () => {
+            fs_1.readFileSync.mockReturnValue(Buffer.from("Hello file"));
+            const result = await (0, parser_1.parse)("test.txt");
+            (0, vitest_1.expect)(result).toEqual({
+                type: "text",
+                text: "Hello file",
+                metadata: {
+                    length: 10,
+                    lines: 1,
+                },
+            });
+            (0, vitest_1.expect)(fs_1.readFileSync).toHaveBeenCalledWith("test.txt");
+        });
+    });
+    (0, vitest_1.describe)("JSON Parsing", () => {
+        (0, vitest_1.it)("should parse JSON from buffer", async () => {
+            const data = { key: "value" };
+            const buffer = Buffer.from(JSON.stringify(data));
+            const result = await (0, parser_1.parse)(buffer, {}, "test.json");
+            (0, vitest_1.expect)(result.type).toBe("json");
+            (0, vitest_1.expect)(result.data).toEqual(data);
+        });
+        (0, vitest_1.it)("should detect JSON without extension", async () => {
+            const data = { key: "value" };
+            const buffer = Buffer.from(JSON.stringify(data));
+            const result = await (0, parser_1.parse)(buffer);
+            (0, vitest_1.expect)(result.type).toBe("json");
+        });
+    });
+    (0, vitest_1.describe)("CSV Parsing", () => {
+        (0, vitest_1.it)("should parse CSV from buffer", async () => {
+            const csv = "name,age\nAlice,30\nBob,25";
+            const buffer = Buffer.from(csv);
+            const result = await (0, parser_1.parse)(buffer, {}, "test.csv");
+            (0, vitest_1.expect)(result.type).toBe("csv");
+            (0, vitest_1.expect)(Array.isArray(result.data)).toBe(true);
+            const data = result.data;
+            (0, vitest_1.expect)(data).toHaveLength(2);
+            (0, vitest_1.expect)(data[0]).toEqual({ name: "Alice", age: "30" });
+        });
+    });
+    (0, vitest_1.describe)("PDF Parsing", () => {
+        (0, vitest_1.it)("should parse PDF using pdf-parse", async () => {
+            const mockData = {
+                numpages: 1,
+                info: {},
+                metadata: {},
+                version: "1.0",
+                text: "PDF content",
+            };
+            pdf_parse_1.default.mockResolvedValue(mockData);
+            const buffer = Buffer.from("%PDF-1.5"); // Magic bytes for PDF
+            const result = await (0, parser_1.parse)(buffer);
+            (0, vitest_1.expect)(result.type).toBe("pdf");
+            (0, vitest_1.expect)(result.text).toBe("PDF content");
+            (0, vitest_1.expect)(pdf_parse_1.default).toHaveBeenCalledWith(buffer);
+        });
+    });
+    (0, vitest_1.describe)("DOCX Parsing", () => {
+        (0, vitest_1.it)("should parse DOCX using mammoth", async () => {
+            const mockResult = { value: "DOCX content", messages: [] };
+            // Mock convertToHtml since that's what the parser uses first
+            mammoth_1.default.convertToHtml.mockResolvedValue(mockResult);
+            // Magic bytes for DOCX (PK zip header)
+            const buffer = Buffer.from("504b0304", "hex");
+            // We need to hint extension or provide magic bytes that match zip/docx
+            // The parser checks magic bytes '504b' -> 'docx'
+            const result = await (0, parser_1.parse)(buffer);
+            (0, vitest_1.expect)(result.type).toBe("docx");
+            (0, vitest_1.expect)(result.text).toBe("DOCX content");
+        });
+    });
+    (0, vitest_1.describe)("Image OCR", () => {
+        (0, vitest_1.it)("should parse image using tesseract", async () => {
+            const mockWorker = {
+                reinitialize: vitest_1.vi.fn(),
+                recognize: vitest_1.vi.fn().mockResolvedValue({
+                    data: {
+                        text: "OCR Text",
+                        confidence: 90,
+                    },
+                }),
+                terminate: vitest_1.vi.fn(),
+            };
+            tesseract_js_1.createWorker.mockResolvedValue(mockWorker);
+            // Magic bytes for PNG
+            const buffer = Buffer.from("89504e47", "hex");
+            const result = await (0, parser_1.parse)(buffer);
+            (0, vitest_1.expect)(result.type).toBe("image");
+            (0, vitest_1.expect)(result.text).toBe("OCR Text");
+            (0, vitest_1.expect)(mockWorker.recognize).toHaveBeenCalledWith(buffer);
+            (0, vitest_1.expect)(mockWorker.terminate).toHaveBeenCalled();
+        });
+    });
+    (0, vitest_1.describe)("Error Handling", () => {
+        (0, vitest_1.it)("should throw error for unsupported file type", async () => {
+            // Empty buffer returns "unknown" type because it fails the text check (trim())
+            const buffer = Buffer.from("");
+            await (0, vitest_1.expect)((0, parser_1.parse)(buffer)).rejects.toThrow("Unsupported file type: unknown");
+        });
+        (0, vitest_1.it)("should propagate parsing errors", async () => {
+            const buffer = Buffer.from("{ invalid json");
+            // Expect the specific error from the JSON parser, not the generic wrapper
+            await (0, vitest_1.expect)((0, parser_1.parse)(buffer, {}, "test.json")).rejects.toThrow("Failed to parse JSON file");
+        });
+    });
+});

package/dist/modules/reddit.d.ts

@@ -0,0 +1,21 @@
+interface RedditPost {
+    title: string;
+    id: string;
+    url: string;
+    permalink: string;
+    author: string;
+    score: number;
+    num_comments: number;
+    created_utc: number;
+    selftext?: string;
+    subreddit: string;
+    is_self: boolean;
+}
+export declare function getSubredditHot(subreddit: string, limit?: number): Promise<RedditPost[]>;
+export declare function searchReddit(query: string, options?: {
+    sort?: 'hot' | 'new' | 'top' | 'relevance';
+    limit?: number;
+    type?: 'posts' | 'comments';
+}): Promise<RedditPost[]>;
+export declare function getPostFromUrl(url: string): Promise<RedditPost>;
+export {};

package/dist/modules/reddit.js

@@ -0,0 +1,107 @@
+"use strict";
+// reddit.ts - handles reddit stuff like getting posts n searching
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getSubredditHot = getSubredditHot;
+exports.searchReddit = searchReddit;
+exports.getPostFromUrl = getPostFromUrl;
+// Get hot posts from a subreddit
+async function getSubredditHot(subreddit, limit = 25) {
+    try {
+        const url = `https://www.reddit.com/r/${subreddit}/hot.json?limit=${limit}`;
+        const response = await fetch(url);
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        const data = await response.json();
+        return data.data.children.map((post) => ({
+            title: post.data.title,
+            id: post.data.id,
+            url: post.data.url,
+            permalink: `https://reddit.com${post.data.permalink}`,
+            author: post.data.author,
+            score: post.data.score,
+            num_comments: post.data.num_comments,
+            created_utc: post.data.created_utc,
+            selftext: post.data.selftext,
+            subreddit: post.data.subreddit,
+            is_self: post.data.is_self
+        }));
+    }
+    catch (error) {
+        throw {
+            message: `failed to get subreddit posts: ${error instanceof Error ? error.message : String(error)}`,
+            code: 'REDDIT_ERROR',
+            originalError: error
+        };
+    }
+}
+// Search Reddit posts
+async function searchReddit(query, options = {}) {
+    try {
+        const params = new URLSearchParams({
+            q: query,
+            sort: options.sort || 'hot',
+            type: options.type || 'posts',
+            limit: String(options.limit || 25)
+        });
+        const url = `https://www.reddit.com/search.json?${params.toString()}`;
+        const response = await fetch(url);
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        const data = await response.json();
+        return data.data.children.map((post) => ({
+            title: post.data.title,
+            id: post.data.id,
+            url: post.data.url,
+            permalink: `https://reddit.com${post.data.permalink}`,
+            author: post.data.author,
+            score: post.data.score,
+            num_comments: post.data.num_comments,
+            created_utc: post.data.created_utc,
+            selftext: post.data.selftext,
+            subreddit: post.data.subreddit,
+            is_self: post.data.is_self
+        }));
+    }
+    catch (error) {
+        throw {
+            message: `reddit search failed: ${error instanceof Error ? error.message : String(error)}`,
+            code: 'REDDIT_SEARCH_ERROR',
+            originalError: error
+        };
+    }
+}
+// Get post info by URL
+async function getPostFromUrl(url) {
+    try {
+        // Convert post URL to .json URL
+        const jsonUrl = url.replace(/\/?$/, '.json');
+        const response = await fetch(jsonUrl);
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        const data = await response.json();
+        const post = data[0].data.children[0].data;
+        return {
+            title: post.title,
+            id: post.id,
+            url: post.url,
+            permalink: `https://reddit.com${post.permalink}`,
+            author: post.author,
+            score: post.score,
+            num_comments: post.num_comments,
+            created_utc: post.created_utc,
+            selftext: post.selftext,
+            subreddit: post.subreddit,
+            is_self: post.is_self
+        };
+    }
+    catch (error) {
+        throw {
+            message: `failed to get reddit post: ${error instanceof Error ? error.message : String(error)}`,
+            code: 'REDDIT_POST_ERROR',
+            originalError: error
+        };
+    }
+}

package/dist/modules/scrape.d.ts

@@ -0,0 +1,16 @@
+import { ScraperOptions, WebpageContent } from "../types";
+export interface NormalizeContentParams {
+    url: string;
+    html: string;
+    title?: string;
+    siteName?: string;
+    fallbackFavicon?: string;
+    skipReadability?: boolean;
+}
+export declare function normalizeContent(params: NormalizeContentParams): WebpageContent;
+export declare function getWebpageContent(url: string, options?: ({
+    usePuppeteer?: boolean;
+} & ScraperOptions) | boolean): Promise<WebpageContent>;
+export declare function getWebpageText(url: string, options?: {
+    usePuppeteer?: boolean;
+} & ScraperOptions): Promise<string>;