ai-search-indexer 1.0.0

package/README.md

@@ -0,0 +1,84 @@
+# @website-grabber/indexer
+
+Website content indexer using Mozilla Readability and Playwright. Extracts only relevant content from web pages, filtering out navigation, ads, and UI elements.
+
+## Features
+
+- 🚀 Smart content extraction with Mozilla Readability
+- 🎯 Full SPA support (Angular, React, Vue, etc.)
+- 📄 Extracts articles, documentation, and code examples
+- ⚡ Powered by Playwright for accurate rendering
+
+## Installation
+
+```bash
+npm install @website-grabber/indexer
+```
+
+## Usage
+
+```javascript
+import { WebsiteIndexer } from '@website-grabber/indexer';
+
+const indexer = new WebsiteIndexer('./context.json');
+
+await indexer.indexUrls([
+    'https://angular.dev/overview',
+    'https://reactjs.org/docs',
+    'https://vuejs.org/guide'
+]);
+
+indexer.displayStats();
+```
+
+## API
+
+### WebsiteIndexer
+
+```javascript
+const indexer = new WebsiteIndexer(contextFile);
+```
+
+**Methods:**
+
+- `indexUrls(urls: string[]): Promise<void>` - Index multiple URLs
+- `indexUrl(page, url: string): Promise<object>` - Index a single URL
+- `saveContext(): void` - Save indexed content to file
+- `loadContext(): object` - Load indexed content from file
+- `displayStats(): void` - Display indexing statistics
+
+## Output Format
+
+Indexed content is saved as JSON:
+
+```json
+{
+  "indexed_content": [
+    {
+      "url": "https://example.com",
+      "title": "Page Title",
+      "description": "Meta description",
+      "content": "Extracted main content...",
+      "html_content": "<div>HTML with code examples</div>",
+      "excerpt": "Brief excerpt...",
+      "word_count": 2500,
+      "indexed_at": "2026-02-12T18:00:00.000Z",
+      "site_name": "Example Site"
+    }
+  ],
+  "metadata": {
+    "last_updated": "2026-02-12T18:00:00.000Z",
+    "total_pages": 1,
+    "version": "1.0"
+  }
+}
+```
+
+## Requirements
+
+- Node.js 18+
+- Playwright (automatically installs browser)
+
+## License
+
+MIT

package/examples/index-sites.js

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+import { WebsiteIndexer } from '../src/index.js';
+
+async function main() {
+    const urls = process.argv.slice(2);
+
+    if (urls.length === 0) {
+        console.log('Usage: node examples/index-sites.js <url1> <url2> ...');
+        console.log('Example: node examples/index-sites.js https://example.com');
+        process.exit(1);
+    }
+
+    console.log(`Starting indexer for ${urls.length} URL(s)...\n`);
+
+    const indexer = new WebsiteIndexer('./context.json');
+    await indexer.indexUrls(urls);
+    indexer.displayStats();
+
+    console.log('Indexing complete!');
+}
+
+main().catch(console.error);

package/examples/index-wikipedia.js

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+import { WebsiteIndexer } from '../src/index.js';
+import { resolve, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+async function main() {
+    const wikipediaUrl = 'https://de.wikipedia.org/wiki/Eisenbahnunfall_im_Gotthard-Basistunnel';
+
+    console.log(`Starting indexer for Wikipedia article...\n`);
+    console.log(`URL: ${wikipediaUrl}\n`);
+
+    // Save context to a location accessible by the web components demo
+    const contextPath = resolve(__dirname, '../../web-components/demo/context.json');
+
+    const indexer = new WebsiteIndexer(contextPath);
+    await indexer.indexUrls([wikipediaUrl]);
+    indexer.displayStats();
+
+    console.log(`\nIndexing complete!`);
+    console.log(`Context saved to: ${contextPath}`);
+}
+
+main().catch(console.error);

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "ai-search-indexer",
+  "version": "1.0.0",
+  "description": "Website content indexer using Mozilla Readability and Playwright",
+  "type": "module",
+  "main": "src/index.js",
+  "exports": {
+    ".": "./src/index.js"
+  },
+  "keywords": [
+    "indexer",
+    "readability",
+    "playwright",
+    "content-extraction",
+    "web-scraping"
+  ],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@mozilla/readability": "^0.5.0",
+    "playwright": "^1.58.2"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "scripts": {
+    "example": "node examples/index-sites.js"
+  }
+}

package/src/index.js

@@ -0,0 +1 @@
+export { WebsiteIndexer } from './indexer.js';

package/src/indexer.js

@@ -0,0 +1,185 @@
+import { chromium } from 'playwright';
+import { readFileSync, writeFileSync } from 'fs';
+
+/**
+ * Indexes web pages and extracts their content using Mozilla Readability
+ * Supports Angular, React, Vue and other SPAs
+ */
+export class WebsiteIndexer {
+    constructor(contextFile = './context.json') {
+        this.contextFile = contextFile;
+        this.context = this.loadContext();
+    }
+
+    loadContext() {
+        try {
+            const data = readFileSync(this.contextFile, 'utf8');
+            return JSON.parse(data);
+        } catch (error) {
+            console.error('Error loading context file:', error);
+            return {
+                indexed_content: [],
+                metadata: {
+                    last_updated: null,
+                    total_pages: 0,
+                    version: '1.0'
+                }
+            };
+        }
+    }
+
+    saveContext() {
+        this.context.metadata.last_updated = new Date().toISOString();
+        this.context.metadata.total_pages = this.context.indexed_content.length;
+
+        try {
+            writeFileSync(
+                this.contextFile,
+                JSON.stringify(this.context, null, 2),
+                'utf8'
+            );
+            console.log(`Context saved successfully. Total pages: ${this.context.metadata.total_pages}`);
+        } catch (error) {
+            console.error('Error saving context file:', error);
+        }
+    }
+
+    /**
+     * Extract main content using Mozilla Readability via Playwright
+     * Extracts only relevant long-form content and code examples
+     * Works with Angular, React, Vue and other SPAs by using rendered DOM
+     */
+    async extractMainContent(page) {
+        try {
+            // Add Readability script to the page and extract content
+            await page.addScriptTag({
+                path: './node_modules/@mozilla/readability/Readability.js'
+            });
+
+            // Extract content using Readability in the browser context
+            const article = await page.evaluate(() => {
+                const documentClone = document.cloneNode(true);
+                const reader = new window.Readability(documentClone);
+                return reader.parse();
+            });
+
+            if (!article) {
+                return null;
+            }
+
+            return {
+                title: article.title,
+                textContent: article.textContent,
+                htmlContent: article.content,
+                excerpt: article.excerpt,
+                length: article.length,
+                siteName: article.siteName
+            };
+        } catch (error) {
+            console.error('Error extracting content with Readability:', error.message);
+            return null;
+        }
+    }
+
+    /**
+     * Index a single URL
+     */
+    async indexUrl(page, url) {
+        try {
+            console.log(`Indexing: ${url}`);
+
+            await page.goto(url, {
+                waitUntil: 'networkidle',
+                timeout: 30000
+            });
+
+            // Wait for Angular/React/Vue apps to fully render
+            // Wait for network to be idle and a bit more for client-side rendering
+            await page.waitForTimeout(3000);
+
+            // Extract main content using Readability on the rendered page
+            const article = await this.extractMainContent(page);
+
+            if (!article) {
+                console.log(`No main content found for: ${url}`);
+                return null;
+            }
+
+            // Get metadata
+            const metadata = await page.evaluate(() => {
+                return {
+                    description: document.querySelector('meta[name="description"]')?.content || '',
+                };
+            });
+
+            // Create indexed entry with only relevant content
+            const indexedEntry = {
+                url: url,
+                title: article.title,
+                description: metadata.description || article.excerpt,
+                content: article.textContent.substring(0, 15000), // Store clean text content
+                html_content: article.htmlContent, // Store HTML for code examples
+                excerpt: article.excerpt,
+                indexed_at: new Date().toISOString(),
+                word_count: article.length,
+                site_name: article.siteName
+            };
+
+            // Check if URL already exists and update or add
+            const existingIndex = this.context.indexed_content.findIndex(item => item.url === url);
+            if (existingIndex >= 0) {
+                this.context.indexed_content[existingIndex] = indexedEntry;
+                console.log(`✓ Updated: ${url} (${article.length} words)`);
+            } else {
+                this.context.indexed_content.push(indexedEntry);
+                console.log(`✓ Added: ${url} (${article.length} words)`);
+            }
+
+            return indexedEntry;
+
+        } catch (error) {
+            console.error(`✗ Error indexing ${url}:`, error.message);
+            return null;
+        }
+    }
+
+    /**
+     * Index multiple URLs
+     */
+    async indexUrls(urls) {
+        const browser = await chromium.launch({
+            headless: true
+        });
+
+        try {
+            const page = await browser.newPage();
+
+            for (const url of urls) {
+                await this.indexUrl(page, url);
+            }
+
+            this.saveContext();
+
+        } catch (error) {
+            console.error('Error during indexing:', error);
+        } finally {
+            await browser.close();
+        }
+    }
+
+    /**
+     * Display statistics
+     */
+    displayStats() {
+        console.log('\n=== Indexing Statistics ===');
+        console.log(`Total pages indexed: ${this.context.metadata.total_pages}`);
+        console.log(`Last updated: ${this.context.metadata.last_updated}`);
+
+        if (this.context.indexed_content.length > 0) {
+            const totalWords = this.context.indexed_content.reduce((sum, item) => sum + item.word_count, 0);
+            console.log(`Total words indexed: ${totalWords}`);
+            console.log(`Average words per page: ${Math.round(totalWords / this.context.indexed_content.length)}`);
+        }
+        console.log('==========================\n');
+    }
+}