@airoom/nextmin-node 2.0.1 → 2.0.3

package/dist/services/AggregateService.js

@@ -0,0 +1,87 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AggregateService = void 0;
+const Events_1 = require("../utils/Events");
+const Logger_1 = __importDefault(require("../utils/Logger"));
+class AggregateService {
+    constructor(ctx) {
+        this.ctx = ctx;
+        this.init();
+    }
+    init() {
+        Events_1.events.on(Events_1.Events.AFTER_CREATE, (payload) => this.handleUpdate(payload.modelName, payload.result, 'create'));
+        Events_1.events.on(Events_1.Events.AFTER_UPDATE, (payload) => this.handleUpdate(payload.modelName, payload.result, 'update'));
+        Events_1.events.on(Events_1.Events.AFTER_DELETE, (payload) => this.handleUpdate(payload.modelName, payload.result, 'delete'));
+        Logger_1.default.info('AggregateService', 'Initialized and listening for CRUD events');
+    }
+    async handleUpdate(sourceModelName, doc, action) {
+        if (!doc)
+            return;
+        // Find SCHEMAS that have aggregates dependent on sourceModelName
+        const allSchemas = Object.values(this.ctx.liveSchemas);
+        for (const targetSchema of allSchemas) {
+            if (!targetSchema.aggregates)
+                continue;
+            const ags = targetSchema.aggregates.filter((a) => a.sourceModel.toLowerCase() === sourceModelName.toLowerCase());
+            if (ags.length === 0)
+                continue;
+            for (const ag of ags) {
+                const parentId = doc[ag.foreignField];
+                if (!parentId)
+                    continue;
+                // Recalculate and update target document (Doctor)
+                await this.recalculate(targetSchema.modelName, parentId, ag);
+            }
+        }
+    }
+    async recalculate(targetModelName, targetId, ag) {
+        try {
+            const sourceModel = this.ctx.getModel(ag.sourceModel.toLowerCase());
+            const targetModel = this.ctx.getModel(targetModelName.toLowerCase());
+            const filter = { [ag.foreignField]: targetId };
+            let result = 0;
+            if (ag.op === 'count') {
+                result = await sourceModel.count(filter, true);
+            }
+            else if (ag.op === 'avg' || ag.op === 'sum') {
+                const adapter = this.ctx.dbAdapter;
+                if (adapter.dataSource.options.type === 'mongodb') {
+                    const repo = adapter.dataSource.getMongoRepository(ag.sourceModel);
+                    const aggregation = [
+                        { $match: adapter.transformQuery(filter, sourceModel.schema) },
+                        { $group: { _id: null, val: { [`$${ag.op}`]: `$${ag.field}` } } },
+                    ];
+                    const res = await repo.aggregate(aggregation).toArray();
+                    result = res[0]?.val || 0;
+                }
+                else {
+                    // SQL
+                    const repo = adapter.repositories.get(ag.sourceModel.toLowerCase());
+                    const qb = repo.createQueryBuilder('entity');
+                    qb.select(`${ag.op === 'avg' ? 'AVG' : 'SUM'}(entity.${ag.field})`, 'val');
+                    qb.where(adapter.transformQuery(filter, sourceModel.schema));
+                    const res = await qb.getRawOne();
+                    result = parseFloat(res?.val || res?.VAL || 0);
+                }
+            }
+            // Update target document (e.g. Doctor)
+            const updateData = {};
+            const parts = ag.targetField.split('.');
+            let curr = updateData;
+            for (let i = 0; i < parts.length - 1; i++) {
+                curr[parts[i]] = {};
+                curr = curr[parts[i]];
+            }
+            curr[parts[parts.length - 1]] = result;
+            await targetModel.update(targetId, updateData, true);
+            Logger_1.default.debug('AggregateService', `Updated ${targetModelName}:${targetId} aggregate ${ag.targetField} to ${result}`);
+        }
+        catch (err) {
+            Logger_1.default.error('AggregateService', `Failed to recalculate aggregate for ${targetModelName}:${targetId}`, err);
+        }
+    }
+}
+exports.AggregateService = AggregateService;

package/dist/services/IndexingService.d.ts

@@ -0,0 +1,24 @@
+import { ChatWidgetConfig } from '../api/apiRouter';
+export declare class IndexingService {
+    private chatWidget;
+    private force;
+    private dbAdapter?;
+    private itemKeyToId;
+    constructor(chatWidget: ChatWidgetConfig, force?: boolean, dbAdapter?: any);
+    private getManifestPath;
+    private readManifest;
+    private writeManifest;
+    private getScrapedPagesDir;
+    private writeScrapedPageLocal;
+    private deleteScrapedPageLocal;
+    private fetchSitemapUrls;
+    crawlSite(siteUrl: string, startPaths?: string[], concurrency?: number): Promise<Map<string, string>>;
+    private extractInternalLinks;
+    private indexDatabaseSchemas;
+    runIndex(): Promise<void>;
+}
+export declare function reindex(config: {
+    chatWidget: ChatWidgetConfig;
+    force?: boolean;
+    dbAdapter?: any;
+}): Promise<void>;

package/dist/services/IndexingService.js

@@ -0,0 +1,555 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IndexingService = void 0;
+exports.reindex = reindex;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const crypto_1 = __importDefault(require("crypto"));
+const https_1 = __importDefault(require("https"));
+const http_1 = __importDefault(require("http"));
+const Logger_1 = __importDefault(require("../utils/Logger"));
+const OpenAIService_1 = require("./OpenAIService");
+// Custom HTTPS agent that skips TLS verification — needed for sites with
+// expired or self-signed certificates (e.g. Cloudflare Pages custom domains).
+const insecureHttpsAgent = new https_1.default.Agent({ rejectUnauthorized: false });
+/**
+ * Fetch a URL using Node's built-in http/https modules, bypassing TLS errors.
+ * Returns a Response-compatible object with ok, status, headers, text() and json().
+ */
+function fetchInsecure(url, options = {}) {
+    return new Promise((resolve, reject) => {
+        const parsedUrl = new URL(url);
+        const isHttps = parsedUrl.protocol === 'https:';
+        const lib = isHttps ? https_1.default : http_1.default;
+        const reqOptions = {
+            hostname: parsedUrl.hostname,
+            port: parsedUrl.port || (isHttps ? 443 : 80),
+            path: parsedUrl.pathname + parsedUrl.search,
+            method: 'GET',
+            headers: options.headers || {},
+            agent: isHttps ? insecureHttpsAgent : undefined,
+            timeout: options.timeout || 30000,
+        };
+        const req = lib.request(reqOptions, (res) => {
+            const chunks = [];
+            res.on('data', (chunk) => chunks.push(chunk));
+            res.on('end', () => {
+                const body = Buffer.concat(chunks).toString('utf-8');
+                const statusCode = res.statusCode || 0;
+                const headers = res.headers;
+                resolve({
+                    ok: statusCode >= 200 && statusCode < 300,
+                    status: statusCode,
+                    headers: {
+                        get: (k) => {
+                            const v = headers[k.toLowerCase()];
+                            return Array.isArray(v) ? v[0] : (v ?? null);
+                        }
+                    },
+                    text: async () => body,
+                });
+            });
+            res.on('error', reject);
+        });
+        req.on('timeout', () => {
+            req.destroy();
+            reject(new Error(`Request to ${url} timed out after ${options.timeout || 30000}ms`));
+        });
+        req.on('error', reject);
+        req.end();
+    });
+}
+// Helper for fetch with timeout — wraps fetchInsecure with a consistent interface.
+async function fetchWithTimeout(url, options = {}) {
+    // Use global fetch if it has been stubbed (like in unit tests)
+    if (typeof global !== 'undefined' && typeof global.fetch === 'function') {
+        try {
+            return await global.fetch(url, options);
+        }
+        catch (err) {
+            // Fallback to fetchInsecure if global fetch fails
+        }
+    }
+    return fetchInsecure(url, options);
+}
+// Simple HTML text stripper
+function stripHtml(html) {
+    // Preprocess links to preserve their text and destination in Markdown format
+    const processed = html.replace(/<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (match, href, text) => {
+        const cleanText = text.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
+        if (!cleanText)
+            return '';
+        return ` [${cleanText}](${href}) `;
+    });
+    return processed
+        .replace(/<script[^>]*>([\s\S]*?)<\/script>/gi, ' ')
+        .replace(/<style[^>]*>([\s\S]*?)<\/style>/gi, ' ')
+        .replace(/<[^>]*>/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+class IndexingService {
+    constructor(chatWidget, force = false, dbAdapter) {
+        this.itemKeyToId = new Map();
+        this.chatWidget = chatWidget;
+        this.force = force;
+        this.dbAdapter = dbAdapter;
+    }
+    getManifestPath() {
+        const dir = path_1.default.join(process.cwd(), '.nextmin');
+        if (!fs_1.default.existsSync(dir))
+            fs_1.default.mkdirSync(dir, { recursive: true });
+        return path_1.default.join(dir, 'indexing-manifest.json');
+    }
+    readManifest() {
+        const filePath = this.getManifestPath();
+        if (!fs_1.default.existsSync(filePath))
+            return {};
+        try {
+            const content = fs_1.default.readFileSync(filePath, 'utf-8');
+            const parsed = JSON.parse(content);
+            return parsed.hashes || {};
+        }
+        catch {
+            return {};
+        }
+    }
+    writeManifest(hashes) {
+        const filePath = this.getManifestPath();
+        try {
+            fs_1.default.writeFileSync(filePath, JSON.stringify({ lastIndexed: new Date().toISOString(), hashes }, null, 2), 'utf-8');
+        }
+        catch (err) {
+            Logger_1.default.error('IndexingService', 'Failed to write indexing-manifest.json', err);
+        }
+    }
+    getScrapedPagesDir() {
+        const dir = path_1.default.join(process.cwd(), '.nextmin', 'scraped-pages');
+        if (!fs_1.default.existsSync(dir))
+            fs_1.default.mkdirSync(dir, { recursive: true });
+        return dir;
+    }
+    writeScrapedPageLocal(key, content) {
+        const dir = this.getScrapedPagesDir();
+        const safeName = key.replace(/[^a-zA-Z0-9_\-.]/g, '_');
+        const filePath = path_1.default.join(dir, `${safeName}.txt`);
+        fs_1.default.writeFileSync(filePath, content, 'utf-8');
+        return filePath;
+    }
+    deleteScrapedPageLocal(key) {
+        const dir = this.getScrapedPagesDir();
+        const safeName = key.replace(/[^a-zA-Z0-9_\-.]/g, '_');
+        const filePath = path_1.default.join(dir, `${safeName}.txt`);
+        if (fs_1.default.existsSync(filePath)) {
+            try {
+                fs_1.default.unlinkSync(filePath);
+            }
+            catch (e) {
+                Logger_1.default.warn('IndexingService', `Failed to delete local scraped file ${filePath}`, e);
+            }
+        }
+    }
+    // ---------- Web Page Scraping / Crawler API ----------
+    async fetchSitemapUrls(sitemapUrl, baseUrl, visited = new Set()) {
+        if (visited.has(sitemapUrl))
+            return [];
+        visited.add(sitemapUrl);
+        Logger_1.default.info('IndexingService', `Fetching sitemap: ${sitemapUrl}`);
+        try {
+            const res = await fetchWithTimeout(sitemapUrl, {
+                headers: { 'User-Agent': 'NextMinCrawler/1.0' },
+                timeout: 30000
+            });
+            if (!res.ok) {
+                Logger_1.default.warn('IndexingService', `Sitemap fetch failed (${res.status}): ${sitemapUrl}`);
+                return [];
+            }
+            const xml = await res.text();
+            const locRegex = /<loc>([^<]+)<\/loc>/gi;
+            let match;
+            const subSitemaps = [];
+            const paths = [];
+            const siteOrigin = new URL(baseUrl).origin;
+            const isSitemapIndex = xml.includes('<sitemapindex') || xml.includes('<sitemap>');
+            while ((match = locRegex.exec(xml)) !== null) {
+                const loc = match[1].trim().replace(/&amp;/g, '&');
+                try {
+                    const urlObj = new URL(loc);
+                    if (urlObj.origin === siteOrigin) {
+                        let path = urlObj.pathname;
+                        if (urlObj.search)
+                            path += urlObj.search;
+                        if (path !== '/' && path.endsWith('/')) {
+                            path = path.slice(0, -1);
+                        }
+                        if (isSitemapIndex) {
+                            subSitemaps.push(loc);
+                        }
+                        else {
+                            paths.push(path);
+                        }
+                    }
+                }
+                catch {
+                    // Ignore invalid URLs
+                }
+            }
+            if (subSitemaps.length > 0) {
+                const results = await Promise.all(subSitemaps.map(subUrl => this.fetchSitemapUrls(subUrl, baseUrl, visited)));
+                return [...paths, ...results.flat()];
+            }
+            return paths;
+        }
+        catch (err) {
+            Logger_1.default.error('IndexingService', `Error fetching sitemap ${sitemapUrl}`, err);
+            return [];
+        }
+    }
+    async crawlSite(siteUrl, startPaths = ['/'], concurrency = 5) {
+        const visited = new Set();
+        const results = new Map();
+        const base = siteUrl.endsWith('/') ? siteUrl.slice(0, -1) : siteUrl;
+        // 1. Try to discover URLs via Sitemap first
+        let sitemapPaths = null;
+        const sitemapUrl = this.chatWidget.sitemapUrl || `${base}/sitemap.xml`;
+        Logger_1.default.info('IndexingService', `Checking for sitemap at ${sitemapUrl}...`);
+        try {
+            const discovered = await this.fetchSitemapUrls(sitemapUrl, base);
+            if (discovered && discovered.length > 0) {
+                sitemapPaths = [...new Set(discovered)];
+                Logger_1.default.info('IndexingService', `Discovered ${sitemapPaths.length} URLs from sitemap.`);
+            }
+            else {
+                Logger_1.default.info('IndexingService', `No URLs found in sitemap or sitemap is unreachable.`);
+            }
+        }
+        catch (err) {
+            Logger_1.default.warn('IndexingService', `Failed to discover sitemap URLs. Falling back to recursive crawler.`, err);
+        }
+        const useSitemap = sitemapPaths !== null && sitemapPaths.length > 0;
+        const queue = useSitemap ? sitemapPaths : [...startPaths];
+        Logger_1.default.info('IndexingService', `Starting concurrent web scrape of ${base} with ${concurrency} threads (${useSitemap ? 'Sitemap Mode' : 'Link-following Mode'})...`);
+        const activeFetches = new Set();
+        const runWorker = async () => {
+            while (true) {
+                let currentPath = null;
+                for (let i = 0; i < queue.length; i++) {
+                    const p = queue[i];
+                    if (!visited.has(p) && !activeFetches.has(p)) {
+                        currentPath = p;
+                        queue.splice(i, 1);
+                        break;
+                    }
+                }
+                if (currentPath === null) {
+                    if (activeFetches.size === 0 && queue.length === 0) {
+                        break;
+                    }
+                    await new Promise(resolve => setTimeout(resolve, 50));
+                    continue;
+                }
+                activeFetches.add(currentPath);
+                const fullUrl = `${base}${currentPath}`;
+                try {
+                    Logger_1.default.info('IndexingService', `Fetching page for scraping: ${fullUrl}`);
+                    const res = await fetchWithTimeout(fullUrl, {
+                        headers: {
+                            'User-Agent': 'NextMinCrawler/1.0'
+                        },
+                        timeout: 30000
+                    });
+                    visited.add(currentPath);
+                    activeFetches.delete(currentPath);
+                    if (!res.ok) {
+                        Logger_1.default.warn('IndexingService', `Failed to fetch page ${fullUrl} (${res.status})`);
+                        continue;
+                    }
+                    const contentType = res.headers.get('content-type') || '';
+                    if (!contentType.includes('text/html')) {
+                        Logger_1.default.debug('IndexingService', `Skipping non-HTML URL: ${fullUrl} (${contentType})`);
+                        continue;
+                    }
+                    const html = await res.text();
+                    const text = stripHtml(html);
+                    if (text && text.length >= 20) {
+                        results.set(currentPath, text);
+                    }
+                    // ONLY extract and follow links if we are NOT using the sitemap list
+                    if (!useSitemap) {
+                        const links = this.extractInternalLinks(html, currentPath, base);
+                        for (const link of links) {
+                            if (!visited.has(link) && !activeFetches.has(link) && !queue.includes(link)) {
+                                queue.push(link);
+                            }
+                        }
+                    }
+                }
+                catch (err) {
+                    visited.add(currentPath);
+                    activeFetches.delete(currentPath);
+                    Logger_1.default.error('IndexingService', `Error crawling page ${fullUrl}`, err);
+                }
+                await new Promise(resolve => setTimeout(resolve, 100));
+            }
+        };
+        const workers = Array.from({ length: concurrency }, () => runWorker());
+        await Promise.all(workers);
+        Logger_1.default.info('IndexingService', `Scrape complete. Scraped ${results.size} pages.`);
+        return results;
+    }
+    extractInternalLinks(html, currentPath, baseUrl) {
+        const links = [];
+        const hrefRegex = /<a[^>]+href=["']([^"']+)["']/gi;
+        let match;
+        while ((match = hrefRegex.exec(html)) !== null) {
+            let href = match[1].trim();
+            if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.toLowerCase().startsWith('javascript:')) {
+                continue;
+            }
+            try {
+                const resolved = new URL(href, `${baseUrl}${currentPath}`);
+                if (resolved.origin === new URL(baseUrl).origin) {
+                    let path = resolved.pathname;
+                    if (resolved.search)
+                        path += resolved.search;
+                    if (path !== '/' && path.endsWith('/')) {
+                        path = path.slice(0, -1);
+                    }
+                    const isAsset = /\.(png|jpe?g|gif|svg|css|js|ico|pdf|zip|mp4|json|xml)$/i.test(resolved.pathname);
+                    if (!isAsset) {
+                        links.push(path);
+                    }
+                }
+            }
+            catch {
+                // Ignore invalid URL
+            }
+        }
+        return [...new Set(links)];
+    }
+    // ---------- Database Schema Indexing ----------
+    async indexDatabaseSchemas() {
+        const results = new Map();
+        if (!this.dbAdapter)
+            return results;
+        const schemas = this.chatWidget.whitelabelSchemas || [];
+        if (schemas.length === 0)
+            return results;
+        Logger_1.default.info('IndexingService', `Indexing ${schemas.length} database schemas: ${schemas.join(', ')}...`);
+        const siteUrl = this.chatWidget.siteUrl || 'https://doctors24.bd';
+        for (const schemaName of schemas) {
+            try {
+                const collectionName = schemaName.toLowerCase();
+                let docs = [];
+                if (typeof this.dbAdapter.read === 'function') {
+                    docs = await this.dbAdapter.read(collectionName, {}, 50000, 0);
+                }
+                else if (this.dbAdapter.db && typeof this.dbAdapter.db.collection === 'function') {
+                    docs = await this.dbAdapter.db.collection(collectionName).find({}).toArray();
+                }
+                if (!Array.isArray(docs))
+                    continue;
+                Logger_1.default.info('IndexingService', `Schema '${schemaName}': found ${docs.length} database records.`);
+                for (const doc of docs) {
+                    if (!doc)
+                        continue;
+                    const slug = doc.slug || doc._id || doc.id;
+                    if (!slug)
+                        continue;
+                    let entityType = schemaName.replace(/s$/i, '').toLowerCase();
+                    if (entityType === 'specialitie' || entityType === 'specialty')
+                        entityType = 'speciality';
+                    const canonicalPath = `/${entityType}/${slug}`;
+                    const fullUrl = `${siteUrl}${canonicalPath}`;
+                    const title = doc.fullName || doc.name || doc.title || doc.headline || `${schemaName} Record`;
+                    let apptText = '';
+                    if (doc.appointments) {
+                        try {
+                            const apps = typeof doc.appointments === 'string' ? JSON.parse(doc.appointments) : doc.appointments;
+                            if (Array.isArray(apps)) {
+                                apptText = apps
+                                    .map((a) => `- Hospital/Chamber: ${a.hospital?.name || a.name || ''}\n  Address: ${a.hospital?.address || a.hospital?.physicalLocation || a.address || ''}\n  Visiting Hours: ${a.visitingHours || ''}\n  Phone / Appointment: ${a.phone || a.hospital?.telephone || doc.phone || ''}`)
+                                    .join('\n\n');
+                            }
+                        }
+                        catch { }
+                    }
+                    let formattedDetails = '';
+                    if (doc.degree || doc.fellowshipsOrTrainings)
+                        formattedDetails += `- Qualifications: ${doc.degree || ''} ${doc.fellowshipsOrTrainings || ''}\n`;
+                    if (doc.speciality || doc.specialty)
+                        formattedDetails += `- Specialty: ${doc.speciality || doc.specialty || ''}\n`;
+                    if (doc.designationAndDepartment || doc.designation)
+                        formattedDetails += `- Designation: ${doc.designationAndDepartment || doc.designation || ''}\n`;
+                    if (doc.workPlace || doc.workplace)
+                        formattedDetails += `- Workplace: ${doc.workPlace || doc.workplace || ''}\n`;
+                    if (doc.phone || doc.telephone)
+                        formattedDetails += `- Primary Phone: ${doc.phone || doc.telephone || ''}\n`;
+                    if (doc.address || doc.physicalLocation)
+                        formattedDetails += `- Address: ${doc.address || doc.physicalLocation || ''}\n`;
+                    const aboutText = stripHtml(doc.aboutYou || doc.aboutHospital || doc.description || doc.body || doc.content || '');
+                    const content = `DIRECTORY DOCUMENT
+PAGE_TITLE: ${title} | ${schemaName}
+CANONICAL_URL: ${fullUrl}
+
+INFORMATION:
+- Name/Title: ${title}
+${formattedDetails}
+${apptText ? `\nCHAMBER & APPOINTMENT DETAILS:\n${apptText}\n` : ''}
+${aboutText ? `\nABOUT:\n${aboutText}\n` : ''}
+EXPLICIT ROUTING INSTRUCTION FOR ASSISTANT:
+When users ask about this item or entity, always direct them to canonical URL: ${fullUrl}
+`;
+                    results.set(canonicalPath, content);
+                }
+            }
+            catch (err) {
+                Logger_1.default.error('IndexingService', `Failed to index database schema '${schemaName}'`, err);
+            }
+        }
+        return results;
+    }
+    // ---------- Entrypoint ----------
+    async runIndex() {
+        if (!this.chatWidget.enabled) {
+            Logger_1.default.info('IndexingService', 'Chat widget is disabled. Skipping index.');
+            return;
+        }
+        Logger_1.default.info('IndexingService', 'Starting LOCAL-ONLY indexing pipeline...');
+        const activeKeys = new Set();
+        const manifestHashes = this.force ? {} : this.readManifest();
+        const currentManifest = { ...manifestHashes };
+        let hasErrors = false;
+        const allPageEntries = new Map();
+        // 1. Scrape Site Pages (Crawler)
+        if (this.chatWidget.siteUrl && this.chatWidget.sourceType !== 'db') {
+            const paths = this.chatWidget.scrapePaths || ['/'];
+            try {
+                const concurrency = this.chatWidget.concurrency || 5;
+                const scrapedPages = await this.crawlSite(this.chatWidget.siteUrl, paths, concurrency);
+                for (const [k, v] of scrapedPages.entries())
+                    allPageEntries.set(k, v);
+            }
+            catch (err) {
+                Logger_1.default.error('IndexingService', 'Site crawl step failed', err);
+                hasErrors = true;
+            }
+        }
+        // 2. Index Database Schemas (DB Exporter)
+        if (this.dbAdapter && (this.chatWidget.sourceType === 'db' || this.chatWidget.sourceType === 'both' || !this.chatWidget.sourceType)) {
+            try {
+                const dbPages = await this.indexDatabaseSchemas();
+                for (const [k, v] of dbPages.entries())
+                    allPageEntries.set(k, v);
+            }
+            catch (err) {
+                Logger_1.default.error('IndexingService', 'Database schema indexing failed', err);
+                hasErrors = true;
+            }
+        }
+        if (allPageEntries.size > 0) {
+            Logger_1.default.info('IndexingService', `Processing ${allPageEntries.size} total page entries...`);
+            const concurrency = this.chatWidget.concurrency || 5;
+            try {
+                let processedCount = 0;
+                let skippedCount = 0;
+                const entries = Array.from(allPageEntries.entries());
+                let index = 0;
+                // Concurrent Indexing Workers
+                const indexWorker = async () => {
+                    while (true) {
+                        let entry = null;
+                        if (index < entries.length) {
+                            entry = entries[index++];
+                        }
+                        if (!entry)
+                            break;
+                        const [pathname, text] = entry;
+                        try {
+                            const key = `page-${pathname.replace(/[^a-zA-Z0-9_\-]/g, '_') || 'index'}`;
+                            activeKeys.add(key);
+                            const hash = crypto_1.default.createHash('md5').update(text).digest('hex');
+                            if (manifestHashes[key] === hash) {
+                                skippedCount++;
+                                currentManifest[key] = hash;
+                                continue;
+                            }
+                            // Save locally
+                            this.writeScrapedPageLocal(key, text);
+                            processedCount++;
+                            // Incrementally write manifest to disk to save progress immediately
+                            currentManifest[key] = hash;
+                            this.writeManifest(currentManifest);
+                            await new Promise(resolve => setTimeout(resolve, 50)); // Gentle throttle
+                        }
+                        catch (err) {
+                            Logger_1.default.error('IndexingService', `Failed to index scraped page ${pathname}`, err);
+                            hasErrors = true;
+                        }
+                    }
+                };
+                Logger_1.default.info('IndexingService', `Processing in parallel using ${concurrency} concurrent workers...`);
+                const indexWorkers = Array.from({ length: concurrency }, () => indexWorker());
+                await Promise.all(indexWorkers);
+                Logger_1.default.info('IndexingService', `Completed indexing scraped pages: ${processedCount} updated, ${skippedCount} unchanged (loaded from cache).`);
+            }
+            catch (err) {
+                Logger_1.default.error('IndexingService', 'Site crawl step failed', err);
+                hasErrors = true;
+            }
+        }
+        // 2. Prune deleted files
+        if (!hasErrors) {
+            for (const key of Object.keys(manifestHashes)) {
+                if (!activeKeys.has(key)) {
+                    Logger_1.default.info('IndexingService', `Pruning deleted document: ${key}`);
+                    // Remove local file
+                    this.deleteScrapedPageLocal(key);
+                }
+            }
+        }
+        else {
+            Logger_1.default.info('IndexingService', 'Skipping pruning stage due to errors encountered during crawling/indexing.');
+        }
+        // 3. Update final local manifest
+        const finalManifest = {};
+        for (const key of activeKeys) {
+            if (currentManifest[key]) {
+                finalManifest[key] = currentManifest[key];
+            }
+        }
+        // If errors occurred, preserve unvisited entries in the manifest
+        if (hasErrors) {
+            for (const key of Object.keys(manifestHashes)) {
+                if (!activeKeys.has(key)) {
+                    finalManifest[key] = manifestHashes[key];
+                }
+            }
+        }
+        this.writeManifest(finalManifest);
+        // Sync newly indexed files to OpenAI Vector Store if enabled and API Key is set
+        if (this.chatWidget.enableOpenai !== false && process.env.OPENAI_API_KEY) {
+            try {
+                Logger_1.default.info('IndexingService', 'Triggering OpenAI Vector Store synchronization...');
+                const openAI = new OpenAIService_1.OpenAIService();
+                await openAI.syncFilesToVectorStore(this.force);
+            }
+            catch (err) {
+                Logger_1.default.error('IndexingService', `OpenAI file sync failed: ${err.message || err}`);
+                hasErrors = true;
+            }
+        }
+        if (hasErrors) {
+            throw new Error('Indexing completed with errors. Please check the logs.');
+        }
+        Logger_1.default.info('IndexingService', `Incremental indexing complete.`);
+    }
+}
+exports.IndexingService = IndexingService;
+async function reindex(config) {
+    const indexer = new IndexingService(config.chatWidget, config.force, config.dbAdapter);
+    await indexer.runIndex();
+}