@bndynet/ragbox 0.1.0

package/dist/src/folder-index/watch.js

@@ -0,0 +1,550 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startWatchFolder = startWatchFolder;
+exports.watchFolder = watchFolder;
+const chokidar_1 = __importDefault(require("chokidar"));
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+const config_1 = require("./config");
+const indexer_1 = require("./indexer");
+const manifest_1 = require("./manifest");
+const path_utils_1 = require("./path-utils");
+const scan_1 = require("./scan");
+const WATCH_IGNORED = /(^|[/\\])(node_modules|\.git|\.pageindex|dist|build)([/\\]|$)/;
+class WatchIndexRunError extends Error {
+    attempt;
+    maxAttempts;
+    constructor(error, attempt, maxAttempts) {
+        super(errorMessage(error));
+        this.name = "WatchIndexRunError";
+        this.attempt = attempt;
+        this.maxAttempts = maxAttempts;
+    }
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+async function pathExists(filePath) {
+    try {
+        await promises_1.default.access(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function toIndexCounts(result) {
+    return {
+        total: result.manifest.documents.length,
+        ready: result.ready,
+        failed: result.failed,
+        added: result.added,
+        modified: result.modified,
+        retryFailed: result.retryFailed,
+        unchanged: result.unchanged,
+        deleted: result.deleted
+    };
+}
+function createWatchEvent(event) {
+    return {
+        version: 1,
+        timestamp: new Date().toISOString(),
+        ...event
+    };
+}
+function reportWatchProgress(options, event, sendWebhook = true) {
+    try {
+        options.watchProgress?.(event);
+    }
+    catch {
+        // Watch progress reporting must never change watch behavior.
+    }
+    if (!sendWebhook || !options.watchWebhookUrl) {
+        return;
+    }
+    void (async () => {
+        try {
+            const response = await fetch(options.watchWebhookUrl, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json"
+                },
+                body: JSON.stringify(event)
+            });
+            if (!response.ok) {
+                throw new Error(`Webhook returned ${response.status}`);
+            }
+        }
+        catch (error) {
+            reportWatchProgress(options, createWatchEvent({
+                type: "watch-webhook-failed",
+                rootDir: event.rootDir,
+                outputDir: event.outputDir,
+                url: options.watchWebhookUrl,
+                error: errorMessage(error)
+            }), false);
+        }
+    })();
+}
+async function acquireLock(lockFile, rootDir, outputDir) {
+    await promises_1.default.mkdir(node_path_1.default.dirname(lockFile), { recursive: true });
+    let handle;
+    try {
+        handle = await promises_1.default.open(lockFile, "wx");
+        await handle.writeFile(`${JSON.stringify({
+            version: 1,
+            pid: process.pid,
+            rootDir,
+            outputDir,
+            startedAt: new Date().toISOString()
+        }, null, 2)}\n`, "utf8");
+    }
+    catch (error) {
+        if (error.code === "EEXIST") {
+            throw new Error(`Watch lock already exists: ${lockFile}`);
+        }
+        throw error;
+    }
+    finally {
+        await handle?.close();
+    }
+    return async () => {
+        await promises_1.default.rm(lockFile, { force: true });
+    };
+}
+async function prepareStagingOutput(outputDir, stagingOutputDir) {
+    await promises_1.default.rm(stagingOutputDir, { force: true, recursive: true });
+    if (await pathExists(outputDir)) {
+        await promises_1.default.cp(outputDir, stagingOutputDir, { recursive: true });
+        return;
+    }
+    await promises_1.default.mkdir(stagingOutputDir, { recursive: true });
+}
+async function promoteStagingOutput(outputDir, stagingOutputDir) {
+    const backupDir = `${outputDir}.previous-${process.pid}-${Date.now()}`;
+    if (!(await pathExists(outputDir))) {
+        await promises_1.default.rename(stagingOutputDir, outputDir);
+        return;
+    }
+    await promises_1.default.rm(backupDir, { force: true, recursive: true });
+    await promises_1.default.rename(outputDir, backupDir);
+    try {
+        await promises_1.default.rename(stagingOutputDir, outputDir);
+        await promises_1.default.rm(backupDir, { force: true, recursive: true });
+    }
+    catch (error) {
+        try {
+            if (await pathExists(backupDir)) {
+                await promises_1.default.rm(outputDir, { force: true, recursive: true });
+                await promises_1.default.rename(backupDir, outputDir);
+            }
+        }
+        catch {
+            // Preserve the original promotion error; rollback best-effort failed.
+        }
+        throw error;
+    }
+}
+function withOutputDir(result, outputDir) {
+    return {
+        ...result,
+        outputDir,
+        manifestPath: node_path_1.default.join(outputDir, manifest_1.MANIFEST_FILE),
+        rootTreePath: node_path_1.default.join(outputDir, manifest_1.ROOT_TREE_FILE)
+    };
+}
+async function startWatchFolder(folder, options = {}) {
+    const rootDir = node_path_1.default.resolve(folder);
+    const config = (0, config_1.loadPageIndexConfig)(options);
+    const outputDir = (0, manifest_1.resolvePageIndexDir)(rootDir, config.outputDir);
+    const lockFile = config.watchLockFile ? node_path_1.default.resolve(config.watchLockFile) : undefined;
+    const healthFile = config.watchHealthFile ? node_path_1.default.resolve(config.watchHealthFile) : undefined;
+    const retryAttempts = Math.max(0, config.watchRetryAttempts ?? 0);
+    const retryDelayMs = Math.max(0, config.watchRetryDelayMs ?? 1000);
+    const maxAttempts = retryAttempts + 1;
+    const debounceMs = Math.max(0, config.watchDebounceMs ?? 500);
+    const stagingOutputDir = config.watchStaging
+        ? node_path_1.default.resolve(config.watchStagingOutputDir ?? `${outputDir}.staging`)
+        : undefined;
+    const ignoredOutputDirs = [outputDir, stagingOutputDir]
+        .filter((value) => Boolean(value))
+        .filter((candidate) => (0, path_utils_1.isStrictSubPath)(rootDir, candidate));
+    const hasStructuredProgress = Boolean(config.watchProgress);
+    const startedAt = new Date().toISOString();
+    let watcher;
+    let timer;
+    let running = false;
+    let pending = false;
+    let stopped = false;
+    let stopReported = false;
+    let readySettled = false;
+    let closeStarted;
+    let releaseLock;
+    let lastSuccessAt;
+    let lastFailureAt;
+    let lastHealthResult;
+    let lastHealthError;
+    let resolveReady;
+    let resolveClosed;
+    const ready = new Promise((resolve) => {
+        resolveReady = resolve;
+    });
+    const closed = new Promise((resolve) => {
+        resolveClosed = resolve;
+    });
+    async function writeHealth(status, reason, fields = {}) {
+        if (!healthFile) {
+            return;
+        }
+        const updatedAt = new Date().toISOString();
+        if (fields.result) {
+            lastHealthResult = fields.result;
+        }
+        if (fields.error) {
+            lastHealthError = fields.error;
+        }
+        const health = {
+            version: 1,
+            ok: status === "ready",
+            status,
+            rootDir,
+            outputDir,
+            pid: process.pid,
+            startedAt,
+            updatedAt,
+            lastSuccessAt,
+            lastFailureAt,
+            reason,
+            result: fields.result ?? lastHealthResult,
+            error: fields.error ?? lastHealthError
+        };
+        try {
+            await (0, manifest_1.atomicWriteJson)(healthFile, health);
+            reportWatchProgress(config, createWatchEvent({
+                type: "watch-health",
+                rootDir,
+                outputDir,
+                healthFile,
+                status,
+                ok: health.ok
+            }));
+        }
+        catch (error) {
+            reportWatchProgress(config, createWatchEvent({
+                type: "watch-health-failed",
+                rootDir,
+                outputDir,
+                healthFile,
+                error: errorMessage(error)
+            }));
+        }
+    }
+    async function finishClose() {
+        if (closeStarted) {
+            return await closeStarted;
+        }
+        closeStarted = (async () => {
+            if (timer) {
+                clearTimeout(timer);
+                timer = undefined;
+            }
+            if (watcher) {
+                await watcher.close();
+                watcher = undefined;
+            }
+            if (!stopReported) {
+                stopReported = true;
+                await writeHealth("stopped");
+                reportWatchProgress(config, createWatchEvent({
+                    type: "watch-stop",
+                    rootDir,
+                    outputDir
+                }));
+            }
+            if (releaseLock && lockFile) {
+                await releaseLock();
+                releaseLock = undefined;
+                reportWatchProgress(config, createWatchEvent({
+                    type: "watch-lock-released",
+                    rootDir,
+                    outputDir,
+                    lockFile
+                }));
+            }
+            resolveClosed();
+        })();
+        return await closeStarted;
+    }
+    async function close() {
+        stopped = true;
+        finishReady({ ok: false, error: "Watch closed before initial index completed" });
+        if (!running) {
+            await finishClose();
+        }
+        return await closed;
+    }
+    function finishReady(value) {
+        if (!readySettled) {
+            readySettled = true;
+            resolveReady(value);
+        }
+    }
+    function isIgnored(candidatePath) {
+        if (WATCH_IGNORED.test(candidatePath)) {
+            return true;
+        }
+        if (ignoredOutputDirs.length === 0) {
+            return false;
+        }
+        const absoluteCandidatePath = node_path_1.default.isAbsolute(candidatePath) ? candidatePath : node_path_1.default.resolve(rootDir, candidatePath);
+        return ignoredOutputDirs.some((ignoredOutputDir) => (0, path_utils_1.isSubPath)(ignoredOutputDir, absoluteCandidatePath));
+    }
+    async function runIndexAttempt(reason, attempt) {
+        reportWatchProgress(config, createWatchEvent({
+            type: "watch-index-start",
+            rootDir,
+            outputDir,
+            reason,
+            attempt,
+            maxAttempts
+        }));
+        await writeHealth("indexing", reason);
+        if (!stagingOutputDir) {
+            return await (0, indexer_1.indexFolder)(rootDir, config);
+        }
+        await prepareStagingOutput(outputDir, stagingOutputDir);
+        const stagedResult = await (0, indexer_1.indexFolder)(rootDir, {
+            ...config,
+            outputDir: stagingOutputDir
+        });
+        if (stagedResult.failed > 0) {
+            return stagedResult;
+        }
+        await promoteStagingOutput(outputDir, stagingOutputDir);
+        reportWatchProgress(config, createWatchEvent({
+            type: "watch-output-promoted",
+            rootDir,
+            outputDir,
+            stagingOutputDir
+        }));
+        return withOutputDir(stagedResult, outputDir);
+    }
+    async function reportRetry(reason, attempt, error) {
+        reportWatchProgress(config, createWatchEvent({
+            type: "watch-index-retry",
+            rootDir,
+            outputDir,
+            reason,
+            attempt,
+            maxAttempts,
+            delayMs: retryDelayMs,
+            error
+        }));
+        if (retryDelayMs > 0) {
+            await sleep(retryDelayMs);
+        }
+    }
+    async function runIndexWithRetry(reason) {
+        for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+            try {
+                const result = await runIndexAttempt(reason, attempt);
+                if (result.failed > 0) {
+                    const counts = toIndexCounts(result);
+                    const message = `Index completed with ${result.failed} failed document(s)`;
+                    reportWatchProgress(config, createWatchEvent({
+                        type: "watch-index-partial-failure",
+                        rootDir,
+                        outputDir,
+                        reason,
+                        attempt,
+                        maxAttempts,
+                        failed: result.failed,
+                        result: counts
+                    }));
+                    if (attempt < maxAttempts && !stopped) {
+                        await reportRetry(reason, attempt, message);
+                        continue;
+                    }
+                    if (stagingOutputDir) {
+                        throw new Error(`${message}; staging output was not promoted`);
+                    }
+                }
+                return {
+                    attempt,
+                    maxAttempts,
+                    result
+                };
+            }
+            catch (error) {
+                if (attempt < maxAttempts && !stopped) {
+                    await reportRetry(reason, attempt, errorMessage(error));
+                    continue;
+                }
+                throw new WatchIndexRunError(error, attempt, maxAttempts);
+            }
+        }
+        throw new WatchIndexRunError("Index did not run", maxAttempts, maxAttempts);
+    }
+    async function runIndex(reason) {
+        if (running) {
+            pending = true;
+            return;
+        }
+        if (stopped) {
+            return;
+        }
+        running = true;
+        pending = false;
+        try {
+            const run = await runIndexWithRetry(reason);
+            const counts = toIndexCounts(run.result);
+            const healthStatus = run.result.failed > 0 ? "degraded" : "ready";
+            if (healthStatus === "ready") {
+                lastSuccessAt = new Date().toISOString();
+                lastHealthError = undefined;
+            }
+            else {
+                lastFailureAt = new Date().toISOString();
+                lastHealthError = `Index completed with ${run.result.failed} failed document(s)`;
+            }
+            await writeHealth(healthStatus, reason, { result: counts, error: lastHealthError });
+            reportWatchProgress(config, createWatchEvent({
+                type: "watch-index-done",
+                rootDir,
+                outputDir,
+                reason,
+                attempt: run.attempt,
+                maxAttempts: run.maxAttempts,
+                result: counts,
+                manifestPath: run.result.manifestPath,
+                rootTreePath: run.result.rootTreePath
+            }));
+            if (reason === "initial") {
+                finishReady({ ok: true, result: run.result });
+            }
+            if (!hasStructuredProgress) {
+                console.log(`Indexed ${rootDir}: ready=${run.result.ready}, failed=${run.result.failed}, added=${run.result.added}, modified=${run.result.modified}, deleted=${run.result.deleted}, unchanged=${run.result.unchanged}`);
+            }
+        }
+        catch (error) {
+            const message = errorMessage(error);
+            const attempt = error instanceof WatchIndexRunError ? error.attempt : maxAttempts;
+            const failedMaxAttempts = error instanceof WatchIndexRunError ? error.maxAttempts : maxAttempts;
+            lastFailureAt = new Date().toISOString();
+            lastHealthError = message;
+            await writeHealth("failed", reason, { error: message });
+            reportWatchProgress(config, createWatchEvent({
+                type: "watch-index-failed",
+                rootDir,
+                outputDir,
+                reason,
+                attempt,
+                maxAttempts: failedMaxAttempts,
+                error: message
+            }));
+            if (reason === "initial") {
+                finishReady({ ok: false, error: message });
+            }
+            if (!hasStructuredProgress) {
+                console.error(`Index failed: ${message}`);
+            }
+        }
+        finally {
+            running = false;
+            if (pending && !stopped) {
+                await runIndex("change");
+            }
+            else if (stopped) {
+                await finishClose();
+            }
+        }
+    }
+    function scheduleIndex(eventName, changedPath) {
+        if (stopped) {
+            return;
+        }
+        if (!(0, scan_1.isIncludedPath)(changedPath, { exclude: config.exclude, include: config.include })) {
+            return;
+        }
+        reportWatchProgress(config, createWatchEvent({
+            type: "watch-file-event",
+            rootDir,
+            outputDir,
+            eventName,
+            path: changedPath
+        }));
+        if (!hasStructuredProgress) {
+            console.log(`${eventName}: ${changedPath}`);
+        }
+        if (timer) {
+            clearTimeout(timer);
+        }
+        timer = setTimeout(() => {
+            void runIndex("change");
+        }, debounceMs);
+    }
+    if (lockFile) {
+        releaseLock = await acquireLock(lockFile, rootDir, outputDir);
+        reportWatchProgress(config, createWatchEvent({
+            type: "watch-lock-acquired",
+            rootDir,
+            outputDir,
+            lockFile
+        }));
+    }
+    await writeHealth("starting");
+    reportWatchProgress(config, createWatchEvent({
+        type: "watch-start",
+        rootDir,
+        outputDir,
+        pid: process.pid
+    }));
+    void (async () => {
+        await runIndex("initial");
+        if (stopped) {
+            await finishClose();
+            return;
+        }
+        watcher = chokidar_1.default.watch(["**/*.md", "**/*.mdx"], {
+            cwd: rootDir,
+            ignored: isIgnored,
+            ignoreInitial: true,
+            persistent: true
+        });
+        watcher
+            .on("add", (changedPath) => scheduleIndex("add", changedPath))
+            .on("change", (changedPath) => scheduleIndex("change", changedPath))
+            .on("unlink", (changedPath) => scheduleIndex("unlink", changedPath));
+        if (stopped) {
+            await finishClose();
+        }
+    })();
+    return {
+        rootDir,
+        outputDir,
+        ready,
+        closed,
+        close
+    };
+}
+async function watchFolder(folder, options = {}) {
+    const handle = await startWatchFolder(folder, options);
+    const stop = () => {
+        void handle.close();
+    };
+    process.once("SIGINT", stop);
+    process.once("SIGTERM", stop);
+    try {
+        await handle.closed;
+    }
+    finally {
+        process.off("SIGINT", stop);
+        process.off("SIGTERM", stop);
+    }
+}

package/dist/src/index.d.ts

@@ -0,0 +1,6 @@
+export { createIndex, inspectIndex, queryIndex, validateIndex, watchIndex } from "./sdk";
+export { startServe } from "./serve";
+export type { RagboxConfig, RagboxConfigSource, RagboxIndexConfig, RagboxLlmConfig, RagboxPageIndexConfig } from "./config-file";
+export type { CreateIndexOptions, CreateIndexResult, IndexCounts, InspectIndexDocument, InspectIndexResult, LlmChatRequest, LlmClient, QueryIndexOptions, QueryResult, SdkOptions, ValidateIndexResult, ValidationIssue, WatchIndexHandle, WatchIndexOptions, WatchIndexReadyResult } from "./sdk";
+export type { ServeHandle, ServeHealthResult, ServeIndexesResult, ServeIndexSummary, ServeOptions } from "./serve";
+export * as advanced from "./advanced";

package/dist/src/index.js

@@ -0,0 +1,45 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.advanced = exports.startServe = exports.watchIndex = exports.validateIndex = exports.queryIndex = exports.inspectIndex = exports.createIndex = void 0;
+var sdk_1 = require("./sdk");
+Object.defineProperty(exports, "createIndex", { enumerable: true, get: function () { return sdk_1.createIndex; } });
+Object.defineProperty(exports, "inspectIndex", { enumerable: true, get: function () { return sdk_1.inspectIndex; } });
+Object.defineProperty(exports, "queryIndex", { enumerable: true, get: function () { return sdk_1.queryIndex; } });
+Object.defineProperty(exports, "validateIndex", { enumerable: true, get: function () { return sdk_1.validateIndex; } });
+Object.defineProperty(exports, "watchIndex", { enumerable: true, get: function () { return sdk_1.watchIndex; } });
+var serve_1 = require("./serve");
+Object.defineProperty(exports, "startServe", { enumerable: true, get: function () { return serve_1.startServe; } });
+exports.advanced = __importStar(require("./advanced"));

package/dist/src/sdk.d.ts

@@ -0,0 +1,101 @@
+import { DocumentRecord, IndexCounts, IndexProgressEvent, LlmClient, Manifest, QueryResult, RootTreeNode, WatchProgressEvent } from "./folder-index/types";
+export type { IndexCounts, IndexProgressEvent, LlmChatRequest, LlmClient, QueryResult, WatchProgressEvent } from "./folder-index/types";
+export type SdkOptions = {
+    apiKey?: string;
+    baseUrl?: string;
+    configPath?: string;
+    model?: string;
+    outputDir?: string;
+    source?: string;
+    env?: NodeJS.ProcessEnv;
+    llmClient?: LlmClient;
+};
+export type CreateIndexOptions = SdkOptions & {
+    exclude?: string[];
+    include?: string[];
+    pageIndexCli?: string;
+    pageIndexPython?: string;
+    pageIndexOutputArg?: string;
+    pageIndexExtraArgs?: string[];
+    concurrency?: number;
+    onProgress?: (event: IndexProgressEvent) => void;
+};
+export type QueryIndexOptions = SdkOptions & {
+    trace?: boolean;
+};
+export type WatchIndexOptions = CreateIndexOptions & {
+    debounceMs?: number;
+    healthFile?: string;
+    lockFile?: string;
+    onEvent?: (event: WatchProgressEvent) => void;
+    retryAttempts?: number;
+    retryDelayMs?: number;
+    staging?: boolean;
+    stagingOutputDir?: string;
+    webhookUrl?: string;
+};
+export type CreateIndexResult = {
+    version: 1;
+    rootDir: string;
+    outputDir: string;
+    manifestPath: string;
+    rootTreePath: string;
+    generatedAt: string;
+    counts: IndexCounts;
+    manifest: Manifest;
+    rootTree: RootTreeNode;
+};
+export type WatchIndexReadyResult = {
+    ok: true;
+    result: CreateIndexResult;
+} | {
+    ok: false;
+    error: string;
+};
+export type WatchIndexHandle = {
+    rootDir: string;
+    outputDir: string;
+    ready: Promise<WatchIndexReadyResult>;
+    closed: Promise<void>;
+    close: () => Promise<void>;
+};
+export type InspectIndexDocument = {
+    docId: string;
+    path: string;
+    title: string;
+    status: DocumentRecord["status"];
+    indexPath: string;
+    summary?: string;
+    size: number;
+    mtimeMs: number;
+};
+export type InspectIndexResult = {
+    version: 1;
+    target: string;
+    rootDir: string;
+    outputDir: string;
+    manifestPath: string;
+    rootTreePath: string;
+    generatedAt: string;
+    counts: IndexCounts;
+    documents: InspectIndexDocument[];
+};
+export type ValidationIssue = {
+    code: string;
+    message: string;
+    path?: string;
+    docId?: string;
+};
+export type ValidateIndexResult = {
+    version: 1;
+    target: string;
+    ok: boolean;
+    errors: ValidationIssue[];
+    warnings: ValidationIssue[];
+    inspect?: InspectIndexResult;
+};
+export declare function createIndex(folder: string, options?: CreateIndexOptions): Promise<CreateIndexResult>;
+export declare function queryIndex(target: string, question: string, options?: QueryIndexOptions): Promise<QueryResult>;
+export declare function watchIndex(folder: string, options?: WatchIndexOptions): Promise<WatchIndexHandle>;
+export declare function inspectIndex(target: string): Promise<InspectIndexResult>;
+export declare function validateIndex(target: string): Promise<ValidateIndexResult>;