@bndynet/ragbox 0.1.0

package/dist/src/serve.js

@@ -0,0 +1,466 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startServe = startServe;
+const node_http_1 = __importDefault(require("node:http"));
+const node_url_1 = require("node:url");
+const config_file_1 = require("./config-file");
+const multi_query_1 = require("./folder-index/multi-query");
+const query_1 = require("./folder-index/query");
+const sdk_1 = require("./sdk");
+const DEFAULT_HOST = "127.0.0.1";
+const DEFAULT_PORT = 8787;
+const MAX_JSON_BODY_BYTES = 1024 * 1024;
+class ServeHttpError extends Error {
+    status;
+    code;
+    constructor(status, code, message) {
+        super(message);
+        this.name = "ServeHttpError";
+        this.status = status;
+        this.code = code;
+    }
+}
+function mergeDefined(...values) {
+    const merged = {};
+    for (const value of values) {
+        for (const [key, nestedValue] of Object.entries(value)) {
+            if (nestedValue !== undefined) {
+                merged[key] = nestedValue;
+            }
+        }
+    }
+    return merged;
+}
+function parsePositivePort(value, fallback) {
+    if (!value) {
+        return fallback;
+    }
+    const parsed = Number.parseInt(value, 10);
+    if (!Number.isFinite(parsed) || parsed < 0 || parsed > 65535) {
+        throw new Error(`Invalid serve port: ${value}`);
+    }
+    return parsed;
+}
+function parseSourceNames(source) {
+    if (Array.isArray(source)) {
+        return source.map((name) => name.trim()).filter(Boolean);
+    }
+    return (source ?? "")
+        .split(",")
+        .map((name) => name.trim())
+        .filter(Boolean);
+}
+function isJsonObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function queryOptionsFromServeOptions(configOptions, options, trace) {
+    return mergeDefined({
+        ...configOptions
+    }, {
+        apiKey: options.apiKey,
+        baseUrl: options.baseUrl,
+        env: options.env,
+        llmClient: options.llmClient,
+        model: options.model,
+        trace
+    });
+}
+async function loadBaseQueryOptions(options, trace) {
+    const resolved = await (0, config_file_1.resolveRagboxConfig)({
+        configPath: options.configPath
+    });
+    return queryOptionsFromServeOptions(resolved.pageIndexOptions, options, trace);
+}
+async function resolveTargets(options, request = {}) {
+    if (request.target) {
+        return [
+            {
+                target: request.target,
+                options: await loadBaseQueryOptions(options, request.trace)
+            }
+        ];
+    }
+    const allSources = request.allSources ?? options.allSources;
+    let sourceNames = parseSourceNames(request.source ?? options.source);
+    if (allSources) {
+        const { config } = await (0, config_file_1.readRagboxConfig)(options.configPath);
+        sourceNames = (0, config_file_1.listRagboxConfigSourceNames)(config);
+        if (sourceNames.length === 0) {
+            throw new ServeHttpError(400, "invalid_request", "No configured sources found. Add docs or sources to ragbox.config.json.");
+        }
+    }
+    if (sourceNames.length === 0 && !options.target) {
+        const { config } = await (0, config_file_1.readRagboxConfig)(options.configPath);
+        const configuredSourceNames = (0, config_file_1.listRagboxConfigSourceNames)(config);
+        if (configuredSourceNames.length > 0) {
+            sourceNames = configuredSourceNames;
+        }
+    }
+    if (sourceNames.length > 0) {
+        const targets = [];
+        for (const sourceName of sourceNames) {
+            const resolved = await (0, config_file_1.resolveRagboxConfig)({
+                configPath: options.configPath,
+                source: sourceName
+            });
+            const target = resolved.pageIndexOptions.outputDir ?? resolved.rootDir;
+            if (!target) {
+                throw new ServeHttpError(400, "invalid_request", `Source does not define outputDir or rootDir: ${sourceName}`);
+            }
+            targets.push({
+                source: sourceName,
+                target,
+                options: queryOptionsFromServeOptions(resolved.pageIndexOptions, options, request.trace)
+            });
+        }
+        return targets;
+    }
+    if (options.target) {
+        return [
+            {
+                target: options.target,
+                options: await loadBaseQueryOptions(options, request.trace)
+            }
+        ];
+    }
+    throw new ServeHttpError(400, "invalid_request", "Missing query target. Pass a target, --source, --all-sources, or configure sources.");
+}
+async function buildIndexes(targets) {
+    const indexes = [];
+    for (const target of targets) {
+        const validation = await (0, sdk_1.validateIndex)(target.target);
+        indexes.push({
+            source: target.source,
+            target: target.target,
+            ok: validation.ok,
+            generatedAt: validation.inspect?.generatedAt,
+            counts: validation.inspect?.counts,
+            errors: validation.errors,
+            warnings: validation.warnings
+        });
+    }
+    return {
+        version: 1,
+        indexes
+    };
+}
+function healthFromIndexes(startedAt, lastReloadAt, indexes) {
+    const ready = indexes.indexes.filter((index) => index.ok).length;
+    const failed = indexes.indexes.length - ready;
+    const ok = indexes.indexes.length > 0 && failed === 0;
+    const status = ok ? "ready" : ready > 0 ? "degraded" : "error";
+    return {
+        version: 1,
+        ok,
+        status,
+        uptimeMs: Date.now() - startedAt,
+        lastReloadAt,
+        indexes: {
+            total: indexes.indexes.length,
+            ready,
+            failed
+        }
+    };
+}
+function rootFromHealth(health, authRequired) {
+    return {
+        version: 1,
+        name: "ragbox",
+        status: health.status,
+        ok: health.ok,
+        health,
+        endpoints: [
+            {
+                method: "GET",
+                path: "/",
+                authRequired: false,
+                description: "Service entrypoint and endpoint list."
+            },
+            {
+                method: "GET",
+                path: "/health",
+                authRequired: false,
+                description: "Readiness and index health summary."
+            },
+            {
+                method: "GET",
+                path: "/indexes",
+                authRequired,
+                description: "Validated index snapshot."
+            },
+            {
+                method: "POST",
+                path: "/query",
+                authRequired,
+                description: "Ask questions about the configured knowledge sources."
+            },
+            {
+                method: "POST",
+                path: "/reload",
+                authRequired,
+                description: "Reload configured index snapshots."
+            }
+        ]
+    };
+}
+function writeJson(response, status, value) {
+    response.writeHead(status, {
+        "Content-Type": "application/json; charset=utf-8"
+    });
+    response.end(`${JSON.stringify(value, null, 2)}\n`);
+}
+function writeError(response, status, code, message) {
+    writeJson(response, status, {
+        version: 1,
+        error: {
+            code,
+            message
+        }
+    });
+}
+function methodNotAllowed(response) {
+    writeError(response, 405, "method_not_allowed", "Method not allowed.");
+}
+function notFound(response) {
+    writeError(response, 404, "not_found", "Route not found.");
+}
+function readJsonBody(request) {
+    return new Promise((resolve, reject) => {
+        let body = "";
+        let bytes = 0;
+        request.on("data", (chunk) => {
+            bytes += chunk.length;
+            if (bytes > MAX_JSON_BODY_BYTES) {
+                reject(new ServeHttpError(400, "invalid_request", "JSON body is too large."));
+                request.destroy();
+                return;
+            }
+            body += chunk.toString("utf8");
+        });
+        request.on("end", () => {
+            try {
+                const parsed = body.trim() ? JSON.parse(body) : {};
+                if (!isJsonObject(parsed)) {
+                    reject(new ServeHttpError(400, "invalid_request", "Expected a JSON object."));
+                    return;
+                }
+                resolve(parsed);
+            }
+            catch (error) {
+                reject(new ServeHttpError(400, "invalid_request", `Invalid JSON body: ${error instanceof Error ? error.message : String(error)}`));
+            }
+        });
+        request.on("error", reject);
+    });
+}
+function authorizationHeader(request) {
+    const header = request.headers.authorization;
+    return Array.isArray(header) ? header[0] : header;
+}
+function assertAuthorized(request, authToken) {
+    if (!authToken) {
+        return;
+    }
+    if (authorizationHeader(request) !== `Bearer ${authToken}`) {
+        throw new ServeHttpError(401, "unauthorized", "Missing or invalid bearer token.");
+    }
+}
+function requestSource(value) {
+    if (typeof value === "string") {
+        return value;
+    }
+    if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
+        return value;
+    }
+    if (value === undefined) {
+        return undefined;
+    }
+    throw new ServeHttpError(400, "invalid_request", "source must be a string or string array.");
+}
+function requestTarget(value) {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (typeof value === "string" && value.trim()) {
+        return value;
+    }
+    throw new ServeHttpError(400, "invalid_request", "target must be a non-empty string.");
+}
+function requestBoolean(value, name) {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (typeof value === "boolean") {
+        return value;
+    }
+    throw new ServeHttpError(400, "invalid_request", `${name} must be a boolean.`);
+}
+function statusForThrownError(error) {
+    if (error instanceof ServeHttpError) {
+        return {
+            status: error.status,
+            code: error.code,
+            message: error.message
+        };
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    if (error instanceof query_1.QueryStageError &&
+        (error.stage === "select-documents" || error.stage === "select-nodes" || error.stage === "answer")) {
+        return {
+            status: 502,
+            code: "upstream_error",
+            message
+        };
+    }
+    if (/LLM request failed|OPENAI_API_KEY|chat completions/i.test(message)) {
+        return {
+            status: 502,
+            code: "upstream_error",
+            message
+        };
+    }
+    return {
+        status: 500,
+        code: "internal_error",
+        message
+    };
+}
+async function queryTargets(targets, question, options) {
+    if (targets.length === 0) {
+        throw new ServeHttpError(400, "invalid_request", "At least one query source is required.");
+    }
+    if (targets.length === 1) {
+        return await (0, query_1.queryFolder)(targets[0].target, question, targets[0].options);
+    }
+    const multiTargets = targets.map((target) => ({
+        name: target.source ?? target.target,
+        target: target.target,
+        options: target.options
+    }));
+    return await (0, multi_query_1.queryMultipleIndexes)(multiTargets, question, targets[0].options ?? await loadBaseQueryOptions(options));
+}
+async function startServe(options = {}) {
+    const env = options.env ?? process.env;
+    const host = options.host ?? env.RAGBOX_SERVE_HOST ?? DEFAULT_HOST;
+    const port = options.port ?? parsePositivePort(env.RAGBOX_SERVE_PORT, DEFAULT_PORT);
+    const authToken = options.authToken ?? env.RAGBOX_SERVE_TOKEN;
+    const serverOptions = {
+        ...options,
+        authToken,
+        env,
+        host,
+        port
+    };
+    const startedAt = Date.now();
+    let defaultTargets = await resolveTargets(serverOptions);
+    let lastReloadAt = new Date().toISOString();
+    let indexes = await buildIndexes(defaultTargets);
+    async function reload() {
+        defaultTargets = await resolveTargets(serverOptions);
+        indexes = await buildIndexes(defaultTargets);
+        lastReloadAt = new Date().toISOString();
+        return indexes;
+    }
+    const server = node_http_1.default.createServer((request, response) => {
+        void (async () => {
+            const requestUrl = new node_url_1.URL(request.url ?? "/", `http://${request.headers.host ?? `${host}:${port}`}`);
+            const route = requestUrl.pathname.replace(/\/+$/, "") || "/";
+            if (route === "/") {
+                if (request.method !== "GET") {
+                    methodNotAllowed(response);
+                    return;
+                }
+                const health = healthFromIndexes(startedAt, lastReloadAt, indexes);
+                writeJson(response, 200, rootFromHealth(health, Boolean(authToken)));
+                return;
+            }
+            if (route === "/health") {
+                if (request.method !== "GET") {
+                    methodNotAllowed(response);
+                    return;
+                }
+                const health = healthFromIndexes(startedAt, lastReloadAt, indexes);
+                writeJson(response, health.ok ? 200 : 503, health);
+                return;
+            }
+            assertAuthorized(request, authToken);
+            if (route === "/indexes") {
+                if (request.method !== "GET") {
+                    methodNotAllowed(response);
+                    return;
+                }
+                writeJson(response, 200, indexes);
+                return;
+            }
+            if (route === "/reload") {
+                if (request.method !== "POST") {
+                    methodNotAllowed(response);
+                    return;
+                }
+                writeJson(response, 200, await reload());
+                return;
+            }
+            if (route === "/query") {
+                if (request.method !== "POST") {
+                    methodNotAllowed(response);
+                    return;
+                }
+                const body = await readJsonBody(request);
+                const question = typeof body.question === "string" && body.question.trim() ? body.question : undefined;
+                if (!question) {
+                    throw new ServeHttpError(400, "invalid_request", "question must be a non-empty string.");
+                }
+                const target = requestTarget(body.target);
+                const source = requestSource(body.source);
+                const allSources = requestBoolean(body.allSources, "allSources");
+                const trace = requestBoolean(body.trace, "trace");
+                if (target && (source || allSources)) {
+                    throw new ServeHttpError(400, "invalid_request", "target cannot be combined with source or allSources.");
+                }
+                const targets = target || source || allSources
+                    ? await resolveTargets(serverOptions, { allSources, source, target, trace })
+                    : defaultTargets.map((resolvedTarget) => ({
+                        ...resolvedTarget,
+                        options: queryOptionsFromServeOptions(resolvedTarget.options, serverOptions, trace)
+                    }));
+                writeJson(response, 200, await queryTargets(targets, question, serverOptions));
+                return;
+            }
+            notFound(response);
+        })().catch((error) => {
+            const result = statusForThrownError(error);
+            writeError(response, result.status, result.code, result.message);
+        });
+    });
+    await new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(port, host, () => {
+            server.off("error", reject);
+            resolve();
+        });
+    });
+    const address = server.address();
+    const resolvedHost = address.address === "::" ? "localhost" : address.address;
+    const resolvedPort = address.port;
+    return {
+        url: `http://${resolvedHost}:${resolvedPort}`,
+        host: resolvedHost,
+        port: resolvedPort,
+        server,
+        reload,
+        close: async () => {
+            await new Promise((resolve, reject) => {
+                server.close((error) => {
+                    if (error) {
+                        reject(error);
+                        return;
+                    }
+                    resolve();
+                });
+            });
+        }
+    };
+}

package/dist/src/setup-pageindex.d.ts

@@ -0,0 +1,30 @@
+export type SetupPageIndexOptions = {
+    configPath?: string;
+    cwd?: string;
+    dir?: string;
+    gitignore?: boolean;
+    install?: boolean;
+    python?: string;
+    ref?: string;
+    repo?: string;
+    writeConfig?: boolean;
+};
+export type SetupPageIndexResult = {
+    version: 1;
+    command: "setup pageindex";
+    pageIndexDir: string;
+    cliPath: string;
+    pythonPath?: string;
+    venvDir?: string;
+    configPath?: string;
+    gitignorePath?: string;
+    actions: {
+        checkedOutRef?: string;
+        cloned: boolean;
+        installedDependencies: boolean;
+        reusedExisting: boolean;
+        updatedGitignore: boolean;
+        wroteConfig: boolean;
+    };
+};
+export declare function setupPageIndex(options?: SetupPageIndexOptions): Promise<SetupPageIndexResult>;

package/dist/src/setup-pageindex.js

@@ -0,0 +1,184 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setupPageIndex = setupPageIndex;
+const node_child_process_1 = require("node:child_process");
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+const config_file_1 = require("./config-file");
+const MAX_CAPTURED_OUTPUT = 32 * 1024;
+const DEFAULT_PAGEINDEX_REPO = "https://github.com/VectifyAI/PageIndex.git";
+const DEFAULT_PAGEINDEX_DIR = ".ragbox/PageIndex";
+const DEFAULT_PAGEINDEX_VENV_DIR = ".ragbox/pageindex-venv";
+const DEFAULT_PYTHON = "python3";
+const GITIGNORE_ENTRY = ".ragbox/";
+function appendCapturedOutput(current, chunk) {
+    const next = current + chunk.toString("utf8");
+    return next.length > MAX_CAPTURED_OUTPUT ? next.slice(-MAX_CAPTURED_OUTPUT) : next;
+}
+function commandFailure(message, stdout, stderr) {
+    const details = [
+        stdout.trim() ? `STDOUT:\n${stdout.trim()}` : undefined,
+        stderr.trim() ? `STDERR:\n${stderr.trim()}` : undefined
+    ].filter(Boolean);
+    return new Error(details.length ? `${message}\n${details.join("\n")}` : message);
+}
+async function runCommand(command, args, options) {
+    await new Promise((resolve, reject) => {
+        let stdout = "";
+        let stderr = "";
+        const child = (0, node_child_process_1.spawn)(command, args, {
+            cwd: options.cwd,
+            env: {
+                ...process.env,
+                ...options.env
+            },
+            stdio: ["ignore", "pipe", "pipe"]
+        });
+        child.stdout.on("data", (chunk) => {
+            stdout = appendCapturedOutput(stdout, chunk);
+        });
+        child.stderr.on("data", (chunk) => {
+            stderr = appendCapturedOutput(stderr, chunk);
+        });
+        child.on("error", (error) => {
+            reject(error.code === "ENOENT" ? new Error(options.missingMessage) : error);
+        });
+        child.on("close", (code) => {
+            if (code === 0) {
+                resolve();
+                return;
+            }
+            reject(commandFailure(`${options.failureMessage} (exit code ${code ?? "unknown"})`, stdout, stderr));
+        });
+    });
+}
+async function pathExists(filePath) {
+    try {
+        await promises_1.default.access(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function hasPageIndexEntrypoint(pageIndexDir) {
+    try {
+        const stat = await promises_1.default.stat(node_path_1.default.join(pageIndexDir, "run_pageindex.py"));
+        return stat.isFile();
+    }
+    catch {
+        return false;
+    }
+}
+function venvPythonPath(venvDir) {
+    return process.platform === "win32" ? node_path_1.default.join(venvDir, "Scripts", "python.exe") : node_path_1.default.join(venvDir, "bin", "python");
+}
+async function ensureGitignoreEntry(cwd) {
+    const gitignorePath = node_path_1.default.join(cwd, ".gitignore");
+    let current = "";
+    try {
+        current = await promises_1.default.readFile(gitignorePath, "utf8");
+    }
+    catch (error) {
+        const code = error.code;
+        if (code !== "ENOENT") {
+            throw error;
+        }
+    }
+    const lines = current.split(/\r?\n/).map((line) => line.trim());
+    if (lines.includes(GITIGNORE_ENTRY)) {
+        return { gitignorePath, updated: false };
+    }
+    const separator = current && !current.endsWith("\n") ? "\n" : "";
+    await promises_1.default.writeFile(gitignorePath, `${current}${separator}${GITIGNORE_ENTRY}\n`, "utf8");
+    return { gitignorePath, updated: true };
+}
+async function ensurePageIndexSource(pageIndexDir, repo, ref, env) {
+    if (await pathExists(pageIndexDir)) {
+        if (!(await hasPageIndexEntrypoint(pageIndexDir))) {
+            throw new Error(`PageIndex directory already exists but run_pageindex.py was not found: ${pageIndexDir}. Pass --dir to use another location or remove the existing directory.`);
+        }
+        return { cloned: false, reusedExisting: true };
+    }
+    await promises_1.default.mkdir(node_path_1.default.dirname(pageIndexDir), { recursive: true });
+    await runCommand("git", ["clone", repo, pageIndexDir], {
+        env,
+        failureMessage: `Failed to clone PageIndex from ${repo}`,
+        missingMessage: "git is required to install PageIndex. Install git or pass --dir pointing to an existing PageIndex checkout."
+    });
+    if (ref) {
+        await runCommand("git", ["-C", pageIndexDir, "checkout", ref], {
+            env,
+            failureMessage: `Failed to checkout PageIndex ref ${ref}`,
+            missingMessage: "git is required to checkout a PageIndex ref."
+        });
+    }
+    if (!(await hasPageIndexEntrypoint(pageIndexDir))) {
+        throw new Error(`PageIndex repo does not contain run_pageindex.py: ${pageIndexDir}`);
+    }
+    return { checkedOutRef: ref, cloned: true, reusedExisting: false };
+}
+async function installPageIndexDependencies(pageIndexDir, venvDir, python, env) {
+    const requirementsPath = node_path_1.default.join(pageIndexDir, "requirements.txt");
+    if (!(await pathExists(requirementsPath))) {
+        throw new Error(`PageIndex requirements.txt was not found: ${requirementsPath}`);
+    }
+    await promises_1.default.mkdir(node_path_1.default.dirname(venvDir), { recursive: true });
+    await runCommand(python, ["-m", "venv", venvDir], {
+        env,
+        failureMessage: `Failed to create PageIndex virtual environment at ${venvDir}`,
+        missingMessage: `Python executable was not found: ${python}`
+    });
+    const pythonPath = venvPythonPath(venvDir);
+    await runCommand(pythonPath, ["-m", "pip", "install", "--upgrade", "-r", requirementsPath], {
+        env,
+        failureMessage: "Failed to install PageIndex Python dependencies",
+        missingMessage: `Virtual environment Python was not found after creation: ${pythonPath}`
+    });
+    return pythonPath;
+}
+async function setupPageIndex(options = {}) {
+    const cwd = node_path_1.default.resolve(options.cwd ?? process.cwd());
+    const repo = options.repo ?? DEFAULT_PAGEINDEX_REPO;
+    const pageIndexDir = node_path_1.default.resolve(cwd, options.dir ?? DEFAULT_PAGEINDEX_DIR);
+    const venvDir = node_path_1.default.resolve(cwd, DEFAULT_PAGEINDEX_VENV_DIR);
+    const install = options.install ?? true;
+    const writeConfig = options.writeConfig ?? true;
+    const updateGitignore = options.gitignore ?? true;
+    const source = await ensurePageIndexSource(pageIndexDir, repo, options.ref);
+    const cliPath = node_path_1.default.join(pageIndexDir, "run_pageindex.py");
+    let pythonPath;
+    if (install) {
+        pythonPath = await installPageIndexDependencies(pageIndexDir, venvDir, options.python ?? DEFAULT_PYTHON);
+    }
+    const configPath = writeConfig
+        ? await (0, config_file_1.writePageIndexSetupConfig)({
+            cliPath,
+            configPath: options.configPath,
+            cwd,
+            pythonPath
+        })
+        : undefined;
+    const gitignore = updateGitignore ? await ensureGitignoreEntry(cwd) : undefined;
+    return {
+        version: 1,
+        command: "setup pageindex",
+        pageIndexDir,
+        cliPath,
+        pythonPath,
+        venvDir: install ? venvDir : undefined,
+        configPath,
+        gitignorePath: gitignore?.gitignorePath,
+        actions: {
+            checkedOutRef: source.checkedOutRef,
+            cloned: source.cloned,
+            installedDependencies: install,
+            reusedExisting: source.reusedExisting,
+            updatedGitignore: gitignore?.updated ?? false,
+            wroteConfig: Boolean(configPath)
+        }
+    };
+}