@bndynet/ragbox 0.1.0

package/dist/src/folder-index/manifest.js

@@ -0,0 +1,158 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FILE_STATE_FILE = exports.ROOT_TREE_FILE = exports.MANIFEST_FILE = exports.INDEXES_DIR = exports.PAGEINDEX_DIR = void 0;
+exports.createEmptyManifest = createEmptyManifest;
+exports.resolvePageIndexDir = resolvePageIndexDir;
+exports.getPageIndexPath = getPageIndexPath;
+exports.resolveDocumentIndexPath = resolveDocumentIndexPath;
+exports.readManifest = readManifest;
+exports.diffManifest = diffManifest;
+exports.recordFromScannedFile = recordFromScannedFile;
+exports.atomicWriteJson = atomicWriteJson;
+exports.writeManifest = writeManifest;
+exports.writeFileState = writeFileState;
+exports.removeDeletedIndexFiles = removeDeletedIndexFiles;
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+const path_utils_1 = require("./path-utils");
+exports.PAGEINDEX_DIR = ".pageindex";
+exports.INDEXES_DIR = "indexes";
+exports.MANIFEST_FILE = "manifest.json";
+exports.ROOT_TREE_FILE = "root-tree.json";
+exports.FILE_STATE_FILE = node_path_1.default.join("state", "file-state.json");
+function createEmptyManifest(rootDir) {
+    return {
+        version: 1,
+        rootDir: (0, path_utils_1.normalizeAbsolutePath)(rootDir),
+        generatedAt: new Date().toISOString(),
+        documents: []
+    };
+}
+function resolvePageIndexDir(rootDir, outputDir) {
+    return node_path_1.default.resolve(outputDir ?? node_path_1.default.join(rootDir, exports.PAGEINDEX_DIR));
+}
+function getPageIndexPath(rootDir, relativePath, outputDir) {
+    return node_path_1.default.join(resolvePageIndexDir(rootDir, outputDir), relativePath);
+}
+function resolveDocumentIndexPath(rootDir, indexPath, outputDir) {
+    if (node_path_1.default.isAbsolute(indexPath)) {
+        return indexPath;
+    }
+    const normalizedIndexPath = (0, path_utils_1.normalizeRelativePath)(indexPath);
+    if (normalizedIndexPath === exports.PAGEINDEX_DIR || normalizedIndexPath.startsWith(`${exports.PAGEINDEX_DIR}/`)) {
+        return node_path_1.default.join(rootDir, normalizedIndexPath);
+    }
+    return node_path_1.default.join(resolvePageIndexDir(rootDir, outputDir), normalizedIndexPath);
+}
+async function readManifest(rootDir, outputDir) {
+    const manifestPath = getPageIndexPath(rootDir, exports.MANIFEST_FILE, outputDir);
+    try {
+        const raw = await promises_1.default.readFile(manifestPath, "utf8");
+        const manifest = JSON.parse(raw);
+        return {
+            ...manifest,
+            documents: Array.isArray(manifest.documents) ? manifest.documents : []
+        };
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            return createEmptyManifest(rootDir);
+        }
+        throw error;
+    }
+}
+function diffManifest(previous, scannedFiles) {
+    const previousByPath = new Map(previous.documents.map((record) => [record.path, record]));
+    const scannedByPath = new Map(scannedFiles.map((file) => [file.path, file]));
+    const added = [];
+    const modified = [];
+    const retryFailed = [];
+    const unchanged = [];
+    const deleted = [];
+    for (const scannedFile of scannedFiles) {
+        const previousRecord = previousByPath.get(scannedFile.path);
+        if (!previousRecord) {
+            added.push(scannedFile);
+            continue;
+        }
+        if (previousRecord.status === "failed") {
+            retryFailed.push(scannedFile);
+            continue;
+        }
+        if (previousRecord.contentHash !== scannedFile.contentHash) {
+            modified.push(scannedFile);
+            continue;
+        }
+        unchanged.push(scannedFile);
+    }
+    for (const record of previous.documents) {
+        if (!scannedByPath.has(record.path)) {
+            deleted.push(record);
+        }
+    }
+    return {
+        added,
+        modified,
+        retryFailed,
+        unchanged,
+        deleted,
+        toIndex: [...added, ...modified, ...retryFailed]
+    };
+}
+function recordFromScannedFile(scannedFile, fields = {}) {
+    return {
+        docId: scannedFile.docId,
+        path: scannedFile.path,
+        absolutePath: scannedFile.absolutePath,
+        contentHash: scannedFile.contentHash,
+        size: scannedFile.size,
+        mtimeMs: scannedFile.mtimeMs,
+        title: scannedFile.title,
+        indexPath: scannedFile.indexPath,
+        status: fields.status ?? "ready",
+        summary: fields.summary,
+        error: fields.error
+    };
+}
+async function atomicWriteJson(filePath, value) {
+    await promises_1.default.mkdir(node_path_1.default.dirname(filePath), { recursive: true });
+    const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+    await promises_1.default.writeFile(tempPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
+    await promises_1.default.rename(tempPath, filePath);
+}
+async function writeManifest(rootDir, manifest, outputDir) {
+    await atomicWriteJson(getPageIndexPath(rootDir, exports.MANIFEST_FILE, outputDir), manifest);
+}
+async function writeFileState(rootDir, manifest, outputDir) {
+    const state = {
+        version: 1,
+        generatedAt: manifest.generatedAt,
+        files: manifest.documents.map((record) => ({
+            path: record.path,
+            absolutePath: record.absolutePath,
+            docId: record.docId,
+            contentHash: record.contentHash,
+            size: record.size,
+            mtimeMs: record.mtimeMs,
+            indexPath: record.indexPath,
+            status: record.status,
+            error: record.error
+        }))
+    };
+    await atomicWriteJson(getPageIndexPath(rootDir, exports.FILE_STATE_FILE, outputDir), state);
+}
+async function removeDeletedIndexFiles(rootDir, deletedRecords, outputDir) {
+    await Promise.all(deletedRecords.map(async (record) => {
+        try {
+            await promises_1.default.rm(resolveDocumentIndexPath(rootDir, record.indexPath, outputDir), { force: true });
+        }
+        catch (error) {
+            if (error.code !== "ENOENT") {
+                throw error;
+            }
+        }
+    }));
+}

package/dist/src/folder-index/multi-query.d.ts

@@ -0,0 +1,45 @@
+import { PageIndexOptions, QueryResult, QuerySource } from "./types";
+export type MultiQueryTarget = {
+    name: string;
+    target: string;
+    options?: PageIndexOptions;
+};
+export type MultiQuerySource = QuerySource & {
+    source: string;
+    originalReference: string;
+};
+export type MultiQuerySourceResult = QueryResult & {
+    source: string;
+};
+export type MultiQueryResult = {
+    version: 1;
+    target: "multiple";
+    sourcesQueried: string[];
+    question: string;
+    model: string;
+    answer: string;
+    contextBytes: number;
+    contextTokens: number;
+    results: MultiQuerySourceResult[];
+    sources: MultiQuerySource[];
+    warnings: string[];
+    timingsMs: {
+        query: number;
+        answer: number;
+        total: number;
+    };
+    trace?: {
+        version: 1;
+        context: {
+            sourceCount: number;
+            bytes: number;
+            tokens: number;
+        };
+        answer: {
+            promptBytes: number;
+            responseBytes: number;
+        };
+        failures: [];
+    };
+};
+export declare function queryMultipleIndexes(targets: MultiQueryTarget[], question: string, answerOptions?: PageIndexOptions): Promise<MultiQueryResult>;

package/dist/src/folder-index/multi-query.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.queryMultipleIndexes = queryMultipleIndexes;
+const config_1 = require("./config");
+const llm_client_1 = require("./llm-client");
+const query_1 = require("./query");
+function byteLength(value) {
+    return Buffer.byteLength(value, "utf8");
+}
+function estimateTokenCount(value) {
+    const trimmed = value.trim();
+    return trimmed ? Math.ceil(trimmed.length / 4) : 0;
+}
+function elapsedSince(startedAt) {
+    return Date.now() - startedAt;
+}
+function sourceReference(source, reference) {
+    return `${source}:${reference}`;
+}
+function answerContext(results) {
+    const parts = results.flatMap((result) => result.sources.map((source) => `Source: ${sourceReference(result.source, source.reference)}\n${source.text}`));
+    return parts.length > 0 ? parts.join("\n\n---\n\n") : "(no relevant context found)";
+}
+function sourceAnswerSummary(results) {
+    return results
+        .map((result) => {
+        const warnings = result.warnings.length > 0 ? `\nWarnings:\n${result.warnings.map((warning) => `- ${warning}`).join("\n")}` : "";
+        return `Source: ${result.source}\nAnswer:\n${result.answer}${warnings}`;
+    })
+        .join("\n\n---\n\n");
+}
+async function queryMultipleIndexes(targets, question, answerOptions = {}) {
+    if (targets.length === 0) {
+        throw new Error("At least one query source is required.");
+    }
+    const totalStartedAt = Date.now();
+    const queryStartedAt = Date.now();
+    const results = [];
+    for (const target of targets) {
+        results.push({
+            ...(await (0, query_1.queryFolder)(target.target, question, target.options ?? answerOptions)),
+            source: target.name
+        });
+    }
+    const queryMs = elapsedSince(queryStartedAt);
+    const warnings = results.flatMap((result) => result.warnings.map((warning) => `[${result.source}] ${warning}`));
+    const sources = results.flatMap((result) => result.sources.map((source) => ({
+        ...source,
+        source: result.source,
+        originalReference: source.reference,
+        reference: sourceReference(result.source, source.reference)
+    })));
+    const prompt = `Answer the user question using only the provided multi-source context.
+Synthesize across sources when they complement each other.
+If sources conflict, call that out briefly.
+If the context is insufficient, say that you could not find enough information in the available documentation.
+Do not expose implementation details about how the documentation was found or prepared.
+Use source references in the form source:path#node_id when possible.
+
+User question:
+${question}
+
+Per-source draft answers:
+${sourceAnswerSummary(results)}
+
+Multi-source context:
+${answerContext(results)}`;
+    const context = answerContext(results);
+    const contextBytes = byteLength(context);
+    const contextTokens = estimateTokenCount(context);
+    const answerStartedAt = Date.now();
+    const answer = await (0, llm_client_1.chatCompletion)([{ role: "user", content: prompt }], answerOptions);
+    const answerMs = elapsedSince(answerStartedAt);
+    return {
+        version: 1,
+        target: "multiple",
+        sourcesQueried: targets.map((target) => target.name),
+        question,
+        model: (0, config_1.loadPageIndexConfig)(answerOptions).model,
+        answer,
+        contextBytes,
+        contextTokens,
+        results,
+        sources,
+        warnings,
+        timingsMs: {
+            query: queryMs,
+            answer: answerMs,
+            total: elapsedSince(totalStartedAt)
+        },
+        ...(answerOptions.trace
+            ? {
+                trace: {
+                    version: 1,
+                    context: {
+                        sourceCount: sources.length,
+                        bytes: contextBytes,
+                        tokens: contextTokens
+                    },
+                    answer: {
+                        promptBytes: byteLength(prompt),
+                        responseBytes: byteLength(answer)
+                    },
+                    failures: []
+                }
+            }
+            : {})
+    };
+}

package/dist/src/folder-index/pageindex-runner.d.ts

@@ -0,0 +1,3 @@
+import { PageIndexOptions } from "./types";
+export declare function runPageIndex(inputPath: string, outputPath: string, options?: PageIndexOptions): Promise<void>;
+export declare function readPageIndexSummary(indexPath: string): Promise<string | undefined>;

package/dist/src/folder-index/pageindex-runner.js

@@ -0,0 +1,218 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runPageIndex = runPageIndex;
+exports.readPageIndexSummary = readPageIndexSummary;
+const node_child_process_1 = require("node:child_process");
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+const node_os_1 = __importDefault(require("node:os"));
+const config_1 = require("./config");
+const path_utils_1 = require("./path-utils");
+const MAX_CAPTURED_OUTPUT = 64 * 1024;
+const DEFAULT_MARKDOWN_ARGS = ["--if-add-node-text", "yes", "--if-add-node-id", "yes"];
+const unsupportedOutputArgs = new Set();
+class PageIndexRunError extends Error {
+    stdout;
+    stderr;
+    constructor(message, stdout, stderr) {
+        super(message);
+        this.stdout = stdout;
+        this.stderr = stderr;
+    }
+}
+function appendCapturedOutput(current, chunk) {
+    const next = current + chunk.toString("utf8");
+    return next.length > MAX_CAPTURED_OUTPUT ? next.slice(-MAX_CAPTURED_OUTPUT) : next;
+}
+function pageIndexRunError(code, stdout, stderr) {
+    const trimmedStdout = stdout.trim();
+    const trimmedStderr = stderr.trim();
+    return new PageIndexRunError(`PageIndex failed with exit code ${code ?? "unknown"}\nSTDOUT:\n${trimmedStdout}\nSTDERR:\n${trimmedStderr}`, trimmedStdout, trimmedStderr);
+}
+function unsupportedOutputArgKey(pythonPath, cliPath, outputArg) {
+    return `${pythonPath}\0${cliPath}\0${outputArg}`;
+}
+function outputArgWasRejected(error, outputArg) {
+    if (!(error instanceof PageIndexRunError)) {
+        return false;
+    }
+    const output = `${error.stdout}\n${error.stderr}`;
+    return output.includes("unrecognized arguments") && output.includes(outputArg);
+}
+async function fileUpdatedAfter(filePath, startedAtMs) {
+    try {
+        const stat = await promises_1.default.stat(filePath);
+        return stat.mtimeMs >= startedAtMs - 1000;
+    }
+    catch {
+        return false;
+    }
+}
+async function findJsonFiles(rootDir) {
+    const results = [];
+    async function walk(currentDir) {
+        let entries;
+        try {
+            entries = await promises_1.default.readdir(currentDir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const absolutePath = node_path_1.default.join(currentDir, entry.name);
+            if (entry.isDirectory()) {
+                await walk(absolutePath);
+            }
+            else if (entry.isFile() && entry.name.endsWith(".json")) {
+                results.push(absolutePath);
+            }
+        }
+    }
+    await walk(rootDir);
+    return results;
+}
+async function locatePageIndexResult(searchRoots, startedAtMs) {
+    const uniqueRoots = [...new Set(searchRoots)];
+    const candidates = [];
+    for (const root of uniqueRoots) {
+        const files = await findJsonFiles(root);
+        for (const filePath of files) {
+            const stat = await promises_1.default.stat(filePath);
+            if (stat.mtimeMs >= startedAtMs - 1000) {
+                candidates.push({ filePath, mtimeMs: stat.mtimeMs });
+            }
+        }
+    }
+    candidates.sort((left, right) => right.mtimeMs - left.mtimeMs);
+    return candidates[0]?.filePath;
+}
+async function runPageIndex(inputPath, outputPath, options = {}) {
+    const config = (0, config_1.loadPageIndexConfig)(options);
+    if (!config.cliPath) {
+        throw new Error("PAGEINDEX_CLI is required to run PageIndex");
+    }
+    const cliPath = node_path_1.default.resolve(config.cliPath);
+    const absoluteInputPath = node_path_1.default.resolve(inputPath);
+    const absoluteOutputPath = node_path_1.default.resolve(outputPath);
+    const tempDir = await promises_1.default.mkdtemp(node_path_1.default.join(node_os_1.default.tmpdir(), "ragbox-"));
+    await promises_1.default.mkdir(node_path_1.default.dirname(absoluteOutputPath), { recursive: true });
+    try {
+        const runOnce = async (outputArg) => {
+            const startedAtMs = Date.now();
+            const args = [cliPath, "--md_path", absoluteInputPath, "--model", config.model, ...DEFAULT_MARKDOWN_ARGS];
+            if (outputArg) {
+                args.push(outputArg, absoluteOutputPath);
+            }
+            if (config.extraArgs?.length) {
+                args.push(...config.extraArgs);
+            }
+            await new Promise((resolve, reject) => {
+                let stdout = "";
+                let stderr = "";
+                const child = (0, node_child_process_1.spawn)(config.pythonPath, args, {
+                    cwd: tempDir,
+                    env: {
+                        ...process.env,
+                        ...config.env,
+                        OPENAI_BASE_URL: config.baseUrl,
+                        OPENAI_API_KEY: config.apiKey ?? process.env.OPENAI_API_KEY ?? ""
+                    }
+                });
+                child.stdout.on("data", (chunk) => {
+                    stdout = appendCapturedOutput(stdout, chunk);
+                });
+                child.stderr.on("data", (chunk) => {
+                    stderr = appendCapturedOutput(stderr, chunk);
+                });
+                child.on("error", reject);
+                child.on("close", (code) => {
+                    if (code === 0) {
+                        resolve();
+                        return;
+                    }
+                    reject(pageIndexRunError(code, stdout, stderr));
+                });
+            });
+            return startedAtMs;
+        };
+        let outputArg = config.outputArg;
+        const outputArgKey = outputArg ? unsupportedOutputArgKey(config.pythonPath, cliPath, outputArg) : undefined;
+        if (outputArgKey && unsupportedOutputArgs.has(outputArgKey)) {
+            outputArg = undefined;
+        }
+        let startedAtMs;
+        try {
+            startedAtMs = await runOnce(outputArg);
+        }
+        catch (error) {
+            if (!outputArg || !outputArgKey || !outputArgWasRejected(error, outputArg)) {
+                throw error;
+            }
+            unsupportedOutputArgs.add(outputArgKey);
+            startedAtMs = await runOnce(undefined);
+        }
+        const cliDir = node_path_1.default.dirname(cliPath);
+        const inputDir = node_path_1.default.dirname(absoluteInputPath);
+        if (await fileUpdatedAfter(absoluteOutputPath, startedAtMs)) {
+            return;
+        }
+        const searchRoots = [
+            node_path_1.default.join(tempDir, "results"),
+            tempDir,
+            node_path_1.default.join(cliDir, "results"),
+            node_path_1.default.join(inputDir, "results"),
+            node_path_1.default.join(process.cwd(), "results")
+        ];
+        const resultPath = await locatePageIndexResult(searchRoots, startedAtMs);
+        if (!resultPath) {
+            throw new Error("PageIndex completed but no generated JSON result was found");
+        }
+        if (node_path_1.default.resolve(resultPath) !== absoluteOutputPath) {
+            await promises_1.default.copyFile(resultPath, absoluteOutputPath);
+        }
+    }
+    finally {
+        if ((0, path_utils_1.isSubPath)(node_os_1.default.tmpdir(), tempDir)) {
+            await promises_1.default.rm(tempDir, { recursive: true, force: true });
+        }
+    }
+}
+function findSummary(value) {
+    if (!value || typeof value !== "object") {
+        return undefined;
+    }
+    const record = value;
+    if (typeof record.summary === "string" && record.summary.trim()) {
+        return record.summary.trim();
+    }
+    for (const key of ["root", "tree", "document"]) {
+        const nested = findSummary(record[key]);
+        if (nested) {
+            return nested;
+        }
+    }
+    for (const key of ["children", "nodes"]) {
+        const children = record[key];
+        if (Array.isArray(children)) {
+            for (const child of children) {
+                const nested = findSummary(child);
+                if (nested) {
+                    return nested;
+                }
+            }
+        }
+    }
+    return undefined;
+}
+async function readPageIndexSummary(indexPath) {
+    try {
+        const raw = await promises_1.default.readFile(indexPath, "utf8");
+        return findSummary(JSON.parse(raw));
+    }
+    catch {
+        return undefined;
+    }
+}

package/dist/src/folder-index/path-utils.d.ts

@@ -0,0 +1,5 @@
+export declare function toPosixPath(inputPath: string): string;
+export declare function normalizeRelativePath(inputPath: string, rootDir?: string): string;
+export declare function normalizeAbsolutePath(inputPath: string): string;
+export declare function isSubPath(parentDir: string, candidatePath: string): boolean;
+export declare function isStrictSubPath(parentDir: string, candidatePath: string): boolean;

package/dist/src/folder-index/path-utils.js

@@ -0,0 +1,33 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toPosixPath = toPosixPath;
+exports.normalizeRelativePath = normalizeRelativePath;
+exports.normalizeAbsolutePath = normalizeAbsolutePath;
+exports.isSubPath = isSubPath;
+exports.isStrictSubPath = isStrictSubPath;
+const node_path_1 = __importDefault(require("node:path"));
+function toPosixPath(inputPath) {
+    return inputPath.replace(/\\/g, "/");
+}
+function normalizeRelativePath(inputPath, rootDir) {
+    const relativePath = rootDir ? node_path_1.default.relative(rootDir, inputPath) : inputPath;
+    return toPosixPath(relativePath)
+        .replace(/^\.\//, "")
+        .split("/")
+        .filter((part) => part.length > 0 && part !== ".")
+        .join("/");
+}
+function normalizeAbsolutePath(inputPath) {
+    return toPosixPath(node_path_1.default.resolve(inputPath));
+}
+function isSubPath(parentDir, candidatePath) {
+    const relativePath = node_path_1.default.relative(parentDir, candidatePath);
+    return relativePath === "" || (!relativePath.startsWith("..") && !node_path_1.default.isAbsolute(relativePath));
+}
+function isStrictSubPath(parentDir, candidatePath) {
+    const relativePath = node_path_1.default.relative(parentDir, candidatePath);
+    return relativePath !== "" && !relativePath.startsWith("..") && !node_path_1.default.isAbsolute(relativePath);
+}

package/dist/src/folder-index/query.d.ts

@@ -0,0 +1,19 @@
+import { PageIndexOptions, QueryFailureStage, QueryResult } from "./types";
+type JsonObject = Record<string, unknown>;
+export type QueryIndexLocation = {
+    rootDir: string;
+    outputDir?: string;
+    manifestPath: string;
+    rootTreePath: string;
+};
+export declare class QueryStageError extends Error {
+    readonly stage: QueryFailureStage;
+    readonly cause: unknown;
+    constructor(stage: QueryFailureStage, error: unknown);
+}
+export declare function stripText<T>(value: T): T;
+export declare function buildNodeMap(tree: unknown): Map<string, JsonObject>;
+export declare function extractNodeTextFromMarkdown(node: JsonObject, tree: unknown, markdown: string): string | undefined;
+export declare function resolveQueryIndexLocation(target: string): Promise<QueryIndexLocation>;
+export declare function queryFolder(target: string, question: string, options?: PageIndexOptions): Promise<QueryResult>;
+export {};