@bndynet/ragbox 0.1.0

package/dist/src/folder-index/scan.d.ts

@@ -0,0 +1,14 @@
+import { ScannedFile } from "./types";
+type ScanMarkdownFilesOptions = {
+    excludedDirs?: string[];
+    exclude?: string[];
+    include?: string[];
+};
+export declare function isMarkdownDocument(filePath: string): boolean;
+export declare function createDocId(relativePath: string): string;
+export declare function docIdToIndexFileName(docId: string): string;
+export declare function createIndexPath(docId: string): string;
+export declare function isIncludedPath(relativePath: string, options?: Pick<ScanMarkdownFilesOptions, "exclude" | "include">): boolean;
+export declare function deriveMarkdownTitle(filePath: string): Promise<string>;
+export declare function scanMarkdownFiles(rootDir: string, options?: ScanMarkdownFilesOptions): Promise<ScannedFile[]>;
+export {};

package/dist/src/folder-index/scan.js

@@ -0,0 +1,152 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isMarkdownDocument = isMarkdownDocument;
+exports.createDocId = createDocId;
+exports.docIdToIndexFileName = docIdToIndexFileName;
+exports.createIndexPath = createIndexPath;
+exports.isIncludedPath = isIncludedPath;
+exports.deriveMarkdownTitle = deriveMarkdownTitle;
+exports.scanMarkdownFiles = scanMarkdownFiles;
+const node_crypto_1 = require("node:crypto");
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+const node_readline_1 = __importDefault(require("node:readline"));
+const node_fs_1 = require("node:fs");
+const hash_1 = require("./hash");
+const manifest_1 = require("./manifest");
+const path_utils_1 = require("./path-utils");
+const DEFAULT_EXCLUDED_DIRS = new Set(["node_modules", ".git", ".pageindex", "dist", "build"]);
+function isMarkdownDocument(filePath) {
+    const ext = node_path_1.default.extname(filePath).toLowerCase();
+    return ext === ".md" || ext === ".mdx";
+}
+function createDocId(relativePath) {
+    const normalizedPath = (0, path_utils_1.normalizeRelativePath)(relativePath);
+    const digest = (0, node_crypto_1.createHash)("sha1").update(normalizedPath).digest("hex");
+    return `doc:${digest}`;
+}
+function docIdToIndexFileName(docId) {
+    return `${docId.replace(/[^a-zA-Z0-9_-]/g, "_")}.pageindex.json`;
+}
+function createIndexPath(docId) {
+    return [manifest_1.INDEXES_DIR, docIdToIndexFileName(docId)].join("/");
+}
+function escapeRegex(value) {
+    return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
+}
+function globToRegExp(pattern) {
+    const normalizedPattern = (0, path_utils_1.normalizeRelativePath)(pattern).replace(/^\/+/, "");
+    let regex = "^";
+    let index = 0;
+    while (index < normalizedPattern.length) {
+        const char = normalizedPattern[index];
+        const next = normalizedPattern[index + 1];
+        const afterNext = normalizedPattern[index + 2];
+        if (char === "*" && next === "*" && afterNext === "/") {
+            regex += "(?:.*/)?";
+            index += 3;
+            continue;
+        }
+        if (char === "*" && next === "*") {
+            regex += ".*";
+            index += 2;
+            continue;
+        }
+        if (char === "*") {
+            regex += "[^/]*";
+            index += 1;
+            continue;
+        }
+        if (char === "?") {
+            regex += "[^/]";
+            index += 1;
+            continue;
+        }
+        regex += escapeRegex(char);
+        index += 1;
+    }
+    return new RegExp(`${regex}$`);
+}
+function matchesPattern(relativePath, patterns) {
+    if (!patterns?.length) {
+        return false;
+    }
+    const normalizedPath = (0, path_utils_1.normalizeRelativePath)(relativePath);
+    return patterns.some((pattern) => globToRegExp(pattern).test(normalizedPath));
+}
+function isIncludedPath(relativePath, options = {}) {
+    const normalizedPath = (0, path_utils_1.normalizeRelativePath)(relativePath);
+    const includePatterns = options.include?.length ? options.include : undefined;
+    if (includePatterns && !matchesPattern(normalizedPath, includePatterns)) {
+        return false;
+    }
+    return !matchesPattern(normalizedPath, options.exclude);
+}
+async function deriveMarkdownTitle(filePath) {
+    const stream = (0, node_fs_1.createReadStream)(filePath, { encoding: "utf8" });
+    const reader = node_readline_1.default.createInterface({
+        input: stream,
+        crlfDelay: Infinity
+    });
+    try {
+        for await (const line of reader) {
+            const match = /^#\s+(.+?)\s*#*\s*$/.exec(line.trim());
+            if (match?.[1]) {
+                return match[1].trim();
+            }
+        }
+    }
+    finally {
+        reader.close();
+        stream.destroy();
+    }
+    return node_path_1.default.basename(filePath, node_path_1.default.extname(filePath));
+}
+async function scanMarkdownFiles(rootDir, options = {}) {
+    const absoluteRoot = node_path_1.default.resolve(rootDir);
+    const excludedDirs = (options.excludedDirs ?? []).map((dir) => node_path_1.default.resolve(dir));
+    const files = [];
+    function isExcludedDirectory(entryName, absolutePath) {
+        const relativePath = (0, path_utils_1.normalizeRelativePath)(absolutePath, absoluteRoot);
+        return (DEFAULT_EXCLUDED_DIRS.has(entryName) ||
+            excludedDirs.some((excludedDir) => (0, path_utils_1.isSubPath)(excludedDir, absolutePath)) ||
+            matchesPattern(relativePath, options.exclude) ||
+            matchesPattern(`${relativePath}/`, options.exclude));
+    }
+    async function walk(currentDir) {
+        const entries = await promises_1.default.readdir(currentDir, { withFileTypes: true });
+        for (const entry of entries) {
+            const absolutePath = node_path_1.default.join(currentDir, entry.name);
+            if (entry.isDirectory()) {
+                if (!isExcludedDirectory(entry.name, absolutePath)) {
+                    await walk(absolutePath);
+                }
+                continue;
+            }
+            if (!entry.isFile() || !isMarkdownDocument(entry.name)) {
+                continue;
+            }
+            const stat = await promises_1.default.stat(absolutePath);
+            const relativePath = (0, path_utils_1.normalizeRelativePath)(absolutePath, absoluteRoot);
+            if (!isIncludedPath(relativePath, options)) {
+                continue;
+            }
+            const docId = createDocId(relativePath);
+            files.push({
+                docId,
+                path: relativePath,
+                absolutePath: (0, path_utils_1.normalizeAbsolutePath)(absolutePath),
+                contentHash: await (0, hash_1.hashFile)(absolutePath),
+                size: stat.size,
+                mtimeMs: stat.mtimeMs,
+                title: await deriveMarkdownTitle(absolutePath),
+                indexPath: createIndexPath(docId)
+            });
+        }
+    }
+    await walk(absoluteRoot);
+    return files.sort((left, right) => left.path.localeCompare(right.path));
+}

package/dist/src/folder-index/types.d.ts

@@ -0,0 +1,368 @@
+export type DocumentStatus = "ready" | "failed" | "deleted";
+export type Manifest = {
+    version: 1;
+    rootDir: string;
+    generatedAt: string;
+    documents: DocumentRecord[];
+};
+export type DocumentRecord = {
+    docId: string;
+    path: string;
+    absolutePath: string;
+    contentHash: string;
+    size: number;
+    mtimeMs: number;
+    title: string;
+    summary?: string;
+    indexPath: string;
+    status: DocumentStatus;
+    error?: string;
+};
+export type RootTreeNode = {
+    node_id: string;
+    type: "root" | "directory" | "document";
+    title: string;
+    summary?: string;
+    path?: string;
+    index_path?: string;
+    children?: RootTreeNode[];
+};
+export type ChatMessage = {
+    role: "system" | "user" | "assistant";
+    content: string;
+};
+export type LlmChatRequest = {
+    messages: ChatMessage[];
+    model: string;
+    temperature: number;
+};
+export type LlmClient = {
+    chatCompletion: (request: LlmChatRequest) => Promise<string>;
+};
+export type PageIndexOptions = {
+    pythonPath?: string;
+    cliPath?: string;
+    model?: string;
+    baseUrl?: string;
+    apiKey?: string;
+    llmClient?: LlmClient;
+    concurrency?: number;
+    exclude?: string[];
+    include?: string[];
+    outputDir?: string;
+    outputArg?: string;
+    extraArgs?: string[];
+    env?: NodeJS.ProcessEnv;
+    progress?: (event: IndexProgressEvent) => void;
+    trace?: boolean;
+    watchDebounceMs?: number;
+    watchHealthFile?: string;
+    watchLockFile?: string;
+    watchProgress?: (event: WatchProgressEvent) => void;
+    watchRetryAttempts?: number;
+    watchRetryDelayMs?: number;
+    watchStaging?: boolean;
+    watchStagingOutputDir?: string;
+    watchWebhookUrl?: string;
+};
+export type IndexProgressEvent = {
+    type: "scan";
+    rootDir: string;
+    outputDir: string;
+    total: number;
+    toIndex: number;
+    unchanged: number;
+    deleted: number;
+} | {
+    type: "index-start";
+    path: string;
+    index: number;
+    total: number;
+} | {
+    type: "index-done";
+    path: string;
+    index: number;
+    total: number;
+    summary?: string;
+} | {
+    type: "index-failed";
+    path: string;
+    index: number;
+    total: number;
+    error: string;
+} | {
+    type: "write";
+    manifestPath: string;
+    rootTreePath: string;
+};
+export type WatchProgressEvent = {
+    version: 1;
+    timestamp: string;
+    type: "watch-start";
+    rootDir: string;
+    outputDir: string;
+    pid?: number;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-lock-acquired";
+    rootDir: string;
+    outputDir: string;
+    lockFile: string;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-lock-released";
+    rootDir: string;
+    outputDir: string;
+    lockFile: string;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-file-event";
+    rootDir: string;
+    outputDir: string;
+    eventName: "add" | "change" | "unlink";
+    path: string;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-index-start";
+    rootDir: string;
+    outputDir: string;
+    reason: "initial" | "change";
+    attempt: number;
+    maxAttempts: number;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-index-retry";
+    rootDir: string;
+    outputDir: string;
+    reason: "initial" | "change";
+    attempt: number;
+    maxAttempts: number;
+    delayMs: number;
+    error: string;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-index-done";
+    rootDir: string;
+    outputDir: string;
+    reason: "initial" | "change";
+    attempt: number;
+    maxAttempts: number;
+    result: IndexCounts;
+    manifestPath: string;
+    rootTreePath: string;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-index-partial-failure";
+    rootDir: string;
+    outputDir: string;
+    reason: "initial" | "change";
+    attempt: number;
+    maxAttempts: number;
+    failed: number;
+    result: IndexCounts;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-index-failed";
+    rootDir: string;
+    outputDir: string;
+    reason: "initial" | "change";
+    attempt: number;
+    maxAttempts: number;
+    error: string;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-output-promoted";
+    rootDir: string;
+    outputDir: string;
+    stagingOutputDir: string;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-health";
+    rootDir: string;
+    outputDir: string;
+    healthFile: string;
+    status: WatchHealthStatus;
+    ok: boolean;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-health-failed";
+    rootDir: string;
+    outputDir: string;
+    healthFile: string;
+    error: string;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-webhook-failed";
+    rootDir: string;
+    outputDir: string;
+    url: string;
+    error: string;
+} | {
+    version: 1;
+    timestamp: string;
+    type: "watch-stop";
+    rootDir: string;
+    outputDir: string;
+};
+export type WatchHealthStatus = "starting" | "indexing" | "ready" | "degraded" | "failed" | "stopped";
+export type WatchHealthFile = {
+    version: 1;
+    ok: boolean;
+    status: WatchHealthStatus;
+    rootDir: string;
+    outputDir: string;
+    pid: number;
+    startedAt: string;
+    updatedAt: string;
+    lastSuccessAt?: string;
+    lastFailureAt?: string;
+    reason?: "initial" | "change";
+    result?: IndexCounts;
+    error?: string;
+};
+export type ScannedFile = {
+    docId: string;
+    path: string;
+    absolutePath: string;
+    contentHash: string;
+    size: number;
+    mtimeMs: number;
+    title: string;
+    indexPath: string;
+};
+export type ManifestDiff = {
+    added: ScannedFile[];
+    modified: ScannedFile[];
+    retryFailed: ScannedFile[];
+    unchanged: ScannedFile[];
+    deleted: DocumentRecord[];
+    toIndex: ScannedFile[];
+};
+export type IndexCounts = {
+    total: number;
+    ready: number;
+    failed: number;
+    added: number;
+    modified: number;
+    retryFailed: number;
+    unchanged: number;
+    deleted: number;
+};
+export type IndexFolderResult = {
+    manifest: Manifest;
+    rootTree: RootTreeNode;
+    outputDir: string;
+    manifestPath: string;
+    rootTreePath: string;
+    added: number;
+    modified: number;
+    retryFailed: number;
+    unchanged: number;
+    deleted: number;
+    failed: number;
+    ready: number;
+};
+export type QuerySelectedDocument = {
+    docId: string;
+    available: boolean;
+    path?: string;
+    title?: string;
+    status?: DocumentStatus;
+    indexPath?: string;
+    selectionReason: "selected_by_document_planner";
+    skipReason?: "missing_root_tree_document" | "missing_manifest_record" | "missing_index_path" | "document_not_ready";
+};
+export type QuerySelectedNode = {
+    docId: string;
+    path: string;
+    nodeId: string;
+    found: boolean;
+    hasText: boolean;
+    reference?: string;
+    selectionReason: "selected_by_node_planner" | "matched_query_text";
+    skipReason?: "node_not_found" | "missing_text";
+    textBytes?: number;
+};
+export type QuerySource = {
+    path: string;
+    nodeId: string;
+    reference: string;
+    text: string;
+};
+export type QueryTimings = {
+    resolve: number;
+    selectDocuments: number;
+    selectNodes: number;
+    answer: number;
+    total: number;
+};
+export type QueryFailureStage = "resolve" | "read-index" | "select-documents" | "read-document-index" | "select-nodes" | "extract-context" | "answer";
+export type QueryTraceFailure = {
+    stage: QueryFailureStage;
+    code: string;
+    message: string;
+    docId?: string;
+    nodeId?: string;
+    path?: string;
+    reference?: string;
+};
+export type QueryDocumentSelectionTrace = {
+    promptBytes: number;
+    responseBytes: number;
+    rawResponse: string;
+    selectedDocumentIds: string[];
+};
+export type QueryNodeSelectionTrace = {
+    docId: string;
+    path: string;
+    promptBytes: number;
+    responseBytes: number;
+    rawResponse: string;
+    selectedNodeIds: string[];
+};
+export type QueryContextTrace = {
+    sourceCount: number;
+    bytes: number;
+    tokens: number;
+};
+export type QueryAnswerTrace = {
+    promptBytes: number;
+    responseBytes: number;
+};
+export type QueryTrace = {
+    version: 1;
+    documentSelection?: QueryDocumentSelectionTrace;
+    nodeSelections: QueryNodeSelectionTrace[];
+    context: QueryContextTrace;
+    answer?: QueryAnswerTrace;
+    failures: QueryTraceFailure[];
+};
+export type QueryResult = {
+    version: 1;
+    target: string;
+    rootDir: string;
+    outputDir: string;
+    question: string;
+    model: string;
+    answer: string;
+    contextBytes: number;
+    contextTokens: number;
+    selectedDocuments: QuerySelectedDocument[];
+    selectedNodes: QuerySelectedNode[];
+    sources: QuerySource[];
+    warnings: string[];
+    timingsMs: QueryTimings;
+    trace?: QueryTrace;
+};

package/dist/src/folder-index/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/src/folder-index/watch.d.ts

@@ -0,0 +1,17 @@
+import { IndexFolderResult, PageIndexOptions } from "./types";
+export type WatchFolderReadyResult = {
+    ok: true;
+    result: IndexFolderResult;
+} | {
+    ok: false;
+    error: string;
+};
+export type WatchFolderHandle = {
+    rootDir: string;
+    outputDir: string;
+    ready: Promise<WatchFolderReadyResult>;
+    closed: Promise<void>;
+    close: () => Promise<void>;
+};
+export declare function startWatchFolder(folder: string, options?: PageIndexOptions): Promise<WatchFolderHandle>;
+export declare function watchFolder(folder: string, options?: PageIndexOptions): Promise<void>;