abb-opencode-local-rag 0.1.0

package/dist/utils/list-sources.js

@@ -0,0 +1,50 @@
+// Pure `sources` classifier shared by the MCP `list_files` handler
+// (`src/server/index.ts`) and the `list` CLI (`src/cli/list.ts`). Extracted
+// from the byte-for-byte-identical inline blocks in both surfaces so the
+// raw-data / real-file scope branch lives in one place, mirroring the
+// drift-avoidance rationale behind the shared `scope-match.ts` matcher.
+//
+// `sources` are ingested entries whose identity key matched no scanned file:
+// content ingested via `ingest_data` (raw-data paths) plus orphaned DB entries
+// for real files not currently on disk under a scanned root.
+import { extractSourceFromPath, isManagedRawDataPath } from './raw-data-utils.js';
+import { matchesAnyScope } from './scope-match.js';
+/**
+ * Classify the ingested entries that matched no scanned file into `sources`.
+ *
+ * Base filter (always): keep only entries whose `key` is absent from
+ * `matchedKeys`.
+ *
+ * When `scope` is present (non-empty): raw-data entries
+ * Managed raw-data entries have no filesystem path under a base directory, so
+ * they are always emitted regardless of scope; real-file entries are kept only
+ * when their stored `filePath` is under scope (`matchesAnyScope`) — a real-file
+ * entry outside scope is dropped so it appears in neither `files[]` (already
+ * pruned from the scan) nor `sources[]` (no orphan misclassification), while an
+ * unmatched real-file entry under scope remains an orphan source.
+ *
+ * When `scope` is absent (or empty): behavior is unchanged from the pre-extraction
+ * inline block — every unmatched entry is emitted, raw-data as `{ source }` when
+ * `extractSourceFromPath` yields one, else as `{ filePath }`.
+ */
+export function classifyIngestedSources(ingestedKeyed, matchedKeys, dbPath, scope) {
+    const scopePrefixes = scope && scope.length > 0 ? scope : undefined;
+    return ingestedKeyed
+        .filter(({ key }) => !matchedKeys.has(key))
+        .filter(({ entry }) => {
+        if (!scopePrefixes)
+            return true;
+        // Raw-data sources are exempt from scope; real-file entries respect it.
+        return (isManagedRawDataPath(entry.filePath, dbPath) ||
+            matchesAnyScope(entry.filePath, scopePrefixes));
+    })
+        .map(({ entry }) => {
+        if (isManagedRawDataPath(entry.filePath, dbPath)) {
+            const source = extractSourceFromPath(entry.filePath);
+            if (source)
+                return { source, chunkCount: entry.chunkCount, timestamp: entry.timestamp };
+        }
+        return { filePath: entry.filePath, chunkCount: entry.chunkCount, timestamp: entry.timestamp };
+    });
+}
+//# sourceMappingURL=list-sources.js.map

package/dist/utils/list-sources.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"list-sources.js","sourceRoot":"","sources":["../../src/utils/list-sources.ts"],"names":[],"mappings":"AAAA,mEAAmE;AACnE,4EAA4E;AAC5E,yEAAyE;AACzE,sEAAsE;AACtE,wEAAwE;AACxE,EAAE;AACF,6EAA6E;AAC7E,+EAA+E;AAC/E,6DAA6D;AAE7D,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAA;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAqBlD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,uBAAuB,CACrC,aAA4C,EAC5C,WAAgC,EAChC,MAAc,EACd,KAAgB;IAEhB,MAAM,aAAa,GAAG,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAA;IAEnE,OAAO,aAAa;SACjB,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1C,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;QACpB,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAA;QAC/B,wEAAwE;QACxE,OAAO,CACL,oBAAoB,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC;YAC5C,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC,CAC/C,CAAA;IACH,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;QACjB,IAAI,oBAAoB,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;YACjD,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YACpD,IAAI,MAAM;gBAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAA;QACzF,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAA;IAC/F,CAAC,CAAC,CAAA;AACN,CAAC"}

package/dist/utils/raw-data-utils.d.ts

@@ -0,0 +1,131 @@
+/**
+ * Encode string to URL-safe base64 (base64url)
+ * - Replaces + with -
+ * - Replaces / with _
+ * - Removes padding (=)
+ *
+ * @param str - String to encode
+ * @returns URL-safe base64 encoded string
+ */
+export declare function encodeBase64Url(str: string): string;
+/**
+ * Decode URL-safe base64 (base64url) to string
+ *
+ * @param base64url - URL-safe base64 encoded string
+ * @returns Decoded string
+ */
+export declare function decodeBase64Url(base64url: string): string;
+/**
+ * Normalize source URL by removing query string and fragment
+ * Only normalizes HTTP(S) URLs. Other sources (e.g., "clipboard://...") are returned as-is
+ *
+ * @param source - Source identifier (URL or custom ID)
+ * @returns Normalized source
+ */
+export declare function normalizeSource(source: string): string;
+/** Formats accepted by ingest_data at runtime and in its public schema. */
+export declare const CONTENT_FORMATS: readonly ['text', 'html', 'markdown'];
+export type ContentFormat = (typeof CONTENT_FORMATS)[number];
+/**
+ * Get raw-data directory path
+ *
+ * @param dbPath - LanceDB database path
+ * @returns Raw-data directory path
+ */
+export declare function getRawDataDir(dbPath: string): string;
+/**
+ * Generate raw-data file path from source and format
+ * Path format: {dbPath}/raw-data/{base64url(normalizedSource)}.{ext}
+ *
+ * @param dbPath - LanceDB database path
+ * @param source - Source identifier (URL or custom ID)
+ * @param format - Content format
+ * @returns Generated file path
+ */
+export declare function generateRawDataPath(dbPath: string, source: string): string;
+/**
+ * Save content to raw-data directory
+ * Creates directory if it doesn't exist
+ *
+ * @param dbPath - LanceDB database path
+ * @param source - Source identifier (URL or custom ID)
+ * @param content - Content to save
+ * @param format - Content format
+ * @returns Saved file path
+ */
+export declare function saveRawData(dbPath: string, source: string, content: string): Promise<string>;
+/** True only for managed raw-data content files under this database. */
+export declare function isManagedRawDataPath(filePath: string, dbPath: string): boolean;
+/**
+ * Lexical containment in `<dbPath>/raw-data/`. Safe for cleanup gates
+ * (`unlink` does not follow symlinks). Use {@link isPathInRawDataDir}
+ * when the result controls `readFile`.
+ */
+export declare function isPathInRawDataDirLexical(filePath: string, dbPath: string): boolean;
+/**
+ * Lexical containment plus `realpath` so a symlink under raw-data
+ * pointing outside cannot route a read through the raw-data fast-path.
+ * Fail-closed on `realpath` errors.
+ */
+export declare function isPathInRawDataDir(filePath: string, dbPath: string): Promise<boolean>;
+/**
+ * Extract original source from raw-data file path
+ * Returns null if not a raw-data path
+ *
+ * @param filePath - Raw-data file path
+ * @returns Original source or null
+ */
+export declare function extractSourceFromPath(filePath: string): string | null;
+/**
+ * Metadata stored alongside each raw-data .md file as a .meta.json sidecar
+ */
+export interface RawDataMeta {
+    title: string | null;
+    source: string;
+    format: ContentFormat;
+}
+/**
+ * Generate the .meta.json sidecar path for a given .md file path
+ * Replaces the trailing `.md` extension with `.meta.json`
+ *
+ * @param mdPath - Path to the .md raw-data file
+ * @returns Path to the corresponding .meta.json file
+ */
+export declare function generateMetaJsonPath(mdPath: string): string;
+/**
+ * Save metadata as a JSON sidecar file alongside a raw-data .md file
+ *
+ * @param mdPath - Path to the .md raw-data file
+ * @param meta - Metadata to persist
+ */
+export declare function saveMetaJson(mdPath: string, meta: RawDataMeta): Promise<void>;
+/**
+ * Load metadata from a .meta.json sidecar file
+ * Returns null when the sidecar file does not exist (ENOENT).
+ * All other read errors are re-thrown (fail-fast).
+ *
+ * @param mdPath - Path to the .md raw-data file
+ * @returns Parsed metadata or null if file does not exist
+ */
+export declare function loadMetaJson(mdPath: string): Promise<RawDataMeta | null>;
+/**
+ * True when a filesystem error means the target path does not exist.
+ */
+export declare function isEnoent(error: unknown): boolean;
+/**
+ * Pre-unlink existence of the on-disk raw-data artifacts for a target .md
+ * path: the raw-data file itself and its .meta.json sidecar.
+ */
+export interface RawDataArtifactExistence {
+    rawDataExisted: boolean;
+    metaExisted: boolean;
+}
+/**
+ * Report which raw-data artifacts exist for a target path. Single source for
+ * the delete `existed` signal so the MCP server and CLI delete paths cannot
+ * drift. Call BEFORE unlinking — it reports pre-unlink state.
+ *
+ * @param targetPath - Raw-data .md file path
+ */
+export declare function checkRawDataArtifacts(targetPath: string): Promise<RawDataArtifactExistence>;
+//# sourceMappingURL=raw-data-utils.d.ts.map

package/dist/utils/raw-data-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"raw-data-utils.d.ts","sourceRoot":"","sources":["../../src/utils/raw-data-utils.ts"],"names":[],"mappings":"AAUA;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAMnD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAUzD;AAMD;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAatD;AAMD,2EAA2E;AAC3E,eAAO,MAAM,eAAe,YAAI,MAAM,EAAE,MAAM,EAAE,UAAU,CAAU,CAAA;AACpE,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,CAAA;AAQ5D;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAK1E;AAMD;;;;;;;;;GASG;AACH,wBAAsB,WAAW,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAUjB;AAMD,wEAAwE;AACxE,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAI9E;AAUD;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAInF;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAS3F;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAkBrE;AAMD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,aAAa,CAAA;CACtB;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAGnF;AAED;;;;;;;GAOG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAW9E;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAIhD;AAkBD;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACvC,cAAc,EAAE,OAAO,CAAA;IACvB,WAAW,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAIjG"}

package/dist/utils/raw-data-utils.js

@@ -0,0 +1,255 @@
+// Raw Data Utilities for ingest_data tool
+// Handles: base64url encoding, source normalization, file saving, source extraction
+import { access, mkdir, readFile, realpath, writeFile } from 'node:fs/promises';
+import { basename, dirname, join, resolve, sep } from 'node:path';
+// ============================================
+// Base64URL Encoding/Decoding
+// ============================================
+/**
+ * Encode string to URL-safe base64 (base64url)
+ * - Replaces + with -
+ * - Replaces / with _
+ * - Removes padding (=)
+ *
+ * @param str - String to encode
+ * @returns URL-safe base64 encoded string
+ */
+export function encodeBase64Url(str) {
+    return Buffer.from(str, 'utf-8')
+        .toString('base64')
+        .replace(/\+/g, '-')
+        .replace(/\//g, '_')
+        .replace(/=+$/, '');
+}
+/**
+ * Decode URL-safe base64 (base64url) to string
+ *
+ * @param base64url - URL-safe base64 encoded string
+ * @returns Decoded string
+ */
+export function decodeBase64Url(base64url) {
+    // Convert base64url to standard base64
+    let base64 = base64url.replace(/-/g, '+').replace(/_/g, '/');
+    // Add padding if needed
+    while (base64.length % 4 !== 0) {
+        base64 += '=';
+    }
+    return Buffer.from(base64, 'base64').toString('utf-8');
+}
+// ============================================
+// Source Normalization
+// ============================================
+/**
+ * Normalize source URL by removing query string and fragment
+ * Only normalizes HTTP(S) URLs. Other sources (e.g., "clipboard://...") are returned as-is
+ *
+ * @param source - Source identifier (URL or custom ID)
+ * @returns Normalized source
+ */
+export function normalizeSource(source) {
+    try {
+        const parsed = new URL(source);
+        // Only normalize HTTP(S) URLs
+        if (parsed.protocol === 'http:' || parsed.protocol === 'https:') {
+            return `${parsed.origin}${parsed.pathname}`;
+        }
+        // Non-HTTP URLs (clipboard://, etc.) are returned as-is
+        return source;
+    }
+    catch {
+        // Not a valid URL, return as-is
+        return source;
+    }
+}
+// ============================================
+// Format Utilities
+// ============================================
+/** Formats accepted by ingest_data at runtime and in its public schema. */
+export const CONTENT_FORMATS = ['text', 'html', 'markdown'];
+const RAW_DATA_EXTENSION = 'md';
+// ============================================
+// Path Generation
+// ============================================
+/**
+ * Get raw-data directory path
+ *
+ * @param dbPath - LanceDB database path
+ * @returns Raw-data directory path
+ */
+export function getRawDataDir(dbPath) {
+    return join(dbPath, 'raw-data');
+}
+/**
+ * Generate raw-data file path from source and format
+ * Path format: {dbPath}/raw-data/{base64url(normalizedSource)}.{ext}
+ *
+ * @param dbPath - LanceDB database path
+ * @param source - Source identifier (URL or custom ID)
+ * @param format - Content format
+ * @returns Generated file path
+ */
+export function generateRawDataPath(dbPath, source) {
+    const normalizedSource = normalizeSource(source);
+    const encoded = encodeBase64Url(normalizedSource);
+    // Use resolve to ensure absolute path (required by validateFilePath)
+    return resolve(getRawDataDir(dbPath), `${encoded}.${RAW_DATA_EXTENSION}`);
+}
+// ============================================
+// File Operations
+// ============================================
+/**
+ * Save content to raw-data directory
+ * Creates directory if it doesn't exist
+ *
+ * @param dbPath - LanceDB database path
+ * @param source - Source identifier (URL or custom ID)
+ * @param content - Content to save
+ * @param format - Content format
+ * @returns Saved file path
+ */
+export async function saveRawData(dbPath, source, content) {
+    const filePath = generateRawDataPath(dbPath, source);
+    // Ensure directory exists
+    await mkdir(dirname(filePath), { recursive: true });
+    // Write content to file
+    await writeFile(filePath, content, 'utf-8');
+    return filePath;
+}
+// ============================================
+// Path Detection and Source Extraction
+// ============================================
+/** True only for managed raw-data content files under this database. */
+export function isManagedRawDataPath(filePath, dbPath) {
+    return (isPathInRawDataDirLexical(filePath, dbPath) && /^[A-Za-z0-9_-]+\.md$/.test(basename(filePath)));
+}
+/**
+ * Case-normalize a path for prefix containment. Windows filesystems are
+ * case-insensitive by default, so the boundary check must mirror that.
+ */
+function caseNormalize(p) {
+    return process.platform === 'win32' ? p.toLowerCase() : p;
+}
+/**
+ * Lexical containment in `<dbPath>/raw-data/`. Safe for cleanup gates
+ * (`unlink` does not follow symlinks). Use {@link isPathInRawDataDir}
+ * when the result controls `readFile`.
+ */
+export function isPathInRawDataDirLexical(filePath, dbPath) {
+    const target = caseNormalize(resolve(filePath));
+    const rawDir = caseNormalize(resolve(getRawDataDir(dbPath)));
+    return target === rawDir || target.startsWith(rawDir + sep);
+}
+/**
+ * Lexical containment plus `realpath` so a symlink under raw-data
+ * pointing outside cannot route a read through the raw-data fast-path.
+ * Fail-closed on `realpath` errors.
+ */
+export async function isPathInRawDataDir(filePath, dbPath) {
+    if (!isPathInRawDataDirLexical(filePath, dbPath))
+        return false;
+    try {
+        const realTarget = caseNormalize(await realpath(resolve(filePath)));
+        const realRaw = caseNormalize(await realpath(resolve(getRawDataDir(dbPath))));
+        return realTarget === realRaw || realTarget.startsWith(realRaw + sep);
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Extract original source from raw-data file path
+ * Returns null if not a raw-data path
+ *
+ * @param filePath - Raw-data file path
+ * @returns Original source or null
+ */
+export function extractSourceFromPath(filePath) {
+    const normalized = filePath.replace(/\\/g, '/');
+    const rawDataMarker = '/raw-data/';
+    const rawDataIndex = normalized.indexOf(rawDataMarker);
+    if (rawDataIndex === -1) {
+        return null;
+    }
+    const fileName = filePath.slice(rawDataIndex + rawDataMarker.length);
+    const dotIndex = fileName.lastIndexOf('.');
+    if (dotIndex === -1) {
+        return null;
+    }
+    const encoded = fileName.slice(0, dotIndex);
+    return decodeBase64Url(encoded);
+}
+/**
+ * Generate the .meta.json sidecar path for a given .md file path
+ * Replaces the trailing `.md` extension with `.meta.json`
+ *
+ * @param mdPath - Path to the .md raw-data file
+ * @returns Path to the corresponding .meta.json file
+ */
+export function generateMetaJsonPath(mdPath) {
+    return mdPath.replace(/\.md$/, '.meta.json');
+}
+/**
+ * Save metadata as a JSON sidecar file alongside a raw-data .md file
+ *
+ * @param mdPath - Path to the .md raw-data file
+ * @param meta - Metadata to persist
+ */
+export async function saveMetaJson(mdPath, meta) {
+    const metaPath = generateMetaJsonPath(mdPath);
+    await writeFile(metaPath, JSON.stringify(meta, null, 2), 'utf-8');
+}
+/**
+ * Load metadata from a .meta.json sidecar file
+ * Returns null when the sidecar file does not exist (ENOENT).
+ * All other read errors are re-thrown (fail-fast).
+ *
+ * @param mdPath - Path to the .md raw-data file
+ * @returns Parsed metadata or null if file does not exist
+ */
+export async function loadMetaJson(mdPath) {
+    const metaPath = generateMetaJsonPath(mdPath);
+    try {
+        const content = await readFile(metaPath, 'utf-8');
+        return JSON.parse(content);
+    }
+    catch (error) {
+        if (isEnoent(error)) {
+            return null;
+        }
+        throw error;
+    }
+}
+/**
+ * True when a filesystem error means the target path does not exist.
+ */
+export function isEnoent(error) {
+    return (error instanceof Error && 'code' in error && error.code === 'ENOENT');
+}
+// ============================================
+// Raw-Data Artifact Existence
+// ============================================
+/**
+ * Whether a path exists. access()-based, never throws.
+ */
+async function pathExists(path) {
+    try {
+        await access(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Report which raw-data artifacts exist for a target path. Single source for
+ * the delete `existed` signal so the MCP server and CLI delete paths cannot
+ * drift. Call BEFORE unlinking — it reports pre-unlink state.
+ *
+ * @param targetPath - Raw-data .md file path
+ */
+export async function checkRawDataArtifacts(targetPath) {
+    const rawDataExisted = await pathExists(targetPath);
+    const metaExisted = await pathExists(generateMetaJsonPath(targetPath));
+    return { rawDataExisted, metaExisted };
+}
+//# sourceMappingURL=raw-data-utils.js.map

package/dist/utils/raw-data-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"raw-data-utils.js","sourceRoot":"","sources":["../../src/utils/raw-data-utils.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,oFAAoF;AAEpF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC/E,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAA;AAEjE,+CAA+C;AAC/C,8BAA8B;AAC9B,+CAA+C;AAE/C;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC;SAC7B,QAAQ,CAAC,QAAQ,CAAC;SAClB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;AACvB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,SAAiB;IAC/C,uCAAuC;IACvC,IAAI,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAE5D,wBAAwB;IACxB,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,CAAA;IACf,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;AACxD,CAAC;AAED,+CAA+C;AAC/C,uBAAuB;AACvB,+CAA+C;AAE/C;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAA;QAC9B,8BAA8B;QAC9B,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAChE,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;QAC7C,CAAC;QACD,wDAAwD;QACxD,OAAO,MAAM,CAAA;IACf,CAAC;IAAC,MAAM,CAAC;QACP,gCAAgC;QAChC,OAAO,MAAM,CAAA;IACf,CAAC;AACH,CAAC;AAED,+CAA+C;AAC/C,mBAAmB;AACnB,+CAA+C;AAE/C,2EAA2E;AAC3E,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAU,CAAA;AAGpE,MAAM,kBAAkB,GAAG,IAAI,CAAA;AAE/B,+CAA+C;AAC/C,kBAAkB;AAClB,+CAA+C;AAE/C;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,MAAc;IAC1C,OAAO,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;AACjC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAc,EAAE,MAAc;IAChE,MAAM,gBAAgB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;IACjD,qEAAqE;IACrE,OAAO,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,GAAG,OAAO,IAAI,kBAAkB,EAAE,CAAC,CAAA;AAC3E,CAAC;AAED,+CAA+C;AAC/C,kBAAkB;AAClB,+CAA+C;AAE/C;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAc,EACd,MAAc,EACd,OAAe;IAEf,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAEpD,0BAA0B;IAC1B,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEnD,wBAAwB;IACxB,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAE3C,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,+CAA+C;AAC/C,uCAAuC;AACvC,+CAA+C;AAE/C,wEAAwE;AACxE,MAAM,UAAU,oBAAoB,CAAC,QAAgB,EAAE,MAAc;IACnE,OAAO,CACL,yBAAyB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAC/F,CAAA;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAC3D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,QAAgB,EAAE,MAAc;IACxE,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAA;IAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAC5D,OAAO,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,CAAA;AAC7D,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB,EAAE,MAAc;IACvE,IAAI,CAAC,yBAAyB,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAE,OAAO,KAAK,CAAA;IAC9D,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;QACnE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC7E,OAAO,UAAU,KAAK,OAAO,IAAI,UAAU,CAAC,UAAU,CAAC,OAAO,GAAG,GAAG,CAAC,CAAA;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAC/C,MAAM,aAAa,GAAG,YAAY,CAAA;IAClC,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,CAAA;IAEtD,IAAI,YAAY,KAAK,CAAC,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAA;IACb,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,YAAY,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IACpE,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IAE1C,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;QACpB,OAAO,IAAI,CAAA;IACb,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAA;IAC3C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAA;AACjC,CAAC;AAeD;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAc;IACjD,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;AAC9C,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,IAAiB;IAClE,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAA;IAC7C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;AACnE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc;IAC/C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAA;IAC7C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;QACjD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAgB,CAAA;IAC3C,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,OAAO,CACL,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,CAChG,CAAA;AACH,CAAC;AAED,+CAA+C;AAC/C,8BAA8B;AAC9B,+CAA+C;AAE/C;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;QAClB,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAWD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,UAAkB;IAC5D,MAAM,cAAc,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAA;IACnD,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,CAAA;IACtE,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,CAAA;AACxC,CAAC"}

package/dist/utils/scan.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Canonical identity key for the `list`/`list_files` cross-reference: a file's
+ * realpath, falling back to the input path when realpath fails (orphaned or
+ * raw-data entries). Matching ingested DB entries against scanned files by this
+ * key recognizes the same physical file across symlinked spellings (prefix or
+ * alias). Storage, lookup, and display still use the normal resolve() path —
+ * realpath here is the file-identity comparison, not a user-facing value.
+ */
+export declare function realpathForMatch(filePath: string): Promise<string>;
+/**
+ * Canonical form of one explicitly requested path: its parent chain resolved
+ * through symbolic links, with the final component appended verbatim. `null` when
+ * the parent chain cannot be resolved at all — absent, or a directory this
+ * process may not traverse — which a caller must treat as "not contained",
+ * because telling those cases apart would report the state of paths outside its
+ * configured roots.
+ *
+ * Only the parent chain is resolved, because the requested entry itself is judged
+ * by {@link classifyRequestedPath}'s `lstat`: a symbolic link named directly
+ * inside a root is an in-root entry that is refused as a link, not a path to be
+ * reported by whatever it points at.
+ *
+ * `realpath` here is the containment (security) boundary, the same role it plays
+ * in `DocumentParser.validateFilePath` — never a spelling anything is stored,
+ * looked up, or displayed under. Those stay `resolve()`-only.
+ */
+export declare function canonicalizeRequestedPath(path: string): Promise<string | null>;
+/** Why the collect predicates below refuse a path. */
+export type ScanRejection = 
+/** A symbolic link, which is never followed. */
+'symlink'
+/** Under a configured excluded prefix (the database or cache directory). */
+ | 'excluded'
+/** A regular file whose extension is not a supported document type. */
+ | 'unsupported'
+/** Neither a regular file nor a directory (socket, FIFO, device, …). */
+ | 'irregular';
+/** What a path is, once the collect predicates have judged it. */
+export type ScanEntryKind = 'file' | 'directory' | ScanRejection;
+/** The `Dirent` / `Stats` subset the collect predicates read. */
+interface EntryTypeFacts {
+    isSymbolicLink(): boolean;
+    isDirectory(): boolean;
+    isFile(): boolean;
+}
+/**
+ * The collect predicates of {@link bfsCollectSupportedFiles} as one decision, so
+ * a discovered directory entry and an explicitly requested path
+ * ({@link classifyRequestedPath}) are judged by exactly the same rules instead of
+ * by two implementations that can drift.
+ *
+ * Evaluation order is part of the contract and matches the walk: a symbolic link
+ * is reported as a link even under an excluded prefix, and a directory is
+ * accepted without any extension test.
+ *
+ * `platform` is a parameter rather than a direct `process.platform` read — the
+ * same reason `toSyncPathKey` takes one: the Windows exclusion semantics
+ * ({@link isUnderExcludedPrefix}) must be provable on a macOS/Linux machine. The
+ * default leaves every call site unchanged.
+ *
+ * Both `Dirent` (from `readdir`) and `Stats` (from `lstat`) satisfy
+ * {@link EntryTypeFacts} structurally.
+ */
+export declare function classifyScanEntry(fullPath: string, entry: EntryTypeFacts, excludePaths: readonly string[], platform?: NodeJS.Platform): ScanEntryKind;
+/**
+ * Classify one explicitly requested path with {@link classifyScanEntry}, so a
+ * path a caller names is subject to the same predicates as a path the walker
+ * discovers.
+ *
+ * `lstat` rather than `stat`, so a symbolic link is reported as a link instead of
+ * as whatever it points at; and `lstat` rather than any read, so a caller can
+ * refuse the path before its bytes cost anything — reading a FIFO blocks forever,
+ * and reading through a link reaches outside the configured roots.
+ *
+ * Any stat failure is `'missing'`: an unreachable path and an absent one are the
+ * same non-answer to "what is here".
+ */
+export declare function classifyRequestedPath(path: string, excludePaths: readonly string[], platform?: NodeJS.Platform): Promise<ScanEntryKind | 'missing'>;
+/** A directory that could not be read during the scan. */
+export interface UnreadableDir {
+    dirPath: string;
+    /** Node error `code` (e.g. `EACCES`), or `'UNKNOWN'` when unavailable. */
+    code: string;
+}
+/** Structured result of a bounded directory scan. */
+export interface DirScanResult {
+    /** Supported files found under the root, in BFS-discovery order (unsorted). */
+    files: string[];
+    /** Directories skipped because `readdir` failed (caller decides how to warn). */
+    unreadableDirs: UnreadableDir[];
+    /**
+     * Each entry is the first unvisited directory of a branch pruned for
+     * exceeding `maxDepth` — the directory that was reached but never read. That
+     * path and every descendant of it is unobserved by this scan; its ancestors
+     * and fully visited siblings are not listed.
+     */
+    depthLimitedDirs: string[];
+    /** Full paths of directory entries skipped because they are symbolic links. */
+    skippedSymlinks: string[];
+    /** True if any branch was pruned for exceeding `maxDepth`. */
+    depthLimited: boolean;
+}
+/**
+ * Bounded BFS scan of a single root, collecting every supported file up to
+ * `maxDepth` levels deep, counted from `rootPath` itself. Symlinks are skipped
+ * (never followed) and recorded in `skippedSymlinks`; paths under any
+ * `excludePaths` prefix are filtered out. A per-directory `readdir` failure is
+ * captured into `unreadableDirs` and does not abort the scan (best-effort per
+ * directory); a branch pruned at `maxDepth` is captured into `depthLimitedDirs`.
+ *
+ * When `scope` is provided (non-empty), the predicate is pushed into the
+ * traversal: a directory is visited only if it is in-scope or an ancestor of
+ * some scope prefix, and a file is collected only if it is in-scope. A root that
+ * intersects no prefix is skipped without any `readdir`. An absent/empty `scope`
+ * leaves traversal and collection byte-for-byte unchanged.
+ *
+ * `platform` only selects how the exclusion comparison treats case (see
+ * {@link classifyScanEntry}); it defaults to the host, so every existing call is
+ * unchanged.
+ *
+ * Does not sort, dedupe, or emit warnings — callers handle those so their
+ * existing output contracts are preserved.
+ */
+export declare function bfsCollectSupportedFiles(rootPath: string, excludePaths: readonly string[], maxDepth?: number, scope?: string[], platform?: NodeJS.Platform): Promise<DirScanResult>;
+export {};
+//# sourceMappingURL=scan.d.ts.map

package/dist/utils/scan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scan.d.ts","sourceRoot":"","sources":["../../src/utils/scan.ts"],"names":[],"mappings":"AAyBA;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMxE;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAMpF;AAED,sDAAsD;AACtD,MAAM,MAAM,aAAa;AACvB,gDAAgD;AAC9C,SAAS;AACX,4EAA4E;GAC1E,UAAU;AACZ,uEAAuE;GACrE,aAAa;AACf,wEAAwE;GACtE,WAAW,CAAA;AAEf,kEAAkE;AAClE,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,WAAW,GAAG,aAAa,CAAA;AAEhE,iEAAiE;AACjE,UAAU,cAAc;IACtB,cAAc,IAAI,OAAO,CAAA;IACzB,WAAW,IAAI,OAAO,CAAA;IACtB,MAAM,IAAI,OAAO,CAAA;CAClB;AA+BD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,cAAc,EACrB,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,QAAQ,GAAE,MAAM,CAAC,QAA2B,GAC3C,aAAa,CAMf;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,QAAQ,GAAE,MAAM,CAAC,QAA2B,GAC3C,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAMpC;AAED,0DAA0D;AAC1D,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,0EAA0E;IAC1E,IAAI,EAAE,MAAM,CAAA;CACb;AAED,qDAAqD;AACrD,MAAM,WAAW,aAAa;IAC5B,+EAA+E;IAC/E,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,iFAAiF;IACjF,cAAc,EAAE,aAAa,EAAE,CAAA;IAC/B;;;;;OAKG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,+EAA+E;IAC/E,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,8DAA8D;IAC9D,YAAY,EAAE,OAAO,CAAA;CACtB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,QAAQ,GAAE,MAAuB,EACjC,KAAK,CAAC,EAAE,MAAM,EAAE,EAChB,QAAQ,GAAE,MAAM,CAAC,QAA2B,GAC3C,OAAO,CAAC,aAAa,CAAC,CAgExB"}