@ssmeo/mcp-local-rag 0.18.6

package/dist/utils/limits.js

@@ -0,0 +1,28 @@
+// Cross-cutting numeric limits shared across CLI and MCP server entry points.
+// Dependency-free leaf module so any layer can import it without coupling.
+/**
+ * Maximum directory recursion depth when scanning a base directory or ingest
+ * target. Applied identically by the CLI `ingest`/`list` walkers and the MCP
+ * server's `list_files` scan so the boundary is consistent everywhere.
+ */
+export const MAX_SCAN_DEPTH = 10;
+/**
+ * Default maximum file size for ingestion, in bytes (100 MB). Used when neither
+ * the CLI `--max-file-size` flag nor the `MAX_FILE_SIZE` env var is provided.
+ */
+export const DEFAULT_MAX_FILE_SIZE = 104_857_600;
+/**
+ * Hard upper bound (inclusive) for the configurable max file size, in bytes
+ * (500 MB). Values above this are rejected by `validateMaxFileSize`.
+ */
+export const MAX_FILE_SIZE_LIMIT = 524_288_000;
+/** Inclusive result-count range shared by CLI, MCP validation, and VectorStore. */
+export const MIN_QUERY_LIMIT = 1;
+export const MAX_QUERY_LIMIT = 20;
+/** Maximum number of adjacent chunks accepted on either side of a target. */
+export const MAX_NEIGHBOR_COUNT = 50;
+/** Inclusive upper bound for the configurable minimum chunk length. */
+export const MAX_CHUNK_MIN_LENGTH = 10_000;
+/** Maximum encoded bytes for one visual attachment returned with a chunk. */
+export const MAX_VISUAL_RENDITION_BYTES = 512 * 1024;
+//# sourceMappingURL=limits.js.map

package/dist/utils/limits.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"limits.js","sourceRoot":"","sources":["../../src/utils/limits.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,2EAA2E;AAE3E;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,EAAE,CAAA;AAEhC;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,WAAW,CAAA;AAEhD;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAA;AAE9C,mFAAmF;AACnF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAA;AAChC,MAAM,CAAC,MAAM,eAAe,GAAG,EAAE,CAAA;AAEjC,6EAA6E;AAC7E,MAAM,CAAC,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAEpC,uEAAuE;AACvE,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAA;AAE1C,6EAA6E;AAC7E,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,GAAG,IAAI,CAAA"}

package/dist/utils/list-sources.d.ts

@@ -0,0 +1,38 @@
+/**
+ * An ingested entry paired with its identity key (`realpathForMatch` of the
+ * stored `filePath`). Only the fields the classifier reads are required, so the
+ * caller's richer row type is consumed structurally.
+ */
+export interface KeyedIngestedEntry {
+    entry: {
+        filePath: string;
+        chunkCount: number;
+        timestamp: string;
+    };
+    key: string;
+}
+/**
+ * A classified source: a raw-data entry restored to its original `source`, or a
+ * real-file / orphaned entry keyed by `filePath`. Structurally compatible with
+ * the `SourceEntry` union both surfaces already return (no type-move refactor).
+ */
+export type ClassifiedSource = {
+    source: string;
+    chunkCount: number;
+    timestamp: string;
+} | {
+    filePath: string;
+    chunkCount: number;
+    timestamp: string;
+};
+/**
+ * Classify the ingested entries that matched no scanned file into `sources`.
+ *
+ * Under a `scope`, a managed raw-data entry is always emitted — it has no
+ * filesystem path to be in or out of scope — while a real-file entry is kept
+ * only when its stored path is in scope. An out-of-scope real file therefore
+ * appears in neither `files[]` nor `sources[]`, rather than being
+ * misreported as an orphan.
+ */
+export declare function classifyIngestedSources(ingestedKeyed: readonly KeyedIngestedEntry[], matchedKeys: ReadonlySet<string>, dbPath: string, scope?: string[]): ClassifiedSource[];
+//# sourceMappingURL=list-sources.d.ts.map

package/dist/utils/list-sources.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"list-sources.d.ts","sourceRoot":"","sources":["../../src/utils/list-sources.ts"],"names":[],"mappings":"AAUA;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAA;IAClE,GAAG,EAAE,MAAM,CAAA;CACZ;AAED;;;;GAIG;AACH,MAAM,MAAM,gBAAgB,GACxB;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACzD;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAA;AAE/D;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CACrC,aAAa,EAAE,SAAS,kBAAkB,EAAE,EAC5C,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,EAChC,MAAM,EAAE,MAAM,EACd,KAAK,CAAC,EAAE,MAAM,EAAE,GACf,gBAAgB,EAAE,CAwBpB"}

package/dist/utils/list-sources.js

@@ -0,0 +1,40 @@
+// Pure `sources` classifier shared by the MCP `list_files` handler and the
+// `list` CLI, so the raw-data / real-file scope branch lives in one place.
+//
+// `sources` are ingested entries whose identity key matched no scanned file:
+// content from `ingest_data`, plus orphaned DB entries for files no longer on
+// disk under a scanned root.
+import { extractSourceFromPath, isManagedRawDataPath } from './raw-data-utils.js';
+import { matchesAnyScope } from './scope-match.js';
+/**
+ * Classify the ingested entries that matched no scanned file into `sources`.
+ *
+ * Under a `scope`, a managed raw-data entry is always emitted — it has no
+ * filesystem path to be in or out of scope — while a real-file entry is kept
+ * only when its stored path is in scope. An out-of-scope real file therefore
+ * appears in neither `files[]` nor `sources[]`, rather than being
+ * misreported as an orphan.
+ */
+export function classifyIngestedSources(ingestedKeyed, matchedKeys, dbPath, scope) {
+    const scopePrefixes = scope && scope.length > 0 ? scope : undefined;
+    return ingestedKeyed
+        .filter(({ key }) => !matchedKeys.has(key))
+        .filter(({ entry }) => {
+        if (!scopePrefixes) {
+            return true;
+        }
+        // Raw-data sources are exempt from scope; real-file entries respect it.
+        return (isManagedRawDataPath(entry.filePath, dbPath) ||
+            matchesAnyScope(entry.filePath, scopePrefixes));
+    })
+        .map(({ entry }) => {
+        if (isManagedRawDataPath(entry.filePath, dbPath)) {
+            const source = extractSourceFromPath(entry.filePath);
+            if (source) {
+                return { source, chunkCount: entry.chunkCount, timestamp: entry.timestamp };
+            }
+        }
+        return { filePath: entry.filePath, chunkCount: entry.chunkCount, timestamp: entry.timestamp };
+    });
+}
+//# sourceMappingURL=list-sources.js.map

package/dist/utils/list-sources.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"list-sources.js","sourceRoot":"","sources":["../../src/utils/list-sources.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,2EAA2E;AAC3E,EAAE;AACF,6EAA6E;AAC7E,8EAA8E;AAC9E,6BAA6B;AAE7B,OAAO,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAA;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAqBlD;;;;;;;;GAQG;AACH,MAAM,UAAU,uBAAuB,CACrC,aAA4C,EAC5C,WAAgC,EAChC,MAAc,EACd,KAAgB;IAEhB,MAAM,aAAa,GAAG,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAA;IAEnE,OAAO,aAAa;SACjB,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1C,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;QACpB,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,OAAO,IAAI,CAAA;QACb,CAAC;QACD,wEAAwE;QACxE,OAAO,CACL,oBAAoB,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC;YAC5C,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC,CAC/C,CAAA;IACH,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;QACjB,IAAI,oBAAoB,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;YACjD,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YACpD,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAA;YAC7E,CAAC;QACH,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAA;IAC/F,CAAC,CAAC,CAAA;AACN,CAAC"}

package/dist/utils/raw-data-utils.d.ts

@@ -0,0 +1,76 @@
+/** Encode to URL-safe base64, so a source can be used as a filename. */
+export declare function encodeBase64Url(str: string): string;
+/** Decode URL-safe base64 (base64url). */
+export declare function decodeBase64Url(base64url: string): string;
+/**
+ * Drop the query string and fragment from an HTTP(S) source, so the same page
+ * maps to one path. A non-URL source (`clipboard://...`) passes through.
+ */
+export declare function normalizeSource(source: string): string;
+/** Formats accepted by ingest_data at runtime and in its public schema. */
+export declare const CONTENT_FORMATS: readonly ['text', 'html', 'markdown'];
+export type ContentFormat = (typeof CONTENT_FORMATS)[number];
+/** Raw-data directory for a given LanceDB path. */
+export declare function getRawDataDir(dbPath: string): string;
+/** `{dbPath}/raw-data/{base64url(normalizedSource)}.{ext}` */
+export declare function generateRawDataPath(dbPath: string, source: string): string;
+/** Save content under raw-data, creating the directory if needed. */
+export declare function saveRawData(dbPath: string, source: string, content: string): Promise<string>;
+/** True only for managed raw-data content files under this database. */
+export declare function isManagedRawDataPath(filePath: string, dbPath: string): boolean;
+/**
+ * Lexical containment in `<dbPath>/raw-data/`. Safe for cleanup gates
+ * (`unlink` does not follow symlinks). Use {@link isPathInRawDataDir}
+ * when the result controls `readFile`.
+ */
+export declare function isPathInRawDataDirLexical(filePath: string, dbPath: string): boolean;
+/**
+ * Lexical containment plus `realpath` so a symlink under raw-data
+ * pointing outside cannot route a read through the raw-data fast-path.
+ * Fail-closed on `realpath` errors.
+ */
+export declare function isPathInRawDataDir(filePath: string, dbPath: string): Promise<boolean>;
+/** Original source of a raw-data path, or `null` when it is not one. */
+export declare function extractSourceFromPath(filePath: string): string | null;
+/**
+ * Metadata stored alongside each raw-data .md file as a .meta.json sidecar
+ */
+/** What {@link saveMetaJson} writes. */
+export interface RawDataMeta {
+    title: string | null;
+    source: string;
+    format: ContentFormat;
+}
+/**
+ * What re-ingest reads back. Only `title` is consumed, so a sidecar missing
+ * `source` or `format` still yields a usable title rather than failing an
+ * otherwise valid ingest.
+ */
+export interface RawDataMetaRead {
+    title: string | null;
+}
+/** Replaces the trailing `.md` with `.meta.json`. */
+export declare function generateMetaJsonPath(mdPath: string): string;
+/** Save metadata as a JSON sidecar beside a raw-data .md file. */
+export declare function saveMetaJson(mdPath: string, meta: RawDataMeta): Promise<void>;
+/** `null` when the sidecar does not exist; any other read error is re-thrown. */
+export declare function loadMetaJson(mdPath: string): Promise<RawDataMetaRead | null>;
+/**
+ * True when a filesystem error means the target path does not exist.
+ */
+export declare function isEnoent(error: unknown): boolean;
+/**
+ * Pre-unlink existence of the on-disk raw-data artifacts for a target .md
+ * path: the raw-data file itself and its .meta.json sidecar.
+ */
+export interface RawDataArtifactExistence {
+    rawDataExisted: boolean;
+    metaExisted: boolean;
+}
+/**
+ * Report which raw-data artifacts exist. Single source for the delete
+ * `existed` signal, so the MCP and CLI paths cannot drift. Call BEFORE
+ * unlinking — it reports pre-unlink state.
+ */
+export declare function checkRawDataArtifacts(targetPath: string): Promise<RawDataArtifactExistence>;
+//# sourceMappingURL=raw-data-utils.d.ts.map

package/dist/utils/raw-data-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"raw-data-utils.d.ts","sourceRoot":"","sources":["../../src/utils/raw-data-utils.ts"],"names":[],"mappings":"AAYA,wEAAwE;AACxE,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAMnD;AAED,0CAA0C;AAC1C,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAUzD;AAMD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAatD;AAMD,2EAA2E;AAC3E,eAAO,MAAM,eAAe,YAAI,MAAM,EAAE,MAAM,EAAE,UAAU,CAAU,CAAA;AACpE,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,CAAA;AAQ5D,mDAAmD;AACnD,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,8DAA8D;AAC9D,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAK1E;AAMD,qEAAqE;AACrE,wBAAsB,WAAW,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAUjB;AAMD,wEAAwE;AACxE,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAI9E;AAUD;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAInF;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAW3F;AAED,wEAAwE;AACxE,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAkBrE;AAMD;;GAEG;AACH,wCAAwC;AACxC,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,aAAa,CAAA;CACtB;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;CACrB;AAYD,qDAAqD;AACrD,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED,kEAAkE;AAClE,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAGnF;AAED,iFAAiF;AACjF,wBAAsB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAWlF;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEhD;AAkBD;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACvC,cAAc,EAAE,OAAO,CAAA;IACvB,WAAW,EAAE,OAAO,CAAA;CACrB;AAED;;;;GAIG;AACH,wBAAsB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAIjG"}

package/dist/utils/raw-data-utils.js

@@ -0,0 +1,202 @@
+// Raw Data Utilities for ingest_data tool
+// Handles: base64url encoding, source normalization, file saving, source extraction
+import { access, mkdir, readFile, realpath, writeFile } from 'node:fs/promises';
+import { basename, dirname, join, posix, resolve, sep } from 'node:path';
+import { errorCode, isRecord } from './type-guards.js';
+// ============================================
+// Base64URL Encoding/Decoding
+// ============================================
+/** Encode to URL-safe base64, so a source can be used as a filename. */
+export function encodeBase64Url(str) {
+    return Buffer.from(str, 'utf-8')
+        .toString('base64')
+        .replace(/\+/g, '-')
+        .replace(/\//g, '_')
+        .replace(/=+$/, '');
+}
+/** Decode URL-safe base64 (base64url). */
+export function decodeBase64Url(base64url) {
+    // Convert base64url to standard base64
+    let base64 = base64url.replace(/-/g, '+').replace(/_/g, '/');
+    // Add padding if needed
+    while (base64.length % 4 !== 0) {
+        base64 += '=';
+    }
+    return Buffer.from(base64, 'base64').toString('utf-8');
+}
+// ============================================
+// Source Normalization
+// ============================================
+/**
+ * Drop the query string and fragment from an HTTP(S) source, so the same page
+ * maps to one path. A non-URL source (`clipboard://...`) passes through.
+ */
+export function normalizeSource(source) {
+    try {
+        const parsed = new URL(source);
+        // Only normalize HTTP(S) URLs
+        if (parsed.protocol === 'http:' || parsed.protocol === 'https:') {
+            return `${parsed.origin}${parsed.pathname}`;
+        }
+        // Non-HTTP URLs (clipboard://, etc.) are returned as-is
+        return source;
+    }
+    catch {
+        // Not a valid URL, return as-is
+        return source;
+    }
+}
+// ============================================
+// Format Utilities
+// ============================================
+/** Formats accepted by ingest_data at runtime and in its public schema. */
+export const CONTENT_FORMATS = ['text', 'html', 'markdown'];
+const RAW_DATA_EXTENSION = 'md';
+// ============================================
+// Path Generation
+// ============================================
+/** Raw-data directory for a given LanceDB path. */
+export function getRawDataDir(dbPath) {
+    return join(dbPath, 'raw-data');
+}
+/** `{dbPath}/raw-data/{base64url(normalizedSource)}.{ext}` */
+export function generateRawDataPath(dbPath, source) {
+    const normalizedSource = normalizeSource(source);
+    const encoded = encodeBase64Url(normalizedSource);
+    // Use resolve to ensure absolute path (required by validateFilePath)
+    return resolve(getRawDataDir(dbPath), `${encoded}.${RAW_DATA_EXTENSION}`);
+}
+// ============================================
+// File Operations
+// ============================================
+/** Save content under raw-data, creating the directory if needed. */
+export async function saveRawData(dbPath, source, content) {
+    const filePath = generateRawDataPath(dbPath, source);
+    // Ensure directory exists
+    await mkdir(dirname(filePath), { recursive: true });
+    // Write content to file
+    await writeFile(filePath, content, 'utf-8');
+    return filePath;
+}
+// ============================================
+// Path Detection and Source Extraction
+// ============================================
+/** True only for managed raw-data content files under this database. */
+export function isManagedRawDataPath(filePath, dbPath) {
+    return (isPathInRawDataDirLexical(filePath, dbPath) && /^[A-Za-z0-9_-]+\.md$/.test(basename(filePath)));
+}
+/**
+ * Case-normalize a path for prefix containment. Windows filesystems are
+ * case-insensitive by default, so the boundary check must mirror that.
+ */
+function caseNormalize(p) {
+    return process.platform === 'win32' ? p.toLowerCase() : p;
+}
+/**
+ * Lexical containment in `<dbPath>/raw-data/`. Safe for cleanup gates
+ * (`unlink` does not follow symlinks). Use {@link isPathInRawDataDir}
+ * when the result controls `readFile`.
+ */
+export function isPathInRawDataDirLexical(filePath, dbPath) {
+    const target = caseNormalize(resolve(filePath));
+    const rawDir = caseNormalize(resolve(getRawDataDir(dbPath)));
+    return target === rawDir || target.startsWith(rawDir + sep);
+}
+/**
+ * Lexical containment plus `realpath` so a symlink under raw-data
+ * pointing outside cannot route a read through the raw-data fast-path.
+ * Fail-closed on `realpath` errors.
+ */
+export async function isPathInRawDataDir(filePath, dbPath) {
+    if (!isPathInRawDataDirLexical(filePath, dbPath)) {
+        return false;
+    }
+    try {
+        const realTarget = caseNormalize(await realpath(resolve(filePath)));
+        const realRaw = caseNormalize(await realpath(resolve(getRawDataDir(dbPath))));
+        return realTarget === realRaw || realTarget.startsWith(realRaw + sep);
+    }
+    catch {
+        return false;
+    }
+}
+/** Original source of a raw-data path, or `null` when it is not one. */
+export function extractSourceFromPath(filePath) {
+    const normalized = filePath.replace(/\\/g, '/');
+    const rawDataMarker = '/raw-data/';
+    const rawDataIndex = normalized.indexOf(rawDataMarker);
+    if (rawDataIndex === -1) {
+        return null;
+    }
+    const fileName = posix.basename(normalized);
+    const dotIndex = fileName.lastIndexOf('.');
+    if (dotIndex === -1) {
+        return null;
+    }
+    const encoded = fileName.slice(0, dotIndex);
+    return decodeBase64Url(encoded);
+}
+/**
+ * Read one persisted sidecar. The file is plain JSON on disk, so a malformed or
+ * hand-edited value is a real possibility and each field is checked rather than
+ * assumed.
+ */
+function toRawDataMetaRead(parsed) {
+    const title = isRecord(parsed) ? parsed['title'] : undefined;
+    return { title: typeof title === 'string' ? title : null };
+}
+/** Replaces the trailing `.md` with `.meta.json`. */
+export function generateMetaJsonPath(mdPath) {
+    return mdPath.replace(/\.md$/, '.meta.json');
+}
+/** Save metadata as a JSON sidecar beside a raw-data .md file. */
+export async function saveMetaJson(mdPath, meta) {
+    const metaPath = generateMetaJsonPath(mdPath);
+    await writeFile(metaPath, JSON.stringify(meta, null, 2), 'utf-8');
+}
+/** `null` when the sidecar does not exist; any other read error is re-thrown. */
+export async function loadMetaJson(mdPath) {
+    const metaPath = generateMetaJsonPath(mdPath);
+    try {
+        const content = await readFile(metaPath, 'utf-8');
+        return toRawDataMetaRead(JSON.parse(content));
+    }
+    catch (error) {
+        if (isEnoent(error)) {
+            return null;
+        }
+        throw error;
+    }
+}
+/**
+ * True when a filesystem error means the target path does not exist.
+ */
+export function isEnoent(error) {
+    return errorCode(error) === 'ENOENT';
+}
+// ============================================
+// Raw-Data Artifact Existence
+// ============================================
+/**
+ * Whether a path exists. access()-based, never throws.
+ */
+async function pathExists(path) {
+    try {
+        await access(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Report which raw-data artifacts exist. Single source for the delete
+ * `existed` signal, so the MCP and CLI paths cannot drift. Call BEFORE
+ * unlinking — it reports pre-unlink state.
+ */
+export async function checkRawDataArtifacts(targetPath) {
+    const rawDataExisted = await pathExists(targetPath);
+    const metaExisted = await pathExists(generateMetaJsonPath(targetPath));
+    return { rawDataExisted, metaExisted };
+}
+//# sourceMappingURL=raw-data-utils.js.map

package/dist/utils/raw-data-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"raw-data-utils.js","sourceRoot":"","sources":["../../src/utils/raw-data-utils.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,oFAAoF;AAEpF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC/E,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAA;AAExE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAEtD,+CAA+C;AAC/C,8BAA8B;AAC9B,+CAA+C;AAE/C,wEAAwE;AACxE,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC;SAC7B,QAAQ,CAAC,QAAQ,CAAC;SAClB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;AACvB,CAAC;AAED,0CAA0C;AAC1C,MAAM,UAAU,eAAe,CAAC,SAAiB;IAC/C,uCAAuC;IACvC,IAAI,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAE5D,wBAAwB;IACxB,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,CAAA;IACf,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;AACxD,CAAC;AAED,+CAA+C;AAC/C,uBAAuB;AACvB,+CAA+C;AAE/C;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAA;QAC9B,8BAA8B;QAC9B,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAChE,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;QAC7C,CAAC;QACD,wDAAwD;QACxD,OAAO,MAAM,CAAA;IACf,CAAC;IAAC,MAAM,CAAC;QACP,gCAAgC;QAChC,OAAO,MAAM,CAAA;IACf,CAAC;AACH,CAAC;AAED,+CAA+C;AAC/C,mBAAmB;AACnB,+CAA+C;AAE/C,2EAA2E;AAC3E,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAU,CAAA;AAGpE,MAAM,kBAAkB,GAAG,IAAI,CAAA;AAE/B,+CAA+C;AAC/C,kBAAkB;AAClB,+CAA+C;AAE/C,mDAAmD;AACnD,MAAM,UAAU,aAAa,CAAC,MAAc;IAC1C,OAAO,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;AACjC,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,mBAAmB,CAAC,MAAc,EAAE,MAAc;IAChE,MAAM,gBAAgB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;IACjD,qEAAqE;IACrE,OAAO,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,GAAG,OAAO,IAAI,kBAAkB,EAAE,CAAC,CAAA;AAC3E,CAAC;AAED,+CAA+C;AAC/C,kBAAkB;AAClB,+CAA+C;AAE/C,qEAAqE;AACrE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAc,EACd,MAAc,EACd,OAAe;IAEf,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAEpD,0BAA0B;IAC1B,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEnD,wBAAwB;IACxB,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAE3C,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,+CAA+C;AAC/C,uCAAuC;AACvC,+CAA+C;AAE/C,wEAAwE;AACxE,MAAM,UAAU,oBAAoB,CAAC,QAAgB,EAAE,MAAc;IACnE,OAAO,CACL,yBAAyB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAC/F,CAAA;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAC3D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,QAAgB,EAAE,MAAc;IACxE,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAA;IAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAC5D,OAAO,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,CAAA;AAC7D,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB,EAAE,MAAc;IACvE,IAAI,CAAC,yBAAyB,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;QACjD,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;QACnE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC7E,OAAO,UAAU,KAAK,OAAO,IAAI,UAAU,CAAC,UAAU,CAAC,OAAO,GAAG,GAAG,CAAC,CAAA;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACpD,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAC/C,MAAM,aAAa,GAAG,YAAY,CAAA;IAClC,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,CAAA;IAEtD,IAAI,YAAY,KAAK,CAAC,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAA;IACb,CAAC;IAED,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IAE1C,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;QACpB,OAAO,IAAI,CAAA;IACb,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAA;IAC3C,OAAO,eAAe,CAAC,OAAO,CAAC,CAAA;AACjC,CAAC;AAyBD;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,MAAe;IACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAC5D,OAAO,EAAE,KAAK,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;AAC5D,CAAC;AAED,qDAAqD;AACrD,MAAM,UAAU,oBAAoB,CAAC,MAAc;IACjD,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;AAC9C,CAAC;AAED,kEAAkE;AAClE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc,EAAE,IAAiB;IAClE,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAA;IAC7C,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;AACnE,CAAC;AAED,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,MAAc;IAC/C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAA;IAC7C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;QACjD,OAAO,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAA;IAC/C,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,OAAO,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAA;AACtC,CAAC;AAED,+CAA+C;AAC/C,8BAA8B;AAC9B,+CAA+C;AAE/C;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;QAClB,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAWD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,UAAkB;IAC5D,MAAM,cAAc,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAA;IACnD,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,CAAA;IACtE,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,CAAA;AACxC,CAAC"}

package/dist/utils/scan.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Canonical identity key for the `list`/`list_files` cross-reference: a file's
+ * realpath, falling back to the input path when realpath fails (orphaned or
+ * raw-data entries). Matching ingested DB entries against scanned files by this
+ * key recognizes the same physical file across symlinked spellings (prefix or
+ * alias). Storage, lookup, and display still use the normal resolve() path —
+ * realpath here is the file-identity comparison, not a user-facing value.
+ */
+export declare function realpathForMatch(filePath: string): Promise<string>;
+/**
+ * Canonical form of one explicitly requested path: its parent chain resolved
+ * through symbolic links, with the final component appended verbatim. `null` when
+ * the parent chain cannot be resolved at all — absent, or a directory this
+ * process may not traverse — which a caller must treat as "not contained",
+ * because telling those cases apart would report the state of paths outside its
+ * configured roots.
+ *
+ * Only the parent chain is resolved, because the requested entry itself is judged
+ * by {@link classifyRequestedPath}'s `lstat`: a symbolic link named directly
+ * inside a root is an in-root entry that is refused as a link, not a path to be
+ * reported by whatever it points at.
+ *
+ * `realpath` here is the containment (security) boundary, the same role it plays
+ * in `DocumentParser.validateFilePath` — never a spelling anything is stored,
+ * looked up, or displayed under. Those stay `resolve()`-only.
+ */
+export declare function canonicalizeRequestedPath(path: string): Promise<string | null>;
+/** Why the collect predicates below refuse a path. */
+export type ScanRejection = 
+/** A symbolic link, which is never followed. */
+'symlink'
+/** Under a configured excluded prefix (the database or cache directory). */
+ | 'excluded'
+/** A regular file whose extension is not a supported document type. */
+ | 'unsupported'
+/** Neither a regular file nor a directory (socket, FIFO, device, …). */
+ | 'irregular';
+/** What a path is, once the collect predicates have judged it. */
+export type ScanEntryKind = 'file' | 'directory' | ScanRejection;
+/** The `Dirent` / `Stats` subset the collect predicates read. */
+interface EntryTypeFacts {
+    isSymbolicLink(): boolean;
+    isDirectory(): boolean;
+    isFile(): boolean;
+}
+/**
+ * The collect predicates of {@link bfsCollectSupportedFiles} as one decision,
+ * so a discovered entry and an explicitly requested path are judged by the same
+ * rules rather than by two implementations that drift.
+ *
+ * Evaluation order is part of the contract: a symbolic link is reported as a
+ * link even under an excluded prefix, and a directory needs no extension test.
+ *
+ * `platform` is a parameter so the Windows exclusion semantics are provable on
+ * a POSIX host. Both `Dirent` and `Stats` satisfy {@link EntryTypeFacts}.
+ */
+export declare function classifyScanEntry(fullPath: string, entry: EntryTypeFacts, excludePaths: readonly string[], platform?: NodeJS.Platform): ScanEntryKind;
+/**
+ * Classify one explicitly requested path with {@link classifyScanEntry}, so a
+ * path a caller names is subject to the same predicates as a path the walker
+ * discovers.
+ *
+ * `lstat` rather than `stat`, so a symbolic link is reported as a link instead of
+ * as whatever it points at; and `lstat` rather than any read, so a caller can
+ * refuse the path before its bytes cost anything — reading a FIFO blocks forever,
+ * and reading through a link reaches outside the configured roots.
+ *
+ * Any stat failure is `'missing'`: an unreachable path and an absent one are the
+ * same non-answer to "what is here".
+ */
+export declare function classifyRequestedPath(path: string, excludePaths: readonly string[], platform?: NodeJS.Platform): Promise<ScanEntryKind | 'missing'>;
+/** A directory that could not be read during the scan. */
+export interface UnreadableDir {
+    dirPath: string;
+    /** Node error `code` (e.g. `EACCES`), or `'UNKNOWN'` when unavailable. */
+    code: string;
+}
+/** Structured result of a bounded directory scan. */
+export interface DirScanResult {
+    /** Supported files found under the root, in BFS-discovery order (unsorted). */
+    files: string[];
+    /** Directories skipped because `readdir` failed (caller decides how to warn). */
+    unreadableDirs: UnreadableDir[];
+    /**
+     * Each entry is the first unvisited directory of a branch pruned for
+     * exceeding `maxDepth` — the directory that was reached but never read. That
+     * path and every descendant of it is unobserved by this scan; its ancestors
+     * and fully visited siblings are not listed.
+     */
+    depthLimitedDirs: string[];
+    /** Full paths of directory entries skipped because they are symbolic links. */
+    skippedSymlinks: string[];
+    /** True if any branch was pruned for exceeding `maxDepth`. */
+    depthLimited: boolean;
+}
+export interface BfsCollectOptions {
+    /** Traversal bound; defaults to {@link MAX_SCAN_DEPTH}. */
+    maxDepth?: number;
+    /** Absolute prefixes to restrict traversal and collection to. */
+    scope?: string[] | undefined;
+    /** Selects case sensitivity of the exclusion comparison; defaults to the host. */
+    platform?: NodeJS.Platform;
+}
+/**
+ * Bounded BFS scan of a single root, depth counted from `rootPath` itself.
+ * Symlinks are never followed. An unreadable directory is recorded and does
+ * not abort the scan.
+ *
+ * Does not sort, dedupe, or emit warnings — each caller owns its own output
+ * contract for those.
+ */
+export declare function bfsCollectSupportedFiles(rootPath: string, excludePaths: readonly string[], options?: BfsCollectOptions): Promise<DirScanResult>;
+export {};
+//# sourceMappingURL=scan.d.ts.map

package/dist/utils/scan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scan.d.ts","sourceRoot":"","sources":["../../src/utils/scan.ts"],"names":[],"mappings":"AAkBA;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMxE;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAMpF;AAED,sDAAsD;AACtD,MAAM,MAAM,aAAa;AACvB,gDAAgD;AAC9C,SAAS;AACX,4EAA4E;GAC1E,UAAU;AACZ,uEAAuE;GACrE,aAAa;AACf,wEAAwE;GACtE,WAAW,CAAA;AAEf,kEAAkE;AAClE,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,WAAW,GAAG,aAAa,CAAA;AAEhE,iEAAiE;AACjE,UAAU,cAAc;IACtB,cAAc,IAAI,OAAO,CAAA;IACzB,WAAW,IAAI,OAAO,CAAA;IACtB,MAAM,IAAI,OAAO,CAAA;CAClB;AA+BD;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,cAAc,EACrB,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,QAAQ,GAAE,MAAM,CAAC,QAA2B,GAC3C,aAAa,CAcf;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,QAAQ,GAAE,MAAM,CAAC,QAA2B,GAC3C,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAMpC;AAED,0DAA0D;AAC1D,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,0EAA0E;IAC1E,IAAI,EAAE,MAAM,CAAA;CACb;AAED,qDAAqD;AACrD,MAAM,WAAW,aAAa;IAC5B,+EAA+E;IAC/E,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,iFAAiF;IACjF,cAAc,EAAE,aAAa,EAAE,CAAA;IAC/B;;;;;OAKG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,+EAA+E;IAC/E,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,8DAA8D;IAC9D,YAAY,EAAE,OAAO,CAAA;CACtB;AAmDD,MAAM,WAAW,iBAAiB;IAChC,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAA;IAC5B,kFAAkF;IAClF,QAAQ,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAA;CAC3B;AAED;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,aAAa,CAAC,CAoDxB"}