mcard-js 2.1.12 → 2.1.13

package/dist/util/FileIO.js

@@ -0,0 +1,203 @@
+import * as crypto from 'crypto';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { ContentTypeInterpreter } from '../model/ContentTypeInterpreter';
+/**
+ * Stream-read bytes up to byte_cap, decode, and soft wrap.
+ */
+export async function streamReadNormalizedText(filePath, options) {
+    const { byteCap, wrapWidth } = options;
+    const sha = crypto.createHash('sha256');
+    let totalSize = 0;
+    let producedText = '';
+    let currentLen = 0;
+    const handle = await fs.open(filePath, 'r');
+    try {
+        const buffer = new Uint8Array(8192);
+        let remaining = byteCap;
+        // Check if TextDecoder supports replacement (it does by default fatal: false)
+        const decoder = new TextDecoder('utf-8', { fatal: false });
+        // Use position variable manually to be safe with handle.read
+        let position = 0;
+        while (remaining > 0) {
+            const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), position);
+            if (bytesRead === 0)
+                break;
+            position += bytesRead;
+            const chunk = buffer.subarray(0, bytesRead);
+            sha.update(chunk);
+            totalSize += bytesRead;
+            remaining -= bytesRead;
+            const s = decoder.decode(chunk, { stream: true });
+            for (const ch of s) {
+                if (ch === '\r')
+                    continue;
+                producedText += ch;
+                if (ch === '\n') {
+                    currentLen = 0;
+                }
+                else {
+                    currentLen++;
+                    if (wrapWidth > 0 && currentLen >= wrapWidth) {
+                        producedText += '\n';
+                        currentLen = 0;
+                    }
+                }
+            }
+        }
+        // Flush decoder
+        const s = decoder.decode();
+        for (const ch of s) {
+            if (ch === '\r')
+                continue;
+            producedText += ch;
+            if (ch === '\n') {
+                currentLen = 0;
+            }
+            else {
+                currentLen++;
+                if (wrapWidth > 0 && currentLen >= wrapWidth) {
+                    producedText += '\n';
+                    currentLen = 0;
+                }
+            }
+        }
+    }
+    finally {
+        await handle.close();
+    }
+    return {
+        text: producedText,
+        originalSize: totalSize,
+        originalSha256Prefix: sha.digest('hex').substring(0, 16)
+    };
+}
+// Constants
+const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
+const READ_TIMEOUT_MS = 5000;
+/**
+ * Check if a file is likely to cause processing issues.
+ */
+export async function isProblematicFile(filePath) {
+    try {
+        const stats = await fs.stat(filePath);
+        if (stats.size === 0)
+            return false;
+        // Skip hidden files/directories (starting with .)
+        if (path.basename(filePath).startsWith('.'))
+            return true;
+        if (stats.size > MAX_FILE_SIZE)
+            return true;
+        const ext = path.extname(filePath);
+        const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(ext);
+        if (isKnownType && stats.size > 1024 * 1024)
+            return true; // >1MB known type skip
+        // Sample content
+        const handle = await fs.open(filePath, 'r');
+        try {
+            const buffer = new Uint8Array(32 * 1024);
+            const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+            const sample = buffer.subarray(0, bytesRead);
+            if (ContentTypeInterpreter.isUnstructuredBinary(sample))
+                return true;
+            if (ContentTypeInterpreter.hasPathologicalLines(sample, isKnownType))
+                return true;
+        }
+        finally {
+            await handle.close();
+        }
+        return false;
+    }
+    catch {
+        return true;
+    }
+}
+/**
+ * Safely read a file with limits and timeouts.
+ */
+export async function readFileSafely(filePath, options = {}) {
+    const stats = await fs.stat(filePath);
+    if (stats.size > MAX_FILE_SIZE)
+        throw new Error(`File too large: ${stats.size}`);
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), READ_TIMEOUT_MS);
+    try {
+        const handle = await fs.open(filePath, 'r');
+        try {
+            const buffer = new Uint8Array(stats.size);
+            await handle.read(buffer, 0, stats.size, 0);
+            return buffer;
+        }
+        finally {
+            await handle.close();
+        }
+    }
+    catch (e) {
+        if (e.name === 'AbortError')
+            throw new Error(`Read timeout for ${filePath}`);
+        throw e;
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+/**
+ * List files in directory, filtering out problematic ones.
+ */
+export async function listFiles(dirPath, recursive = false) {
+    let files = [];
+    try {
+        const entries = await fs.readdir(dirPath, { withFileTypes: true });
+        for (const entry of entries) {
+            const fullPath = path.join(dirPath, entry.name);
+            // Skip hidden
+            if (entry.name.startsWith('.'))
+                continue;
+            if (entry.isDirectory()) {
+                if (recursive) {
+                    files = files.concat(await listFiles(fullPath, true));
+                }
+            }
+            else if (entry.isFile()) {
+                if (!(await isProblematicFile(fullPath))) {
+                    files.push(fullPath);
+                }
+            }
+        }
+    }
+    catch (e) {
+        console.warn(`Error listing directory ${dirPath}:`, e);
+    }
+    return files;
+}
+/**
+ * Process a file and return metadata and content.
+ */
+export async function processFileContent(filePath, options = {}) {
+    const rawContent = await readFileSafely(filePath, { allowPathological: options.allowPathological, maxBytes: options.maxBytes });
+    // Detect type using sample
+    const sample = rawContent.subarray(0, 1024 * 1024);
+    const detection = ContentTypeInterpreter.detectContentType(sample, path.extname(filePath));
+    let isBinary = ContentTypeInterpreter.isBinaryContent(sample, detection.mimeType);
+    if (options.forceBinary)
+        isBinary = true;
+    let content = rawContent;
+    if (!isBinary) {
+        try {
+            content = new TextDecoder('utf-8', { fatal: true }).decode(rawContent);
+        }
+        catch {
+            // Fallback to replacement
+            content = new TextDecoder('utf-8', { fatal: false }).decode(rawContent);
+        }
+    }
+    return {
+        content,
+        filename: path.basename(filePath),
+        mimeType: detection.mimeType,
+        extension: detection.extension,
+        isBinary,
+        size: rawContent.length
+    };
+}
+//# sourceMappingURL=FileIO.js.map

package/dist/util/FileIO.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FileIO.js","sourceRoot":"","sources":["../../src/util/FileIO.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAElC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAQzE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,QAAgB,EAChB,OAA+C;IAE/C,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IACvC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,SAAS,GAAG,OAAO,CAAC;QACxB,8EAA8E;QAC9E,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAE3D,6DAA6D;QAC7D,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,OAAO,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,QAAQ,CAAC,CAAC;YAEjG,IAAI,SAAS,KAAK,CAAC;gBAAE,MAAM;YAE3B,QAAQ,IAAI,SAAS,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YAC5C,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClB,SAAS,IAAI,SAAS,CAAC;YACvB,SAAS,IAAI,SAAS,CAAC;YAEvB,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAElD,KAAK,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;gBACjB,IAAI,EAAE,KAAK,IAAI;oBAAE,SAAS;gBAC1B,YAAY,IAAI,EAAE,CAAC;gBACnB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;oBACd,UAAU,GAAG,CAAC,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACJ,UAAU,EAAE,CAAC;oBACb,IAAI,SAAS,GAAG,CAAC,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;wBAC3C,YAAY,IAAI,IAAI,CAAC;wBACrB,UAAU,GAAG,CAAC,CAAC;oBACnB,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QACD,gBAAgB;QAChB,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YACjB,IAAI,EAAE,KAAK,IAAI;gBAAE,SAAS;YAC1B,YAAY,IAAI,EAAE,CAAC;YACnB,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;gBACd,UAAU,GAAG,CAAC,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACJ,UAAU,EAAE,CAAC;gBACb,IAAI,SAAS,GAAG,CAAC,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;oBAC3C,YAAY,IAAI,IAAI,CAAC;oBACrB,UAAU,GAAG,CAAC,CAAC;gBACnB,CAAC;YACL,CAAC;QACL,CAAC;IAEL,CAAC;YAAS,CAAC;QACP,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAED,OAAO;QACH,IAAI,EAAE,YAAY;QAClB,YAAY,EAAE,SAAS;QACvB,oBAAoB,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC;KAC3D,CAAC;AACN,CAAC;AAGD,YAAY;AACZ,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;AAC/C,MAAM,eAAe,GAAG,IAAI,CAAC;AAW7B;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IACpD,IAAI,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAEnC,kDAAkD;QAClD,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAEzD,IAAI,KAAK,CAAC,IAAI,GAAG,aAAa;YAAE,OAAO,IAAI,CAAC;QAE5C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,WAAW,GAAG,sBAAsB,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC;QAEzE,IAAI,WAAW,IAAI,KAAK,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI;YAAE,OAAO,IAAI,CAAC,CAAC,uBAAuB;QAEjF,iBAAiB;QACjB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;YACzC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACrE,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YAE7C,IAAI,sBAAsB,CAAC,oBAAoB,CAAC,MAAM,CAAC;gBAAE,OAAO,IAAI,CAAC;YACrE,IAAI,sBAAsB,CAAC,oBAAoB,CAAC,MAAM,EAAE,WAAW,CAAC;gBAAE,OAAO,IAAI,CAAC;QACtF,CAAC;gBAAS,CAAC;YACP,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAChC,QAAgB,EAChB,UAA8D,EAAE;IAEhE,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,IAAI,GAAG,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IAEjF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,eAAe,CAAC,CAAC;IAEtE,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC5C,OAAO,MAAM,CAAC;QAClB,CAAC;gBAAS,CAAC;YACP,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;QAC7E,MAAM,CAAC,CAAC;IACZ,CAAC;YAAS,CAAC;QACP,YAAY,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAe,EAAE,YAAqB,KAAK;IACvE,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAEhD,cAAc;YACd,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEzC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACtB,IAAI,SAAS,EAAE,CAAC;oBACZ,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC1D,CAAC;YACL,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBACxB,IAAI,CAAC,CAAC,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;oBACvC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,IAAI,CAAC,2BAA2B,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACpC,QAAgB,EAChB,UAAqF,EAAE;IAEvF,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAEhI,2BAA2B;IAC3B,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,sBAAsB,CAAC,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE3F,IAAI,QAAQ,GAAG,sBAAsB,CAAC,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;IAClF,IAAI,OAAO,CAAC,WAAW;QAAE,QAAQ,GAAG,IAAI,CAAC;IAEzC,IAAI,OAAO,GAAwB,UAAU,CAAC;IAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,IAAI,CAAC;YACD,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC3E,CAAC;QAAC,MAAM,CAAC;YACL,0BAA0B;YAC1B,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5E,CAAC;IACL,CAAC;IAED,OAAO;QACH,OAAO;QACP,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACjC,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,SAAS,EAAE,SAAS,CAAC,SAAS;QAC9B,QAAQ;QACR,IAAI,EAAE,UAAU,CAAC,MAAM;KAC1B,CAAC;AACN,CAAC"}

package/dist/util/Loader.d.ts

@@ -0,0 +1,34 @@
+import { CardCollection } from '../model/CardCollection';
+export interface FileProcessingResult {
+    hash: string;
+    contentType?: string;
+    isBinary?: boolean;
+    filename?: string;
+    size?: number;
+    filePath: string;
+    originalSize?: number;
+    originalSha256Prefix?: string;
+    metadataOnly?: boolean;
+}
+export declare function processAndStoreFile(filePath: string, collection: CardCollection, options?: {
+    allowProblematic?: boolean;
+    maxBytesOnProblem?: number;
+    metadataOnly?: boolean;
+    rootPath?: string;
+}): Promise<FileProcessingResult | null>;
+export interface LoaderMetrics {
+    filesCount: number;
+    directoriesCount: number;
+    directoryLevels: number;
+}
+export interface LoaderResponse {
+    metrics: LoaderMetrics;
+    results: FileProcessingResult[];
+}
+export declare function loadFileToCollection(targetPath: string, collection: CardCollection, options?: {
+    recursive?: boolean;
+    includeProblematic?: boolean;
+    maxBytesOnProblem?: number;
+    metadataOnly?: boolean;
+}): Promise<LoaderResponse>;
+//# sourceMappingURL=Loader.d.ts.map

package/dist/util/Loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Loader.d.ts","sourceRoot":"","sources":["../../src/util/Loader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAczD,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,wBAAsB,mBAAmB,CACrC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,cAAc,EAC1B,OAAO,GAAE;IACL,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAChB,GACP,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CA0GtC;AAED,MAAM,WAAW,aAAa;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc;IAC3B,OAAO,EAAE,aAAa,CAAC;IACvB,OAAO,EAAE,oBAAoB,EAAE,CAAC;CACnC;AAED,wBAAsB,oBAAoB,CACtC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,cAAc,EAC1B,OAAO,GAAE;IACL,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,OAAO,CAAC;CACrB,GACP,OAAO,CAAC,cAAc,CAAC,CAkEzB"}

package/dist/util/Loader.js

@@ -0,0 +1,168 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { MCard } from '../model/MCard';
+import { isProblematicFile, processFileContent, streamReadNormalizedText, listFiles } from './FileIO';
+import { ContentTypeInterpreter } from '../model/ContentTypeInterpreter';
+// Constants replicating settings
+const DEFAULT_MAX_PROBLEM_BYTES = 2 * 1024 * 1024; // 2MB
+const WRAP_WIDTH_KNOWN = 1000;
+const WRAP_WIDTH_DEFAULT = 80;
+export async function processAndStoreFile(filePath, collection, options = {}) {
+    const { allowProblematic = false, maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES, metadataOnly = false, rootPath } = options;
+    try {
+        let fileInfo;
+        if (await isProblematicFile(filePath)) {
+            if (!allowProblematic) {
+                console.warn(`Skipping problematic file: ${filePath}`);
+                return null;
+            }
+            const extension = path.extname(filePath).toLowerCase();
+            const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(extension);
+            const wrapWidth = isKnownType ? WRAP_WIDTH_KNOWN : WRAP_WIDTH_DEFAULT;
+            console.warn(`Problematic file detected, processing as safe text: ${filePath}`);
+            try {
+                const streamed = await streamReadNormalizedText(filePath, {
+                    byteCap: maxBytesOnProblem,
+                    wrapWidth
+                });
+                fileInfo = {
+                    content: streamed.text,
+                    filename: path.basename(filePath),
+                    mimeType: 'text/plain',
+                    extension: extension,
+                    isBinary: false,
+                    size: streamed.text.length,
+                    originalSize: streamed.originalSize,
+                    originalSha256Prefix: streamed.originalSha256Prefix,
+                    normalized: true,
+                    wrapWidth
+                };
+            }
+            catch (e) {
+                console.warn(`Safe text processing failed, falling back to capped binary: ${filePath}`);
+                fileInfo = await processFileContent(filePath, {
+                    forceBinary: true,
+                    allowPathological: true,
+                    maxBytes: maxBytesOnProblem
+                });
+            }
+        }
+        else {
+            console.log(`Processing file: ${filePath}`);
+            fileInfo = await processFileContent(filePath);
+        }
+        if (!fileInfo)
+            return null;
+        let mcard = null;
+        const isProblematic = await isProblematicFile(filePath);
+        if (metadataOnly && isProblematic) {
+            mcard = null;
+        }
+        else {
+            mcard = await MCard.create(fileInfo.content);
+            // Handle Logic
+            const handle = path.basename(filePath);
+            try {
+                await collection.addWithHandle(mcard, handle);
+            }
+            catch (e) {
+                let registered = false;
+                if (rootPath) {
+                    const relPath = path.relative(rootPath, filePath);
+                    if (relPath !== handle) {
+                        try {
+                            await collection.addWithHandle(mcard, relPath);
+                            registered = true;
+                        }
+                        catch (e2) {
+                            console.warn(`Handle collision for fallback '${relPath}'`);
+                        }
+                    }
+                }
+                if (!registered) {
+                    try {
+                        await collection.add(mcard);
+                    }
+                    catch (e3) { }
+                }
+            }
+        }
+        const result = {
+            hash: mcard ? mcard.hash : 'METADATA_ONLY',
+            contentType: fileInfo.mimeType,
+            isBinary: fileInfo.isBinary,
+            filename: fileInfo.filename,
+            size: fileInfo.size,
+            filePath: filePath,
+        };
+        if (fileInfo.originalSize !== undefined)
+            result.originalSize = fileInfo.originalSize;
+        if (fileInfo.originalSha256Prefix)
+            result.originalSha256Prefix = fileInfo.originalSha256Prefix;
+        if (metadataOnly && isProblematic)
+            result.metadataOnly = true;
+        return result;
+    }
+    catch (e) {
+        console.error(`Error processing ${filePath}:`, e);
+        return null;
+    }
+}
+export async function loadFileToCollection(targetPath, collection, options = {}) {
+    const { recursive = false, includeProblematic = false, maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES, metadataOnly = false } = options;
+    // Resolve absolute path
+    const resolvedPath = path.resolve(targetPath);
+    const stats = await fs.stat(resolvedPath);
+    const results = [];
+    // Determine files to process
+    let files = [];
+    let rootPath = resolvedPath;
+    if (stats.isFile()) {
+        files = [resolvedPath];
+        rootPath = path.dirname(resolvedPath);
+    }
+    else if (stats.isDirectory()) {
+        files = await listFiles(resolvedPath, recursive);
+        rootPath = resolvedPath;
+    }
+    else {
+        throw new Error(`Path ${targetPath} is not a file or directory`);
+    }
+    // Calculate Metrics
+    const uniqueDirs = new Set();
+    let maxDepth = 0;
+    for (const file of files) {
+        const dir = path.dirname(file);
+        // We track directories relative to root to verify "loaded directories"
+        // If file is directly in root, dir is root.
+        if (dir.startsWith(rootPath)) {
+            uniqueDirs.add(dir);
+            // Depth calculation
+            const rel = path.relative(rootPath, file);
+            // parts length - 1 (filename) gives folder depth. 
+            // e.g. "a.txt" -> 0 depth. "sub/a.txt" -> 1 depth.
+            const parts = rel.split(path.sep);
+            const depth = parts.length - 1;
+            if (depth > maxDepth)
+                maxDepth = depth;
+        }
+    }
+    const metrics = {
+        filesCount: files.length,
+        directoriesCount: uniqueDirs.size,
+        directoryLevels: maxDepth
+    };
+    console.log(`About to process ${files.length} files`);
+    for (const file of files) {
+        const result = await processAndStoreFile(file, collection, {
+            allowProblematic: includeProblematic,
+            maxBytesOnProblem,
+            metadataOnly,
+            rootPath
+        });
+        if (result)
+            results.push(result);
+    }
+    return { metrics, results };
+}
+//# sourceMappingURL=Loader.js.map

package/dist/util/Loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Loader.js","sourceRoot":"","sources":["../../src/util/Loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,OAAO,EACH,iBAAiB,EACjB,kBAAkB,EAClB,wBAAwB,EACxB,SAAS,EACZ,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAEzE,iCAAiC;AACjC,MAAM,yBAAyB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAc9B,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,QAAgB,EAChB,UAA0B,EAC1B,UAKI,EAAE;IAEN,MAAM,EACF,gBAAgB,GAAG,KAAK,EACxB,iBAAiB,GAAG,yBAAyB,EAC7C,YAAY,GAAG,KAAK,EACpB,QAAQ,EACX,GAAG,OAAO,CAAC;IAEZ,IAAI,CAAC;QACD,IAAI,QAAa,CAAC;QAElB,IAAI,MAAM,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC,8BAA8B,QAAQ,EAAE,CAAC,CAAC;gBACvD,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;YACvD,MAAM,WAAW,GAAG,sBAAsB,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC;YAC/E,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,kBAAkB,CAAC;YAEtE,OAAO,CAAC,IAAI,CAAC,uDAAuD,QAAQ,EAAE,CAAC,CAAC;YAEhF,IAAI,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,QAAQ,EAAE;oBACtD,OAAO,EAAE,iBAAiB;oBAC1B,SAAS;iBACZ,CAAC,CAAC;gBAEH,QAAQ,GAAG;oBACP,OAAO,EAAE,QAAQ,CAAC,IAAI;oBACtB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACjC,QAAQ,EAAE,YAAY;oBACtB,SAAS,EAAE,SAAS;oBACpB,QAAQ,EAAE,KAAK;oBACf,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;oBAC1B,YAAY,EAAE,QAAQ,CAAC,YAAY;oBACnC,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB;oBACnD,UAAU,EAAE,IAAI;oBAChB,SAAS;iBACZ,CAAC;YACN,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,OAAO,CAAC,IAAI,CAAC,+DAA+D,QAAQ,EAAE,CAAC,CAAC;gBACxF,QAAQ,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE;oBAC1C,WAAW,EAAE,IAAI;oBACjB,iBAAiB,EAAE,IAAI;oBACvB,QAAQ,EAAE,iBAAiB;iBAC9B,CAAC,CAAC;YACP,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;YAC5C,QAAQ,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,IAAI,KAAK,GAAiB,IAAI,CAAC;QAC/B,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;YAChC,KAAK,GAAG,IAAI,CAAC;QACjB,CAAC;aAAM,CAAC;YACJ,KAAK,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE7C,eAAe;YACf,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC;gBACD,MAAM,UAAU,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAClD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,UAAU,GAAG,KAAK,CAAC;gBACvB,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBAClD,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;wBACrB,IAAI,CAAC;4BACD,MAAM,UAAU,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;4BAC/C,UAAU,GAAG,IAAI,CAAC;wBACtB,CAAC;wBAAC,OAAO,EAAE,EAAE,CAAC;4BACV,OAAO,CAAC,IAAI,CAAC,kCAAkC,OAAO,GAAG,CAAC,CAAC;wBAC/D,CAAC;oBACL,CAAC;gBACL,CAAC;gBACD,IAAI,CAAC,UAAU,EAAE,CAAC;oBACd,IAAI,CAAC;wBAAC,MAAM,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAAC,CAAC;oBAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;gBACvD,CAAC;YACL,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAyB;YACjC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe;YAC1C,WAAW,EAAE,QAAQ,CAAC,QAAQ;YAC9B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,QAAQ,EAAE,QAAQ;SACrB,CAAC;QAEF,IAAI,QAAQ,CAAC,YAAY,KAAK,SAAS;YAAE,MAAM,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC;QACrF,IAAI,QAAQ,CAAC,oBAAoB;YAAE,MAAM,CAAC,oBAAoB,GAAG,QAAQ,CAAC,oBAAoB,CAAC;QAC/F,IAAI,YAAY,IAAI,aAAa;YAAE,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;QAE9D,OAAO,MAAM,CAAC;IAElB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,oBAAoB,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAaD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,UAAkB,EAClB,UAA0B,EAC1B,UAKI,EAAE;IAEN,MAAM,EACF,SAAS,GAAG,KAAK,EACjB,kBAAkB,GAAG,KAAK,EAC1B,iBAAiB,GAAG,yBAAyB,EAC7C,YAAY,GAAG,KAAK,EACvB,GAAG,OAAO,CAAC;IAEZ,wBAAwB;IACxB,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1C,MAAM,OAAO,GAA2B,EAAE,CAAC;IAE3C,6BAA6B;IAC7B,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,QAAQ,GAAG,YAAY,CAAC;IAE5B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;QACjB,KAAK,GAAG,CAAC,YAAY,CAAC,CAAC;QACvB,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC1C,CAAC;SAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;QAC7B,KAAK,GAAG,MAAM,SAAS,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QACjD,QAAQ,GAAG,YAAY,CAAC;IAC5B,CAAC;SAAM,CAAC;QACJ,MAAM,IAAI,KAAK,CAAC,QAAQ,UAAU,6BAA6B,CAAC,CAAC;IACrE,CAAC;IAED,oBAAoB;IACpB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/B,uEAAuE;QACvE,4CAA4C;QAC5C,IAAI,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,oBAAoB;YACpB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC1C,mDAAmD;YACnD,mDAAmD;YACnD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/B,IAAI,KAAK,GAAG,QAAQ;gBAAE,QAAQ,GAAG,KAAK,CAAC;QAC3C,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAkB;QAC3B,UAAU,EAAE,KAAK,CAAC,MAAM;QACxB,gBAAgB,EAAE,UAAU,CAAC,IAAI;QACjC,eAAe,EAAE,QAAQ;KAC5B,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;IAEtD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,UAAU,EAAE;YACvD,gBAAgB,EAAE,kBAAkB;YACpC,iBAAiB;YACjB,YAAY;YACZ,QAAQ;SACX,CAAC,CAAC;QACH,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAChC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "mcard-js",
-    "version": "2.1.12",
+    "version": "2.1.13",
     "description": "MCard - Content-addressable storage with cryptographic hashing, handle resolution, and vector search for Node.js and browsers",
     "type": "module",
     "main": "dist/index.js",
@@ -43,6 +43,7 @@
         "@grafana/faro-web-sdk": "^2.0.2",
         "@grafana/faro-web-tracing": "^2.0.2",
         "idb": "^7.1.1",
+        "mlly": "^1.8.0",
         "sql.js": "^1.13.0",
         "yaml": "^2.8.2"
     },