mcard-js 2.1.41 → 2.1.43

package/dist/chunk-W2XGNMYB.js

@@ -1,355 +0,0 @@
-import {
-  ContentTypeInterpreter,
-  MCard
-} from "./chunk-4UIWXZHY.js";
-import {
-  __export
-} from "./chunk-MLKGABMK.js";
-
-// src/Loader.ts
-var Loader_exports = {};
-__export(Loader_exports, {
-  loadFileToCollection: () => loadFileToCollection,
-  processAndStoreFile: () => processAndStoreFile
-});
-import * as fs2 from "fs/promises";
-import * as path2 from "path";
-
-// src/FileIO.ts
-var FileIO_exports = {};
-__export(FileIO_exports, {
-  isProblematicFile: () => isProblematicFile,
-  listFiles: () => listFiles,
-  processFileContent: () => processFileContent,
-  readFileSafely: () => readFileSafely,
-  streamReadNormalizedText: () => streamReadNormalizedText
-});
-import * as crypto from "crypto";
-import * as fs from "fs/promises";
-import * as path from "path";
-async function streamReadNormalizedText(filePath, options) {
-  const { byteCap, wrapWidth } = options;
-  const sha = crypto.createHash("sha256");
-  let totalSize = 0;
-  let producedText = "";
-  let currentLen = 0;
-  const handle = await fs.open(filePath, "r");
-  try {
-    const buffer = new Uint8Array(8192);
-    let remaining = byteCap;
-    const decoder = new TextDecoder("utf-8", { fatal: false });
-    let position = 0;
-    while (remaining > 0) {
-      const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), position);
-      if (bytesRead === 0) break;
-      position += bytesRead;
-      const chunk = buffer.subarray(0, bytesRead);
-      sha.update(chunk);
-      totalSize += bytesRead;
-      remaining -= bytesRead;
-      const s2 = decoder.decode(chunk, { stream: true });
-      for (const ch of s2) {
-        if (ch === "\r") continue;
-        producedText += ch;
-        if (ch === "\n") {
-          currentLen = 0;
-        } else {
-          currentLen++;
-          if (wrapWidth > 0 && currentLen >= wrapWidth) {
-            producedText += "\n";
-            currentLen = 0;
-          }
-        }
-      }
-    }
-    const s = decoder.decode();
-    for (const ch of s) {
-      if (ch === "\r") continue;
-      producedText += ch;
-      if (ch === "\n") {
-        currentLen = 0;
-      } else {
-        currentLen++;
-        if (wrapWidth > 0 && currentLen >= wrapWidth) {
-          producedText += "\n";
-          currentLen = 0;
-        }
-      }
-    }
-  } finally {
-    await handle.close();
-  }
-  return {
-    text: producedText,
-    originalSize: totalSize,
-    originalSha256Prefix: sha.digest("hex").substring(0, 16)
-  };
-}
-var MAX_FILE_SIZE = 50 * 1024 * 1024;
-var READ_TIMEOUT_MS = 5e3;
-async function isProblematicFile(filePath) {
-  try {
-    const stats = await fs.stat(filePath);
-    if (stats.size === 0) return false;
-    if (path.basename(filePath).startsWith(".")) return true;
-    if (stats.size > MAX_FILE_SIZE) return true;
-    const ext = path.extname(filePath);
-    const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(ext);
-    if (isKnownType && stats.size > 1024 * 1024) return true;
-    const handle = await fs.open(filePath, "r");
-    try {
-      const buffer = new Uint8Array(32 * 1024);
-      const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
-      const sample = buffer.subarray(0, bytesRead);
-      if (ContentTypeInterpreter.isUnstructuredBinary(sample)) return true;
-      if (ContentTypeInterpreter.hasPathologicalLines(sample, isKnownType)) return true;
-    } finally {
-      await handle.close();
-    }
-    return false;
-  } catch {
-    return true;
-  }
-}
-async function readFileSafely(filePath, options = {}) {
-  const stats = await fs.stat(filePath);
-  if (stats.size > MAX_FILE_SIZE) throw new Error(`File too large: ${stats.size}`);
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), READ_TIMEOUT_MS);
-  try {
-    const handle = await fs.open(filePath, "r");
-    try {
-      const buffer = new Uint8Array(stats.size);
-      await handle.read(buffer, 0, stats.size, 0);
-      return buffer;
-    } finally {
-      await handle.close();
-    }
-  } catch (e) {
-    const error = e;
-    if (error.name === "AbortError") throw new Error(`Read timeout for ${filePath}`);
-    throw e;
-  } finally {
-    clearTimeout(timeout);
-  }
-}
-async function listFiles(dirPath, recursive = false) {
-  let files = [];
-  try {
-    const entries = await fs.readdir(dirPath, { withFileTypes: true });
-    for (const entry of entries) {
-      const fullPath = path.join(dirPath, entry.name);
-      if (entry.name.startsWith(".")) continue;
-      if (entry.isDirectory()) {
-        if (recursive) {
-          files = files.concat(await listFiles(fullPath, true));
-        }
-      } else if (entry.isFile()) {
-        if (!await isProblematicFile(fullPath)) {
-          files.push(fullPath);
-        }
-      }
-    }
-  } catch (e) {
-    console.warn(`Error listing directory ${dirPath}:`, e);
-  }
-  return files;
-}
-async function processFileContent(filePath, options = {}) {
-  const rawContent = await readFileSafely(filePath, { allowPathological: options.allowPathological, maxBytes: options.maxBytes });
-  const sample = rawContent.subarray(0, 1024 * 1024);
-  const detection = ContentTypeInterpreter.detectContentType(sample, path.extname(filePath));
-  let isBinary = ContentTypeInterpreter.isBinaryContent(sample, detection.mimeType);
-  if (options.forceBinary) isBinary = true;
-  let content = rawContent;
-  if (!isBinary) {
-    try {
-      content = new TextDecoder("utf-8", { fatal: true }).decode(rawContent);
-    } catch {
-      content = new TextDecoder("utf-8", { fatal: false }).decode(rawContent);
-    }
-  }
-  return {
-    content,
-    filename: path.basename(filePath),
-    mimeType: detection.mimeType,
-    extension: detection.extension,
-    isBinary,
-    size: rawContent.length
-  };
-}
-
-// src/Loader.ts
-var DEFAULT_MAX_PROBLEM_BYTES = 2 * 1024 * 1024;
-var WRAP_WIDTH_KNOWN = 1e3;
-var WRAP_WIDTH_DEFAULT = 80;
-async function processAndStoreFile(filePath, collection, options = {}) {
-  const {
-    allowProblematic = false,
-    maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES,
-    metadataOnly = false,
-    rootPath
-  } = options;
-  try {
-    let fileInfo;
-    if (await isProblematicFile(filePath)) {
-      if (!allowProblematic) {
-        console.warn(`Skipping problematic file: ${filePath}`);
-        return null;
-      }
-      const extension = path2.extname(filePath).toLowerCase();
-      const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(extension);
-      const wrapWidth = isKnownType ? WRAP_WIDTH_KNOWN : WRAP_WIDTH_DEFAULT;
-      console.warn(`Problematic file detected, processing as safe text: ${filePath}`);
-      try {
-        const streamed = await streamReadNormalizedText(filePath, {
-          byteCap: maxBytesOnProblem,
-          wrapWidth
-        });
-        fileInfo = {
-          content: streamed.text,
-          filename: path2.basename(filePath),
-          mimeType: "text/plain",
-          extension,
-          isBinary: false,
-          size: streamed.text.length,
-          originalSize: streamed.originalSize,
-          originalSha256Prefix: streamed.originalSha256Prefix,
-          normalized: true,
-          wrapWidth
-        };
-      } catch (e) {
-        console.warn(`Safe text processing failed, falling back to capped binary: ${filePath}`);
-        fileInfo = await processFileContent(filePath, {
-          forceBinary: true,
-          allowPathological: true,
-          maxBytes: maxBytesOnProblem
-        });
-      }
-    } else {
-      console.log(`Processing file: ${filePath}`);
-      fileInfo = await processFileContent(filePath);
-    }
-    if (!fileInfo) return null;
-    const content = fileInfo.content;
-    if (!content || typeof content === "string" && content.length === 0 || content instanceof Uint8Array && content.length === 0) {
-      if (process.env.DEBUG) {
-        console.log(`Skipping empty file: ${filePath} (empty files cannot be stored as MCards)`);
-      }
-      return {
-        hash: "",
-        contentType: fileInfo.mimeType,
-        isBinary: fileInfo.isBinary,
-        filename: fileInfo.filename,
-        size: 0,
-        filePath
-      };
-    }
-    let mcard = null;
-    const isProblematic = await isProblematicFile(filePath);
-    if (metadataOnly && isProblematic) {
-      mcard = null;
-    } else {
-      mcard = await MCard.create(fileInfo.content);
-      const handle = path2.basename(filePath);
-      try {
-        await collection.addWithHandle(mcard, handle);
-      } catch (e) {
-        let registered = false;
-        if (rootPath) {
-          const relPath = path2.relative(rootPath, filePath);
-          if (relPath !== handle) {
-            try {
-              await collection.addWithHandle(mcard, relPath);
-              registered = true;
-            } catch (e2) {
-              if (process.env.DEBUG) {
-                console.log(
-                  `Handle name '${handle}' already in use (common for files like README.md, LICENSE). MCard stored successfully with hash ${mcard.hash.slice(0, 8)}... (accessible by hash, not by handle)`
-                );
-              }
-            }
-          }
-        }
-        if (!registered) {
-          try {
-            await collection.add(mcard);
-          } catch (e3) {
-          }
-        }
-      }
-    }
-    const result = {
-      hash: mcard ? mcard.hash : "METADATA_ONLY",
-      contentType: fileInfo.mimeType,
-      isBinary: fileInfo.isBinary,
-      filename: fileInfo.filename,
-      size: fileInfo.size,
-      filePath
-    };
-    if (fileInfo.originalSize !== void 0) result.originalSize = fileInfo.originalSize;
-    if (fileInfo.originalSha256Prefix) result.originalSha256Prefix = fileInfo.originalSha256Prefix;
-    if (metadataOnly && isProblematic) result.metadataOnly = true;
-    return result;
-  } catch (e) {
-    console.error(`Error processing ${filePath}:`, e);
-    return null;
-  }
-}
-async function loadFileToCollection(targetPath, collection, options = {}) {
-  const {
-    recursive = false,
-    includeProblematic = false,
-    maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES,
-    metadataOnly = false
-  } = options;
-  const resolvedPath = path2.resolve(targetPath);
-  const stats = await fs2.stat(resolvedPath);
-  const results = [];
-  let files = [];
-  let rootPath = resolvedPath;
-  if (stats.isFile()) {
-    files = [resolvedPath];
-    rootPath = path2.dirname(resolvedPath);
-  } else if (stats.isDirectory()) {
-    files = await listFiles(resolvedPath, recursive);
-    rootPath = resolvedPath;
-  } else {
-    throw new Error(`Path ${targetPath} is not a file or directory`);
-  }
-  const uniqueDirs = /* @__PURE__ */ new Set();
-  let maxDepth = 0;
-  for (const file of files) {
-    const dir = path2.dirname(file);
-    if (dir.startsWith(rootPath)) {
-      uniqueDirs.add(dir);
-      const rel = path2.relative(rootPath, file);
-      const parts = rel.split(path2.sep);
-      const depth = parts.length - 1;
-      if (depth > maxDepth) maxDepth = depth;
-    }
-  }
-  const metrics = {
-    filesCount: files.length,
-    directoriesCount: uniqueDirs.size,
-    directoryLevels: maxDepth
-  };
-  console.log(`About to process ${files.length} files`);
-  for (const file of files) {
-    const result = await processAndStoreFile(file, collection, {
-      allowProblematic: includeProblematic,
-      maxBytesOnProblem,
-      metadataOnly,
-      rootPath
-    });
-    if (result) results.push(result);
-  }
-  return { metrics, results };
-}
-
-export {
-  FileIO_exports,
-  processAndStoreFile,
-  loadFileToCollection,
-  Loader_exports
-};