mcard-js 2.1.49 → 2.1.51

package/dist/chunk-X72XIYSN.js

@@ -0,0 +1,364 @@
+import {
+  ContentTypeInterpreter,
+  MCard
+} from "./chunk-GGQCF7ZK.js";
+import {
+  MAX_FILE_SIZE,
+  READ_TIMEOUT_MS
+} from "./chunk-PKLONZCF.js";
+import {
+  __export
+} from "./chunk-PNKVD2UK.js";
+
+// src/Loader.ts
+var Loader_exports = {};
+__export(Loader_exports, {
+  loadFileToCollection: () => loadFileToCollection,
+  processAndStoreFile: () => processAndStoreFile
+});
+import * as fs2 from "fs/promises";
+import * as path2 from "path";
+
+// src/FileIO.ts
+var FileIO_exports = {};
+__export(FileIO_exports, {
+  isProblematicFile: () => isProblematicFile,
+  listFiles: () => listFiles,
+  processFileContent: () => processFileContent,
+  readFileSafely: () => readFileSafely,
+  streamReadNormalizedText: () => streamReadNormalizedText
+});
+import * as crypto from "crypto";
+import * as fs from "fs/promises";
+import * as path from "path";
+async function streamReadNormalizedText(filePath, options) {
+  const { byteCap, wrapWidth } = options;
+  const sha = crypto.createHash("sha256");
+  let totalSize = 0;
+  let producedText = "";
+  let currentLen = 0;
+  const handle = await fs.open(filePath, "r");
+  try {
+    const buffer = new Uint8Array(8192);
+    let remaining = byteCap;
+    const decoder = new TextDecoder("utf-8", { fatal: false });
+    let position = 0;
+    while (remaining > 0) {
+      const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), position);
+      if (bytesRead === 0) break;
+      position += bytesRead;
+      const chunk = buffer.subarray(0, bytesRead);
+      sha.update(chunk);
+      totalSize += bytesRead;
+      remaining -= bytesRead;
+      const s2 = decoder.decode(chunk, { stream: true });
+      for (const ch of s2) {
+        if (ch === "\r") continue;
+        producedText += ch;
+        if (ch === "\n") {
+          currentLen = 0;
+        } else {
+          currentLen++;
+          if (wrapWidth > 0 && currentLen >= wrapWidth) {
+            producedText += "\n";
+            currentLen = 0;
+          }
+        }
+      }
+    }
+    const s = decoder.decode();
+    for (const ch of s) {
+      if (ch === "\r") continue;
+      producedText += ch;
+      if (ch === "\n") {
+        currentLen = 0;
+      } else {
+        currentLen++;
+        if (wrapWidth > 0 && currentLen >= wrapWidth) {
+          producedText += "\n";
+          currentLen = 0;
+        }
+      }
+    }
+  } finally {
+    await handle.close();
+  }
+  return {
+    text: producedText,
+    originalSize: totalSize,
+    originalSha256Prefix: sha.digest("hex").substring(0, 16)
+  };
+}
+var MAX_FILE_SIZE2 = MAX_FILE_SIZE;
+var READ_TIMEOUT_MS2 = READ_TIMEOUT_MS;
+async function isProblematicFile(filePath) {
+  try {
+    const stats = await fs.stat(filePath);
+    if (stats.size === 0) return false;
+    if (path.basename(filePath).startsWith(".")) return true;
+    if (stats.size > MAX_FILE_SIZE2) return true;
+    const ext = path.extname(filePath);
+    const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(ext);
+    if (isKnownType && stats.size > 1024 * 1024) return true;
+    const handle = await fs.open(filePath, "r");
+    try {
+      const buffer = new Uint8Array(32 * 1024);
+      const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+      const sample = buffer.subarray(0, bytesRead);
+      if (ContentTypeInterpreter.isUnstructuredBinary(sample)) return true;
+      if (ContentTypeInterpreter.hasPathologicalLines(sample, isKnownType)) return true;
+    } finally {
+      await handle.close();
+    }
+    return false;
+  } catch {
+    return true;
+  }
+}
+async function readFileSafely(filePath, options = {}) {
+  const stats = await fs.stat(filePath);
+  if (stats.size > MAX_FILE_SIZE2) throw new Error(`File too large: ${stats.size}`);
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), READ_TIMEOUT_MS2);
+  try {
+    const handle = await fs.open(filePath, "r");
+    try {
+      const buffer = new Uint8Array(stats.size);
+      await handle.read(buffer, 0, stats.size, 0);
+      return buffer;
+    } finally {
+      await handle.close();
+    }
+  } catch (e) {
+    const error = e;
+    if (error.name === "AbortError") throw new Error(`Read timeout for ${filePath}`);
+    throw e;
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+async function listFiles(dirPath, recursive = false) {
+  let files = [];
+  try {
+    const entries = await fs.readdir(dirPath, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = path.join(dirPath, entry.name);
+      if (entry.name.startsWith(".")) continue;
+      if (entry.isDirectory()) {
+        if (recursive) {
+          files = files.concat(await listFiles(fullPath, true));
+        }
+      } else if (entry.isFile()) {
+        if (!await isProblematicFile(fullPath)) {
+          files.push(fullPath);
+        }
+      }
+    }
+  } catch (e) {
+    console.warn(`Error listing directory ${dirPath}:`, e);
+  }
+  return files;
+}
+async function processFileContent(filePath, options = {}) {
+  const rawContent = await readFileSafely(filePath, { allowPathological: options.allowPathological, maxBytes: options.maxBytes });
+  const sample = rawContent.subarray(0, 1024 * 1024);
+  const detection = ContentTypeInterpreter.detectContentType(sample, path.extname(filePath));
+  let isBinary = ContentTypeInterpreter.isBinaryContent(sample, detection.mimeType);
+  if (options.forceBinary) isBinary = true;
+  let content = rawContent;
+  if (!isBinary) {
+    try {
+      content = new TextDecoder("utf-8", { fatal: true }).decode(rawContent);
+    } catch {
+      content = new TextDecoder("utf-8", { fatal: false }).decode(rawContent);
+    }
+  }
+  return {
+    content,
+    filename: path.basename(filePath),
+    mimeType: detection.mimeType,
+    extension: detection.extension,
+    isBinary,
+    size: rawContent.length
+  };
+}
+
+// src/Loader.ts
+var DEFAULT_MAX_PROBLEM_BYTES = 2 * 1024 * 1024;
+var WRAP_WIDTH_KNOWN = 1e3;
+var WRAP_WIDTH_DEFAULT = 80;
+var Logger = {
+  info: (...args) => console.log("[Loader]", ...args),
+  warn: (...args) => console.warn("[Loader]", ...args),
+  error: (...args) => console.error("[Loader]", ...args),
+  debug: (...args) => {
+    if (process.env.DEBUG) console.log("[Loader]", ...args);
+  }
+};
+async function processAndStoreFile(filePath, collection, options = {}) {
+  const {
+    allowProblematic = false,
+    maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES,
+    metadataOnly = false,
+    rootPath
+  } = options;
+  try {
+    let fileInfo;
+    if (await isProblematicFile(filePath)) {
+      if (!allowProblematic) {
+        Logger.warn(`Skipping problematic file: ${filePath}`);
+        return null;
+      }
+      const extension = path2.extname(filePath).toLowerCase();
+      const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(extension);
+      const wrapWidth = isKnownType ? WRAP_WIDTH_KNOWN : WRAP_WIDTH_DEFAULT;
+      Logger.warn(`Problematic file detected, processing as safe text: ${filePath}`);
+      try {
+        const streamed = await streamReadNormalizedText(filePath, {
+          byteCap: maxBytesOnProblem,
+          wrapWidth
+        });
+        fileInfo = {
+          content: streamed.text,
+          filename: path2.basename(filePath),
+          mimeType: "text/plain",
+          extension,
+          isBinary: false,
+          size: streamed.text.length,
+          originalSize: streamed.originalSize,
+          originalSha256Prefix: streamed.originalSha256Prefix,
+          normalized: true,
+          wrapWidth
+        };
+      } catch (e) {
+        Logger.warn(`Safe text processing failed, falling back to capped binary: ${filePath}`, e);
+        fileInfo = await processFileContent(filePath, {
+          forceBinary: true,
+          allowPathological: true,
+          maxBytes: maxBytesOnProblem
+        });
+      }
+    } else {
+      Logger.info(`Processing file: ${filePath}`);
+      fileInfo = await processFileContent(filePath);
+    }
+    if (!fileInfo) return null;
+    const content = fileInfo.content;
+    if (!content || typeof content === "string" && content.length === 0 || content instanceof Uint8Array && content.length === 0) {
+      Logger.debug(`Skipping empty file: ${filePath} (empty files cannot be stored as MCards)`);
+      return {
+        hash: "",
+        contentType: fileInfo.mimeType,
+        isBinary: fileInfo.isBinary,
+        filename: fileInfo.filename,
+        size: 0,
+        filePath
+      };
+    }
+    let mcard = null;
+    const isProblematic = await isProblematicFile(filePath);
+    if (metadataOnly && isProblematic) {
+      mcard = null;
+    } else {
+      mcard = await MCard.create(fileInfo.content);
+      const handle = path2.basename(filePath);
+      try {
+        await collection.addWithHandle(mcard, handle);
+      } catch (e) {
+        let registered = false;
+        if (rootPath) {
+          const relPath = path2.relative(rootPath, filePath);
+          if (relPath !== handle) {
+            try {
+              await collection.addWithHandle(mcard, relPath);
+              registered = true;
+            } catch (e2) {
+              Logger.debug(
+                `Handle name '${handle}' already in use (common for files like README.md, LICENSE). MCard stored successfully with hash ${mcard.hash.slice(0, 8)}... (accessible by hash, not by handle)`
+              );
+            }
+          }
+        }
+        if (!registered) {
+          try {
+            await collection.add(mcard);
+          } catch (e3) {
+            Logger.warn(`Hash fallback also failed for ${handle}:`, e3);
+          }
+        }
+      }
+    }
+    const result = {
+      hash: mcard ? mcard.hash : "METADATA_ONLY",
+      contentType: fileInfo.mimeType,
+      isBinary: fileInfo.isBinary,
+      filename: fileInfo.filename,
+      size: fileInfo.size,
+      filePath
+    };
+    if (fileInfo.originalSize !== void 0) result.originalSize = fileInfo.originalSize;
+    if (fileInfo.originalSha256Prefix) result.originalSha256Prefix = fileInfo.originalSha256Prefix;
+    if (metadataOnly && isProblematic) result.metadataOnly = true;
+    return result;
+  } catch (e) {
+    Logger.error(`Error processing ${filePath}:`, e);
+    return null;
+  }
+}
+async function loadFileToCollection(targetPath, collection, options = {}) {
+  const {
+    recursive = false,
+    includeProblematic = false,
+    maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES,
+    metadataOnly = false
+  } = options;
+  const resolvedPath = path2.resolve(targetPath);
+  const stats = await fs2.stat(resolvedPath);
+  const results = [];
+  let files = [];
+  let rootPath = resolvedPath;
+  if (stats.isFile()) {
+    files = [resolvedPath];
+    rootPath = path2.dirname(resolvedPath);
+  } else if (stats.isDirectory()) {
+    files = await listFiles(resolvedPath, recursive);
+    rootPath = resolvedPath;
+  } else {
+    throw new Error(`Path ${targetPath} is not a file or directory`);
+  }
+  const uniqueDirs = /* @__PURE__ */ new Set();
+  let maxDepth = 0;
+  for (const file of files) {
+    const dir = path2.dirname(file);
+    if (dir.startsWith(rootPath)) {
+      uniqueDirs.add(dir);
+      const rel = path2.relative(rootPath, file);
+      const parts = rel.split(path2.sep);
+      const depth = parts.length - 1;
+      if (depth > maxDepth) maxDepth = depth;
+    }
+  }
+  const metrics = {
+    filesCount: files.length,
+    directoriesCount: uniqueDirs.size,
+    directoryLevels: maxDepth
+  };
+  Logger.info(`About to process ${files.length} files`);
+  for (const file of files) {
+    const result = await processAndStoreFile(file, collection, {
+      allowProblematic: includeProblematic,
+      maxBytesOnProblem,
+      metadataOnly,
+      rootPath
+    });
+    if (result) results.push(result);
+  }
+  return { metrics, results };
+}
+
+export {
+  FileIO_exports,
+  processAndStoreFile,
+  loadFileToCollection,
+  Loader_exports
+};

package/dist/chunk-XETU7TV4.js

@@ -0,0 +1,112 @@
+import {
+  AbstractSqlEngine,
+  CORE_SCHEMAS
+} from "./chunk-GIKMCG4D.js";
+import {
+  MCard
+} from "./chunk-GGQCF7ZK.js";
+import {
+  DEFAULT_SQLJS_WASM_URL
+} from "./chunk-ETJWXHKZ.js";
+
+// src/storage/engines/SqliteWasmEngine.ts
+var SqliteWasmEngine = class extends AbstractSqlEngine {
+  db = null;
+  SQL = null;
+  /**
+   * Initialize the database
+   * @param wasmUrl URL to sql-wasm.wasm file (optional, defaults to CDN)
+   * @param existingData Optional existing database as Uint8Array
+   */
+  async init(wasmUrl, existingData) {
+    const initSqlJs = (await import("sql.js")).default;
+    const SQL = await initSqlJs({
+      locateFile: (file) => wasmUrl || `${DEFAULT_SQLJS_WASM_URL}${file}`
+    });
+    this.SQL = SQL;
+    this.db = existingData ? new SQL.Database(existingData) : new SQL.Database();
+    this.db.run(CORE_SCHEMAS.card);
+    this.db.run(CORE_SCHEMAS.handleRegistry);
+    this.db.run(CORE_SCHEMAS.handleHistory);
+    this.db.run(CORE_SCHEMAS.handleIndex);
+  }
+  ensureDb() {
+    if (!this.db) throw new Error("Database not initialized. Call init() first.");
+    return this.db;
+  }
+  // ======================================================================
+  // AbstractSqlEngine primitives
+  // ======================================================================
+  async queryRows(sql, ...params) {
+    const db = this.ensureDb();
+    const stmt = db.prepare(sql);
+    if (params.length > 0) stmt.bind(params);
+    const results = [];
+    while (stmt.step()) {
+      const row = stmt.getAsObject();
+      results.push(row);
+    }
+    stmt.free();
+    return results;
+  }
+  async execSql(sql, ...params) {
+    const db = this.ensureDb();
+    if (params.length === 0) {
+      db.run(sql);
+    } else {
+      db.run(sql, params);
+    }
+    return 0;
+  }
+  // ======================================================================
+  // Row → MCard conversion (sql.js returns Uint8Array for BLOBs)
+  // ======================================================================
+  rowToCard(row) {
+    const rawContent = row.content;
+    const content = rawContent instanceof Uint8Array ? rawContent : typeof rawContent === "string" ? new TextEncoder().encode(rawContent) : new Uint8Array(0);
+    return MCard.fromData(content, String(row.hash), String(row.g_time));
+  }
+  // ======================================================================
+  // sql.js-specific helpers
+  // ======================================================================
+  /**
+   * Export database as Uint8Array (for persistence)
+   */
+  export() {
+    return this.ensureDb().export();
+  }
+  /**
+   * Get raw sql.js Database for use with VectorStore adapter.
+   */
+  getRawDb() {
+    return this.ensureDb();
+  }
+  // =========== pruneHandleHistory override (needs count before delete) ===========
+  async pruneHandleHistory(handle, options = {}) {
+    const { validateHandle } = await import("./Handle-3N4QOA3U.js");
+    const db = this.ensureDb();
+    const normalized = validateHandle(handle);
+    if (options.deleteAll) {
+      const stmt = db.prepare("SELECT COUNT(*) FROM handle_history WHERE handle = ?");
+      stmt.bind([normalized]);
+      stmt.step();
+      const count = stmt.get()[0];
+      stmt.free();
+      db.run("DELETE FROM handle_history WHERE handle = ?", [normalized]);
+      return count;
+    } else if (options.olderThan) {
+      const stmt = db.prepare("SELECT COUNT(*) FROM handle_history WHERE handle = ? AND changed_at < ?");
+      stmt.bind([normalized, options.olderThan]);
+      stmt.step();
+      const count = stmt.get()[0];
+      stmt.free();
+      db.run("DELETE FROM handle_history WHERE handle = ? AND changed_at < ?", [normalized, options.olderThan]);
+      return count;
+    }
+    return 0;
+  }
+};
+
+export {
+  SqliteWasmEngine
+};