mcard-js 2.1.47 → 2.1.49

package/dist/chunk-HWBEGVEN.js

@@ -0,0 +1,364 @@
+import {
+  MAX_FILE_SIZE,
+  READ_TIMEOUT_MS
+} from "./chunk-J3IDWMDN.js";
+import {
+  ContentTypeInterpreter,
+  MCard
+} from "./chunk-GGQCF7ZK.js";
+import {
+  __export
+} from "./chunk-PNKVD2UK.js";
+
+// src/Loader.ts
+var Loader_exports = {};
+__export(Loader_exports, {
+  loadFileToCollection: () => loadFileToCollection,
+  processAndStoreFile: () => processAndStoreFile
+});
+import * as fs2 from "fs/promises";
+import * as path2 from "path";
+
+// src/FileIO.ts
+var FileIO_exports = {};
+__export(FileIO_exports, {
+  isProblematicFile: () => isProblematicFile,
+  listFiles: () => listFiles,
+  processFileContent: () => processFileContent,
+  readFileSafely: () => readFileSafely,
+  streamReadNormalizedText: () => streamReadNormalizedText
+});
+import * as crypto from "crypto";
+import * as fs from "fs/promises";
+import * as path from "path";
+async function streamReadNormalizedText(filePath, options) {
+  const { byteCap, wrapWidth } = options;
+  const sha = crypto.createHash("sha256");
+  let totalSize = 0;
+  let producedText = "";
+  let currentLen = 0;
+  const handle = await fs.open(filePath, "r");
+  try {
+    const buffer = new Uint8Array(8192);
+    let remaining = byteCap;
+    const decoder = new TextDecoder("utf-8", { fatal: false });
+    let position = 0;
+    while (remaining > 0) {
+      const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), position);
+      if (bytesRead === 0) break;
+      position += bytesRead;
+      const chunk = buffer.subarray(0, bytesRead);
+      sha.update(chunk);
+      totalSize += bytesRead;
+      remaining -= bytesRead;
+      const s2 = decoder.decode(chunk, { stream: true });
+      for (const ch of s2) {
+        if (ch === "\r") continue;
+        producedText += ch;
+        if (ch === "\n") {
+          currentLen = 0;
+        } else {
+          currentLen++;
+          if (wrapWidth > 0 && currentLen >= wrapWidth) {
+            producedText += "\n";
+            currentLen = 0;
+          }
+        }
+      }
+    }
+    const s = decoder.decode();
+    for (const ch of s) {
+      if (ch === "\r") continue;
+      producedText += ch;
+      if (ch === "\n") {
+        currentLen = 0;
+      } else {
+        currentLen++;
+        if (wrapWidth > 0 && currentLen >= wrapWidth) {
+          producedText += "\n";
+          currentLen = 0;
+        }
+      }
+    }
+  } finally {
+    await handle.close();
+  }
+  return {
+    text: producedText,
+    originalSize: totalSize,
+    originalSha256Prefix: sha.digest("hex").substring(0, 16)
+  };
+}
+var MAX_FILE_SIZE2 = MAX_FILE_SIZE;
+var READ_TIMEOUT_MS2 = READ_TIMEOUT_MS;
+async function isProblematicFile(filePath) {
+  try {
+    const stats = await fs.stat(filePath);
+    if (stats.size === 0) return false;
+    if (path.basename(filePath).startsWith(".")) return true;
+    if (stats.size > MAX_FILE_SIZE2) return true;
+    const ext = path.extname(filePath);
+    const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(ext);
+    if (isKnownType && stats.size > 1024 * 1024) return true;
+    const handle = await fs.open(filePath, "r");
+    try {
+      const buffer = new Uint8Array(32 * 1024);
+      const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+      const sample = buffer.subarray(0, bytesRead);
+      if (ContentTypeInterpreter.isUnstructuredBinary(sample)) return true;
+      if (ContentTypeInterpreter.hasPathologicalLines(sample, isKnownType)) return true;
+    } finally {
+      await handle.close();
+    }
+    return false;
+  } catch {
+    return true;
+  }
+}
+async function readFileSafely(filePath, options = {}) {
+  const stats = await fs.stat(filePath);
+  if (stats.size > MAX_FILE_SIZE2) throw new Error(`File too large: ${stats.size}`);
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), READ_TIMEOUT_MS2);
+  try {
+    const handle = await fs.open(filePath, "r");
+    try {
+      const buffer = new Uint8Array(stats.size);
+      await handle.read(buffer, 0, stats.size, 0);
+      return buffer;
+    } finally {
+      await handle.close();
+    }
+  } catch (e) {
+    const error = e;
+    if (error.name === "AbortError") throw new Error(`Read timeout for ${filePath}`);
+    throw e;
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+async function listFiles(dirPath, recursive = false) {
+  let files = [];
+  try {
+    const entries = await fs.readdir(dirPath, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = path.join(dirPath, entry.name);
+      if (entry.name.startsWith(".")) continue;
+      if (entry.isDirectory()) {
+        if (recursive) {
+          files = files.concat(await listFiles(fullPath, true));
+        }
+      } else if (entry.isFile()) {
+        if (!await isProblematicFile(fullPath)) {
+          files.push(fullPath);
+        }
+      }
+    }
+  } catch (e) {
+    console.warn(`Error listing directory ${dirPath}:`, e);
+  }
+  return files;
+}
+async function processFileContent(filePath, options = {}) {
+  const rawContent = await readFileSafely(filePath, { allowPathological: options.allowPathological, maxBytes: options.maxBytes });
+  const sample = rawContent.subarray(0, 1024 * 1024);
+  const detection = ContentTypeInterpreter.detectContentType(sample, path.extname(filePath));
+  let isBinary = ContentTypeInterpreter.isBinaryContent(sample, detection.mimeType);
+  if (options.forceBinary) isBinary = true;
+  let content = rawContent;
+  if (!isBinary) {
+    try {
+      content = new TextDecoder("utf-8", { fatal: true }).decode(rawContent);
+    } catch {
+      content = new TextDecoder("utf-8", { fatal: false }).decode(rawContent);
+    }
+  }
+  return {
+    content,
+    filename: path.basename(filePath),
+    mimeType: detection.mimeType,
+    extension: detection.extension,
+    isBinary,
+    size: rawContent.length
+  };
+}
+
+// src/Loader.ts
+var DEFAULT_MAX_PROBLEM_BYTES = 2 * 1024 * 1024;
+var WRAP_WIDTH_KNOWN = 1e3;
+var WRAP_WIDTH_DEFAULT = 80;
+var Logger = {
+  info: (...args) => console.log("[Loader]", ...args),
+  warn: (...args) => console.warn("[Loader]", ...args),
+  error: (...args) => console.error("[Loader]", ...args),
+  debug: (...args) => {
+    if (process.env.DEBUG) console.log("[Loader]", ...args);
+  }
+};
+async function processAndStoreFile(filePath, collection, options = {}) {
+  const {
+    allowProblematic = false,
+    maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES,
+    metadataOnly = false,
+    rootPath
+  } = options;
+  try {
+    let fileInfo;
+    if (await isProblematicFile(filePath)) {
+      if (!allowProblematic) {
+        Logger.warn(`Skipping problematic file: ${filePath}`);
+        return null;
+      }
+      const extension = path2.extname(filePath).toLowerCase();
+      const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(extension);
+      const wrapWidth = isKnownType ? WRAP_WIDTH_KNOWN : WRAP_WIDTH_DEFAULT;
+      Logger.warn(`Problematic file detected, processing as safe text: ${filePath}`);
+      try {
+        const streamed = await streamReadNormalizedText(filePath, {
+          byteCap: maxBytesOnProblem,
+          wrapWidth
+        });
+        fileInfo = {
+          content: streamed.text,
+          filename: path2.basename(filePath),
+          mimeType: "text/plain",
+          extension,
+          isBinary: false,
+          size: streamed.text.length,
+          originalSize: streamed.originalSize,
+          originalSha256Prefix: streamed.originalSha256Prefix,
+          normalized: true,
+          wrapWidth
+        };
+      } catch (e) {
+        Logger.warn(`Safe text processing failed, falling back to capped binary: ${filePath}`, e);
+        fileInfo = await processFileContent(filePath, {
+          forceBinary: true,
+          allowPathological: true,
+          maxBytes: maxBytesOnProblem
+        });
+      }
+    } else {
+      Logger.info(`Processing file: ${filePath}`);
+      fileInfo = await processFileContent(filePath);
+    }
+    if (!fileInfo) return null;
+    const content = fileInfo.content;
+    if (!content || typeof content === "string" && content.length === 0 || content instanceof Uint8Array && content.length === 0) {
+      Logger.debug(`Skipping empty file: ${filePath} (empty files cannot be stored as MCards)`);
+      return {
+        hash: "",
+        contentType: fileInfo.mimeType,
+        isBinary: fileInfo.isBinary,
+        filename: fileInfo.filename,
+        size: 0,
+        filePath
+      };
+    }
+    let mcard = null;
+    const isProblematic = await isProblematicFile(filePath);
+    if (metadataOnly && isProblematic) {
+      mcard = null;
+    } else {
+      mcard = await MCard.create(fileInfo.content);
+      const handle = path2.basename(filePath);
+      try {
+        await collection.addWithHandle(mcard, handle);
+      } catch (e) {
+        let registered = false;
+        if (rootPath) {
+          const relPath = path2.relative(rootPath, filePath);
+          if (relPath !== handle) {
+            try {
+              await collection.addWithHandle(mcard, relPath);
+              registered = true;
+            } catch (e2) {
+              Logger.debug(
+                `Handle name '${handle}' already in use (common for files like README.md, LICENSE). MCard stored successfully with hash ${mcard.hash.slice(0, 8)}... (accessible by hash, not by handle)`
+              );
+            }
+          }
+        }
+        if (!registered) {
+          try {
+            await collection.add(mcard);
+          } catch (e3) {
+            Logger.warn(`Hash fallback also failed for ${handle}:`, e3);
+          }
+        }
+      }
+    }
+    const result = {
+      hash: mcard ? mcard.hash : "METADATA_ONLY",
+      contentType: fileInfo.mimeType,
+      isBinary: fileInfo.isBinary,
+      filename: fileInfo.filename,
+      size: fileInfo.size,
+      filePath
+    };
+    if (fileInfo.originalSize !== void 0) result.originalSize = fileInfo.originalSize;
+    if (fileInfo.originalSha256Prefix) result.originalSha256Prefix = fileInfo.originalSha256Prefix;
+    if (metadataOnly && isProblematic) result.metadataOnly = true;
+    return result;
+  } catch (e) {
+    Logger.error(`Error processing ${filePath}:`, e);
+    return null;
+  }
+}
+async function loadFileToCollection(targetPath, collection, options = {}) {
+  const {
+    recursive = false,
+    includeProblematic = false,
+    maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES,
+    metadataOnly = false
+  } = options;
+  const resolvedPath = path2.resolve(targetPath);
+  const stats = await fs2.stat(resolvedPath);
+  const results = [];
+  let files = [];
+  let rootPath = resolvedPath;
+  if (stats.isFile()) {
+    files = [resolvedPath];
+    rootPath = path2.dirname(resolvedPath);
+  } else if (stats.isDirectory()) {
+    files = await listFiles(resolvedPath, recursive);
+    rootPath = resolvedPath;
+  } else {
+    throw new Error(`Path ${targetPath} is not a file or directory`);
+  }
+  const uniqueDirs = /* @__PURE__ */ new Set();
+  let maxDepth = 0;
+  for (const file of files) {
+    const dir = path2.dirname(file);
+    if (dir.startsWith(rootPath)) {
+      uniqueDirs.add(dir);
+      const rel = path2.relative(rootPath, file);
+      const parts = rel.split(path2.sep);
+      const depth = parts.length - 1;
+      if (depth > maxDepth) maxDepth = depth;
+    }
+  }
+  const metrics = {
+    filesCount: files.length,
+    directoriesCount: uniqueDirs.size,
+    directoryLevels: maxDepth
+  };
+  Logger.info(`About to process ${files.length} files`);
+  for (const file of files) {
+    const result = await processAndStoreFile(file, collection, {
+      allowProblematic: includeProblematic,
+      maxBytesOnProblem,
+      metadataOnly,
+      rootPath
+    });
+    if (result) results.push(result);
+  }
+  return { metrics, results };
+}
+
+export {
+  FileIO_exports,
+  processAndStoreFile,
+  loadFileToCollection,
+  Loader_exports
+};

package/dist/chunk-ISY5LYLF.js

@@ -0,0 +1,217 @@
+import {
+  AbstractSqlEngine,
+  initCoreSchemas
+} from "./chunk-FPW33LMV.js";
+import {
+  createPage
+} from "./chunk-3EIBJPNF.js";
+import {
+  Handle_exports,
+  init_Handle
+} from "./chunk-ADV52544.js";
+import {
+  DEFAULT_PAGE_SIZE,
+  SQLITE_BUSY_TIMEOUT_MS
+} from "./chunk-J3IDWMDN.js";
+import {
+  MCard
+} from "./chunk-GGQCF7ZK.js";
+import {
+  __toCommonJS
+} from "./chunk-PNKVD2UK.js";
+
+// src/storage/engines/SqliteNodeEngine.ts
+var SqliteNodeEngine = class _SqliteNodeEngine extends AbstractSqlEngine {
+  db;
+  dbPath;
+  /**
+   * Convert a database row into an MCard instance.
+   */
+  rowToCard(row) {
+    const rawContent = row.content;
+    const content = rawContent instanceof Buffer ? new Uint8Array(rawContent) : new Uint8Array(rawContent);
+    return MCard.fromData(content, String(row.hash), String(row.g_time));
+  }
+  /**
+   * Create a new SqliteNodeEngine via async factory.
+   * @param dbPath Path to database file, or ':memory:' for in-memory database
+   */
+  static async create(dbPath = ":memory:") {
+    const engine = new _SqliteNodeEngine(dbPath);
+    await engine.init();
+    return engine;
+  }
+  constructor(dbPath) {
+    super();
+    this.dbPath = dbPath;
+  }
+  async init() {
+    try {
+      const mod = await import("better-sqlite3");
+      const Database = mod.default || mod;
+      this.db = new Database(this.dbPath);
+      this.setupDatabase();
+    } catch (e) {
+      console.error(`SqliteNodeEngine: Failed to load better-sqlite3: ${e.message}`);
+      throw e;
+    }
+  }
+  /**
+   * Initialize database schema
+   */
+  setupDatabase() {
+    this.db.pragma("journal_mode = WAL");
+    this.db.pragma(`busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);
+    this.db.pragma("synchronous = NORMAL");
+    initCoreSchemas(this.db);
+  }
+  // ======================================================================
+  // AbstractSqlEngine primitives
+  // ======================================================================
+  async queryRows(sql, ...params) {
+    const stmt = this.db.prepare(sql);
+    return stmt.all(...params);
+  }
+  async execSql(sql, ...params) {
+    if (params.length === 0) {
+      this.db.exec(sql);
+      return 0;
+    }
+    const stmt = this.db.prepare(sql);
+    const result = stmt.run(...params);
+    return result.changes;
+  }
+  /**
+   * Close the database connection
+   */
+  close() {
+    this.db.close();
+  }
+  /**
+   * Get the database path
+   */
+  getDbPath() {
+    return this.dbPath;
+  }
+  // =========== Card Operations (overrides to use Buffer.from for better-sqlite3) ===========
+  async add(card) {
+    const stmt = this.db.prepare(
+      "INSERT OR REPLACE INTO card (hash, content, g_time) VALUES (?, ?, ?)"
+    );
+    stmt.run(card.hash, Buffer.from(card.content), card.g_time);
+    return card.hash;
+  }
+  // =========== Sync card operations (for performance-critical callers) ===========
+  addSync(card) {
+    const stmt = this.db.prepare(
+      "INSERT OR REPLACE INTO card (hash, content, g_time) VALUES (?, ?, ?)"
+    );
+    stmt.run(card.hash, Buffer.from(card.content), card.g_time);
+    return card.hash;
+  }
+  getSync(hash) {
+    const stmt = this.db.prepare("SELECT hash, content, g_time FROM card WHERE hash = ?");
+    const row = stmt.get(hash);
+    if (!row) return null;
+    return this.rowToCard(row);
+  }
+  deleteSync(hash) {
+    const stmt = this.db.prepare("DELETE FROM card WHERE hash = ?");
+    const result = stmt.run(hash);
+    return result.changes > 0;
+  }
+  getPageSync(pageNumber = 1, pageSize = DEFAULT_PAGE_SIZE) {
+    if (pageNumber < 1) throw new Error("Page number must be >= 1");
+    if (pageSize < 1) throw new Error("Page size must be >= 1");
+    const totalItems = this.countSync();
+    const offset = (pageNumber - 1) * pageSize;
+    const stmt = this.db.prepare(
+      "SELECT hash, content, g_time FROM card ORDER BY g_time DESC LIMIT ? OFFSET ?"
+    );
+    const rows = stmt.all(pageSize, offset);
+    const items = rows.map((row) => this.rowToCard(row));
+    return createPage(items, totalItems, pageNumber, pageSize);
+  }
+  countSync() {
+    const stmt = this.db.prepare("SELECT COUNT(*) as cnt FROM card");
+    const row = stmt.get();
+    return row.cnt;
+  }
+  clearSync() {
+    this.db.exec("DELETE FROM handle_history");
+    this.db.exec("DELETE FROM handle_registry");
+    this.db.exec("DELETE FROM card");
+  }
+  // =========== Additional sync search operations ===========
+  searchByString(searchString, pageNumber = 1, pageSize = DEFAULT_PAGE_SIZE) {
+    if (pageNumber < 1) throw new Error("Page number must be >= 1");
+    if (pageSize < 1) throw new Error("Page size must be >= 1");
+    const pattern = `%${searchString}%`;
+    const offset = (pageNumber - 1) * pageSize;
+    const countStmt = this.db.prepare(`
+            SELECT COUNT(*) as cnt FROM card
+            WHERE hash LIKE ? OR g_time LIKE ? OR CAST(content AS TEXT) LIKE ?
+        `);
+    const countRow = countStmt.get(pattern, pattern, pattern);
+    const totalItems = countRow.cnt;
+    const stmt = this.db.prepare(`
+            SELECT hash, content, g_time FROM card
+            WHERE hash LIKE ? OR g_time LIKE ? OR CAST(content AS TEXT) LIKE ?
+            ORDER BY g_time DESC LIMIT ? OFFSET ?
+        `);
+    const rows = stmt.all(pattern, pattern, pattern, pageSize, offset);
+    const items = rows.map((row) => this.rowToCard(row));
+    return createPage(items, totalItems, pageNumber, pageSize);
+  }
+  searchByContent(searchPattern, pageNumber = 1, pageSize = DEFAULT_PAGE_SIZE) {
+    if (pageNumber < 1) throw new Error("Page number must be >= 1");
+    if (pageSize < 1) throw new Error("Page size must be >= 1");
+    if (!searchPattern || typeof searchPattern === "string" && searchPattern.length === 0) {
+      throw new Error("Search pattern cannot be empty");
+    }
+    const searchBytes = typeof searchPattern === "string" ? Buffer.from(searchPattern, "utf-8") : Buffer.from(searchPattern);
+    const offset = (pageNumber - 1) * pageSize;
+    const countStmt = this.db.prepare("SELECT COUNT(*) as cnt FROM card WHERE INSTR(content, ?) > 0");
+    const countRow = countStmt.get(searchBytes);
+    const totalItems = countRow.cnt;
+    const stmt = this.db.prepare(`
+            SELECT hash, content, g_time FROM card
+            WHERE INSTR(content, ?) > 0
+            ORDER BY g_time DESC LIMIT ? OFFSET ?
+        `);
+    const rows = stmt.all(searchBytes, pageSize, offset);
+    const items = rows.map((row) => this.rowToCard(row));
+    return createPage(items, totalItems, pageNumber, pageSize);
+  }
+  getAllCards() {
+    const stmt = this.db.prepare("SELECT hash, content, g_time FROM card ORDER BY g_time DESC");
+    const rows = stmt.all();
+    const items = rows.map((row) => this.rowToCard(row));
+    return createPage(items, items.length, 1, items.length || 1);
+  }
+  // =========== Unique sync method (not in AbstractSqlEngine) ===========
+  deleteHistoryEntrySync(handle, previousHash) {
+    const { validateHandle } = (init_Handle(), __toCommonJS(Handle_exports));
+    const normalized = validateHandle(handle);
+    const performStitch = this.db.transaction(() => {
+      const rOut = this.db.prepare(
+        "SELECT id, previous_hash FROM handle_history WHERE handle = ? AND previous_hash = ? ORDER BY id DESC LIMIT 1"
+      ).get(normalized, previousHash);
+      if (!rOut) return;
+      const rIn = this.db.prepare(
+        "SELECT id, previous_hash FROM handle_history WHERE handle = ? AND id < ? ORDER BY id DESC LIMIT 1"
+      ).get(normalized, rOut.id);
+      if (rIn) {
+        this.db.prepare("UPDATE handle_history SET previous_hash = ? WHERE id = ?").run(rIn.previous_hash, rOut.id);
+        this.db.prepare("DELETE FROM handle_history WHERE id = ?").run(rIn.id);
+      } else {
+        this.db.prepare("DELETE FROM handle_history WHERE id = ?").run(rOut.id);
+      }
+    });
+    performStitch();
+  }
+};
+
+export {
+  SqliteNodeEngine
+};