mcard-js 2.1.41 → 2.1.43

package/dist/chunk-AVXJOPUE.js

@@ -1,354 +0,0 @@
-import {
-  ContentTypeInterpreter,
-  MCard
-} from "./chunk-QPZ2HNSU.js";
-import {
-  __export
-} from "./chunk-MLKGABMK.js";
-
-// src/Loader.ts
-var Loader_exports = {};
-__export(Loader_exports, {
-  loadFileToCollection: () => loadFileToCollection,
-  processAndStoreFile: () => processAndStoreFile
-});
-import * as fs2 from "fs/promises";
-import * as path2 from "path";
-
-// src/FileIO.ts
-var FileIO_exports = {};
-__export(FileIO_exports, {
-  isProblematicFile: () => isProblematicFile,
-  listFiles: () => listFiles,
-  processFileContent: () => processFileContent,
-  readFileSafely: () => readFileSafely,
-  streamReadNormalizedText: () => streamReadNormalizedText
-});
-import * as crypto from "crypto";
-import * as fs from "fs/promises";
-import * as path from "path";
-async function streamReadNormalizedText(filePath, options) {
-  const { byteCap, wrapWidth } = options;
-  const sha = crypto.createHash("sha256");
-  let totalSize = 0;
-  let producedText = "";
-  let currentLen = 0;
-  const handle = await fs.open(filePath, "r");
-  try {
-    const buffer = new Uint8Array(8192);
-    let remaining = byteCap;
-    const decoder = new TextDecoder("utf-8", { fatal: false });
-    let position = 0;
-    while (remaining > 0) {
-      const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), position);
-      if (bytesRead === 0) break;
-      position += bytesRead;
-      const chunk = buffer.subarray(0, bytesRead);
-      sha.update(chunk);
-      totalSize += bytesRead;
-      remaining -= bytesRead;
-      const s2 = decoder.decode(chunk, { stream: true });
-      for (const ch of s2) {
-        if (ch === "\r") continue;
-        producedText += ch;
-        if (ch === "\n") {
-          currentLen = 0;
-        } else {
-          currentLen++;
-          if (wrapWidth > 0 && currentLen >= wrapWidth) {
-            producedText += "\n";
-            currentLen = 0;
-          }
-        }
-      }
-    }
-    const s = decoder.decode();
-    for (const ch of s) {
-      if (ch === "\r") continue;
-      producedText += ch;
-      if (ch === "\n") {
-        currentLen = 0;
-      } else {
-        currentLen++;
-        if (wrapWidth > 0 && currentLen >= wrapWidth) {
-          producedText += "\n";
-          currentLen = 0;
-        }
-      }
-    }
-  } finally {
-    await handle.close();
-  }
-  return {
-    text: producedText,
-    originalSize: totalSize,
-    originalSha256Prefix: sha.digest("hex").substring(0, 16)
-  };
-}
-var MAX_FILE_SIZE = 50 * 1024 * 1024;
-var READ_TIMEOUT_MS = 5e3;
-async function isProblematicFile(filePath) {
-  try {
-    const stats = await fs.stat(filePath);
-    if (stats.size === 0) return false;
-    if (path.basename(filePath).startsWith(".")) return true;
-    if (stats.size > MAX_FILE_SIZE) return true;
-    const ext = path.extname(filePath);
-    const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(ext);
-    if (isKnownType && stats.size > 1024 * 1024) return true;
-    const handle = await fs.open(filePath, "r");
-    try {
-      const buffer = new Uint8Array(32 * 1024);
-      const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
-      const sample = buffer.subarray(0, bytesRead);
-      if (ContentTypeInterpreter.isUnstructuredBinary(sample)) return true;
-      if (ContentTypeInterpreter.hasPathologicalLines(sample, isKnownType)) return true;
-    } finally {
-      await handle.close();
-    }
-    return false;
-  } catch {
-    return true;
-  }
-}
-async function readFileSafely(filePath, options = {}) {
-  const stats = await fs.stat(filePath);
-  if (stats.size > MAX_FILE_SIZE) throw new Error(`File too large: ${stats.size}`);
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), READ_TIMEOUT_MS);
-  try {
-    const handle = await fs.open(filePath, "r");
-    try {
-      const buffer = new Uint8Array(stats.size);
-      await handle.read(buffer, 0, stats.size, 0);
-      return buffer;
-    } finally {
-      await handle.close();
-    }
-  } catch (e) {
-    if (e.name === "AbortError") throw new Error(`Read timeout for ${filePath}`);
-    throw e;
-  } finally {
-    clearTimeout(timeout);
-  }
-}
-async function listFiles(dirPath, recursive = false) {
-  let files = [];
-  try {
-    const entries = await fs.readdir(dirPath, { withFileTypes: true });
-    for (const entry of entries) {
-      const fullPath = path.join(dirPath, entry.name);
-      if (entry.name.startsWith(".")) continue;
-      if (entry.isDirectory()) {
-        if (recursive) {
-          files = files.concat(await listFiles(fullPath, true));
-        }
-      } else if (entry.isFile()) {
-        if (!await isProblematicFile(fullPath)) {
-          files.push(fullPath);
-        }
-      }
-    }
-  } catch (e) {
-    console.warn(`Error listing directory ${dirPath}:`, e);
-  }
-  return files;
-}
-async function processFileContent(filePath, options = {}) {
-  const rawContent = await readFileSafely(filePath, { allowPathological: options.allowPathological, maxBytes: options.maxBytes });
-  const sample = rawContent.subarray(0, 1024 * 1024);
-  const detection = ContentTypeInterpreter.detectContentType(sample, path.extname(filePath));
-  let isBinary = ContentTypeInterpreter.isBinaryContent(sample, detection.mimeType);
-  if (options.forceBinary) isBinary = true;
-  let content = rawContent;
-  if (!isBinary) {
-    try {
-      content = new TextDecoder("utf-8", { fatal: true }).decode(rawContent);
-    } catch {
-      content = new TextDecoder("utf-8", { fatal: false }).decode(rawContent);
-    }
-  }
-  return {
-    content,
-    filename: path.basename(filePath),
-    mimeType: detection.mimeType,
-    extension: detection.extension,
-    isBinary,
-    size: rawContent.length
-  };
-}
-
-// src/Loader.ts
-var DEFAULT_MAX_PROBLEM_BYTES = 2 * 1024 * 1024;
-var WRAP_WIDTH_KNOWN = 1e3;
-var WRAP_WIDTH_DEFAULT = 80;
-async function processAndStoreFile(filePath, collection, options = {}) {
-  const {
-    allowProblematic = false,
-    maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES,
-    metadataOnly = false,
-    rootPath
-  } = options;
-  try {
-    let fileInfo;
-    if (await isProblematicFile(filePath)) {
-      if (!allowProblematic) {
-        console.warn(`Skipping problematic file: ${filePath}`);
-        return null;
-      }
-      const extension = path2.extname(filePath).toLowerCase();
-      const isKnownType = ContentTypeInterpreter.isKnownLongLineExtension(extension);
-      const wrapWidth = isKnownType ? WRAP_WIDTH_KNOWN : WRAP_WIDTH_DEFAULT;
-      console.warn(`Problematic file detected, processing as safe text: ${filePath}`);
-      try {
-        const streamed = await streamReadNormalizedText(filePath, {
-          byteCap: maxBytesOnProblem,
-          wrapWidth
-        });
-        fileInfo = {
-          content: streamed.text,
-          filename: path2.basename(filePath),
-          mimeType: "text/plain",
-          extension,
-          isBinary: false,
-          size: streamed.text.length,
-          originalSize: streamed.originalSize,
-          originalSha256Prefix: streamed.originalSha256Prefix,
-          normalized: true,
-          wrapWidth
-        };
-      } catch (e) {
-        console.warn(`Safe text processing failed, falling back to capped binary: ${filePath}`);
-        fileInfo = await processFileContent(filePath, {
-          forceBinary: true,
-          allowPathological: true,
-          maxBytes: maxBytesOnProblem
-        });
-      }
-    } else {
-      console.log(`Processing file: ${filePath}`);
-      fileInfo = await processFileContent(filePath);
-    }
-    if (!fileInfo) return null;
-    const content = fileInfo.content;
-    if (!content || typeof content === "string" && content.length === 0 || content instanceof Uint8Array && content.length === 0) {
-      if (process.env.DEBUG) {
-        console.log(`Skipping empty file: ${filePath} (empty files cannot be stored as MCards)`);
-      }
-      return {
-        hash: "",
-        contentType: fileInfo.mimeType,
-        isBinary: fileInfo.isBinary,
-        filename: fileInfo.filename,
-        size: 0,
-        filePath
-      };
-    }
-    let mcard = null;
-    const isProblematic = await isProblematicFile(filePath);
-    if (metadataOnly && isProblematic) {
-      mcard = null;
-    } else {
-      mcard = await MCard.create(fileInfo.content);
-      const handle = path2.basename(filePath);
-      try {
-        await collection.addWithHandle(mcard, handle);
-      } catch (e) {
-        let registered = false;
-        if (rootPath) {
-          const relPath = path2.relative(rootPath, filePath);
-          if (relPath !== handle) {
-            try {
-              await collection.addWithHandle(mcard, relPath);
-              registered = true;
-            } catch (e2) {
-              if (process.env.DEBUG) {
-                console.log(
-                  `Handle name '${handle}' already in use (common for files like README.md, LICENSE). MCard stored successfully with hash ${mcard.hash.slice(0, 8)}... (accessible by hash, not by handle)`
-                );
-              }
-            }
-          }
-        }
-        if (!registered) {
-          try {
-            await collection.add(mcard);
-          } catch (e3) {
-          }
-        }
-      }
-    }
-    const result = {
-      hash: mcard ? mcard.hash : "METADATA_ONLY",
-      contentType: fileInfo.mimeType,
-      isBinary: fileInfo.isBinary,
-      filename: fileInfo.filename,
-      size: fileInfo.size,
-      filePath
-    };
-    if (fileInfo.originalSize !== void 0) result.originalSize = fileInfo.originalSize;
-    if (fileInfo.originalSha256Prefix) result.originalSha256Prefix = fileInfo.originalSha256Prefix;
-    if (metadataOnly && isProblematic) result.metadataOnly = true;
-    return result;
-  } catch (e) {
-    console.error(`Error processing ${filePath}:`, e);
-    return null;
-  }
-}
-async function loadFileToCollection(targetPath, collection, options = {}) {
-  const {
-    recursive = false,
-    includeProblematic = false,
-    maxBytesOnProblem = DEFAULT_MAX_PROBLEM_BYTES,
-    metadataOnly = false
-  } = options;
-  const resolvedPath = path2.resolve(targetPath);
-  const stats = await fs2.stat(resolvedPath);
-  const results = [];
-  let files = [];
-  let rootPath = resolvedPath;
-  if (stats.isFile()) {
-    files = [resolvedPath];
-    rootPath = path2.dirname(resolvedPath);
-  } else if (stats.isDirectory()) {
-    files = await listFiles(resolvedPath, recursive);
-    rootPath = resolvedPath;
-  } else {
-    throw new Error(`Path ${targetPath} is not a file or directory`);
-  }
-  const uniqueDirs = /* @__PURE__ */ new Set();
-  let maxDepth = 0;
-  for (const file of files) {
-    const dir = path2.dirname(file);
-    if (dir.startsWith(rootPath)) {
-      uniqueDirs.add(dir);
-      const rel = path2.relative(rootPath, file);
-      const parts = rel.split(path2.sep);
-      const depth = parts.length - 1;
-      if (depth > maxDepth) maxDepth = depth;
-    }
-  }
-  const metrics = {
-    filesCount: files.length,
-    directoriesCount: uniqueDirs.size,
-    directoryLevels: maxDepth
-  };
-  console.log(`About to process ${files.length} files`);
-  for (const file of files) {
-    const result = await processAndStoreFile(file, collection, {
-      allowProblematic: includeProblematic,
-      maxBytesOnProblem,
-      metadataOnly,
-      rootPath
-    });
-    if (result) results.push(result);
-  }
-  return { metrics, results };
-}
-
-export {
-  FileIO_exports,
-  processAndStoreFile,
-  loadFileToCollection,
-  Loader_exports
-};

package/dist/chunk-B7VEU4S2.js

@@ -1,204 +0,0 @@
-import {
-  CORE_SCHEMAS,
-  validateHandle
-} from "./chunk-U3FWUPD3.js";
-import {
-  MCard
-} from "./chunk-QPZ2HNSU.js";
-
-// src/storage/SqliteWasmEngine.ts
-var SqliteWasmEngine = class {
-  db = null;
-  SQL = null;
-  /**
-   * Initialize the database
-   * @param wasmUrl URL to sql-wasm.wasm file (optional, defaults to CDN)
-   * @param existingData Optional existing database as Uint8Array
-   */
-  async init(wasmUrl, existingData) {
-    const initSqlJs = (await import("sql.js")).default;
-    const SQL = await initSqlJs({
-      locateFile: (file) => wasmUrl || `https://sql.js.org/dist/${file}`
-    });
-    this.SQL = SQL;
-    this.db = existingData ? new SQL.Database(existingData) : new SQL.Database();
-    this.db.run(CORE_SCHEMAS.card);
-    this.db.run(CORE_SCHEMAS.handleRegistry);
-    this.db.run(CORE_SCHEMAS.handleHistory);
-    this.db.run(CORE_SCHEMAS.handleIndex);
-  }
-  ensureDb() {
-    if (!this.db) throw new Error("Database not initialized. Call init() first.");
-    return this.db;
-  }
-  /**
-   * Export database as Uint8Array (for persistence)
-   */
-  export() {
-    return this.ensureDb().export();
-  }
-  /**
-   * Get raw sql.js Database for use with VectorStore adapter.
-   * 
-   * Example:
-   *   const vecStore = createWasmVectorStore(engine.getRawDb());
-   */
-  getRawDb() {
-    return this.ensureDb();
-  }
-  // =========== Card Operations ===========
-  async add(card) {
-    const db = this.ensureDb();
-    db.run(
-      "INSERT OR REPLACE INTO card (hash, content, g_time) VALUES (?, ?, ?)",
-      [card.hash, card.content, card.g_time]
-    );
-    return card.hash;
-  }
-  async get(hash) {
-    const db = this.ensureDb();
-    const result = db.exec(`SELECT hash, content, g_time FROM card WHERE hash = '${hash}'`);
-    if (result.length === 0 || result[0].values.length === 0) return null;
-    const [h, content, g_time] = result[0].values[0];
-    return MCard.fromData(new Uint8Array(content), h, g_time);
-  }
-  async delete(hash) {
-    this.ensureDb().run("DELETE FROM card WHERE hash = ?", [hash]);
-  }
-  async getPage(pageNumber, pageSize) {
-    const db = this.ensureDb();
-    const totalItems = await this.count();
-    const totalPages = Math.ceil(totalItems / pageSize);
-    const offset = (pageNumber - 1) * pageSize;
-    const result = db.exec(`SELECT hash, content, g_time FROM card LIMIT ${pageSize} OFFSET ${offset}`);
-    const items = (result[0]?.values || []).map(
-      ([h, content, g_time]) => MCard.fromData(new Uint8Array(content), h, g_time)
-    );
-    return {
-      items,
-      totalItems,
-      pageNumber,
-      pageSize,
-      totalPages,
-      hasNext: pageNumber < totalPages,
-      hasPrevious: pageNumber > 1
-    };
-  }
-  async count() {
-    const result = this.ensureDb().exec("SELECT COUNT(*) FROM card");
-    return result[0]?.values[0]?.[0] || 0;
-  }
-  async searchByHash(hashPrefix) {
-    const db = this.ensureDb();
-    const stmt = db.prepare("SELECT hash, content, g_time FROM card WHERE hash LIKE ?");
-    stmt.bind([`${hashPrefix}%`]);
-    const items = [];
-    while (stmt.step()) {
-      const row = stmt.get();
-      items.push(MCard.fromData(new Uint8Array(row[1]), row[0], row[2]));
-    }
-    stmt.free();
-    return items;
-  }
-  async search(query, pageNumber, pageSize) {
-    const db = this.ensureDb();
-    const offset = (pageNumber - 1) * pageSize;
-    const pattern = `%${query}%`;
-    let stmt = db.prepare("SELECT COUNT(*) FROM card WHERE CAST(content AS TEXT) LIKE ?");
-    stmt.bind([pattern]);
-    stmt.step();
-    const totalItems = stmt.get()[0];
-    stmt.free();
-    const totalPages = Math.ceil(totalItems / pageSize);
-    stmt = db.prepare("SELECT hash, content, g_time FROM card WHERE CAST(content AS TEXT) LIKE ? LIMIT ? OFFSET ?");
-    stmt.bind([pattern, pageSize, offset]);
-    const items = [];
-    while (stmt.step()) {
-      const row = stmt.get();
-      items.push(MCard.fromData(new Uint8Array(row[1]), row[0], row[2]));
-    }
-    stmt.free();
-    return {
-      items,
-      totalItems,
-      pageNumber,
-      pageSize,
-      totalPages,
-      hasNext: pageNumber < totalPages,
-      hasPrevious: pageNumber > 1
-    };
-  }
-  async getAll() {
-    const result = this.ensureDb().exec("SELECT hash, content, g_time FROM card");
-    return (result[0]?.values || []).map(
-      ([h, content, g_time]) => MCard.fromData(new Uint8Array(content), h, g_time)
-    );
-  }
-  async clear() {
-    const db = this.ensureDb();
-    db.run("DELETE FROM card");
-    db.run("DELETE FROM handle_registry");
-    db.run("DELETE FROM handle_history");
-  }
-  // =========== Handle Operations ===========
-  async registerHandle(handle, hash) {
-    const db = this.ensureDb();
-    const normalized = validateHandle(handle);
-    const existing = db.exec(`SELECT handle FROM handle_registry WHERE handle = '${normalized}'`);
-    if (existing.length > 0 && existing[0].values.length > 0) {
-      throw new Error(`Handle '${handle}' already exists.`);
-    }
-    const now = (/* @__PURE__ */ new Date()).toISOString();
-    db.run(
-      "INSERT INTO handle_registry (handle, current_hash, created_at, updated_at) VALUES (?, ?, ?, ?)",
-      [normalized, hash, now, now]
-    );
-  }
-  async resolveHandle(handle) {
-    const normalized = validateHandle(handle);
-    const result = this.ensureDb().exec(
-      `SELECT current_hash FROM handle_registry WHERE handle = '${normalized}'`
-    );
-    return result[0]?.values[0]?.[0] || null;
-  }
-  async getByHandle(handle) {
-    const hash = await this.resolveHandle(handle);
-    if (!hash) return null;
-    return this.get(hash);
-  }
-  async updateHandle(handle, newHash) {
-    const db = this.ensureDb();
-    const normalized = validateHandle(handle);
-    const existing = db.exec(
-      `SELECT current_hash FROM handle_registry WHERE handle = '${normalized}'`
-    );
-    if (existing.length === 0 || existing[0].values.length === 0) {
-      throw new Error(`Handle '${handle}' not found.`);
-    }
-    const previousHash = existing[0].values[0][0];
-    const now = (/* @__PURE__ */ new Date()).toISOString();
-    db.run(
-      "INSERT INTO handle_history (handle, previous_hash, changed_at) VALUES (?, ?, ?)",
-      [normalized, previousHash, now]
-    );
-    db.run(
-      "UPDATE handle_registry SET current_hash = ?, updated_at = ? WHERE handle = ?",
-      [newHash, now, normalized]
-    );
-    return previousHash;
-  }
-  async getHandleHistory(handle) {
-    const normalized = validateHandle(handle);
-    const result = this.ensureDb().exec(
-      `SELECT previous_hash, changed_at FROM handle_history WHERE handle = '${normalized}' ORDER BY id DESC`
-    );
-    return (result[0]?.values || []).map(([previousHash, changedAt]) => ({
-      previousHash,
-      changedAt
-    }));
-  }
-};
-
-export {
-  SqliteWasmEngine
-};