mm_scanner 1.0.0 → 1.0.1

package/lib/scanner.js

@@ -3,6 +3,23 @@
 const fs = require('fs');
 const path = require('path');
 
+/* 扫描结果缓存：跨实例复用同一目录的遍历结果 */
+const _scan_cache = new Map();
+/* 目录监听器：缓存键 -> { watcher, timer, pending } */
+const _watchers = new Map();
+/* 处于监听状态的实例：缓存键 -> Set<Scanner> */
+const _instances = new Map();
+/* 缓存条目上限，超出后整体清空 */
+const _CACHE_MAX = 200;
+/* 短时间窗口内直接复用，避免每次重复校验目录与文件的时间戳 */
+const _CACHE_TTL = 2000;
+/* 文件事件防抖窗口，合并一次写入产生的多条事件 */
+const _DEBOUNCE = 100;
+/* 内部同步方法名，仅供监听器调用 */
+const _SYNC = Symbol('mm_scanner.sync');
+/* 内部回调方法名，仅供监听器调用 */
+const _NOTIFY = Symbol('mm_scanner.notify');
+
 /* key 规范化：只统一分隔符、去首尾 /，不再剥扩展名 */
 function normalizeKey(p) {
   return String(p)
@@ -17,6 +34,551 @@ function normalizeExt(ext) {
   return e.startsWith('.') ? e : '.' + e;
 }
 
+/**
+ * 生成扫描缓存键
+ * @param {string} dir 目录绝对路径
+ * @param {Array<string>} extensions 扩展名数组
+ * @param {Array<string>} ignore 忽略目录数组
+ * @returns {string} 缓存键
+ * @private
+ */
+function _cacheKey(dir, extensions, ignore) {
+  return [dir, extensions.join(','), ignore.join(',')].join('\u0001');
+}
+
+/**
+ * 构造单个文件的条目元信息
+ * @param {string} base_dir 扫描根目录绝对路径
+ * @param {string} abs_path 文件绝对路径
+ * @returns {{ key: string, absPath: string, relPath: string, dir: string, name: string, ext: string }} 条目
+ * @private
+ */
+function _createItem(base_dir, abs_path) {
+  const rel_path = path.relative(base_dir, abs_path).replace(/\\/g, '/');
+  const ext = path.extname(rel_path).toLowerCase();
+  const dir_part = path.posix.dirname(rel_path);
+  return {
+    key: normalizeKey(rel_path),
+    absPath: abs_path,
+    relPath: rel_path,
+    dir: dir_part === '.' ? '' : dir_part,
+    name: path.basename(rel_path, ext),
+    ext,
+  };
+}
+
+/**
+ * 判断相对路径是否位于忽略目录内
+ * @param {string} rel_path 相对路径
+ * @param {Array<string>} ignore 忽略目录数组
+ * @returns {boolean} 命中忽略目录返回 true
+ * @private
+ */
+function _isIgnored(rel_path, ignore) {
+  const parts = rel_path.split('/');
+  for (let i = 0; i < parts.length - 1; i++) {
+    if (ignore.includes(parts[i])) return true;
+  }
+  return false;
+}
+
+/**
+ * 递归遍历目录并写入条目
+ * @param {string} dir 当前目录绝对路径
+ * @param {string} base_dir 扫描根目录绝对路径
+ * @param {{ extensions: string[], ignore: string[] }} config 扫描配置
+ * @param {Map<string, object>} files 条目容器
+ * @param {Array<string>} dirs 遍历过的目录，用于记录与缓存校验
+ * @returns {void}
+ * @private
+ */
+function _walkDir(dir, base_dir, config, files, dirs) {
+  dirs.push(dir);
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const full_path = path.join(dir, entry.name);
+
+    if (entry.isDirectory()) {
+      if (config.ignore.includes(entry.name)) continue;
+      _walkDir(full_path, base_dir, config, files, dirs);
+      continue;
+    }
+
+    if (!entry.isFile()) continue;
+
+    const item = _createItem(base_dir, full_path);
+    if (!config.extensions.includes(item.ext)) continue;
+    files.set(item.key, item);
+  }
+}
+
+/**
+ * 获取路径的修改时间
+ * @param {string} target 文件或目录路径
+ * @returns {number} 毫秒时间戳，读取失败返回 -1
+ * @private
+ */
+function _getMtime(target) {
+  try {
+    return fs.statSync(target).mtimeMs;
+  } catch {
+    return -1;
+  }
+}
+
+/**
+ * 校验缓存是否有效
+ * @description 目录时间戳变化表示增删条目，文件时间戳变化表示内容修改
+ * @param {object} cache 缓存项
+ * @returns {boolean} 全部未变化返回 true
+ * @private
+ */
+function _isCacheValid(cache) {
+  for (let i = 0; i < cache.dirs.length; i++) {
+    if (_getMtime(cache.dirs[i]) !== cache.dir_mtimes[i]) {
+      return false;
+    }
+  }
+  for (let i = 0; i < cache.items.length; i++) {
+    if (_getMtime(cache.items[i].absPath) !== cache.file_mtimes[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * 读取仍然有效的缓存
+ * @param {string} key 缓存键
+ * @returns {object|null} 缓存项，不存在或已失效返回 null
+ * @private
+ */
+function _getCache(key) {
+  let cache = _scan_cache.get(key);
+  if (!cache) {
+    return null;
+  }
+  if (cache.watched) {
+    return cache;
+  }
+  let now = Date.now();
+  if (now - cache.time <= _CACHE_TTL) {
+    return cache;
+  }
+  if (_isCacheValid(cache)) {
+    cache.time = now;
+    return cache;
+  }
+  _scan_cache.delete(key);
+  return null;
+}
+
+/**
+ * 读取缓存项，不做有效性校验
+ * @param {string} key 缓存键
+ * @returns {object|null} 缓存项，不存在返回 null
+ * @private
+ */
+function _readCache(key) {
+  return _scan_cache.get(key) || null;
+}
+
+/**
+ * 记录缓存对应的目录与文件时间戳
+ * @param {object} cache 缓存项
+ * @returns {void}
+ * @private
+ */
+function _captureMtimes(cache) {
+  cache.dir_mtimes = [];
+  for (let i = 0; i < cache.dirs.length; i++) {
+    cache.dir_mtimes.push(_getMtime(cache.dirs[i]));
+  }
+  cache.file_mtimes = [];
+  for (let i = 0; i < cache.items.length; i++) {
+    cache.file_mtimes.push(_getMtime(cache.items[i].absPath));
+  }
+  cache.time = Date.now();
+}
+
+/**
+ * 创建缓存项
+ * @param {{ dir: string, extensions: Array<string>, ignore: Array<string>, items: Array<object>, dirs: Array<string> }} options 缓存内容
+ * @param {boolean} watched 是否已开启监听
+ * @returns {object} 缓存项
+ * @private
+ */
+function _createCache(options, watched) {
+  const items = [];
+  for (let i = 0; i < options.items.length; i++) {
+    items.push({ ...options.items[i] });
+  }
+  const cache = {
+    dir: options.dir,
+    extensions: [...options.extensions],
+    ignore: [...options.ignore],
+    items,
+    dirs: [...options.dirs],
+    watched: !!watched,
+    dir_mtimes: [],
+    file_mtimes: [],
+    time: Date.now(),
+  };
+  if (!cache.watched) {
+    _captureMtimes(cache);
+  }
+  return cache;
+}
+
+/**
+ * 写入扫描缓存
+ * @description 已有监听器时保留监听标记，避免监听期间退化回时间戳校验
+ * @param {string} key 缓存键
+ * @param {{ dir: string, extensions: Array<string>, ignore: Array<string>, items: Array<object>, dirs: Array<string> }} options 缓存内容
+ * @returns {void}
+ * @private
+ */
+function _setCache(key, options) {
+  const old = _scan_cache.get(key);
+  const watched = !!(old && old.watched);
+  if (_scan_cache.size >= _CACHE_MAX && !_scan_cache.has(key)) {
+    _clearAll();
+  }
+  _scan_cache.set(key, _createCache(options, watched));
+}
+
+/**
+ * 向缓存中新增或更新一个文件条目
+ * @param {object} cache 缓存项
+ * @param {{ key: string, absPath: string, relPath: string, dir: string, name: string, ext: string }} item 文件条目
+ * @returns {boolean} 新增返回 true，覆盖已有条目返回 false
+ * @private
+ */
+function _addItem(cache, item) {
+  const index = cache.items.findIndex((one) => one.key === item.key);
+  if (index >= 0) {
+    cache.items[index] = item;
+    return false;
+  }
+  cache.items.push(item);
+  return true;
+}
+
+/**
+ * 从缓存中移除指定条目并记录变更
+ * @param {object} cache 缓存项
+ * @param {string} key 条目键
+ * @param {Array<object>} changes 变更收集器
+ * @returns {boolean} 是否移除成功
+ * @private
+ */
+function _delItem(cache, key, changes) {
+  const index = cache.items.findIndex((one) => one.key === key);
+  if (index < 0) return false;
+  const [item] = cache.items.splice(index, 1);
+  changes.push({ type: 'unlink', key, item: { ...item } });
+  return true;
+}
+
+/**
+ * 移除同目录同文件名但扩展名不同的条目
+ * @description 文件改名后扩展名不再受支持时，需要清掉旧条目
+ * @param {object} cache 缓存项
+ * @param {{ key: string, dir: string, name: string }} item 当前文件条目
+ * @param {Array<object>} changes 变更收集器
+ * @returns {void}
+ * @private
+ */
+function _delSameName(cache, item, changes) {
+  for (let i = cache.items.length - 1; i >= 0; i--) {
+    const one = cache.items[i];
+    if (one.key === item.key || one.dir !== item.dir || one.name !== item.name) continue;
+    _delItem(cache, one.key, changes);
+  }
+}
+
+/**
+ * 按文件系统事件更新单个路径
+ * @param {object} cache 缓存项
+ * @param {string} rel_path 相对路径
+ * @param {Array<object>} changes 变更收集器
+ * @returns {boolean} 无法精确定位时返回 false，需要全量刷新
+ * @private
+ */
+function _applyPath(cache, rel_path, changes) {
+  if (!rel_path) return false;
+  if (_isIgnored(rel_path, cache.ignore)) return true;
+
+  const abs_path = path.join(cache.dir, rel_path);
+  let stat = null;
+  try {
+    stat = fs.statSync(abs_path);
+  } catch {
+    stat = null;
+  }
+
+  if (stat && stat.isDirectory()) return false;
+
+  const item = _createItem(cache.dir, abs_path);
+
+  if (!stat) {
+    return _delItem(cache, item.key, changes);
+  }
+
+  if (!stat.isFile()) return true;
+
+  if (!cache.extensions.includes(item.ext)) {
+    _delSameName(cache, item, changes);
+    return true;
+  }
+
+  const added = _addItem(cache, item);
+  changes.push({ type: added ? 'add' : 'change', key: item.key, item: { ...item } });
+  return true;
+}
+
+/**
+ * 对比两份条目集合，生成增量变更
+ * @param {Array<object>} before 变更前的条目
+ * @param {Array<object>} after 变更后的条目
+ * @returns {Array<{ type: string, key: string, item: object }>} 变更列表
+ * @private
+ */
+function _diffItems(before, after) {
+  const before_map = new Map();
+  for (let i = 0; i < before.length; i++) before_map.set(before[i].key, before[i]);
+
+  const after_map = new Map();
+  for (let i = 0; i < after.length; i++) after_map.set(after[i].key, after[i]);
+
+  const changes = [];
+  for (const [key, item] of after_map) {
+    const old = before_map.get(key);
+    if (!old) {
+      changes.push({ type: 'add', key, item: { ...item } });
+      continue;
+    }
+    if (old.absPath !== item.absPath || old.relPath !== item.relPath) {
+      changes.push({ type: 'change', key, item: { ...item } });
+    }
+  }
+  for (const [key, item] of before_map) {
+    if (!after_map.has(key)) changes.push({ type: 'unlink', key, item: { ...item } });
+  }
+  return changes;
+}
+
+/**
+ * 全量重扫缓存对应的目录
+ * @param {object} cache 缓存项
+ * @param {Array<object>} changes 变更收集器
+ * @returns {void}
+ * @private
+ */
+function _refreshCache(cache, changes) {
+  const before = cache.items;
+  const files = new Map();
+  const dirs = [];
+  _walkDir(cache.dir, cache.dir, cache, files, dirs);
+  const after = [...files.values()];
+  cache.items = after;
+  cache.dirs = dirs;
+  cache.time = Date.now();
+
+  const diff = _diffItems(before, after);
+  for (let i = 0; i < diff.length; i++) changes.push(diff[i]);
+}
+
+/**
+ * 把缓存的最新条目与变更同步到监听中的实例
+ * @param {string} key 缓存键
+ * @param {Array<object>} changes 变更列表
+ * @returns {void}
+ * @private
+ */
+function _syncInstances(key, changes) {
+  const cache = _readCache(key);
+  const set = _instances.get(key);
+  if (!cache || !set) return;
+  for (const instance of set) {
+    instance[_SYNC](cache.items, cache.dirs);
+    instance[_NOTIFY](changes);
+  }
+}
+
+/**
+ * 处理防抖窗口内累积的文件事件
+ * @param {string} key 缓存键
+ * @returns {void}
+ * @private
+ */
+function _flushChanges(key) {
+  const entry = _watchers.get(key);
+  if (!entry || !entry.pending.size) return;
+
+  const list = [...entry.pending];
+  entry.pending.clear();
+
+  const cache = _readCache(key);
+  const changes = [];
+  let need_refresh = false;
+  for (let i = 0; i < list.length; i++) {
+    if (!_applyPath(cache, list[i], changes)) {
+      need_refresh = true;
+    }
+  }
+  if (need_refresh) {
+    _refreshCache(cache, changes);
+  }
+  _syncInstances(key, changes);
+}
+
+/**
+ * 记录文件系统事件，按防抖窗口合并后处理
+ * @param {string} key 缓存键
+ * @param {string|null} filename 事件对应的相对路径，为空表示无法定位
+ * @returns {void}
+ * @private
+ */
+function _onWatchEvent(key, filename) {
+  const entry = _watchers.get(key);
+  if (!entry) return;
+
+  entry.pending.add(filename ? normalizeKey(filename) : '');
+  if (entry.timer) return;
+
+  entry.timer = setTimeout(() => {
+    entry.timer = null;
+    _flushChanges(key);
+  }, _DEBOUNCE);
+
+  if (typeof entry.timer.unref === 'function') {
+    entry.timer.unref();
+  }
+}
+
+/**
+ * 为缓存绑定目录监听器
+ * @param {string} key 缓存键
+ * @returns {boolean} 监听是否可用
+ * @private
+ */
+function _bindWatcher(key) {
+  const cache = _readCache(key);
+  if (!cache) return false;
+
+  if (_watchers.has(key)) {
+    cache.watched = true;
+    cache.dir_mtimes = [];
+    cache.file_mtimes = [];
+    return true;
+  }
+
+  let watcher = null;
+  try {
+    watcher = fs.watch(cache.dir, { recursive: true }, (event_type, filename) => {
+      _onWatchEvent(key, filename);
+    });
+  } catch {
+    return false;
+  }
+
+  cache.watched = true;
+  cache.dir_mtimes = [];
+  cache.file_mtimes = [];
+
+  if (typeof watcher.unref === 'function') watcher.unref();
+  watcher.on('error', () => _unbindWatcher(key));
+  _watchers.set(key, { watcher, timer: null, pending: new Set() });
+  return true;
+}
+
+/**
+ * 解除目录监听并恢复时间戳校验
+ * @param {string} key 缓存键
+ * @returns {void}
+ * @private
+ */
+function _unbindWatcher(key) {
+  const entry = _watchers.get(key);
+  if (entry) {
+    if (entry.timer) clearTimeout(entry.timer);
+    try {
+      entry.watcher.close();
+    } catch {
+      // 监听器已关闭时忽略
+    }
+    _watchers.delete(key);
+  }
+
+  _instances.delete(key);
+
+  const cache = _readCache(key);
+  if (cache) {
+    cache.watched = false;
+    _captureMtimes(cache);
+  }
+}
+
+/**
+ * 判断实例是否处于监听状态
+ * @param {string} key 缓存键
+ * @param {object} instance 扫描器实例
+ * @returns {boolean} 已监听返回 true
+ * @private
+ */
+function _hasInstance(key, instance) {
+  const set = _instances.get(key);
+  return !!(set && set.has(instance));
+}
+
+/**
+ * 登记监听中的实例
+ * @param {string} key 缓存键
+ * @param {object} instance 扫描器实例
+ * @returns {void}
+ * @private
+ */
+function _addInstance(key, instance) {
+  let set = _instances.get(key);
+  if (!set) {
+    set = new Set();
+    _instances.set(key, set);
+  }
+  set.add(instance);
+}
+
+/**
+ * 注销监听中的实例，最后一个实例注销时关闭监听器
+ * @param {string} key 缓存键
+ * @param {object} instance 扫描器实例
+ * @returns {void}
+ * @private
+ */
+function _delInstance(key, instance) {
+  const set = _instances.get(key);
+  if (!set) return;
+  set.delete(instance);
+  if (set.size === 0) {
+    _unbindWatcher(key);
+  }
+}
+
+/**
+ * 清空全部缓存、监听器与实例登记
+ * @returns {void}
+ * @private
+ */
+function _clearAll() {
+  const keys = [..._watchers.keys()];
+  for (let i = 0; i < keys.length; i++) {
+    _unbindWatcher(keys[i]);
+  }
+  _instances.clear();
+  _scan_cache.clear();
+}
+
 /**
  * 合并默认配置与用户配置
  * @param {{ extensions: string[], ignore: string[] }} default_config - 默认配置
@@ -51,6 +613,12 @@ class Scanner {
   #files = new Map();
   #baseDir = null;
   #scanned = false;
+  /** 本次遍历过的目录，用于缓存有效性校验 */
+  #dirs = [];
+  /** 本次扫描对应的缓存键 */
+  #cacheKey = null;
+  /** 外部变更回调 */
+  #callback = null;
 
   /**
    * 创建扫描器实例
@@ -62,36 +630,44 @@ class Scanner {
 
   /* -------- 私有 -------- */
 
-  #walk(dir) {
-    const entries = fs.readdirSync(dir, { withFileTypes: true });
-
-    for (const entry of entries) {
-      const fullPath = path.join(dir, entry.name);
-
-      if (entry.isDirectory()) {
-        if (this.config.ignore.includes(entry.name)) continue;
-        this.#walk(fullPath);
-        continue;
-      }
-
-      if (!entry.isFile()) continue;
-
-      const ext = path.extname(entry.name).toLowerCase();
-      if (!this.config.extensions.includes(ext)) continue;
-
-      const relPath = path.relative(this.#baseDir, fullPath).replace(/\\/g, '/');
-      const key = normalizeKey(relPath);              // 保留扩展名，如 'config/app.json'
-      const dirPart = path.posix.dirname(relPath);
+  /**
+   * 用缓存条目填充扫描结果
+   * @param {Array<object>} items 缓存中的条目
+   * @param {Array<string>} dirs 缓存中的目录列表
+   * @returns {void}
+   */
+  #fill(items, dirs) {
+    this.#files.clear();
+    for (const item of items) {
+      this.#files.set(item.key, { ...item });
+    }
+    this.#dirs = [...dirs];
+    this.#scanned = true;
+  }
+
+  /**
+   * 遍历根目录并填充扫描结果
+   * @returns {void}
+   */
+  #walkAndFill() {
+    this.#files.clear();
+    this.#dirs = [];
+    _walkDir(this.#baseDir, this.#baseDir, this.config, this.#files, this.#dirs);
+    this.#scanned = true;
+  }
 
-      this.#files.set(key, {
-        key,
-        absPath: fullPath,
-        relPath,
-        dir: dirPart === '.' ? '' : dirPart,
-        name: path.basename(relPath, ext),
-        ext,
-      });
-    }
+  /**
+   * 构造写入缓存的内容
+   * @returns {{ dir: string, extensions: Array<string>, ignore: Array<string>, items: Array<object>, dirs: Array<string> }} 缓存内容
+   */
+  #snapshot() {
+    return {
+      dir: this.#baseDir,
+      extensions: this.config.extensions,
+      ignore: this.config.ignore,
+      items: [...this.#files.values()],
+      dirs: this.#dirs,
+    };
   }
 
   #ensureReady() {
@@ -130,6 +706,7 @@ class Scanner {
 
   /**
    * 扫描目录（重复调用只生效一次）
+   * @description 优先复用其他实例的扫描缓存，命中时不再遍历目录
    * @param {string} dir
    * @param {{ extensions?: string[], ignore?: string[] }} [options]
    */
@@ -151,8 +728,16 @@ class Scanner {
     }
 
     this.#baseDir = resolved;
-    this.#walk(resolved);
-    this.#scanned = true;
+    this.#cacheKey = _cacheKey(resolved, this.config.extensions, this.config.ignore);
+
+    const cache = _getCache(this.#cacheKey);
+    if (cache) {
+      this.#fill(cache.items, cache.dirs);
+      return this;
+    }
+
+    this.#walkAndFill();
+    _setCache(this.#cacheKey, this.#snapshot());
     return this;
   };
 
@@ -251,13 +836,79 @@ class Scanner {
   /** 重新扫描（开发环境热更新用） */
   rescan = () => {
     if (!this.#baseDir) throw new Error('[scanner] 尚未初始化，无法 rescan');
-    this.#files.clear();
-    this.#walk(this.#baseDir);
+    const cache = _readCache(this.#cacheKey);
+    const before = cache ? cache.items : [];
+    this.#walkAndFill();
+    _setCache(this.#cacheKey, this.#snapshot());
+    _syncInstances(this.#cacheKey, _diffItems(before, this.getAll()));
+    return this;
+  };
+
+  /**
+   * 开启目录监听，文件增删改时增量维护缓存并同步到本实例
+   * @description 监听期间不再按时间戳校验缓存；不支持递归监听的环境会静默降级
+   * @param {(changes: Array<{ type: string, key: string, item: object }>) => void} [callback] 变更回调，type 为 add/change/unlink
+   * @returns {Scanner} 当前实例
+   */
+  watch = (callback) => {
+    this.#ensureReady();
+    if (typeof callback === 'function') this.#callback = callback;
+    if (_hasInstance(this.#cacheKey, this)) return this;
+    _addInstance(this.#cacheKey, this);
+    if (!_bindWatcher(this.#cacheKey)) {
+      _delInstance(this.#cacheKey, this);
+      this.#callback = null;
+    }
+    return this;
+  };
+
+  /**
+   * 关闭本实例的目录监听并注销变更回调，最后一个实例关闭后停止监听
+   * @returns {Scanner} 当前实例
+   */
+  unwatch = () => {
+    this.#callback = null;
+    if (!this.#cacheKey) return this;
+    _delInstance(this.#cacheKey, this);
     return this;
   };
 
+  /**
+   * 清空全部扫描缓存
+   * @returns {void}
+   */
+  static clearCache = () => {
+    _clearAll();
+  };
+
   get size() { return this.#files.size; }
   get baseDir() { return this.#baseDir; }
+  /** 当前实例是否处于监听状态 */
+  get watching() { return this.#cacheKey ? _hasInstance(this.#cacheKey, this) : false; }
+
+  /**
+   * 接收监听器同步的最新条目
+   * @param {Array<object>} items 缓存中的条目
+   * @param {Array<string>} dirs 缓存中的目录列表
+   * @returns {void}
+   */
+  [_SYNC](items, dirs) {
+    this.#fill(items, dirs);
+  }
+
+  /**
+   * 把变更列表交给外部回调
+   * @param {Array<{ type: string, key: string, item: object }>} changes 变更列表
+   * @returns {void}
+   */
+  [_NOTIFY](changes) {
+    if (!this.#callback || !changes.length) return;
+    try {
+      this.#callback(changes.map((one) => ({ ...one, item: { ...one.item } })));
+    } catch (error) {
+      console.error('[scanner] watch 回调执行失败:', error);
+    }
+  }
 }
 
 /* 导出单例 + 类 */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mm_scanner",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "目录扫描器，按扩展名递归扫描目录并缓存文件元信息，支持按 key、扩展名、目录检索文件路径。",
   "repository": {
     "type": "git",