dsh-lost-and-found 0.1.1

package/core/scan.mjs

@@ -0,0 +1,208 @@
+/**
+ * 文件快速寻回 · 扫描器（遍历 + 剪枝 + 增量）
+ *
+ * 铁律：
+ *  - 只读。stat/readdir 之外不碰用户文件；不写 ADS、不生成缩略图、不改时间戳。
+ *  - 目录级剪枝：命中排除目录名或排除路径片段，整棵子树不下钻（不是逐文件过滤）。
+ *  - 不跟随符号链接/junction（避免跨盘递归与死循环，也避免把 junction 后的数据重复计数）。
+ */
+import { opendir, stat } from "node:fs/promises";
+import { join, sep } from "node:path";
+import { SKIP_FILE_RE } from "../config.mjs";
+import { categoryOf, originOf, isNoise } from "./classify.mjs";
+
+const MAX_DEPTH = 40;
+
+/** 组装剪枝判定器；patterns 里 "re:" 前缀表示正则，其余为路径片段包含匹配 */
+export function makeMatcher(excludes) {
+  const dirs = excludes?.dirs instanceof Set ? excludes.dirs : new Set(excludes?.dirs || []);
+  const raw = (excludes?.patterns || []).map((p) => String(p));
+  const plain = raw.filter((p) => !p.startsWith("re:")).map((p) => p.toLowerCase());
+  const regexes = raw.filter((p) => p.startsWith("re:"))
+    .map((p) => { try { return new RegExp(p.slice(3), "i"); } catch { return null; } })
+    .filter(Boolean);
+  return {
+    pruneDir(name, fullPathLower, isRoot) {
+      if (isRoot) return false;
+      if (dirs.has(String(name).toLowerCase())) return true;
+      for (const p of plain) if (fullPathLower.includes(p)) return true;
+      for (const re of regexes) if (re.test(fullPathLower)) return true;
+      return false;
+    },
+    skipFile(name) {
+      for (const re of SKIP_FILE_RE) if (re.test(name)) return true;
+      return false;
+    },
+  };
+}
+
+function isInside(childLower, parentLower) {
+  if (!parentLower) return false;
+  const a = childLower.replace(/[\\/]+$/, "");
+  const b = parentLower.replace(/[\\/]+$/, "");
+  if (a === b) return true;
+  return a.startsWith(b + sep) || a.startsWith(b + "/");
+}
+
+/**
+ * 遍历扫描目录，回调每个符合「出现时间 >= 该目录自己的 sinceMs」的文件。
+ *
+ * 两个关键点（都是修 BUG 的核心）：
+ *  1. 每个目录带自己的 sinceMs（root.sinceMs），不再共用一个全局窗口——
+ *     否则后加入的目录只能看到全局锚点之后 6 小时的文件，历史文件永久漏收。
+ *  2. 逐目录回报 completed：只有真的走完的目录才允许推进锚点；
+ *     被中断/根目录读不到的目录保持原锚点，下次继续补。
+ *
+ * @returns {{total:number, candidates:number, skippedDirs:number, errors:number, roots:Array}}
+ */
+export async function scanRoots({ roots, excludes, sinceMs, onCandidate, signal, logger }) {
+  const m = makeMatcher(excludes);
+  const stats = { total: 0, candidates: 0, skippedDirs: 0, errors: 0, roots: [] };
+  for (const root of roots || []) {
+    if (signal?.aborted) break;
+    const own = Number(root && root.sinceMs);
+    const since = Number.isFinite(own) ? own : (Number(sinceMs) || 0);
+    const rs = { total: 0, candidates: 0, skippedDirs: 0, errors: 0, readable: true };
+    await walk(root.path, root, 0, since, rs);
+    if (signal?.aborted || !rs.readable) rs.completed = false;
+    stats.total += rs.total;
+    stats.candidates += rs.candidates;
+    stats.skippedDirs += rs.skippedDirs;
+    stats.errors += rs.errors;
+    stats.roots.push({
+      path: root.path, sinceMs: since, total: rs.total, candidates: rs.candidates,
+      errors: rs.errors, completed: rs.completed !== false,
+    });
+  }
+  return stats;
+
+  async function walk(dir, root, depth, since, rs) {
+    if (signal?.aborted || depth > MAX_DEPTH) return;
+    let handle;
+    try {
+      handle = await opendir(dir);
+    } catch {
+      rs.errors++;
+      if (dir === root.path) rs.readable = false; // 根目录都打不开 → 不算扫完，别推进锚点
+      return;
+    }
+    try {
+      for await (const ent of handle) {
+        if (signal?.aborted) return;
+        const full = join(dir, ent.name);
+        const lower = full.toLowerCase();
+        if (ent.isSymbolicLink()) continue;
+        if (ent.isDirectory()) {
+          if (m.pruneDir(ent.name, lower, full === root.path)) {
+            rs.skippedDirs++;
+            continue;
+          }
+          await walk(full, root, depth + 1, since, rs);
+        } else if (ent.isFile()) {
+          if (m.skipFile(ent.name)) continue;
+          let st;
+          try {
+            st = await stat(full);
+          } catch {
+            rs.errors++;
+            continue;
+          }
+          rs.total++;
+          const birth = st.birthtimeMs || st.mtimeMs;
+          const appeared = Math.max(birth, st.mtimeMs);
+          if (appeared < since) continue;
+          rs.candidates++;
+          const dot = ent.name.lastIndexOf(".");
+          const ext = dot > 0 ? ent.name.slice(dot).toLowerCase() : "";
+          onCandidate({
+            path: full,
+            root: root.path,
+            name: ent.name,
+            ext,
+            size: st.size,
+            birth_ms: Math.round(birth),
+            mtime_ms: Math.round(st.mtimeMs),
+            atime_ms: Math.round(st.atimeMs || 0),
+            appeared_ms: Math.round(appeared),
+            category: categoryOf(ext),
+            origin: originOf(full),
+            noise: isNoise({ ext, name: ent.name }),
+            policy: root.policy || "full",
+            first_seen_ms: Date.now(),
+          });
+        }
+      }
+    } catch (e) {
+      rs.errors++;
+      logger?.(`遍历出错 ${dir}: ${e && e.message ? e.message : e}`);
+    }
+  }
+}
+
+/**
+ * 野文件巡查：在白名单之外，找出「最近有新文件出现」的目录，供用户决定是否纳入扫描。
+ * 只统计不索引；限制深度以控制开销。
+ */
+export async function patrolDrives({ drives, excludes, sinceMs, coveredRoots, maxDepth = 4, minFiles = 3, signal }) {
+  const m = makeMatcher(excludes);
+  const covered = (coveredRoots || []).map((p) => String(p).toLowerCase());
+  const found = new Map();
+  const stats = { scannedDirs: 0, skippedDirs: 0, errors: 0 };
+
+  for (const drive of drives) {
+    if (signal?.aborted) break;
+    await walk(drive, 0);
+  }
+
+  async function walk(dir, depth) {
+    if (signal?.aborted || depth > maxDepth) return;
+    const dirLower = dir.toLowerCase();
+    if (covered.some((c) => isInside(dirLower, c))) return; // 已在白名单内，不再巡查
+    let handle;
+    try {
+      handle = await opendir(dir);
+    } catch {
+      stats.errors++;
+      return;
+    }
+    stats.scannedDirs++;
+    let localCount = 0;
+    let sample = null;
+    try {
+      for await (const ent of handle) {
+        if (signal?.aborted) return;
+        const full = join(dir, ent.name);
+        const lower = full.toLowerCase();
+        if (ent.isSymbolicLink()) continue;
+        if (ent.isDirectory()) {
+          if (m.pruneDir(ent.name, lower, false)) {
+            stats.skippedDirs++;
+            continue;
+          }
+          await walk(full, depth + 1);
+        } else if (ent.isFile()) {
+          if (m.skipFile(ent.name)) continue;
+          let st;
+          try {
+            st = await stat(full);
+          } catch {
+            stats.errors++;
+            continue;
+          }
+          const appeared = Math.max(st.birthtimeMs || st.mtimeMs, st.mtimeMs);
+          if (appeared >= sinceMs) {
+            localCount++;
+            if (!sample) sample = ent.name;
+          }
+        }
+      }
+    } catch {
+      stats.errors++;
+    }
+    if (localCount >= minFiles && depth > 0) {
+      found.set(dir, { dir, files: localCount, sample });
+    }
+  }
+
+  return { suggestions: [...found.values()].sort((a, b) => b.files - a.files), stats };
+}

package/core/schedule.mjs

@@ -0,0 +1,71 @@
+/** 文件快速寻回 · 排期与增量锚点 */
+import { lastScanMs, getMeta, rootState } from "../db.mjs";
+
+const DAY = 86400000;
+/** 重叠窗口：防止时钟漂移/写入延迟造成边界漏收 */
+const OVERLAP_MS = 6 * 3600 * 1000;
+
+/** 本次扫描要看「从什么时候起出现」的文件（全局，仅作兼容兜底用） */
+export function computeSinceMs(db, config, now = Date.now()) {
+  const last = lastScanMs(db);
+  if (last > 0) return Math.max(0, last - OVERLAP_MS);
+  const windowDays = Number(config.firstRunWindowDays) || 0;
+  return windowDays > 0 ? now - windowDays * DAY : now;
+}
+
+/**
+ * 单个扫描目录的收录起点（逐目录锚点，这是修掉「按扫描按钮却收不全」的核心）。
+ *
+ * 为什么不能只用全局锚点：全局锚点 = 上次扫描时刻 − 6h，跟"哪个目录"无关。
+ * 于是任何「后加入的目录」首扫只能看到锚点之后 6 小时的文件，它更早的历史全部漏收，
+ * 而且这个漏收是永久的（下次锚点又往后推了）。实测 D:\读书 570 个文件只入库 13 个。
+ *
+ * @param {object} p
+ * @param {object} p.db        已打开的库
+ * @param {{path:string}} p.root 目录
+ * @param {object} p.config    当前配置
+ * @param {number} [p.now]
+ * @param {boolean} [p.forceFull] 用户主动要求全量重扫
+ * @returns {number} sinceMs（0 = 不限时间，全量）
+ */
+export function rootSinceMs({ db, root, config, now = Date.now(), forceFull = false }) {
+  if (forceFull) return 0;
+  const st = rootState(db, root.path);
+  // 从未完整扫过（含老库升级后 first_scan_done 仍为 0）→ 按首扫策略回填
+  if (!st.first_scan_done || !st.last_scan_ms) {
+    const mode = String((config && config.firstScanMode) || "full").toLowerCase();
+    if (mode === "none") return now;                       // 只收今后新增
+    if (mode === "window") {
+      const days = Number(config && config.firstRunWindowDays) || 0;
+      return days > 0 ? now - days * DAY : now;
+    }
+    return 0;                                              // full：收录该目录全部历史
+  }
+  return Math.max(0, st.last_scan_ms - OVERLAP_MS);
+}
+
+/** 是否到了该自动扫描的时间 */
+export function isDue(db, config, now = Date.now()) {
+  const days = Number(config.intervalDays);
+  if (!days || days <= 0) return false;
+  const last = lastScanMs(db);
+  if (!last) return true;
+  return now - last >= days * DAY;
+}
+
+export function nextDueMs(db, config) {
+  const days = Number(config.intervalDays);
+  if (!days || days <= 0) return null;
+  const last = lastScanMs(db);
+  return (last || Date.now()) + days * DAY;
+}
+
+export function lastVerifyMs(db) {
+  const v = getMeta(db, "last_verify_ms");
+  return v ? Number(v) : 0;
+}
+
+/** 每周一次校验扫描 */
+export function verifyDue(db, now = Date.now()) {
+  return now - lastVerifyMs(db) >= 7 * DAY;
+}

package/core/search.mjs

@@ -0,0 +1,217 @@
+/**
+ * 文件快速寻回 · 检索层
+ *
+ * 检索策略（与用户确认过的架构一致）：
+ *  1) SQL 硬过滤：时间范围 / 类型 / 目录 / 来源 / 状态
+ *  2) 多关键词 LIKE 取交（中文无需分词，子串匹配天然可用）
+ *  3) JS 侧命中位置加权打分：文件名 > 关键词标签 > 路径 > 摘要 > 图片描述 > 正文
+ *  4) 对最终结果逐个 stat，标注「还在 / 已不在此处」
+ *  5) 索引不够用时，对扫描目录做一次实时兜底扫描（不写库）
+ *
+ * 为什么不用 FTS5：是否可用取决于 Node 版本 —— 实测系统 node v22.14.0 报
+ * `no such module: fts5`，而 DSH 运行时的 QClaw 自带 node v22.22.3 是带 FTS5 的。
+ * 为在两种环境都能跑，这里不假设 FTS5 存在（将来检测到可用时可作为加速路径）。
+ */
+import { stat } from "node:fs/promises";
+import { scanRoots } from "./scan.mjs";
+import { categoryOf, originOf } from "./classify.mjs";
+
+const LIKE_COLUMNS = ["name", "path", "tags", "summary", "image_desc", "excerpt"];
+
+const WEIGHT = {
+  name: 50,
+  tags: 30,
+  path: 20,
+  summary: 15,
+  image_desc: 10,
+  excerpt: 8,
+};
+
+function escapeLike(s) {
+  return String(s).replace(/[\\%_]/g, (c) => `\\${c}`);
+}
+
+function splitKeywords(input) {
+  if (Array.isArray(input)) return input.map((s) => String(s).trim()).filter(Boolean);
+  return String(input || "")
+    .split(/[\s,，、;；|]+/)
+    .map((s) => s.trim())
+    .filter(Boolean);
+}
+
+/** 兼容「字符串 / 逗号分隔字符串 / 数组」三种入参 */
+function toArray(input) {
+  if (input === undefined || input === null || input === "") return [];
+  if (Array.isArray(input)) return input.map((s) => String(s).trim()).filter(Boolean);
+  return String(input).split(/[,，、;；\s]+/).map((s) => s.trim()).filter(Boolean);
+}
+
+/** 构造 SQL 与参数 */
+function buildQuery(opts) {
+  const where = [];
+  const params = [];
+  const keywords = splitKeywords(opts.keywords);
+
+  if (!opts.includeMissing) where.push("state = 'ok'");
+  if (!opts.includeNoise) where.push("noise = 0");
+
+  if (opts.since) { where.push("appeared_ms >= ?"); params.push(Number(opts.since)); }
+  if (opts.until) { where.push("appeared_ms <= ?"); params.push(Number(opts.until)); }
+
+  const exts = toArray(opts.ext).map((e) => e.toLowerCase()).map((e) => (e.startsWith(".") ? e : `.${e}`));
+  if (exts.length) {
+    where.push(`ext IN (${exts.map(() => "?").join(",")})`);
+    params.push(...exts);
+  }
+
+  const cats = toArray(opts.categories);
+  if (cats.length) {
+    where.push(`category IN (${cats.map(() => "?").join(",")})`);
+    params.push(...cats);
+  }
+
+  const origins = toArray(opts.origins);
+  if (origins.length) {
+    where.push(`origin IN (${origins.map(() => "?").join(",")})`);
+    params.push(...origins);
+  }
+
+  if (opts.folder) {
+    const f = escapeLike(String(opts.folder).replace(/[\\/]+$/, ""));
+    where.push("(path LIKE ? ESCAPE '\\' OR root LIKE ? ESCAPE '\\')");
+    params.push(`${f}%`, `${f}%`);
+  }
+
+  for (const kw of keywords) {
+    const k = `%${escapeLike(kw)}%`;
+    where.push(`(${LIKE_COLUMNS.map((c) => `${c} LIKE ? ESCAPE '\\'`).join(" OR ")})`);
+    for (let i = 0; i < LIKE_COLUMNS.length; i++) params.push(k);
+  }
+
+  const sql = `SELECT * FROM files ${where.length ? `WHERE ${where.join(" AND ")}` : ""}
+               ORDER BY appeared_ms DESC LIMIT ?`;
+  return { sql, params, keywords };
+}
+
+function scoreRow(row, keywords) {
+  const lower = {
+    name: String(row.name || "").toLowerCase(),
+    path: String(row.path || "").toLowerCase(),
+    tags: String(row.tags || "").toLowerCase(),
+    summary: String(row.summary || "").toLowerCase(),
+    image_desc: String(row.image_desc || "").toLowerCase(),
+    excerpt: String(row.excerpt || "").toLowerCase(),
+  };
+  let score = 0;
+  const hitIn = new Set();
+  for (const kw of keywords) {
+    const k = kw.toLowerCase();
+    for (const col of LIKE_COLUMNS) {
+      if (lower[col] && lower[col].includes(k)) {
+        score += WEIGHT[col];
+        hitIn.add(col);
+      }
+    }
+    if (lower.name === k) score += 80;
+  }
+  const age = Date.now() - (row.appeared_ms || 0);
+  if (age < 7 * 86400000) score += 6;
+  else if (age < 30 * 86400000) score += 3;
+  return { score, hitIn: [...hitIn] };
+}
+
+function snippetOf(row, keywords) {
+  for (const col of ["excerpt", "summary", "image_desc"]) {
+    const text = String(row[col] || "");
+    if (!text) continue;
+    const lower = text.toLowerCase();
+    for (const kw of keywords) {
+      const i = lower.indexOf(kw.toLowerCase());
+      if (i >= 0) {
+        const from = Math.max(0, i - 50);
+        const to = Math.min(text.length, i + kw.length + 50);
+        return `${from > 0 ? "…" : ""}${text.slice(from, to).replace(/\s+/g, " ")}${to < text.length ? "…" : ""}`;
+      }
+    }
+    if (col === "excerpt") return text.replace(/\s+/g, " ").slice(0, 100);
+  }
+  return null;
+}
+
+/**
+ * 索引检索。
+ * @returns {Promise<{rows:Array, candidates:number, query:object}>}
+ */
+export async function searchFiles(db, opts = {}) {
+  const limit = Math.min(Math.max(Number(opts.limit) || 20, 1), 100);
+  const candidateLimit = Math.min(Math.max(limit * 25, 300), 3000);
+  const { sql, params, keywords } = buildQuery(opts);
+  const rows = db.prepare(sql).all(...params, candidateLimit);
+
+  const scored = rows.map((row) => {
+    const { score, hitIn } = scoreRow(row, keywords);
+    return { ...row, _score: keywords.length ? score : (row.appeared_ms || 0) / 1e10, _hitIn: hitIn };
+  });
+  scored.sort((a, b) => b._score - a._score);
+  const top = scored.slice(0, limit);
+
+  const out = [];
+  for (const row of top) {
+    let exists = null;
+    try {
+      const st = await stat(row.path);
+      exists = st.isFile();
+    } catch {
+      exists = false;
+    }
+    out.push({ ...row, exists, snippet: snippetOf(row, keywords) });
+  }
+  return { rows: out, candidates: rows.length, query: { ...opts, keywords } };
+}
+
+/**
+ * 实时兜底扫描：不依赖索引、不写库，直接在扫描目录里按名字/时间找。
+ * 用于「索引没命中」或「找的是还没被收录的旧文件」。
+ */
+export async function liveSearch({ roots, excludes, keywords, since, until, ext, limit = 30, includeNoise = false, signal, logger }) {
+  const kws = splitKeywords(keywords).map((s) => s.toLowerCase());
+  const exts = toArray(ext).map((e) => (String(e).startsWith(".") ? String(e) : `.${e}`).toLowerCase());
+  const hits = [];
+  const stats = await scanRoots({
+    roots,
+    excludes,
+    sinceMs: since || 0,
+    signal,
+    logger,
+    onCandidate(rec) {
+      if (until && rec.appeared_ms > until) return;
+      if (exts.length && !exts.includes(rec.ext)) return;
+      if (!includeNoise && rec.noise) return;
+      if (kws.length) {
+        const hay = `${rec.name} ${rec.path}`.toLowerCase();
+        if (!kws.every((k) => hay.includes(k))) return;
+      }
+      hits.push({
+        path: rec.path, name: rec.name, ext: rec.ext, size: rec.size,
+        appeared_ms: rec.appeared_ms, birth_ms: rec.birth_ms, mtime_ms: rec.mtime_ms,
+        category: rec.category || categoryOf(rec.ext), origin: rec.origin || originOf(rec.path),
+        root: rec.root, policy: rec.policy, state: "ok", enrich: "none", noise: rec.noise || 0,
+        summary: null, excerpt: null, image_desc: null, tags: null,
+        _live: true, exists: true, snippet: null, _score: rec.appeared_ms / 1e10,
+      });
+    },
+  });
+  hits.sort((a, b) => b.appeared_ms - a.appeared_ms);
+  return { rows: hits.slice(0, limit), scanned: stats, total: hits.length };
+}
+
+/** 把查询条件翻译成一句人话，便于在回答里说明搜了什么 */
+export function describeQuery(q) {
+  const bits = [];
+  if (q.keywords && q.keywords.length) bits.push(`关键词「${[].concat(q.keywords).join(" ")}」`);
+  if (q.since || q.until) bits.push("时间范围");
+  if (q.ext && q.ext.length) bits.push(`类型 ${[].concat(q.ext).join("/")}`);
+  if (q.folder) bits.push(`范围 ${q.folder}`);
+  if (q.categories && q.categories.length) bits.push(`类别 ${[].concat(q.categories).join("/")}`);
+  return bits.length ? bits.join(" · ") : "全部文件";
+}