chainlesschain 0.162.12 → 0.162.14

package/src/lib/sync-credentials.js

@@ -0,0 +1,225 @@
+/**
+ * CLI sync provider credentials store — Phase 3c follow-up.
+ *
+ * Electron 的 `safeStorage` 在 CLI 没法用，所以这里搭一个 file-based equivalent:
+ *
+ *   - Master key 自动生成 (32 字节 random) 落 `~/.chainlesschain/sync-credentials.key`
+ *     文件 mode 0600 (Unix) / NTFS ACL (Win — fs.chmodSync 容错)
+ *   - 凭据 JSON 编 AES-256-GCM (iv 12B + auth tag 16B)
+ *     落 `~/.chainlesschain/sync-credentials.enc`
+ *   - SENSITIVE_FIELDS 镜像 desktop secure-config-storage.js 用于 sanitize
+ *
+ * 与 desktop sync-credentials.js 同 surface：
+ *   getCredentials / setCredentials / clearCredentials / hasCredentials /
+ *   getCredentialsSanitized / ALLOWED_PROVIDER_IDS
+ *
+ * 威胁模型：root/admin 能读 master key 文件即破，是 CLI 工具合理 baseline
+ * （同 git ~/.netrc / ~/.aws/credentials）。OS keyring 强加密留 v0.2。
+ */
+
+"use strict";
+
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import crypto from "node:crypto";
+
+const SENSITIVE_FIELDS = ["sync.webdav.password", "sync.oss.secretAccessKey"];
+const ALLOWED_PROVIDER_IDS = ["webdav", "oss"];
+const MASK = "********";
+const AES_ALG = "aes-256-gcm";
+const IV_LEN = 12;
+const TAG_LEN = 16;
+const KEY_LEN = 32;
+
+let _ccDirOverride = null;
+
+function _ccDir() {
+  if (_ccDirOverride) return _ccDirOverride;
+  return (
+    process.env.CHAINLESSCHAIN_HOME ||
+    path.join(os.homedir(), ".chainlesschain")
+  );
+}
+
+function _keyPath() {
+  return path.join(_ccDir(), "sync-credentials.key");
+}
+
+function _vaultPath() {
+  return path.join(_ccDir(), "sync-credentials.enc");
+}
+
+function _ensureDir() {
+  const dir = _ccDir();
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+}
+
+function _loadOrCreateMasterKey() {
+  _ensureDir();
+  const kp = _keyPath();
+  if (fs.existsSync(kp)) {
+    const buf = fs.readFileSync(kp);
+    if (buf.length !== KEY_LEN) {
+      throw new Error(
+        `sync-credentials: master key file ${kp} has wrong length ` +
+          `(${buf.length} bytes, expected ${KEY_LEN}). Delete + regenerate manually if intentional.`,
+      );
+    }
+    return buf;
+  }
+  const key = crypto.randomBytes(KEY_LEN);
+  fs.writeFileSync(kp, key);
+  try {
+    fs.chmodSync(kp, 0o600);
+  } catch (_e) {
+    /* non-fatal: NTFS / older fs */
+  }
+  return key;
+}
+
+function _encryptBlob(plainJson) {
+  const key = _loadOrCreateMasterKey();
+  const iv = crypto.randomBytes(IV_LEN);
+  const cipher = crypto.createCipheriv(AES_ALG, key, iv);
+  const enc = Buffer.concat([
+    cipher.update(plainJson, "utf-8"),
+    cipher.final(),
+  ]);
+  const tag = cipher.getAuthTag();
+  // file layout: [iv (12)] [tag (16)] [ciphertext]
+  return Buffer.concat([iv, tag, enc]);
+}
+
+function _decryptBlob(buf) {
+  if (!Buffer.isBuffer(buf) || buf.length < IV_LEN + TAG_LEN + 1) {
+    throw new Error("sync-credentials: enc file too small or invalid");
+  }
+  const key = _loadOrCreateMasterKey();
+  const iv = buf.subarray(0, IV_LEN);
+  const tag = buf.subarray(IV_LEN, IV_LEN + TAG_LEN);
+  const enc = buf.subarray(IV_LEN + TAG_LEN);
+  const decipher = crypto.createDecipheriv(AES_ALG, key, iv);
+  decipher.setAuthTag(tag);
+  return Buffer.concat([decipher.update(enc), decipher.final()]).toString(
+    "utf-8",
+  );
+}
+
+function loadAll() {
+  const vp = _vaultPath();
+  if (!fs.existsSync(vp)) return {};
+  const buf = fs.readFileSync(vp);
+  try {
+    return JSON.parse(_decryptBlob(buf));
+  } catch (err) {
+    throw new Error(
+      `sync-credentials: decrypt failed (${err?.message}). ` +
+        `Vault may be corrupted or master key changed. ` +
+        `Remove ${vp} + reconfigure to recover.`,
+    );
+  }
+}
+
+function saveAll(all) {
+  _ensureDir();
+  fs.writeFileSync(_vaultPath(), _encryptBlob(JSON.stringify(all)));
+  return true;
+}
+
+function assertProviderId(providerId) {
+  if (!ALLOWED_PROVIDER_IDS.includes(providerId)) {
+    throw new Error(
+      `sync-credentials: unknown provider id '${providerId}' ` +
+        `(allowed: ${ALLOWED_PROVIDER_IDS.join(", ")})`,
+    );
+  }
+}
+
+function sanitize(all) {
+  if (!all || typeof all !== "object") return all;
+  const clone = JSON.parse(JSON.stringify(all));
+  for (const dotPath of SENSITIVE_FIELDS) {
+    const parts = dotPath.split(".");
+    let cur = clone;
+    for (let i = 0; i < parts.length - 1; i++) {
+      if (cur && typeof cur === "object" && parts[i] in cur) {
+        cur = cur[parts[i]];
+      } else {
+        cur = null;
+        break;
+      }
+    }
+    const last = parts[parts.length - 1];
+    if (
+      cur &&
+      typeof cur === "object" &&
+      cur[last] != null &&
+      cur[last] !== ""
+    ) {
+      cur[last] = MASK;
+    }
+  }
+  return clone;
+}
+
+function getCredentials(providerId) {
+  assertProviderId(providerId);
+  return loadAll()?.sync?.[providerId] ?? {};
+}
+
+function getCredentialsSanitized(providerId) {
+  assertProviderId(providerId);
+  return sanitize(loadAll())?.sync?.[providerId] ?? {};
+}
+
+function hasCredentials(providerId) {
+  return Object.values(getCredentials(providerId)).some(
+    (v) => v !== null && v !== undefined && v !== "",
+  );
+}
+
+function setCredentials(providerId, creds) {
+  assertProviderId(providerId);
+  if (!creds || typeof creds !== "object") {
+    throw new Error("sync-credentials: creds must be an object");
+  }
+  const all = loadAll();
+  if (!all.sync || typeof all.sync !== "object") all.sync = {};
+  all.sync[providerId] = { ...creds };
+  return saveAll(all);
+}
+
+function clearCredentials(providerId) {
+  assertProviderId(providerId);
+  const all = loadAll();
+  if (all?.sync?.[providerId]) {
+    delete all.sync[providerId];
+    return saveAll(all);
+  }
+  return true;
+}
+
+/** Test seam: override the resolved chainlesschain dir without env leak. */
+function _setCcDirForTest(dir) {
+  _ccDirOverride = dir;
+}
+
+function _resetCcDirForTest() {
+  _ccDirOverride = null;
+}
+
+export {
+  ALLOWED_PROVIDER_IDS,
+  SENSITIVE_FIELDS,
+  MASK,
+  getCredentials,
+  getCredentialsSanitized,
+  hasCredentials,
+  setCredentials,
+  clearCredentials,
+  _setCcDirForTest,
+  _resetCcDirForTest,
+  _keyPath,
+  _vaultPath,
+};

package/src/lib/sync-engine-cli.js

@@ -0,0 +1,406 @@
+/**
+ * CLI sync engine — Phase 3c follow-up Phase 2.
+ *
+ * Provider-agnostic sync engine consolidating the desktop-side
+ * sync-external-store + incremental-walker + markdown-renderer + engine
+ * into ONE focused CLI module. The CLI vault has only KNOWLEDGE_ITEM
+ * tombstones (no mobile sync ResourceTypes) so we drop the filter
+ * parameter and simplify.
+ *
+ * Flow (mirror desktop runWebDAVSync / runOSSSync):
+ *   1. ensureCursor
+ *   2. drain tombstones (delete then deleteFile on remote)
+ *   3. push loop: fetchBatch → putFile → recordPushed → advance cursor
+ *   4. update final cursor state
+ */
+
+"use strict";
+
+const PROGRESS_FLUSH_EVERY = 5;
+const PROGRESS_FLUSH_MS = 500;
+
+// ── markdown renderer (deterministic; mirror desktop) ──────────────
+
+function _cleanTitle(title) {
+  // Note: place `-` AT END of char-class — putting it mid-class triggers
+  // JS "invalid range" silent fallthrough and space stops matching.
+  return (
+    String(title || "untitled")
+      .replace(/[\\/:*?"<>|\s-]/g, "_")
+      .replace(/_+/g, "_")
+      .replace(/^_|_$/g, "")
+      .slice(0, 80) || "untitled"
+  );
+}
+
+function generateFilename(item) {
+  return `${item.id}-${_cleanTitle(item.title)}.md`;
+}
+
+function generateMarkdown(item) {
+  const tags = item.tags ? String(item.tags) : "";
+  const frontMatter = [
+    "---",
+    `id: ${item.id}`,
+    `title: ${JSON.stringify(item.title || "untitled")}`,
+    `type: ${item.type || "note"}`,
+    `created_at: ${item.created_at}`,
+    `updated_at: ${item.updated_at}`,
+    tags ? `tags: ${tags}` : "",
+    "---",
+    "",
+  ]
+    .filter(Boolean)
+    .join("\n");
+  return frontMatter + (item.content || "") + "\n";
+}
+
+// ── store: cursor + tombstone CRUD ─────────────────────────────────
+
+function _parseJsonField(v, fallback) {
+  if (v == null || v === "") return fallback;
+  try {
+    return JSON.parse(v);
+  } catch (_e) {
+    return fallback;
+  }
+}
+
+function getCursor(dbManager, providerId, accountKey = "") {
+  const row = dbManager.get(
+    `SELECT * FROM sync_external_provider_cursor
+     WHERE provider_id = ? AND account_key = ?`,
+    [providerId, accountKey],
+  );
+  if (!row) return undefined;
+  return {
+    providerId: row.provider_id,
+    accountKey: row.account_key,
+    lastSyncAt: row.last_sync_at,
+    lastItemId: row.last_item_id,
+    remoteEtagMap: _parseJsonField(row.remote_etag_map, {}),
+    remoteFilenameMap: _parseJsonField(row.remote_filename_map, {}),
+    lastRunStatus: row.last_run_status,
+    lastRunError: row.last_run_error,
+    lastRunDurationMs: row.last_run_duration_ms,
+    itemsPushed: row.items_pushed,
+    itemsSkipped: row.items_skipped,
+    itemsDeleted: row.items_deleted,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+  };
+}
+
+function ensureCursor(dbManager, providerId, accountKey = "") {
+  const existing = getCursor(dbManager, providerId, accountKey);
+  if (existing) return existing;
+  dbManager.run(
+    `INSERT OR IGNORE INTO sync_external_provider_cursor
+     (provider_id, account_key) VALUES (?, ?)`,
+    [providerId, accountKey],
+  );
+  return getCursor(dbManager, providerId, accountKey);
+}
+
+function updateCursor(dbManager, providerId, patch, accountKey = "") {
+  if (!patch || typeof patch !== "object") return;
+  const fields = [];
+  const params = [];
+  const map = {
+    lastSyncAt: "last_sync_at",
+    lastItemId: "last_item_id",
+    lastRunStatus: "last_run_status",
+    lastRunError: "last_run_error",
+    lastRunDurationMs: "last_run_duration_ms",
+    itemsPushed: "items_pushed",
+    itemsSkipped: "items_skipped",
+    itemsDeleted: "items_deleted",
+  };
+  for (const [k, col] of Object.entries(map)) {
+    if (k in patch) {
+      fields.push(`${col} = ?`);
+      params.push(patch[k]);
+    }
+  }
+  if ("remoteEtagMap" in patch) {
+    fields.push("remote_etag_map = ?");
+    params.push(JSON.stringify(patch.remoteEtagMap));
+  }
+  if ("remoteFilenameMap" in patch) {
+    fields.push("remote_filename_map = ?");
+    params.push(JSON.stringify(patch.remoteFilenameMap));
+  }
+  fields.push("updated_at = ?");
+  params.push(Date.now());
+  params.push(providerId, accountKey);
+  dbManager.run(
+    `UPDATE sync_external_provider_cursor SET ${fields.join(", ")}
+     WHERE provider_id = ? AND account_key = ?`,
+    params,
+  );
+}
+
+function recordPushedItem(
+  dbManager,
+  providerId,
+  itemId,
+  etag,
+  filename,
+  accountKey = "",
+) {
+  const cursor = getCursor(dbManager, providerId, accountKey) || {
+    remoteEtagMap: {},
+    remoteFilenameMap: {},
+  };
+  const etagMap = { ...cursor.remoteEtagMap, [itemId]: etag };
+  const fnMap = { ...cursor.remoteFilenameMap, [itemId]: filename };
+  dbManager.run(
+    `UPDATE sync_external_provider_cursor
+     SET remote_etag_map = ?, remote_filename_map = ?, updated_at = ?
+     WHERE provider_id = ? AND account_key = ?`,
+    [
+      JSON.stringify(etagMap),
+      JSON.stringify(fnMap),
+      Date.now(),
+      providerId,
+      accountKey,
+    ],
+  );
+}
+
+function removeFromMaps(dbManager, providerId, itemId, accountKey = "") {
+  const cursor = getCursor(dbManager, providerId, accountKey);
+  if (!cursor) return;
+  const etagMap = { ...cursor.remoteEtagMap };
+  const fnMap = { ...cursor.remoteFilenameMap };
+  delete etagMap[itemId];
+  delete fnMap[itemId];
+  dbManager.run(
+    `UPDATE sync_external_provider_cursor
+     SET remote_etag_map = ?, remote_filename_map = ?, updated_at = ?
+     WHERE provider_id = ? AND account_key = ?`,
+    [
+      JSON.stringify(etagMap),
+      JSON.stringify(fnMap),
+      Date.now(),
+      providerId,
+      accountKey,
+    ],
+  );
+}
+
+function listTombstones(dbManager, providerId, accountKey = "", limit = 1000) {
+  return dbManager.all(
+    `SELECT * FROM sync_external_tombstones
+     WHERE provider_id = ? AND account_key = ?
+       AND resource_type = 'KNOWLEDGE_ITEM'
+     ORDER BY deleted_at ASC LIMIT ?`,
+    [providerId, accountKey, limit],
+  );
+}
+
+function deleteTombstone(dbManager, id) {
+  dbManager.run(`DELETE FROM sync_external_tombstones WHERE id = ?`, [id]);
+}
+
+function markTombstoneFailed(dbManager, id, errMsg) {
+  dbManager.run(
+    `UPDATE sync_external_tombstones
+     SET retry_count = retry_count + 1, last_error = ?
+     WHERE id = ?`,
+    [String(errMsg || "").slice(0, 500), id],
+  );
+}
+
+// ── walker: incremental batch fetching ─────────────────────────────
+
+function fetchBatch(dbManager, cursor, batchSize = 200) {
+  const after = cursor?.lastSyncAt || 0;
+  const afterId = cursor?.lastItemId || "";
+  return dbManager.all(
+    `SELECT id, title, type, content, tags, created_at, updated_at
+     FROM knowledge_items
+     WHERE updated_at > ? OR (updated_at = ? AND id > ?)
+     ORDER BY updated_at ASC, id ASC
+     LIMIT ?`,
+    [after, after, afterId, batchSize],
+  );
+}
+
+function cursorAfterItem(item) {
+  return { lastSyncAt: item.updated_at, lastItemId: item.id };
+}
+
+function countPending(dbManager, cursor) {
+  const after = cursor?.lastSyncAt || 0;
+  const afterId = cursor?.lastItemId || "";
+  const row = dbManager.get(
+    `SELECT COUNT(*) AS c FROM knowledge_items
+     WHERE updated_at > ? OR (updated_at = ? AND id > ?)`,
+    [after, after, afterId],
+  );
+  return row?.c || 0;
+}
+
+// ── engine: orchestration ──────────────────────────────────────────
+
+async function runSync(deps) {
+  const { dbManager, client, providerId, accountKey = "", onProgress } = deps;
+
+  const t0 = Date.now();
+  let pushed = 0;
+  let skipped = 0;
+  let deleted = 0;
+  let lastError = null;
+  let lastFlushPushed = 0;
+  let lastFlushDeleted = 0;
+  let lastFlushAt = Date.now();
+  let totalPending = 0;
+
+  function maybeFlush(phase) {
+    const now = Date.now();
+    const delta = pushed - lastFlushPushed + (deleted - lastFlushDeleted);
+    if (
+      delta >= PROGRESS_FLUSH_EVERY ||
+      now - lastFlushAt >= PROGRESS_FLUSH_MS
+    ) {
+      onProgress?.({ phase, pushed, skipped, deleted, totalPending });
+      lastFlushPushed = pushed;
+      lastFlushDeleted = deleted;
+      lastFlushAt = now;
+    }
+  }
+
+  function refresh() {
+    return getCursor(dbManager, providerId, accountKey) || {};
+  }
+
+  let cursor = ensureCursor(dbManager, providerId, accountKey);
+  totalPending =
+    countPending(dbManager, cursor) +
+    listTombstones(dbManager, providerId, accountKey, 1000).length;
+  onProgress?.({
+    phase: "start",
+    pushed: 0,
+    skipped: 0,
+    deleted: 0,
+    totalPending,
+  });
+
+  // Drain tombstones
+  const tombs = listTombstones(dbManager, providerId, accountKey, 1000);
+  for (const t of tombs) {
+    const fn = cursor.remoteFilenameMap?.[t.item_id];
+    if (!fn) {
+      deleteTombstone(dbManager, t.id);
+      continue;
+    }
+    const etag = cursor.remoteEtagMap?.[t.item_id] || null;
+    let res;
+    try {
+      res = await client.deleteFile(fn, etag);
+    } catch (err) {
+      markTombstoneFailed(dbManager, t.id, err?.message || String(err));
+      lastError = err?.message || String(err);
+      continue;
+    }
+    if (res.ok) {
+      deleteTombstone(dbManager, t.id);
+      removeFromMaps(dbManager, providerId, t.item_id, accountKey);
+      deleted++;
+      cursor = refresh();
+      maybeFlush("delete");
+    } else if (res.conflict) {
+      markTombstoneFailed(dbManager, t.id, "etag mismatch");
+      skipped++;
+    } else {
+      markTombstoneFailed(dbManager, t.id, res.error || "unknown");
+      lastError = res.error || lastError;
+    }
+  }
+
+  // Push loop
+  pushLoop: while (true) {
+    const batch = fetchBatch(dbManager, cursor, 200);
+    if (batch.length === 0) break;
+    for (const item of batch) {
+      const filename = generateFilename(item);
+      const content = generateMarkdown(item);
+      const etag = cursor.remoteEtagMap?.[item.id] || null;
+      let res;
+      try {
+        res = await client.putFile(filename, content, etag);
+      } catch (err) {
+        lastError = err?.message || String(err);
+        break pushLoop;
+      }
+      if (res.ok) {
+        recordPushedItem(
+          dbManager,
+          providerId,
+          item.id,
+          res.etag,
+          filename,
+          accountKey,
+        );
+        updateCursor(dbManager, providerId, cursorAfterItem(item), accountKey);
+        cursor = refresh();
+        pushed++;
+        maybeFlush("push");
+      } else if (res.conflict) {
+        skipped++;
+        updateCursor(dbManager, providerId, cursorAfterItem(item), accountKey);
+        cursor = refresh();
+      } else {
+        lastError = res.error || `status ${res.status}`;
+        break pushLoop;
+      }
+    }
+  }
+
+  const durationMs = Date.now() - t0;
+  const status = lastError ? "failed" : skipped > 0 ? "conflict" : "success";
+  updateCursor(
+    dbManager,
+    providerId,
+    {
+      lastRunStatus: status,
+      lastRunError: lastError,
+      lastRunDurationMs: durationMs,
+      itemsPushed: pushed,
+      itemsSkipped: skipped,
+      itemsDeleted: deleted,
+    },
+    accountKey,
+  );
+  onProgress?.({ phase: status, pushed, skipped, deleted, totalPending });
+
+  return {
+    success: !lastError,
+    status,
+    pushed,
+    skipped,
+    deleted,
+    durationMs,
+    error: lastError || undefined,
+  };
+}
+
+export {
+  runSync,
+  generateFilename,
+  generateMarkdown,
+  getCursor,
+  ensureCursor,
+  updateCursor,
+  recordPushedItem,
+  removeFromMaps,
+  listTombstones,
+  deleteTombstone,
+  markTombstoneFailed,
+  fetchBatch,
+  cursorAfterItem,
+  countPending,
+  PROGRESS_FLUSH_EVERY,
+  PROGRESS_FLUSH_MS,
+};