@chainlesschain/personal-data-hub 0.2.4 → 0.3.0

package/lib/adapters/git-activity/adapter.js

@@ -0,0 +1,155 @@
+"use strict";
+
+// GitActivityAdapter — yields recent commits from every .git repo found
+// under the user's code roots as a developer-activity timeline.
+//
+// Pure local: shells out `git log` per repo, no remote fetches.
+
+const path = require("node:path");
+
+const {
+  ENTITY_TYPES,
+  EVENT_SUBTYPES,
+  CAPTURED_BY,
+} = require("../../constants");
+
+const {
+  defaultCodeRoots,
+  findGitRepos,
+  listCommits,
+} = require("./git-reader");
+
+const NAME = "git-activity";
+const VERSION = "0.1.0";
+
+class GitActivityAdapter {
+  constructor(opts = {}) {
+    this.name = NAME;
+    this.version = VERSION;
+    this.capabilities = ["sync:git-log-local"];
+    this.extractMode = "file-import";
+    this.rateLimits = { perDay: 48 };
+    this.dataDisclosure = {
+      fields: ["commits:sha,authoredAtMs,authorName,authorEmail,subject,repoName"],
+      sensitivity: "high",
+      legalGate: false,
+      defaultInclude: { commits: true },
+    };
+    this._deps = {
+      defaultRoots: defaultCodeRoots,
+      findRepos: findGitRepos,
+      listCommits,
+    };
+    this._rootsOverride = Array.isArray(opts.codeRoots) ? opts.codeRoots : null;
+  }
+
+  _resolveRoots(opts) {
+    if (Array.isArray(opts?.codeRoots) && opts.codeRoots.length > 0) {
+      return opts.codeRoots;
+    }
+    if (this._rootsOverride && this._rootsOverride.length > 0) return this._rootsOverride;
+    return this._deps.defaultRoots();
+  }
+
+  async authenticate(ctx = {}) {
+    const roots = this._resolveRoots(ctx);
+    if (!roots || roots.length === 0) {
+      return {
+        ok: false,
+        reason: "NO_CODE_ROOTS",
+        message: "no default code roots found (tried C:\\code on Win, ~/code ~/projects on Unix); pass opts.codeRoots",
+      };
+    }
+    const repos = this._deps.findRepos(roots);
+    if (repos.length === 0) {
+      return {
+        ok: false,
+        reason: "NO_GIT_REPOS",
+        message: `no .git directories under any of: ${roots.join(", ")}`,
+      };
+    }
+    return { ok: true, mode: "file-import", codeRoots: roots, repoCount: repos.length };
+  }
+
+  async healthCheck() {
+    const roots = this._resolveRoots({});
+    const ok = roots && roots.length > 0;
+    return { ok, lastChecked: Date.now() };
+  }
+
+  async *sync(opts = {}) {
+    const roots = this._resolveRoots(opts);
+    if (!roots || roots.length === 0) {
+      throw new Error("git-activity.sync: no code roots — pass opts.codeRoots");
+    }
+    const repos = this._deps.findRepos(roots);
+    const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
+    const maxPerRepo =
+      Number.isInteger(opts.maxPerRepo) && opts.maxPerRepo > 0 ? opts.maxPerRepo : 500;
+    const capturedAt = Date.now();
+    let emitted = 0;
+    for (const repoDir of repos) {
+      if (emitted >= limit) return;
+      const commits = this._deps.listCommits(repoDir, { since: opts.since, maxPerRepo });
+      for (const c of commits) {
+        if (emitted >= limit) return;
+        yield {
+          kind: "commit",
+          // sha is globally unique within git's universe — repo path keeps
+          // multi-clone scenarios distinct (same sha in two clones = two rows).
+          originalId: `git-commit:${repoDir}:${c.sha}`,
+          capturedAt,
+          payload: c,
+        };
+        emitted += 1;
+      }
+    }
+  }
+
+  normalize(raw) {
+    const ingestedAt = Date.now();
+    const source = (originalId) => ({
+      adapter: NAME,
+      adapterVersion: VERSION,
+      capturedAt: raw.capturedAt,
+      capturedBy: CAPTURED_BY.SQLITE,
+      originalId,
+    });
+
+    if (raw.kind === "commit") {
+      const p = raw.payload || {};
+      const subj = typeof p.subject === "string" && p.subject ? p.subject : "(no subject)";
+      const event = {
+        id: `event-git-commit-${p.shortSha || p.sha}`,
+        type: ENTITY_TYPES.EVENT,
+        subtype: EVENT_SUBTYPES.OTHER,
+        occurredAt: Number.isInteger(p.authoredAtMs) ? p.authoredAtMs : raw.capturedAt,
+        ingestedAt,
+        source: source(raw.originalId),
+        actor: p.authorName || p.authorEmail || "self",
+        content: {
+          title: subj.length > 100 ? subj.substring(0, 100) + "…" : subj,
+          text: subj,
+        },
+        extra: {
+          kind: "git-commit",
+          sha: p.sha || null,
+          shortSha: p.shortSha || null,
+          repoName: p.repoName || null,
+          repoDir: p.repoDir || null,
+          authorName: p.authorName || null,
+          authorEmail: p.authorEmail || null,
+        },
+      };
+      return { events: [event], persons: [], places: [], items: [], topics: [] };
+    }
+
+    throw new Error(`git-activity.normalize: unknown raw.kind=${raw.kind}`);
+  }
+}
+
+module.exports = {
+  GitActivityAdapter,
+  GIT_ACTIVITY_NAME: NAME,
+  GIT_ACTIVITY_VERSION: VERSION,
+};

package/lib/adapters/git-activity/git-reader.js

@@ -0,0 +1,125 @@
+"use strict";
+
+// git-reader — enumerates `.git` directories under configured code roots
+// and shells out `git log` to extract recent commits. No clone-time
+// metadata, no remote network calls; pure local-filesystem walk.
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+const { execFileSync } = require("node:child_process");
+
+function defaultCodeRoots() {
+  const home = os.homedir();
+  if (process.platform === "win32") {
+    // Most devs on Windows use C:\code\ or ~/code/.
+    const candidates = ["C:\\code", path.join(home, "code"), path.join(home, "projects")];
+    return candidates.filter((d) => {
+      try {
+        return fs.statSync(d).isDirectory();
+      } catch {
+        return false;
+      }
+    });
+  }
+  return [path.join(home, "code"), path.join(home, "projects"), path.join(home, "src")].filter(
+    (d) => {
+      try {
+        return fs.statSync(d).isDirectory();
+      } catch {
+        return false;
+      }
+    },
+  );
+}
+
+// Find every `.git` directory one level under each root. Skips bare /
+// nested repos for v0.1 — keeps the surface area predictable.
+function findGitRepos(roots, opts = {}) {
+  const fsMod = opts.fs || fs;
+  const out = [];
+  for (const root of roots) {
+    let entries;
+    try {
+      entries = fsMod.readdirSync(root, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const e of entries) {
+      if (!e.isDirectory()) continue;
+      const repoDir = path.join(root, e.name);
+      const dotGit = path.join(repoDir, ".git");
+      if (!fsMod.existsSync(dotGit)) continue;
+      out.push(repoDir);
+    }
+  }
+  return out;
+}
+
+// Run git log against one repo. The pipe delimiter is safer than a single
+// char because commit messages may contain tabs or pipes — we add a
+// recognizable sentinel between fields. Bail to [] on any spawn error so
+// one corrupt repo doesn't sink the whole sync.
+const FIELD_SEP = "
"; // SOH — guaranteed absent from commit metadata
+const ROW_SEP = ""; // RS
+
+function listCommits(repoDir, opts = {}) {
+  const sinceMs = Number.isInteger(opts.since) && opts.since > 0 ? opts.since : 0;
+  // git wants ISO-ish dates or relative; use unix-seconds for precision.
+  const sinceArg = sinceMs > 0 ? `--since=@${Math.floor(sinceMs / 1000)}` : "--since=180.days";
+  const maxN = Number.isInteger(opts.maxPerRepo) && opts.maxPerRepo > 0 ? opts.maxPerRepo : 500;
+  const fmt = ["%H", "%aI", "%an", "%ae", "%s"].join(FIELD_SEP) + ROW_SEP;
+  let stdout = "";
+  try {
+    stdout = execFileSync(
+      "git",
+      [
+        "-C",
+        repoDir,
+        "log",
+        sinceArg,
+        `-n${maxN}`,
+        `--pretty=format:${fmt}`,
+        "--no-merges",
+      ],
+      {
+        encoding: "utf-8",
+        timeout: 30_000,
+        windowsHide: true,
+        stdio: ["ignore", "pipe", "ignore"],
+        maxBuffer: 32 * 1024 * 1024, // 32 MB — handles repos with many commits
+      },
+    );
+  } catch {
+    return [];
+  }
+  const repoName = path.basename(repoDir);
+  const out = [];
+  for (const raw of stdout.split(ROW_SEP)) {
+    const line = raw.replace(/^[\r\n]+/, "").replace(/[\r\n]+$/, "");
+    if (!line) continue;
+    const parts = line.split(FIELD_SEP);
+    if (parts.length < 5) continue;
+    const [sha, isoDate, authorName, authorEmail, subject] = parts;
+    const d = new Date(isoDate);
+    const ts = Number.isFinite(d.getTime()) ? d.getTime() : 0;
+    if (ts === 0) continue;
+    out.push({
+      sha,
+      shortSha: sha.substring(0, 8),
+      authoredAtMs: ts,
+      authorName: authorName || "",
+      authorEmail: authorEmail || "",
+      subject: subject || "",
+      repoDir,
+      repoName,
+    });
+  }
+  return out;
+}
+
+module.exports = {
+  defaultCodeRoots,
+  findGitRepos,
+  listCommits,
+};

package/lib/adapters/git-activity/index.js

@@ -0,0 +1,17 @@
+"use strict";
+
+const {
+  GitActivityAdapter,
+  GIT_ACTIVITY_NAME,
+  GIT_ACTIVITY_VERSION,
+} = require("./adapter");
+const reader = require("./git-reader");
+
+module.exports = {
+  GitActivityAdapter,
+  GIT_ACTIVITY_NAME,
+  GIT_ACTIVITY_VERSION,
+  defaultCodeRoots: reader.defaultCodeRoots,
+  findGitRepos: reader.findGitRepos,
+  listCommits: reader.listCommits,
+};

package/lib/adapters/local-files/adapter.js

@@ -0,0 +1,149 @@
+"use strict";
+
+// LocalFilesAdapter — walks user-data roots (Documents / Desktop / Downloads
+// / Pictures / Videos / Music) and surfaces one Event(OTHER) per file. Same
+// "what did I touch and when" shape as win-recent, but rooted in real on-disk
+// files instead of .lnk shortcuts — so it captures files the user created /
+// saved / downloaded even if they never opened them via Explorer.
+//
+// Excludes app cache dirs (xwechat_files / WXWork / node_modules / .git) by
+// default to keep the vault signal-rich.
+
+const {
+  ENTITY_TYPES,
+  EVENT_SUBTYPES,
+  CAPTURED_BY,
+} = require("../../constants");
+
+const {
+  defaultRoots,
+  walkRoots,
+  DEFAULT_EXCLUDES,
+} = require("./file-walker");
+
+const NAME = "local-files";
+const VERSION = "0.1.0";
+
+class LocalFilesAdapter {
+  constructor(opts = {}) {
+    this.name = NAME;
+    this.version = VERSION;
+    this.capabilities = ["sync:local-file-walk"];
+    this.extractMode = "file-import";
+    this.rateLimits = { perDay: 24 };
+    this.dataDisclosure = {
+      fields: ["files:path,name,ext,size,mtimeMs,root"],
+      sensitivity: "high",
+      legalGate: false,
+      defaultInclude: { files: true },
+    };
+    this._deps = {
+      defaultRoots,
+      walkRoots,
+    };
+    this._rootsOverride = Array.isArray(opts.roots) ? opts.roots : null;
+    this._excludesOverride = Array.isArray(opts.excludes) ? opts.excludes : null;
+  }
+
+  _resolveRoots(opts) {
+    if (Array.isArray(opts?.roots) && opts.roots.length > 0) return opts.roots;
+    if (this._rootsOverride) return this._rootsOverride;
+    return this._deps.defaultRoots();
+  }
+
+  _resolveExcludes(opts) {
+    if (Array.isArray(opts?.excludes) && opts.excludes.length > 0) return opts.excludes;
+    if (this._excludesOverride) return this._excludesOverride;
+    return DEFAULT_EXCLUDES;
+  }
+
+  async authenticate(ctx = {}) {
+    const roots = this._resolveRoots(ctx);
+    if (!roots || roots.length === 0) {
+      return {
+        ok: false,
+        reason: "NO_DATA_ROOTS",
+        message: "no default user-data dirs available; pass opts.roots",
+      };
+    }
+    return { ok: true, mode: "file-import", roots };
+  }
+
+  async healthCheck() {
+    return { ok: true, lastChecked: Date.now() };
+  }
+
+  async *sync(opts = {}) {
+    const roots = this._resolveRoots(opts);
+    const excludes = this._resolveExcludes(opts);
+    const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
+    const capturedAt = Date.now();
+    let emitted = 0;
+    for (const row of this._deps.walkRoots(roots, { ...opts, excludes })) {
+      if (emitted >= limit) return;
+      yield {
+        kind: "local-file",
+        originalId: `local-file:${row.path}:${row.mtimeMs}`,
+        capturedAt,
+        payload: row,
+      };
+      emitted += 1;
+    }
+  }
+
+  normalize(raw) {
+    const ingestedAt = Date.now();
+    const source = (originalId) => ({
+      adapter: NAME,
+      adapterVersion: VERSION,
+      capturedAt: raw.capturedAt,
+      capturedBy: CAPTURED_BY.SQLITE,
+      originalId,
+    });
+
+    if (raw.kind === "local-file") {
+      const p = raw.payload || {};
+      const name = typeof p.name === "string" && p.name.length > 0 ? p.name : "(无名)";
+      const titleText = `[file] ${name}`;
+      const event = {
+        id: `event-local-file-${shortHash(raw.originalId)}`,
+        type: ENTITY_TYPES.EVENT,
+        subtype: EVENT_SUBTYPES.OTHER,
+        occurredAt: Number.isInteger(p.mtimeMs) ? p.mtimeMs : raw.capturedAt,
+        ingestedAt,
+        source: source(raw.originalId),
+        actor: "self",
+        content: {
+          title: titleText.length > 100 ? titleText.substring(0, 100) + "…" : titleText,
+          text: typeof p.path === "string" ? p.path : name,
+        },
+        extra: {
+          kind: "local-file",
+          path: typeof p.path === "string" ? p.path : null,
+          name,
+          ext: typeof p.ext === "string" ? p.ext : "",
+          size: Number.isFinite(p.size) ? p.size : null,
+          root: typeof p.root === "string" ? p.root : null,
+        },
+      };
+      return { events: [event], persons: [], places: [], items: [], topics: [] };
+    }
+
+    throw new Error(`local-files.normalize: unknown raw.kind=${raw.kind}`);
+  }
+}
+
+function shortHash(s) {
+  let h = 5381;
+  const str = typeof s === "string" ? s : "";
+  for (let i = 0; i < str.length; i++) {
+    h = ((h << 5) + h + str.charCodeAt(i)) >>> 0;
+  }
+  return h.toString(36).substring(0, 10);
+}
+
+module.exports = {
+  LocalFilesAdapter,
+  LOCAL_FILES_NAME: NAME,
+  LOCAL_FILES_VERSION: VERSION,
+};

package/lib/adapters/local-files/file-walker.js

@@ -0,0 +1,125 @@
+"use strict";
+
+// file-walker — recursive directory walker for user-data roots
+// (Documents / Desktop / Downloads / Pictures / Videos / Music).
+//
+// Emits one row per file with absolute path + mtime + size. Skips:
+//   - any path component matching a default exclude (xwechat_files,
+//     WXWork, node_modules, .git) — these are noisy app caches that
+//     would flood the vault without user value
+//   - hidden files / dirs (leading '.')
+//   - symlinks (avoid loops + permission errors)
+//
+// max depth + max files per root are bounded to keep first-time sync
+// from walking a Downloads dir with 500k files.
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+
+const DEFAULT_EXCLUDES = Object.freeze([
+  "xwechat_files",
+  "WXWork",
+  "node_modules",
+  ".git",
+]);
+
+const DEFAULT_MAX_DEPTH = 6;
+const DEFAULT_MAX_FILES_PER_ROOT = 5000;
+
+function defaultRoots() {
+  const home = os.homedir();
+  if (!home) return [];
+  const candidates = [
+    path.join(home, "Documents"),
+    path.join(home, "Desktop"),
+    path.join(home, "Downloads"),
+    path.join(home, "Pictures"),
+    path.join(home, "Videos"),
+    path.join(home, "Music"),
+  ];
+  return candidates;
+}
+
+function shouldSkip(name, excludes) {
+  if (!name) return true;
+  if (name.startsWith(".")) return true;
+  for (const ex of excludes) {
+    if (name === ex) return true;
+  }
+  return false;
+}
+
+// Walks one root; yields { path, name, ext, size, mtimeMs, root } per file.
+// Uses iterative DFS to bound stack depth and skip whole subtrees on
+// permission errors without aborting the whole walk.
+function* walkRoot(root, opts = {}) {
+  const fsMod = opts.fs || fs;
+  const excludes = Array.isArray(opts.excludes) ? opts.excludes : DEFAULT_EXCLUDES;
+  const maxDepth = Number.isInteger(opts.maxDepth) && opts.maxDepth > 0 ? opts.maxDepth : DEFAULT_MAX_DEPTH;
+  const maxFiles = Number.isInteger(opts.maxFilesPerRoot) && opts.maxFilesPerRoot > 0
+    ? opts.maxFilesPerRoot
+    : DEFAULT_MAX_FILES_PER_ROOT;
+  if (!fsMod.existsSync(root)) return;
+  let count = 0;
+  const stack = [{ dir: root, depth: 0 }];
+  while (stack.length > 0) {
+    const { dir, depth } = stack.pop();
+    if (depth > maxDepth) continue;
+    let entries;
+    try {
+      entries = fsMod.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      if (shouldSkip(entry.name, excludes)) continue;
+      const full = path.join(dir, entry.name);
+      // dirent does not always carry symlink info reliably; skip symlinks
+      // explicitly via lstat to avoid following them.
+      let st;
+      try {
+        st = fsMod.lstatSync(full);
+      } catch {
+        continue;
+      }
+      if (st.isSymbolicLink()) continue;
+      if (st.isDirectory()) {
+        stack.push({ dir: full, depth: depth + 1 });
+        continue;
+      }
+      if (!st.isFile()) continue;
+      if (count >= maxFiles) return;
+      count += 1;
+      yield {
+        path: full,
+        name: entry.name,
+        ext: path.extname(entry.name).toLowerCase().replace(/^\./, ""),
+        size: Number.isFinite(st.size) ? st.size : 0,
+        mtimeMs: Math.floor(st.mtimeMs),
+        root,
+      };
+    }
+  }
+}
+
+// Yields across every configured root in stable order. `since` filters
+// by file mtime so re-syncs only surface recently changed files.
+function* walkRoots(roots, opts = {}) {
+  const sinceMs = Number.isInteger(opts.since) && opts.since > 0 ? opts.since : 0;
+  for (const root of roots) {
+    for (const row of walkRoot(root, opts)) {
+      if (sinceMs > 0 && row.mtimeMs < sinceMs) continue;
+      yield row;
+    }
+  }
+}
+
+module.exports = {
+  DEFAULT_EXCLUDES,
+  DEFAULT_MAX_DEPTH,
+  DEFAULT_MAX_FILES_PER_ROOT,
+  defaultRoots,
+  walkRoot,
+  walkRoots,
+};

package/lib/adapters/local-files/index.js

@@ -0,0 +1,18 @@
+"use strict";
+
+const {
+  LocalFilesAdapter,
+  LOCAL_FILES_NAME,
+  LOCAL_FILES_VERSION,
+} = require("./adapter");
+const walker = require("./file-walker");
+
+module.exports = {
+  LocalFilesAdapter,
+  LOCAL_FILES_NAME,
+  LOCAL_FILES_VERSION,
+  defaultRoots: walker.defaultRoots,
+  walkRoot: walker.walkRoot,
+  walkRoots: walker.walkRoots,
+  DEFAULT_EXCLUDES: walker.DEFAULT_EXCLUDES,
+};