@chainlesschain/personal-data-hub 0.2.3 → 0.3.0

package/lib/adapters/shell-history/adapter.js

@@ -0,0 +1,137 @@
+"use strict";
+
+// ShellHistoryAdapter — surfaces PowerShell + bash + zsh command history
+// as a self-actor event stream. Same shape as VSCode's terminal-command
+// kind, just rooted in shell history files instead of an editor's state DB.
+
+const {
+  ENTITY_TYPES,
+  EVENT_SUBTYPES,
+  CAPTURED_BY,
+} = require("../../constants");
+
+const {
+  defaultHistorySources,
+  readAllHistory,
+} = require("./shell-reader");
+
+const NAME = "shell-history";
+const VERSION = "0.1.0";
+
+class ShellHistoryAdapter {
+  constructor(opts = {}) {
+    this.name = NAME;
+    this.version = VERSION;
+    this.capabilities = ["sync:shell-history-files"];
+    this.extractMode = "file-import";
+    this.rateLimits = { perDay: 96 };
+    this.dataDisclosure = {
+      fields: ["commands:shell,value,sourceIndex,snapshotTs"],
+      sensitivity: "high",
+      legalGate: false,
+      defaultInclude: { commands: true },
+    };
+    this._deps = {
+      defaultSources: defaultHistorySources,
+      readHistory: readAllHistory,
+    };
+    this._sourcesOverride = Array.isArray(opts.sources) ? opts.sources : null;
+  }
+
+  _resolveSources(opts) {
+    if (Array.isArray(opts?.sources) && opts.sources.length > 0) return opts.sources;
+    if (this._sourcesOverride) return this._sourcesOverride;
+    return this._deps.defaultSources();
+  }
+
+  async authenticate(ctx = {}) {
+    const sources = this._resolveSources(ctx);
+    if (!sources || sources.length === 0) {
+      return {
+        ok: false,
+        reason: "NO_HISTORY_SOURCES",
+        message: "no default shell history files on this platform; pass opts.sources",
+      };
+    }
+    return { ok: true, mode: "file-import", sources: sources.map((s) => s.shell) };
+  }
+
+  async healthCheck() {
+    return { ok: true, lastChecked: Date.now() };
+  }
+
+  async *sync(opts = {}) {
+    const sources = this._resolveSources(opts);
+    const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
+    const capturedAt = Date.now();
+    let emitted = 0;
+    for (const row of this._deps.readHistory(sources, { since: opts.since })) {
+      if (emitted >= limit) return;
+      yield {
+        kind: "shell-command",
+        // shell + sourceIndex + hash(value) keeps the same command at the
+        // same position in the same history file dedupable across re-syncs.
+        originalId: `shell-cmd:${row.shell}:${row.sourceIndex}:${hashCommand(row.value)}`,
+        capturedAt,
+        payload: row,
+      };
+      emitted += 1;
+    }
+  }
+
+  normalize(raw) {
+    const ingestedAt = Date.now();
+    const source = (originalId) => ({
+      adapter: NAME,
+      adapterVersion: VERSION,
+      capturedAt: raw.capturedAt,
+      capturedBy: CAPTURED_BY.SQLITE,
+      originalId,
+    });
+
+    if (raw.kind === "shell-command") {
+      const p = raw.payload || {};
+      const cmd = typeof p.value === "string" ? p.value : "";
+      const titleText = `[${p.shell || "?"}] ${cmd}`;
+      const event = {
+        id: `event-shell-cmd-${p.shell || "?"}-${p.sourceIndex}-${shortHash(cmd)}`,
+        type: ENTITY_TYPES.EVENT,
+        subtype: EVENT_SUBTYPES.OTHER,
+        occurredAt: Number.isInteger(p.snapshotTs) ? p.snapshotTs : raw.capturedAt,
+        ingestedAt,
+        source: source(raw.originalId),
+        actor: "self",
+        content: {
+          title: titleText.length > 100 ? titleText.substring(0, 100) + "…" : titleText,
+          text: cmd,
+        },
+        extra: {
+          kind: "shell-command",
+          shell: p.shell || null,
+          file: p.file || null,
+          sourceIndex: Number.isInteger(p.sourceIndex) ? p.sourceIndex : null,
+        },
+      };
+      return { events: [event], persons: [], places: [], items: [], topics: [] };
+    }
+
+    throw new Error(`shell-history.normalize: unknown raw.kind=${raw.kind}`);
+  }
+}
+
+function hashCommand(s) {
+  let h = 5381;
+  for (let i = 0; i < s.length; i++) {
+    h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
+  }
+  return h.toString(36);
+}
+function shortHash(s) {
+  return hashCommand(s).substring(0, 8);
+}
+
+module.exports = {
+  ShellHistoryAdapter,
+  SHELL_HISTORY_NAME: NAME,
+  SHELL_HISTORY_VERSION: VERSION,
+};

package/lib/adapters/shell-history/index.js

@@ -0,0 +1,17 @@
+"use strict";
+
+const {
+  ShellHistoryAdapter,
+  SHELL_HISTORY_NAME,
+  SHELL_HISTORY_VERSION,
+} = require("./adapter");
+const reader = require("./shell-reader");
+
+module.exports = {
+  ShellHistoryAdapter,
+  SHELL_HISTORY_NAME,
+  SHELL_HISTORY_VERSION,
+  defaultHistorySources: reader.defaultHistorySources,
+  readHistoryFile: reader.readHistoryFile,
+  readAllHistory: reader.readAllHistory,
+};

package/lib/adapters/shell-history/shell-reader.js

@@ -0,0 +1,100 @@
+"use strict";
+
+// shell-reader — reads PowerShell + bash command history files. Both
+// formats are dead simple: one command per line. Neither carries
+// per-command timestamps, so we anchor every command to the file mtime
+// and use sourceIndex for stable ordering (same pattern as the VSCode
+// terminal adapter).
+
+const fs = require("node:fs");
+const path = require("node:path");
+const os = require("node:os");
+
+function defaultHistorySources() {
+  const home = os.homedir();
+  const sources = [];
+  if (process.platform === "win32" && process.env.APPDATA) {
+    sources.push({
+      shell: "pwsh",
+      file: path.join(
+        process.env.APPDATA,
+        "Microsoft",
+        "Windows",
+        "PowerShell",
+        "PSReadLine",
+        "ConsoleHost_history.txt",
+      ),
+    });
+  }
+  // bash history exists on Win (Git Bash / WSL) AND Unix.
+  sources.push({ shell: "bash", file: path.join(home, ".bash_history") });
+  // zsh on macOS / Linux defaults.
+  if (process.platform !== "win32") {
+    sources.push({ shell: "zsh", file: path.join(home, ".zsh_history") });
+  }
+  return sources;
+}
+
+// Reads one history file into rows. Skips blank/whitespace-only lines and
+// preserves the original index so we can re-order across syncs.
+function readHistoryFile(source, opts = {}) {
+  const fsMod = opts.fs || fs;
+  if (!fsMod.existsSync(source.file)) return null;
+  let stat;
+  try {
+    stat = fsMod.statSync(source.file);
+  } catch {
+    return null;
+  }
+  let text;
+  try {
+    text = fsMod.readFileSync(source.file, "utf-8");
+  } catch {
+    return null;
+  }
+  const lines = text.split(/\r?\n/);
+  const rows = [];
+  for (let i = 0; i < lines.length; i++) {
+    let value = lines[i];
+    // zsh extended history stores ": <ts>:<dur>;<cmd>" — strip the prefix.
+    if (source.shell === "zsh") {
+      const m = value.match(/^: \d+:\d+;(.*)$/);
+      if (m) value = m[1];
+    }
+    value = value.replace(/[\r\n]+$/, "").trim();
+    if (!value) continue;
+    rows.push({
+      shell: source.shell,
+      file: source.file,
+      value,
+      sourceIndex: i,
+    });
+  }
+  return {
+    shell: source.shell,
+    file: source.file,
+    mtimeMs: Math.floor(stat.mtimeMs),
+    rows,
+  };
+}
+
+// Yields rows across every configured source in (shell, sourceIndex)
+// ascending order. since filter uses file mtime — there's no per-row
+// timestamp in either format.
+function* readAllHistory(sources, opts = {}) {
+  const sinceMs = Number.isInteger(opts.since) && opts.since > 0 ? opts.since : 0;
+  for (const src of sources) {
+    const parsed = readHistoryFile(src, opts);
+    if (!parsed) continue;
+    if (sinceMs > 0 && parsed.mtimeMs < sinceMs) continue;
+    for (const row of parsed.rows) {
+      yield { ...row, snapshotTs: parsed.mtimeMs };
+    }
+  }
+}
+
+module.exports = {
+  defaultHistorySources,
+  readHistoryFile,
+  readAllHistory,
+};

package/lib/adapters/social-kuaishou/index.js

@@ -41,18 +41,23 @@ const fs = require("node:fs");
 const { newId } = require("../../ids");
 const {
   ENTITY_TYPES,
+  PERSON_SUBTYPES,
   EVENT_SUBTYPES,
   CAPTURED_BY,
 } = require("../../constants");
 
 const NAME = "social-kuaishou";
-const VERSION = "0.2.0";
+const VERSION = "0.2.1";
 const SNAPSHOT_SCHEMA_VERSION = 1;
 
+const KIND_PROFILE = "profile";
 const KIND_WATCH = "watch";
 const KIND_COLLECT = "collect";
 const KIND_SEARCH = "search";
+// v0.2.1 — KIND_PROFILE added (mirrors Douyin/Toutiao); v0.3 will add watch/
+// collect/search via NS_sig3. SNAPSHOT_SCHEMA_VERSION stays at 1 — additive.
 const VALID_SNAPSHOT_KINDS = Object.freeze([
+  KIND_PROFILE,
   KIND_WATCH,
   KIND_COLLECT,
   KIND_SEARCH,
@@ -102,6 +107,7 @@ class KuaishouAdapter {
     this.capabilities = [
       "sync:snapshot",
       "sync:sqlite",
+      "parse:kuaishou-profile",
       "parse:kuaishou-photo-history",
       "parse:kuaishou-user-collect",
       "parse:kuaishou-search",
@@ -110,6 +116,7 @@ class KuaishouAdapter {
     this.rateLimits = {};
     this.dataDisclosure = {
       fields: [
+        "kuaishou:profile (user_id / user_name / kuaishou_id / headurl / sex / city)",
         "kuaishou:photo_history (photo_id / caption / view_time / duration / author_id)",
         "kuaishou:user_collect (photo_id / caption / collect_time)",
         "kuaishou:search_record (keyword / search_time)",
@@ -117,6 +124,7 @@ class KuaishouAdapter {
       sensitivity: "medium",
       legalGate: false,
       defaultInclude: {
+        profile: true,
         watch: true,
         collect: true,
         search: true,
@@ -307,6 +315,9 @@ class KuaishouAdapter {
     const kind = raw.kind || raw.payload.kind;
     const p = raw.payload;
 
+    if (kind === KIND_PROFILE) {
+      return normalizeProfile(p, raw, ingestedAt);
+    }
     if (kind === KIND_COLLECT) {
       return normalizeCollect(p, raw, ingestedAt);
     }
@@ -320,6 +331,47 @@ class KuaishouAdapter {
   }
 }
 
+function normalizeProfile(p, raw, ingestedAt) {
+  // v0.2 snapshot-only — produces a person record for the logged-in user
+  // (person-self) carrying kuaishou-uid + kuaishou-id identifiers + profile
+  // metadata in extra. Repeated syncs dedupe on the same id; extra fields
+  // get refreshed.
+  const uid = p.uid || (p.account && p.account.uid) || null;
+  const nickname =
+    p.nickname || (p.account && p.account.displayName) || "(unnamed)";
+  const occurredAt = parseTime(p.capturedAt) || raw.capturedAt || ingestedAt;
+  const source = buildSource(raw, occurredAt, CAPTURED_BY.API);
+  const identifiers = {};
+  if (uid) identifiers["kuaishou-uid"] = [String(uid)];
+  if (p.kuaishouId) identifiers["kuaishou-id"] = [String(p.kuaishouId)];
+  return {
+    events: [],
+    persons: [
+      {
+        id: uid ? `person-kuaishou-${uid}` : `person-kuaishou-self-${newId()}`,
+        type: ENTITY_TYPES.PERSON,
+        subtype: PERSON_SUBTYPES.SELF,
+        names: [nickname],
+        ingestedAt,
+        source,
+        identifiers,
+        extra: {
+          platform: "kuaishou",
+          avatarUrl: p.avatarUrl || null,
+          sex: p.sex || null,
+          city: p.city || null,
+          constellation: p.constellation || null,
+          description: p.description || null,
+          snapshottedAt: occurredAt,
+        },
+      },
+    ],
+    places: [],
+    items: [],
+    topics: [],
+  };
+}
+
 function buildSource(raw, occurredAt, capturedBy) {
   return {
     adapter: NAME,
@@ -486,4 +538,8 @@ module.exports = {
   VERSION,
   SNAPSHOT_SCHEMA_VERSION,
   VALID_SNAPSHOT_KINDS,
+  KIND_PROFILE,
+  KIND_WATCH,
+  KIND_COLLECT,
+  KIND_SEARCH,
 };

package/lib/adapters/social-toutiao/index.js

@@ -44,18 +44,25 @@ const fs = require("node:fs");
 const { newId } = require("../../ids");
 const {
   ENTITY_TYPES,
+  PERSON_SUBTYPES,
   EVENT_SUBTYPES,
   CAPTURED_BY,
 } = require("../../constants");
 
 const NAME = "social-toutiao";
-const VERSION = "0.2.0";
+const VERSION = "0.2.1";
 const SNAPSHOT_SCHEMA_VERSION = 1;
 
+const KIND_PROFILE = "profile";
 const KIND_READ = "read";
 const KIND_COLLECTION = "collection";
 const KIND_SEARCH = "search";
+// v0.2.1 — KIND_PROFILE added (mirrors Douyin); v0.3 will add read/collection
+// /search once _signature path is wired. SNAPSHOT_SCHEMA_VERSION stays at 1:
+// old (events-only) snapshots remain compatible; new profile events are an
+// additive extension.
 const VALID_SNAPSHOT_KINDS = Object.freeze([
+  KIND_PROFILE,
   KIND_READ,
   KIND_COLLECTION,
   KIND_SEARCH,
@@ -106,6 +113,7 @@ class ToutiaoAdapter {
     this.capabilities = [
       "sync:snapshot",
       "sync:sqlite",
+      "parse:toutiao-profile",
       "parse:toutiao-read-history",
       "parse:toutiao-collection",
       "parse:toutiao-search",
@@ -116,6 +124,7 @@ class ToutiaoAdapter {
     this.rateLimits = {};
     this.dataDisclosure = {
       fields: [
+        "toutiao:profile (user_id / screen_name / avatar / mobile / following / followers)",
         "toutiao:read_history (item_id / title / read_time / category)",
         "toutiao:collection_article (item_id / title / save_time)",
         "toutiao:search_history (keyword / search_time)",
@@ -124,6 +133,7 @@ class ToutiaoAdapter {
       sensitivity: "high",
       legalGate: false,
       defaultInclude: {
+        profile: true,
         read: true,
         collection: true,
         search: true,
@@ -317,6 +327,9 @@ class ToutiaoAdapter {
     const kind = raw.kind || raw.payload.kind;
     const p = raw.payload;
 
+    if (kind === KIND_PROFILE) {
+      return normalizeProfile(p, raw, ingestedAt);
+    }
     if (kind === KIND_COLLECTION) {
       return normalizeCollection(p, raw, ingestedAt);
     }
@@ -330,6 +343,47 @@ class ToutiaoAdapter {
   }
 }
 
+function normalizeProfile(p, raw, ingestedAt) {
+  // v0.2 snapshot-only — produces a person record for the logged-in user
+  // (person-self) carrying toutiao-uid identifier + counts in extra.
+  // Repeated syncs dedupe on the same id; extra fields get refreshed.
+  const uid = p.uid || (p.account && p.account.uid) || null;
+  const nickname =
+    p.nickname || (p.account && p.account.displayName) || "(unnamed)";
+  const occurredAt = parseTime(p.capturedAt) || raw.capturedAt || ingestedAt;
+  const source = buildSource(raw, occurredAt, CAPTURED_BY.API);
+  const identifiers = {};
+  if (uid) identifiers["toutiao-uid"] = [String(uid)];
+  if (p.mediaId) identifiers["toutiao-media-id"] = [String(p.mediaId)];
+  return {
+    events: [],
+    persons: [
+      {
+        id: uid ? `person-toutiao-${uid}` : `person-toutiao-self-${newId()}`,
+        type: ENTITY_TYPES.PERSON,
+        subtype: PERSON_SUBTYPES.SELF,
+        names: [nickname],
+        ingestedAt,
+        source,
+        identifiers,
+        extra: {
+          platform: "toutiao",
+          avatarUrl: p.avatarUrl || null,
+          description: p.description || null,
+          mobile: p.mobile || null,
+          followingCount: p.followingCount || 0,
+          followerCount: p.followerCount || 0,
+          mediaId: p.mediaId || null,
+          snapshottedAt: occurredAt,
+        },
+      },
+    ],
+    places: [],
+    items: [],
+    topics: [],
+  };
+}
+
 function buildSource(raw, occurredAt, capturedBy) {
   return {
     adapter: NAME,
@@ -503,4 +557,8 @@ module.exports = {
   VERSION,
   SNAPSHOT_SCHEMA_VERSION,
   VALID_SNAPSHOT_KINDS,
+  KIND_PROFILE,
+  KIND_READ,
+  KIND_COLLECTION,
+  KIND_SEARCH,
 };