@chainlesschain/personal-data-hub 0.4.7 → 0.4.18

package/lib/adapters/_video-base.js

@@ -0,0 +1,331 @@
+/**
+ * _video-base — shared infrastructure for "video watch-history" adapters
+ * (爱奇艺 / 腾讯视频 / etc.), Phase 13+ §12.1 (ROI ⭐⭐ each).
+ *
+ * These platforms expose the same shape of personal data: a paginated list of
+ * videos the user watched (观看记录) + optionally favourited/追剧 (收藏). Rather
+ * than copy ~300 lines per platform (mirroring _document-base / shopping-base /
+ * travel-base), `createVideoAdapter(config)` returns a fully-formed adapter
+ * class with snapshot + cookie-api modes; each platform supplies only its
+ * endpoints + field mapping.
+ *
+ *   1. snapshot mode (opts.inputPath): JSON schemaVersion 1, stateless.
+ *   2. cookie-api mode (opts.account.cookies): fetch watch / favourite lists via
+ *      the injected `fetchFn` (Android in-APK cc → OkHttp; desktop hub →
+ *      Electron WebView net request), paginate. A sign seam (opts.signProvider)
+ *      covers anti-bot tokens; best-effort unsigned. Endpoints overridable via
+ *      opts.watchUrl / opts.favouriteUrl (best-effort, not field-verified —
+ *      FAMILY-23 playbook).
+ *
+ * normalize() emits, per item: a MEDIA event (watch) or LIKE event (favourite)
+ * + a MEDIA item, mirroring netease-music / music-kugou so the vault can both
+ * timeline "我看了 X" and list the video entity.
+ *
+ * Snapshot schema (schemaVersion 1):
+ *   {
+ *     "schemaVersion": 1, "snapshottedAt": <ms>,
+ *     "account": { "userId": "...", "name": "..." },
+ *     "events": [
+ *       { "kind": "watch",     "id": "...", "videoId": "...", "title": "...",
+ *         "category": "movie|tv|variety|anime|...", "episode": "...",
+ *         "channel": "...", "durationSec": N, "capturedAt": <s|ms> },
+ *       { "kind": "favourite", "id": "...", "videoId": "...", "title": "...",
+ *         "category": "...", "capturedAt": <ms> }
+ *     ]
+ *   }
+ */
+
+"use strict";
+
+const fs = require("node:fs");
+const { newId } = require("../ids");
+const {
+  ENTITY_TYPES,
+  EVENT_SUBTYPES,
+  ITEM_SUBTYPES,
+  CAPTURED_BY,
+} = require("../constants");
+
+const SNAPSHOT_SCHEMA_VERSION = 1;
+const KIND_WATCH = "watch";
+const KIND_FAVOURITE = "favourite";
+const VALID_SNAPSHOT_KINDS = Object.freeze([KIND_WATCH, KIND_FAVOURITE]);
+const PAGE_SIZE = 30;
+
+function parseTime(v) {
+  if (Number.isFinite(v)) return v > 1e12 ? v : v >= 1e9 ? v * 1000 : v;
+  if (typeof v === "string") {
+    if (/^\d+$/.test(v)) {
+      const n = parseInt(v, 10);
+      return n > 1e12 ? n : n >= 1e9 ? n * 1000 : n;
+    }
+    const t = Date.parse(v);
+    return Number.isFinite(t) ? t : null;
+  }
+  return null;
+}
+
+/**
+ * @param {object} config
+ * @param {string} config.NAME           e.g. "video-iqiyi"
+ * @param {string} config.VERSION
+ * @param {string} config.platform       e.g. "iqiyi"
+ * @param {string} config.watchUrl       best-effort watch-history endpoint
+ * @param {string} config.favouriteUrl   best-effort favourite/追剧 endpoint
+ * @param {(resp:any)=>any[]} config.extractItems
+ * @param {(raw:any)=>object|null} config.mapItem
+ *        VideoRecord = { videoId, title, category, episode, channel, durationSec, url, occurredAt? }
+ */
+function createVideoAdapter(config) {
+  const { NAME, VERSION, platform, watchUrl, favouriteUrl, extractItems, mapItem } = config;
+  const { CookieAuth } = require("./shopping-base");
+
+  function stableOriginalId(kind, id) {
+    const safe =
+      (typeof id === "string" && id.length > 0 && id) ||
+      (typeof id === "number" && Number.isFinite(id) && String(id)) ||
+      `unknown-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    return `${platform}:${kind}:${safe}`;
+  }
+
+  class VideoAdapter {
+    constructor(opts = {}) {
+      this.account = opts.account || null;
+      this._cookieAuth =
+        opts.account && opts.account.cookies
+          ? new CookieAuth({ platform, cookies: opts.account.cookies })
+          : null;
+      this._fetchFn = typeof opts.fetchFn === "function" ? opts.fetchFn : defaultFetch;
+      this._signProvider =
+        typeof opts.signProvider === "function" ? opts.signProvider : null;
+      this._urls = {
+        watch: opts.watchUrl || watchUrl,
+        favourite: opts.favouriteUrl || favouriteUrl,
+      };
+
+      this.name = NAME;
+      this.version = VERSION;
+      this.capabilities = ["sync:snapshot", "sync:cookie-api", `parse:${platform}-watch`, `parse:${platform}-favourite`];
+      this.extractMode = "web-api";
+      this.rateLimits = {};
+      this.dataDisclosure = {
+        fields: [`${platform}:watch (title / category / episode / channel)`, `${platform}:favourite (title / category)`],
+        sensitivity: "low",
+        legalGate: false,
+        defaultInclude: { watch: true, favourite: true },
+      };
+      this._deps = { fs };
+    }
+
+    async authenticate(ctx = {}) {
+      if (ctx && typeof ctx.inputPath === "string" && ctx.inputPath.length > 0) {
+        try {
+          this._deps.fs.accessSync(ctx.inputPath, this._deps.fs.constants.R_OK);
+        } catch (err) {
+          return { ok: false, reason: "INPUT_PATH_UNREADABLE", message: `snapshot not readable at ${ctx.inputPath}: ${err.message}` };
+        }
+        return { ok: true, mode: "snapshot-file" };
+      }
+      if (this._cookieAuth) {
+        const ok = await this._cookieAuth.validate();
+        if (!ok) return { ok: false, reason: "INVALID_COOKIE", error: "cookies missing" };
+        return { ok: true, account: (this.account && this.account.userId) || null, mode: "cookie" };
+      }
+      return {
+        ok: false,
+        reason: "NO_INPUT",
+        message: `${NAME}.authenticate: needs opts.inputPath (snapshot mode) OR opts.account.cookies (cookie-api mode)`,
+      };
+    }
+
+    async healthCheck() {
+      if (this._cookieAuth) {
+        const r = await this.authenticate();
+        return r.ok ? { ok: true, lastChecked: Date.now() } : { ok: false, reason: r.reason, error: r.error };
+      }
+      return { ok: true, lastChecked: Date.now() };
+    }
+
+    async *sync(opts = {}) {
+      if (typeof opts.inputPath === "string" && opts.inputPath.length > 0) {
+        yield* this._syncViaSnapshot(opts);
+        return;
+      }
+      if (this._cookieAuth) {
+        yield* this._syncViaCookie(opts);
+        return;
+      }
+      throw new Error(`${NAME}.sync: needs opts.inputPath (snapshot mode) OR opts.account.cookies (cookie-api mode)`);
+    }
+
+    async *_syncViaSnapshot(opts) {
+      const raw = this._deps.fs.readFileSync(opts.inputPath, "utf-8");
+      const snapshot = JSON.parse(raw);
+      if (!snapshot || typeof snapshot !== "object" || snapshot.schemaVersion !== SNAPSHOT_SCHEMA_VERSION) {
+        throw new Error(
+          `${NAME}.sync: snapshot schemaVersion mismatch (got ${snapshot && snapshot.schemaVersion}, expected ${SNAPSHOT_SCHEMA_VERSION})`,
+        );
+      }
+      const fallback =
+        Number.isFinite(snapshot.snapshottedAt) && snapshot.snapshottedAt > 0
+          ? Math.floor(snapshot.snapshottedAt)
+          : Date.now();
+      const account = snapshot.account && typeof snapshot.account === "object" ? snapshot.account : null;
+      const include = opts.include || {};
+      const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
+      const events = Array.isArray(snapshot.events) ? snapshot.events : [];
+      let emitted = 0;
+      for (const ev of events) {
+        if (emitted >= limit) return;
+        if (!ev || typeof ev !== "object" || !VALID_SNAPSHOT_KINDS.includes(ev.kind)) continue;
+        if (include[ev.kind] === false) continue;
+        const id = (typeof ev.id === "string" && ev.id) || ev.videoId || null;
+        yield {
+          adapter: NAME,
+          kind: ev.kind,
+          originalId: stableOriginalId(ev.kind, id),
+          capturedAt: parseTime(ev.capturedAt) || fallback,
+          payload: { record: snapshotEventToRecord(ev), kind: ev.kind, account },
+        };
+        emitted += 1;
+      }
+    }
+
+    async *_syncViaCookie(opts = {}) {
+      if (!(await this._cookieAuth.validate())) return;
+      const cookies = this._cookieAuth.toHeader();
+      const include = opts.include || {};
+      const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : Infinity;
+      const maxPages = Number.isInteger(opts.maxPages) && opts.maxPages > 0 ? opts.maxPages : 10;
+
+      const plan = [
+        { kind: KIND_WATCH, url: this._urls.watch },
+        { kind: KIND_FAVOURITE, url: this._urls.favourite },
+      ];
+
+      let emitted = 0;
+      for (const step of plan) {
+        if (include[step.kind] === false) continue;
+        if (!step.url) continue;
+        let page = 1;
+        while (page <= maxPages) {
+          const query = { page, pageSize: PAGE_SIZE };
+          let sign = null;
+          if (this._signProvider) {
+            sign = await this._signProvider({ url: step.url, query, cookies });
+          }
+          const resp = await this._fetchFn({ url: step.url, cookies, query, sign });
+          const items = extractItems(resp) || [];
+          if (!items.length) break;
+          for (const it of items) {
+            const rec = mapItem(it);
+            if (!rec || !rec.videoId) continue;
+            if (emitted >= limit) return;
+            yield {
+              adapter: NAME,
+              kind: step.kind,
+              originalId: stableOriginalId(step.kind, rec.videoId),
+              capturedAt: rec.occurredAt || Date.now(),
+              payload: { record: rec, kind: step.kind },
+            };
+            emitted += 1;
+          }
+          if (items.length < PAGE_SIZE) break;
+          page += 1;
+        }
+      }
+    }
+
+    normalize(raw) {
+      if (!raw || !raw.payload || !raw.payload.record) {
+        throw new Error(`${NAME}.normalize: payload.record missing`);
+      }
+      const kind = raw.kind || raw.payload.kind;
+      const subtype = kind === KIND_FAVOURITE ? EVENT_SUBTYPES.LIKE : EVENT_SUBTYPES.MEDIA;
+      const verb = kind === KIND_FAVOURITE ? "收藏" : "观看";
+      return normalizeVideoRecord(raw.payload.record, raw, platform, NAME, VERSION, subtype, verb);
+    }
+  }
+
+  return VideoAdapter;
+}
+
+function snapshotEventToRecord(ev) {
+  return {
+    videoId: String(ev.videoId || ev.id || "unknown"),
+    title: ev.title || "(未知视频)",
+    category: ev.category || ev.type || null,
+    episode: ev.episode || null,
+    channel: ev.channel || ev.uploader || null,
+    durationSec: Number.isFinite(ev.durationSec) ? ev.durationSec : null,
+    url: ev.url || null,
+    occurredAt: parseTime(ev.capturedAt),
+  };
+}
+
+function normalizeVideoRecord(rec, raw, platform, NAME, VERSION, subtype, verb) {
+  const ingestedAt = Date.now();
+  const occurredAt = rec.occurredAt || raw.capturedAt || ingestedAt;
+  const source = {
+    adapter: NAME,
+    adapterVersion: VERSION,
+    originalId: raw.originalId,
+    capturedAt: raw.capturedAt || occurredAt,
+    capturedBy: CAPTURED_BY.API,
+  };
+  const title = rec.title || "(未知视频)";
+  const epSuffix = rec.episode ? ` ${rec.episode}` : "";
+  const itemId = `item-${platform}-video-${rec.videoId}`;
+  return {
+    events: [
+      {
+        id: newId(),
+        type: ENTITY_TYPES.EVENT,
+        subtype,
+        occurredAt,
+        actor: "person-self",
+        content: { title: `${verb}: ${title}${epSuffix}`, text: title },
+        ingestedAt,
+        source,
+        extra: {
+          platform,
+          videoId: rec.videoId,
+          category: rec.category || null,
+          episode: rec.episode || null,
+          channel: rec.channel || null,
+          durationSec: rec.durationSec != null ? rec.durationSec : null,
+          url: rec.url || null,
+          itemRef: itemId,
+        },
+      },
+    ],
+    items: [
+      {
+        id: itemId,
+        type: ENTITY_TYPES.ITEM,
+        subtype: ITEM_SUBTYPES.MEDIA,
+        name: title,
+        ingestedAt,
+        source,
+        extra: { platform, kind: "video", videoId: rec.videoId, category: rec.category || null, channel: rec.channel || null },
+      },
+    ],
+    persons: [],
+    places: [],
+    topics: [],
+  };
+}
+
+async function defaultFetch(_opts) {
+  throw new Error("video-base: no fetchFn configured for cookie-api mode");
+}
+
+module.exports = {
+  createVideoAdapter,
+  normalizeVideoRecord,
+  parseTime,
+  SNAPSHOT_SCHEMA_VERSION,
+  KIND_WATCH,
+  KIND_FAVOURITE,
+  VALID_SNAPSHOT_KINDS,
+};

package/lib/adapters/doc-baidu-netdisk/index.js

@@ -0,0 +1,91 @@
+/**
+ * §13+ — 百度网盘 (Baidu Netdisk, com.baidu.netdisk) adapter. §12.1 ROI ⭐⭐⭐
+ * "文件 + 外链".
+ *
+ * Thin wrapper over _document-base — a cloud-drive file list is the same shape
+ * as the doc-wps / doc-tencent-docs "own-document list". Baidu Netdisk exposes
+ * the owner's files via pan.baidu.com/api/list (BDUSS cookie); this adapter
+ * supplies the endpoint + field mapping, the base handles snapshot + cookie-api
+ * orchestration + normalize (event POST + item DOCUMENT). Endpoint best-effort +
+ * overridable via opts.listUrl (not field-verified — FAMILY-23 playbook).
+ */
+
+"use strict";
+
+const { createDocumentAdapter, parseTime, SNAPSHOT_SCHEMA_VERSION } = require("../_document-base");
+
+const NAME = "doc-baidu-netdisk";
+const VERSION = "0.1.0";
+
+// Best-effort Baidu Netdisk file-list endpoint. Overridable via opts.listUrl.
+const NETDISK_LIST_URL = "https://pan.baidu.com/api/list";
+
+// Baidu Netdisk `category` codes → normalized docType.
+const CATEGORY_MAP = {
+  1: "video",
+  2: "audio",
+  3: "image",
+  4: "doc",
+  5: "app",
+  6: "other",
+  7: "seed",
+};
+
+function mapNetdiskType(d) {
+  if (d.isdir === 1 || d.isdir === true) return "folder";
+  const cat = d.category != null ? d.category : d.file_category;
+  if (cat != null && CATEGORY_MAP[cat]) return CATEGORY_MAP[cat];
+  const name = String(d.server_filename || d.filename || "").toLowerCase();
+  if (/\.(mp4|mkv|avi|mov)$/.test(name)) return "video";
+  if (/\.(mp3|flac|wav|m4a)$/.test(name)) return "audio";
+  if (/\.(jpg|jpeg|png|gif|webp)$/.test(name)) return "image";
+  if (/\.(docx?|xlsx?|pptx?|pdf|txt)$/.test(name)) return "doc";
+  return "file";
+}
+
+function extractDocs(resp) {
+  if (!resp || typeof resp !== "object") return [];
+  if (Array.isArray(resp.list)) return resp.list;
+  if (Array.isArray(resp.data)) return resp.data;
+  if (resp.data && Array.isArray(resp.data.list)) return resp.data.list;
+  return [];
+}
+
+function mapDoc(d) {
+  if (!d || typeof d !== "object") return null;
+  const docId = d.fs_id || d.fsId || d.id || d.path;
+  if (!docId) return null;
+  return {
+    docId: String(docId),
+    title: d.server_filename || d.filename || d.title || "(未命名)",
+    docType: mapNetdiskType(d),
+    url: d.path || d.dlink || null,
+    createdMs: parseTime(d.server_ctime || d.local_ctime || d.ctime),
+    updatedMs: parseTime(d.server_mtime || d.local_mtime || d.mtime),
+    extra: {
+      size: d.size != null ? d.size : null,
+      isDir: d.isdir === 1 || d.isdir === true ? true : false,
+      path: d.path || null,
+      category: d.category != null ? d.category : null,
+    },
+  };
+}
+
+const BaiduNetdiskAdapter = createDocumentAdapter({
+  NAME,
+  VERSION,
+  platform: "baidu-netdisk",
+  defaultListUrl: NETDISK_LIST_URL,
+  extractDocs,
+  mapDoc,
+});
+
+module.exports = {
+  BaiduNetdiskAdapter,
+  extractDocs,
+  mapDoc,
+  CATEGORY_MAP,
+  NAME,
+  VERSION,
+  SNAPSHOT_SCHEMA_VERSION,
+};

package/lib/adapters/doc-tencent-docs/index.js

@@ -0,0 +1,94 @@
+/**
+ * §13+ — 腾讯文档 (Tencent Docs) adapter. "自创文档列表" (§12.1, ROI ⭐⭐⭐).
+ *
+ * Thin wrapper over _document-base. 腾讯文档 exposes the owner's documents via
+ * docs.qq.com dop-api; this adapter supplies the endpoint + field mapping, the
+ * base handles snapshot + cookie-api orchestration + normalize. Endpoint is
+ * best-effort + overridable via opts.listUrl (docs.qq.com rotates its dop-api;
+ * some calls need a sign — opts.signProvider seam — FAMILY-23 playbook, not
+ * field-verified).
+ */
+
+"use strict";
+
+const { createDocumentAdapter, parseTime, SNAPSHOT_SCHEMA_VERSION } = require("../_document-base");
+
+const NAME = "doc-tencent-docs";
+const VERSION = "0.1.0";
+
+// Best-effort Tencent Docs "my documents" list. Overridable via opts.listUrl.
+const TENCENT_DOCS_LIST_URL = "https://docs.qq.com/dop-api/get/personal/files";
+
+// Tencent Docs type codes → normalized docType.
+const TYPE_MAP = {
+  doc: "doc",
+  document: "doc",
+  sheet: "sheet",
+  spreadsheet: "sheet",
+  slide: "slide",
+  presentation: "slide",
+  pdf: "pdf",
+  form: "form",
+  mind: "mind",
+  1: "doc",
+  2: "sheet",
+  3: "slide",
+};
+
+function mapTencentType(d) {
+  const raw = d.type != null ? d.type : d.docType != null ? d.docType : d.fileType;
+  const key = String(raw == null ? "" : raw).toLowerCase();
+  return TYPE_MAP[key] || TYPE_MAP[raw] || "doc";
+}
+
+function extractDocs(resp) {
+  if (!resp || typeof resp !== "object") return [];
+  if (Array.isArray(resp.files)) return resp.files;
+  if (Array.isArray(resp.list)) return resp.list;
+  const data = resp.data && typeof resp.data === "object" ? resp.data : null;
+  if (data) {
+    if (Array.isArray(data.files)) return data.files;
+    if (Array.isArray(data.list)) return data.list;
+    if (Array.isArray(data.records)) return data.records;
+  }
+  return [];
+}
+
+function mapDoc(d) {
+  if (!d || typeof d !== "object") return null;
+  const docId = d.id || d.fileId || d.file_id || d.docId || d.url;
+  if (!docId) return null;
+  return {
+    docId: String(docId),
+    title: d.title || d.name || d.fileName || "(无标题)",
+    docType: mapTencentType(d),
+    url:
+      d.url ||
+      (d.id ? `https://docs.qq.com/doc/${d.id}` : null),
+    createdMs: parseTime(d.createTime || d.create_time || d.gmtCreate),
+    updatedMs: parseTime(d.lastModifyTime || d.modifyTime || d.updateTime || d.gmtModify),
+    extra: {
+      ownerName: d.ownerName || d.creatorName || null,
+      starred: d.isStar != null ? d.isStar : undefined,
+    },
+  };
+}
+
+const TencentDocsAdapter = createDocumentAdapter({
+  NAME,
+  VERSION,
+  platform: "tencent-docs",
+  defaultListUrl: TENCENT_DOCS_LIST_URL,
+  extractDocs,
+  mapDoc,
+});
+
+module.exports = {
+  TencentDocsAdapter,
+  extractDocs,
+  mapDoc,
+  TYPE_MAP,
+  NAME,
+  VERSION,
+  SNAPSHOT_SCHEMA_VERSION,
+};

package/lib/adapters/doc-wps/index.js

@@ -0,0 +1,77 @@
+/**
+ * §13+ — WPS 云文档 (Kingsoft Office) adapter. "自创文档列表" (§12.1, ROI ⭐⭐⭐).
+ *
+ * Thin wrapper over _document-base. WPS exposes the owner's cloud documents via
+ * drive.wps.cn; this adapter supplies the endpoint + field mapping, the base
+ * handles snapshot + cookie-api orchestration + normalize (event POST + item
+ * DOCUMENT). Endpoint is best-effort + overridable via opts.listUrl (WPS rotates
+ * its drive API — FAMILY-23 playbook, not field-verified).
+ */
+
+"use strict";
+
+const { createDocumentAdapter, parseTime, SNAPSHOT_SCHEMA_VERSION } = require("../_document-base");
+
+const NAME = "doc-wps";
+const VERSION = "0.1.0";
+
+// Best-effort WPS cloud-drive file list. Overridable via opts.listUrl.
+const WPS_LIST_URL = "https://drive.wps.cn/api/v5/groups/special/files";
+
+// WPS doc-type codes/names → normalized docType.
+function mapWpsType(d) {
+  const t = String(d.fname || d.name || "").toLowerCase();
+  if (d.ftype && typeof d.ftype === "string") return d.ftype;
+  if (/\.(xlsx?|et|csv)$/.test(t)) return "sheet";
+  if (/\.(pptx?|dps)$/.test(t)) return "slide";
+  if (/\.pdf$/.test(t)) return "pdf";
+  if (/\.(docx?|wps)$/.test(t)) return "doc";
+  return "doc";
+}
+
+function extractDocs(resp) {
+  if (!resp || typeof resp !== "object") return [];
+  if (Array.isArray(resp.files)) return resp.files;
+  if (Array.isArray(resp.data)) return resp.data;
+  if (resp.data && Array.isArray(resp.data.files)) return resp.data.files;
+  if (Array.isArray(resp.list)) return resp.list;
+  return [];
+}
+
+function mapDoc(d) {
+  if (!d || typeof d !== "object") return null;
+  const docId = d.id || d.fileid || d.file_id || d.fid;
+  if (!docId) return null;
+  return {
+    docId: String(docId),
+    title: d.fname || d.name || d.title || "(无标题)",
+    docType: mapWpsType(d),
+    url:
+      d.url ||
+      (d.id ? `https://www.kdocs.cn/p/${d.id}` : null),
+    createdMs: parseTime(d.ctime || d.create_time || d.created),
+    updatedMs: parseTime(d.mtime || d.modify_time || d.updated || d.utime),
+    extra: {
+      size: d.fsize || d.size || null,
+      groupId: d.group_id || d.groupid || null,
+    },
+  };
+}
+
+const WpsDocAdapter = createDocumentAdapter({
+  NAME,
+  VERSION,
+  platform: "wps",
+  defaultListUrl: WPS_LIST_URL,
+  extractDocs,
+  mapDoc,
+});
+
+module.exports = {
+  WpsDocAdapter,
+  extractDocs,
+  mapDoc,
+  NAME,
+  VERSION,
+  SNAPSHOT_SCHEMA_VERSION,
+};