@chainlesschain/personal-data-hub 0.1.0 → 0.2.1

package/lib/adapters/ai-chat-history/http-client.js

@@ -0,0 +1,211 @@
+/**
+ * HTTP client wrapper used by AI-chat vendor sub-adapters.
+ *
+ * Three responsibilities:
+ *
+ *   1. Inject cookies from a CookieAuthSession (per request, per domain).
+ *   2. Apply per-vendor `rateLimits` — at least minIntervalMs between calls
+ *      and at most perMinute calls in any rolling 60s window. Vendor wiring
+ *      shares one HttpClient instance so all its endpoints obey the same gate.
+ *   3. Retry on 5xx / network-timeout with bounded exponential backoff.
+ *
+ * The actual network call is delegated to `opts.fetch` (defaults to global
+ * fetch in Node 22+). Tests inject a stub that records requests + returns
+ * canned responses, so the whole vendor wiring is verifiable without real
+ * cookies / live servers.
+ *
+ * No external dependency — same pattern as `lib/llm-client.js`.
+ */
+
+"use strict";
+
+const DEFAULT_RATE_LIMITS = Object.freeze({
+  perMinute: 30,
+  minIntervalMs: 1500,
+});
+
+class RateLimitedError extends Error {
+  constructor(retryAfterMs, vendor) {
+    super(`rate limited (vendor=${vendor || "?"}, retryAfterMs=${retryAfterMs})`);
+    this.code = "RATE_LIMITED";
+    this.retryAfterMs = retryAfterMs;
+    this.vendor = vendor;
+  }
+}
+
+class CookieExpiredError extends Error {
+  constructor(vendor, hint) {
+    super(`cookie expired or invalid (vendor=${vendor})${hint ? ": " + hint : ""}`);
+    this.code = "COOKIE_EXPIRED";
+    this.vendor = vendor;
+  }
+}
+
+class HttpClient {
+  /**
+   * @param {object} opts
+   * @param {string} opts.vendor          Vendor name (for error messages + logs).
+   * @param {object} [opts.rateLimits]    { perMinute, minIntervalMs } — defaults to DEFAULT_RATE_LIMITS.
+   * @param {function} [opts.fetch]       Fetch impl override. Defaults to global fetch.
+   * @param {function} [opts.sleep]       Sleep impl override (test seam). Defaults to setTimeout.
+   * @param {function} [opts.now]         Clock override (test seam). Defaults to Date.now.
+   * @param {number}   [opts.maxRetries]  Max retries on 5xx / timeout. Default 3.
+   * @param {number}   [opts.baseBackoffMs] Initial retry backoff. Default 500.
+   * @param {object}   [opts.logger]
+   */
+  constructor(opts = {}) {
+    if (typeof opts.vendor !== "string" || opts.vendor.length === 0) {
+      throw new Error("HttpClient: opts.vendor required");
+    }
+    this.vendor = opts.vendor;
+    this.rateLimits = {
+      ...DEFAULT_RATE_LIMITS,
+      ...(opts.rateLimits || {}),
+    };
+    this._fetch = opts.fetch || (typeof fetch !== "undefined" ? fetch : null);
+    if (!this._fetch) {
+      throw new Error("HttpClient: no fetch available. Node 22+ required, or pass opts.fetch.");
+    }
+    this._sleep = opts.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
+    this._now = opts.now || (() => Date.now());
+    this._maxRetries = Number.isFinite(opts.maxRetries) ? opts.maxRetries : 3;
+    this._baseBackoffMs = Number.isFinite(opts.baseBackoffMs) ? opts.baseBackoffMs : 500;
+    this._logger = opts.logger || { info: () => {}, warn: () => {}, error: () => {} };
+
+    // Sliding window of recent request timestamps for perMinute enforcement.
+    this._recent = [];
+    this._lastCallAt = 0;
+  }
+
+  /**
+   * Make one rate-limited + cookie-injected request. Throws CookieExpiredError
+   * on 401/403, RateLimitedError on 429 after exhausting retries, generic
+   * Error on other failures.
+   *
+   * @param {string} url
+   * @param {object} [reqInit]
+   * @param {CookieAuthSession} [reqInit.session]   Cookies to inject. Must match opts.vendor.
+   * @param {string} [reqInit.matchDomain]          Cookie domain filter.
+   */
+  async request(url, reqInit = {}) {
+    await this._enforceRateLimit();
+
+    const headers = { ...(reqInit.headers || {}) };
+    if (reqInit.session) {
+      if (reqInit.session.vendor !== this.vendor) {
+        throw new Error(
+          `HttpClient(${this.vendor}): session vendor "${reqInit.session.vendor}" mismatch`,
+        );
+      }
+      reqInit.session.applyTo(headers, reqInit.matchDomain);
+    }
+
+    const fetchInit = {
+      method: reqInit.method || "GET",
+      headers,
+      ...(reqInit.body !== undefined ? { body: reqInit.body } : {}),
+      ...(reqInit.signal ? { signal: reqInit.signal } : {}),
+    };
+
+    let lastErr;
+    for (let attempt = 0; attempt <= this._maxRetries; attempt++) {
+      let resp;
+      try {
+        resp = await this._fetch(url, fetchInit);
+      } catch (err) {
+        lastErr = err;
+        if (attempt < this._maxRetries) {
+          await this._sleep(this._backoffMs(attempt));
+          continue;
+        }
+        throw err;
+      }
+
+      if (resp.status === 401 || resp.status === 403) {
+        throw new CookieExpiredError(this.vendor, `HTTP ${resp.status}`);
+      }
+      if (resp.status === 429) {
+        const retryHdr = resp.headers && typeof resp.headers.get === "function"
+          ? resp.headers.get("retry-after")
+          : null;
+        const retryAfterMs = retryHdr ? (Number(retryHdr) * 1000 || this._backoffMs(attempt)) : this._backoffMs(attempt);
+        if (attempt < this._maxRetries) {
+          this._logger.warn(`[${this.vendor}] 429 retry-after=${retryAfterMs}ms attempt=${attempt}`);
+          await this._sleep(retryAfterMs);
+          continue;
+        }
+        throw new RateLimitedError(retryAfterMs, this.vendor);
+      }
+      if (resp.status >= 500 && resp.status < 600) {
+        lastErr = new Error(`HTTP ${resp.status}`);
+        if (attempt < this._maxRetries) {
+          await this._sleep(this._backoffMs(attempt));
+          continue;
+        }
+        throw lastErr;
+      }
+      return resp;
+    }
+    throw lastErr || new Error("HttpClient: exhausted retries");
+  }
+
+  /**
+   * GET shorthand that decodes JSON.
+   */
+  async getJson(url, reqInit = {}) {
+    const resp = await this.request(url, reqInit);
+    if (!resp.ok) {
+      throw new Error(`HttpClient(${this.vendor}) GET ${url} → HTTP ${resp.status}`);
+    }
+    return resp.json();
+  }
+
+  async postJson(url, body, reqInit = {}) {
+    const headers = { "content-type": "application/json", ...(reqInit.headers || {}) };
+    const resp = await this.request(url, { ...reqInit, method: "POST", headers, body: JSON.stringify(body) });
+    if (!resp.ok) {
+      throw new Error(`HttpClient(${this.vendor}) POST ${url} → HTTP ${resp.status}`);
+    }
+    return resp.json();
+  }
+
+  async _enforceRateLimit() {
+    const now = this._now();
+
+    // perMinute window
+    if (this.rateLimits.perMinute > 0) {
+      const cutoff = now - 60_000;
+      this._recent = this._recent.filter((t) => t >= cutoff);
+      if (this._recent.length >= this.rateLimits.perMinute) {
+        const wait = this._recent[0] + 60_000 - now;
+        if (wait > 0) await this._sleep(wait);
+      }
+    }
+
+    // minIntervalMs between consecutive calls
+    if (this.rateLimits.minIntervalMs > 0 && this._lastCallAt > 0) {
+      const since = now - this._lastCallAt;
+      if (since < this.rateLimits.minIntervalMs) {
+        await this._sleep(this.rateLimits.minIntervalMs - since);
+      }
+    }
+
+    const tick = this._now();
+    this._recent.push(tick);
+    this._lastCallAt = tick;
+  }
+
+  _backoffMs(attempt) {
+    // 500 → 1000 → 2000 → 4000 (capped at 4 attempts) + ±20% jitter
+    const base = this._baseBackoffMs * Math.pow(2, attempt);
+    const jitter = base * 0.2 * (Math.random() * 2 - 1);
+    return Math.max(0, Math.floor(base + jitter));
+  }
+}
+
+module.exports = {
+  HttpClient,
+  RateLimitedError,
+  CookieExpiredError,
+  DEFAULT_RATE_LIMITS,
+};

package/lib/adapters/ai-chat-history/index.js

@@ -0,0 +1,28 @@
+"use strict";
+
+const {
+  AIChatHistoryAdapter,
+  SUPPORTED_VENDORS,
+  DEFAULT_VENDOR_SPECS,
+  ADAPTER_NAME,
+  ADAPTER_VERSION,
+} = require("./ai-chat-adapter");
+const { CookieAuthSession } = require("./cookie-auth");
+const { NotImplementedYetError, assertVendorSpec } = require("./vendor-spec");
+const { HttpClient, RateLimitedError, CookieExpiredError } = require("./http-client");
+const schemaMap = require("./schema-map");
+
+module.exports = {
+  AIChatHistoryAdapter,
+  SUPPORTED_VENDORS,
+  DEFAULT_VENDOR_SPECS,
+  ADAPTER_NAME,
+  ADAPTER_VERSION,
+  CookieAuthSession,
+  NotImplementedYetError,
+  HttpClient,
+  RateLimitedError,
+  CookieExpiredError,
+  assertVendorSpec,
+  schemaMap,
+};

package/lib/adapters/ai-chat-history/schema-map.js

@@ -0,0 +1,258 @@
+/**
+ * RawConversation / RawMessage → UnifiedSchema normalized batch.
+ *
+ * Mirrors `docs/design/Adapter_AIChat_History.md` §5.2. This module is
+ * pure-transform: given a RawConversation + its RawMessage[], it returns a
+ * {events, persons, topics, items} bundle ready for the AdapterRegistry to
+ * write into LocalVault.
+ *
+ * Why pure: it's the easiest part to test exhaustively without any HTTP
+ * mocks. Vendor sub-adapters (skeletons in Phase 10.1) only need to produce
+ * Raw* objects matching the documented shape; this module handles the rest.
+ */
+
+"use strict";
+
+const {
+  ENTITY_TYPES,
+  PERSON_SUBTYPES,
+  EVENT_SUBTYPES,
+  ITEM_SUBTYPES,
+  CAPTURED_BY,
+} = require("../../constants");
+const { newId } = require("../../ids");
+
+const SELF_PERSON_ID = "person-self";
+const ADAPTER_NAME = "ai-chat-history";
+const ADAPTER_VERSION = "0.1.0";
+
+function personIdForVendor(vendor) {
+  return `person-ai-${vendor}`;
+}
+
+function topicIdForConversation(vendor, originalConvId) {
+  return `topic-aiconv-${vendor}-${originalConvId}`;
+}
+
+/**
+ * Build the vendor "AI agent" Person entity. The hub upserts these once per
+ * vendor — the consistent `id` form means re-emitting from multiple sync
+ * passes is idempotent.
+ */
+function buildVendorPerson(vendor, displayName) {
+  const now = Date.now();
+  return {
+    id: personIdForVendor(vendor),
+    type: ENTITY_TYPES.PERSON,
+    subtype: PERSON_SUBTYPES.AI_AGENT,
+    names: [displayName],
+    identifiers: { vendor },
+    notes: `${displayName} — 用户的 ${displayName} AI 助手账户`,
+    ingestedAt: now,
+    source: {
+      adapter: ADAPTER_NAME,
+      adapterVersion: ADAPTER_VERSION,
+      originalId: `vendor:${vendor}`,
+      capturedAt: now,
+      capturedBy: CAPTURED_BY.API,
+    },
+  };
+}
+
+/**
+ * Build the conversation Topic entity.
+ */
+function buildConversationTopic(rawConv) {
+  const now = Date.now();
+  return {
+    id: topicIdForConversation(rawConv.vendor, rawConv.originalId),
+    type: ENTITY_TYPES.TOPIC,
+    name: rawConv.title || "(无标题对话)",
+    ingestedAt: now,
+    source: {
+      adapter: ADAPTER_NAME,
+      adapterVersion: ADAPTER_VERSION,
+      originalId: `${rawConv.vendor}:conv:${rawConv.originalId}`,
+      capturedAt: Number(rawConv.updatedAt) || Number(rawConv.createdAt) || now,
+      capturedBy: CAPTURED_BY.API,
+    },
+    extra: {
+      vendor: rawConv.vendor,
+      kind: "ai-conversation",
+      modelName: rawConv.modelName,
+      conversationOriginalId: rawConv.originalId,
+      createdAt: rawConv.createdAt,
+      updatedAt: rawConv.updatedAt,
+      messageCount: rawConv.messageCount,
+      archived: Boolean(rawConv.archived),
+    },
+  };
+}
+
+/**
+ * Build an Event entity for one message.
+ *
+ * - role "user"      → actor = self,             subtype = ai-message
+ * - role "assistant" → actor = person-ai-vendor, subtype = ai-message
+ * - vendor=dreamina + generatedImages → subtype overridden to ai-image-generation
+ */
+function buildMessageEvent(rawMsg, capturedAt) {
+  const vendorPersonId = personIdForVendor(rawMsg.vendor);
+  const actor = rawMsg.role === "user" ? SELF_PERSON_ID : vendorPersonId;
+  const subtype =
+    rawMsg.content && Array.isArray(rawMsg.content.generatedImages) && rawMsg.content.generatedImages.length > 0
+      ? EVENT_SUBTYPES.AI_IMAGE_GENERATION
+      : EVENT_SUBTYPES.AI_MESSAGE;
+  const now = Date.now();
+  // Schema requires positive integer ms timestamps for occurredAt / ingestedAt
+  // / source.capturedAt — see lib/schemas.js validateEvent + validateBaseEntity.
+  const occurredAtMs = Number(rawMsg.createdAt);
+  // Deterministic id keyed on (vendor, originalId) so re-syncs hit
+  // ON CONFLICT(id) DO UPDATE in putEvent and stay idempotent (instead of
+  // throwing on the secondary UNIQUE(source_adapter, source_original_id)
+  // constraint with a fresh newId() each time).
+  return {
+    id: `evt-aichat-${rawMsg.vendor}-${rawMsg.originalId}`,
+    type: ENTITY_TYPES.EVENT,
+    subtype,
+    occurredAt: Number.isFinite(occurredAtMs) && occurredAtMs > 0 ? occurredAtMs : now,
+    actor,
+    participants: [SELF_PERSON_ID, vendorPersonId],
+    content: {
+      text: (rawMsg.content && rawMsg.content.text) || undefined,
+      mediaRefs: rawMsg.content && Array.isArray(rawMsg.content.attachments)
+        ? rawMsg.content.attachments.map((a) => a.url).filter(Boolean)
+        : undefined,
+    },
+    topics: [topicIdForConversation(rawMsg.vendor, rawMsg.conversationId)],
+    ingestedAt: now,
+    source: {
+      adapter: ADAPTER_NAME,
+      adapterVersion: ADAPTER_VERSION,
+      originalId: `${rawMsg.vendor}/${rawMsg.originalId}`,
+      capturedAt: Number(capturedAt) || now,
+      capturedBy: CAPTURED_BY.API,
+    },
+    extra: {
+      vendor: rawMsg.vendor,
+      conversationOriginalId: rawMsg.conversationId,
+      role: rawMsg.role,
+      modelName: rawMsg.modelName,
+      parentMessageId: rawMsg.parentMessageId,
+      toolCalls: rawMsg.content ? rawMsg.content.toolCalls : undefined,
+      generatedImages: rawMsg.content ? rawMsg.content.generatedImages : undefined,
+    },
+  };
+}
+
+/**
+ * Build Item entities for any generated images. Each image becomes a `media`
+ * Item so KG queries like "what images did I generate this month" work.
+ */
+function buildGeneratedImageItems(rawMsg) {
+  if (!rawMsg.content || !Array.isArray(rawMsg.content.generatedImages)) return [];
+  const now = Date.now();
+  const capturedAt = Number(rawMsg.createdAt) || now;
+  return rawMsg.content.generatedImages.map((img, idx) => ({
+    id: `item-aichat-${rawMsg.vendor}-${rawMsg.originalId}-${idx}`,
+    type: ENTITY_TYPES.ITEM,
+    subtype: ITEM_SUBTYPES.MEDIA,
+    name: (img.prompt || "AI image").slice(0, 80),
+    ingestedAt: now,
+    source: {
+      adapter: ADAPTER_NAME,
+      adapterVersion: ADAPTER_VERSION,
+      originalId: `${rawMsg.vendor}:img:${rawMsg.originalId}:${idx}`,
+      capturedAt,
+      capturedBy: CAPTURED_BY.API,
+    },
+    extra: {
+      vendor: rawMsg.vendor,
+      kind: "ai-generated-image",
+      prompt: img.prompt,
+      model: img.model,
+      url: img.url,
+      params: img.params,
+    },
+  }));
+}
+
+/**
+ * Convert one RawConversation + its messages into a NormalizedBatch.
+ *
+ * @param {object} rawConv
+ * @param {Array} rawMessages
+ * @param {object} [vendorMeta]   { displayName: string }
+ * @param {number} [capturedAt]
+ * @returns {{events:Array,persons:Array,places:Array,items:Array,topics:Array}}
+ */
+function conversationToBatch(rawConv, rawMessages, vendorMeta, capturedAt) {
+  if (!rawConv || typeof rawConv !== "object") {
+    throw new Error("conversationToBatch: rawConv required");
+  }
+  if (!Array.isArray(rawMessages)) {
+    throw new Error("conversationToBatch: rawMessages must be an array");
+  }
+  const vendorPerson = buildVendorPerson(rawConv.vendor, (vendorMeta && vendorMeta.displayName) || rawConv.vendor);
+  const topic = buildConversationTopic(rawConv);
+
+  const events = [];
+  const items = [];
+  for (const msg of rawMessages) {
+    if (msg.vendor !== rawConv.vendor) {
+      throw new Error(
+        `conversationToBatch: message vendor "${msg.vendor}" != conv vendor "${rawConv.vendor}"`,
+      );
+    }
+    events.push(buildMessageEvent(msg, capturedAt));
+    items.push(...buildGeneratedImageItems(msg));
+  }
+
+  return {
+    events,
+    persons: [vendorPerson],
+    places: [],
+    items,
+    topics: [topic],
+  };
+}
+
+/**
+ * Merge multiple NormalizedBatch results. Vendor Person is deduped by id
+ * (every batch declares the same `person-ai-<vendor>` so we collapse).
+ */
+function mergeBatches(batches) {
+  const events = [];
+  const personsById = new Map();
+  const places = [];
+  const items = [];
+  const topicsById = new Map();
+  for (const b of batches) {
+    for (const e of b.events) events.push(e);
+    for (const p of b.persons) personsById.set(p.id, p);
+    for (const pl of b.places) places.push(pl);
+    for (const it of b.items) items.push(it);
+    for (const t of b.topics) topicsById.set(t.id, t);
+  }
+  return {
+    events,
+    persons: [...personsById.values()],
+    places,
+    items,
+    topics: [...topicsById.values()],
+  };
+}
+
+module.exports = {
+  SELF_PERSON_ID,
+  ADAPTER_NAME,
+  ADAPTER_VERSION,
+  personIdForVendor,
+  topicIdForConversation,
+  buildVendorPerson,
+  buildConversationTopic,
+  buildMessageEvent,
+  buildGeneratedImageItems,
+  conversationToBatch,
+  mergeBatches,
+};

package/lib/adapters/ai-chat-history/vendor-spec.js

@@ -0,0 +1,86 @@
+/**
+ * VendorSpec — per-AI-vendor contract used by AIChatHistoryAdapter.
+ *
+ * Each of the 8 supported vendors (DeepSeek / Kimi / 通义 / 智谱 / 混元 / 千帆 /
+ * 扣子 / Dreamina) provides one VendorSpec object describing:
+ *
+ *   - Identity (`name`, `displayName`, `androidPackage`)
+ *   - Login surface (`loginUrl`, `cookieDomains`)
+ *   - Sync surface (`listConversations(ctx, opts)`, `listMessages(ctx, convId, opts)`)
+ *   - Health (`validateCookie(ctx)`)
+ *   - Rate limits (`rateLimits`)
+ *
+ * The parent adapter fans out to all configured vendors and reuses one cookie
+ * jar per vendor. Vendor implementations live in `./vendors/<name>.js` and are
+ * SKELETONS in Phase 10.1 — they advertise the right shape and ratelimit so
+ * the AdapterRegistry contract holds, but the actual HTTP wiring is wired in
+ * later (Phase 10.2+). See `Adapter_AIChat_History.md` §6 per-vendor detail.
+ */
+
+"use strict";
+
+const SUPPORTED_VENDORS = Object.freeze([
+  "deepseek",
+  "kimi",
+  "tongyi",
+  "zhipu",
+  "hunyuan",
+  "qianfan",
+  "coze",
+  "dreamina",
+  "doubao",
+]);
+
+class NotImplementedYetError extends Error {
+  constructor(vendor, capability) {
+    super(`vendor "${vendor}" capability "${capability}" not wired yet (Phase 10.2+)`);
+    this.code = "VENDOR_NOT_WIRED";
+    this.vendor = vendor;
+    this.capability = capability;
+  }
+}
+
+function assertVendorSpec(spec) {
+  const errors = [];
+  if (spec == null || typeof spec !== "object") {
+    return { ok: false, errors: ["vendor spec must be an object"] };
+  }
+  if (typeof spec.name !== "string" || !SUPPORTED_VENDORS.includes(spec.name)) {
+    errors.push(`name must be one of ${SUPPORTED_VENDORS.join("|")}`);
+  }
+  if (typeof spec.displayName !== "string" || spec.displayName.length === 0) {
+    errors.push("displayName required");
+  }
+  if (typeof spec.androidPackage !== "string" || spec.androidPackage.length === 0) {
+    errors.push("androidPackage required");
+  }
+  if (typeof spec.loginUrl !== "string" || !/^https:\/\//.test(spec.loginUrl)) {
+    errors.push("loginUrl required and must be https URL");
+  }
+  if (!Array.isArray(spec.cookieDomains) || spec.cookieDomains.length === 0) {
+    errors.push("cookieDomains must be a non-empty array");
+  }
+  if (typeof spec.listConversations !== "function") {
+    errors.push("listConversations must be an async generator function");
+  }
+  if (typeof spec.listMessages !== "function") {
+    errors.push("listMessages must be an async generator function");
+  }
+  if (typeof spec.validateCookie !== "function") {
+    errors.push("validateCookie must be an async function");
+  }
+  if (
+    spec.rateLimits == null
+    || typeof spec.rateLimits !== "object"
+    || typeof spec.rateLimits.perMinute !== "number"
+  ) {
+    errors.push("rateLimits.perMinute (number) required");
+  }
+  return errors.length === 0 ? { ok: true } : { ok: false, errors };
+}
+
+module.exports = {
+  SUPPORTED_VENDORS,
+  NotImplementedYetError,
+  assertVendorSpec,
+};