@chainlesschain/personal-data-hub 0.1.0

package/lib/batch.js

@@ -0,0 +1,123 @@
+/**
+ * NormalizedBatch — what every adapter.normalize() returns to the ingestor
+ *
+ * Mirrors §9.1 of docs/design/Personal_Data_Hub_Architecture.md:
+ *   interface NormalizedBatch {
+ *     events: Event[];
+ *     persons: Person[];
+ *     places: Place[];
+ *     items: Item[];
+ *     topics?: Topic[];
+ *   }
+ *
+ * Helpers here let adapters build batches incrementally + ingestors validate
+ * a whole batch in one call (so a single bad row doesn't kill the whole sync).
+ */
+
+"use strict";
+
+const { validate } = require("./schemas");
+
+function emptyBatch() {
+  return { events: [], persons: [], places: [], items: [], topics: [] };
+}
+
+function mergeBatches(a, b) {
+  return {
+    events: [...(a.events || []), ...(b.events || [])],
+    persons: [...(a.persons || []), ...(b.persons || [])],
+    places: [...(a.places || []), ...(b.places || [])],
+    items: [...(a.items || []), ...(b.items || [])],
+    topics: [...(a.topics || []), ...(b.topics || [])],
+  };
+}
+
+/**
+ * Validate every entity in a batch. Returns:
+ *   {
+ *     valid: boolean,
+ *     entityCount: number,
+ *     errorCount: number,
+ *     errors: Array<{ kind, index, id?, errors: string[] }>,
+ *   }
+ *
+ * The ingestor's policy is: if errorCount > 0, log all bad rows but still
+ * ingest the good ones (don't let a single corrupt row from a third-party
+ * adapter abort the whole sync window).
+ */
+function validateBatch(batch) {
+  if (batch == null || typeof batch !== "object") {
+    return {
+      valid: false,
+      entityCount: 0,
+      errorCount: 1,
+      errors: [{ kind: "batch", index: -1, errors: ["batch must be a plain object"] }],
+    };
+  }
+
+  const errors = [];
+  let entityCount = 0;
+
+  const kinds = ["events", "persons", "places", "items", "topics"];
+  for (const kind of kinds) {
+    const arr = batch[kind];
+    if (arr === undefined) continue;
+    if (!Array.isArray(arr)) {
+      errors.push({ kind, index: -1, errors: [`${kind} must be an array when present`] });
+      continue;
+    }
+    arr.forEach((entity, i) => {
+      entityCount += 1;
+      const result = validate(entity);
+      if (!result.valid) {
+        errors.push({
+          kind,
+          index: i,
+          id: entity && typeof entity === "object" ? entity.id : undefined,
+          errors: result.errors,
+        });
+      }
+    });
+  }
+
+  return {
+    valid: errors.length === 0,
+    entityCount,
+    errorCount: errors.length,
+    errors,
+  };
+}
+
+/**
+ * Partition a batch into "valid" and "invalid" sub-batches.
+ * Lets the ingestor commit valids + spool invalids to a review queue.
+ */
+function partitionBatch(batch) {
+  const valid = emptyBatch();
+  const invalid = emptyBatch();
+  const invalidReasons = [];
+
+  const kinds = ["events", "persons", "places", "items", "topics"];
+  for (const kind of kinds) {
+    const arr = batch[kind] || [];
+    if (!Array.isArray(arr)) continue;
+    arr.forEach((entity, i) => {
+      const result = validate(entity);
+      if (result.valid) {
+        valid[kind].push(entity);
+      } else {
+        invalid[kind].push(entity);
+        invalidReasons.push({ kind, index: i, id: entity?.id, errors: result.errors });
+      }
+    });
+  }
+
+  return { valid, invalid, invalidReasons };
+}
+
+module.exports = {
+  emptyBatch,
+  mergeBatches,
+  validateBatch,
+  partitionBatch,
+};

package/lib/bridges/cc-kg-sink.js

@@ -0,0 +1,264 @@
+/**
+ * CcKgSink — translates hub KG triples (subject/predicate/object|literal)
+ * into ChainlessChain's existing knowledge-graph addEntity + addRelation API.
+ *
+ * Hub triples come in two shapes:
+ *
+ *   Object triples:   { subject: "evt-x", predicate: "by", object: "person-y" }
+ *                     → addRelation(db, { sourceId, targetId, relationType })
+ *
+ *   Literal triples:  { subject: "evt-x", predicate: "subtype", literal: "order" }
+ *                     → accumulate as entity properties at entity creation time
+ *
+ *   Special literal:  { subject: "evt-x", predicate: "rdf:type", literal: "event" }
+ *                     → decides the cc entity type (Person / Event / Concept / ...)
+ *
+ * Hub's 5 entity kinds map onto cc's 7 with this convention:
+ *
+ *   hub      cc                  notes
+ *   ─────    ────────            ──────
+ *   person   Person              direct
+ *   event    Event               direct
+ *   place    Concept             with properties.hubKind = "place"
+ *   item     Concept             with properties.hubKind = "item"
+ *   topic    Concept             with properties.hubKind = "topic"
+ *
+ * Concept is used as a catch-all for hub kinds the cc KG doesn't natively
+ * model. The original kind is preserved in properties.hubKind so future cc
+ * KG schema upgrades (adding Place / Item / Topic) can re-classify.
+ *
+ * Like CcLLMAdapter, this bridge uses dependency injection — caller passes
+ * addEntity + addRelation + db. The bridge has zero static knowledge of
+ * the cc KG module path or module system.
+ *
+ * Two-pass algorithm:
+ *
+ *   Pass 1 — for every distinct subject:
+ *     a. collect literal triples into a property bag + identify primary name
+ *        (first `has-name` wins) + cc type from `rdf:type`
+ *     b. addEntity(db, { id, name, type, properties }) — caller's addEntity
+ *        must be idempotent on duplicate id (the cc impl throws "already
+ *        exists"; we catch that and treat as upsert)
+ *
+ *   Pass 2 — for every object triple, addRelation. Skip if either endpoint
+ *     wasn't seen in pass 1 (avoids dangling-relation errors from cc KG).
+ *
+ * Returns { entitiesUpserted, relationsAdded, errors[] } so the registry
+ * can audit ingest stats.
+ */
+
+"use strict";
+
+const HUB_TO_CC_TYPE = Object.freeze({
+  person: "Person",
+  event: "Event",
+  place: "Concept",
+  item: "Concept",
+  topic: "Concept",
+});
+
+const PROPERTY_TRIPLE_PREDICATES = new Set([
+  "subtype",
+  "occurred-at",
+  "source",
+  "amount-value",
+  "amount-currency",
+  "amount-direction",
+  "address",
+  "category",
+  "located-at",
+  "priced-at",
+  "has-alias",
+  "relation",
+]);
+// `id:<kind>` predicate is handled separately (variable suffix)
+
+const OBJECT_TRIPLE_PREDICATES = new Set([
+  "by",
+  "involves",
+  "happened-at",
+  "about",
+  "topic",
+  "sold-by",
+  "parent",
+  "derived-from",
+]);
+
+class CcKgSink {
+  /**
+   * @param {object} deps
+   * @param {(db: object, config: object) => object} deps.addEntity   cc addEntity
+   * @param {(db: object, config: object) => object} deps.addRelation cc addRelation
+   * @param {object} [deps.db]                                         cc db handle (forwarded)
+   * @param {(label: string, ...args: any[]) => void} [deps.logger]    optional logger for non-fatal errors
+   */
+  constructor(deps) {
+    if (!deps || typeof deps !== "object") {
+      throw new Error("CcKgSink: deps required");
+    }
+    if (typeof deps.addEntity !== "function") {
+      throw new Error("CcKgSink: deps.addEntity(db, config) required");
+    }
+    if (typeof deps.addRelation !== "function") {
+      throw new Error("CcKgSink: deps.addRelation(db, config) required");
+    }
+    this._addEntity = deps.addEntity;
+    this._addRelation = deps.addRelation;
+    this._db = deps.db || null;
+    this._log = typeof deps.logger === "function" ? deps.logger : null;
+    this._seenEntities = new Set(); // de-dup across calls within process lifetime
+  }
+
+  /**
+   * Bound method used as the registry kgSink callback:
+   *   const sink = new CcKgSink({ ... });
+   *   new AdapterRegistry({ vault, kgSink: sink.write.bind(sink) });
+   */
+  async write(triples) {
+    if (!Array.isArray(triples) || triples.length === 0) {
+      return { entitiesUpserted: 0, relationsAdded: 0, errors: [] };
+    }
+
+    // ─── Group triples by subject ─────────────────────────────────────
+    const bySubject = new Map();
+    const objectTriples = [];
+    for (const t of triples) {
+      if (!t || !t.subject || !t.predicate) continue;
+      if (typeof t.object === "string") {
+        objectTriples.push(t);
+        continue;
+      }
+      if (!bySubject.has(t.subject)) bySubject.set(t.subject, []);
+      bySubject.get(t.subject).push(t);
+    }
+
+    const errors = [];
+    let entitiesUpserted = 0;
+    let relationsAdded = 0;
+    const subjectsCreated = new Set();
+
+    // ─── Pass 1: upsert entities ──────────────────────────────────────
+    for (const [subject, subTriples] of bySubject.entries()) {
+      let hubType = null;
+      let primaryName = null;
+      const aliases = [];
+      const properties = {};
+
+      for (const t of subTriples) {
+        const pred = t.predicate;
+        const lit = t.literal;
+        if (pred === "rdf:type") {
+          hubType = typeof lit === "string" ? lit : null;
+          continue;
+        }
+        if (pred === "has-name") {
+          if (primaryName == null) primaryName = String(lit);
+          else aliases.push(String(lit));
+          continue;
+        }
+        if (pred === "has-alias") {
+          aliases.push(String(lit));
+          continue;
+        }
+        if (pred.startsWith("id:")) {
+          properties[pred] = lit;
+          continue;
+        }
+        if (PROPERTY_TRIPLE_PREDICATES.has(pred)) {
+          // Some predicates can repeat (e.g. multiple aliases via separate triples).
+          // Stash as array if duplicate.
+          if (properties[pred] === undefined) {
+            properties[pred] = lit;
+          } else if (Array.isArray(properties[pred])) {
+            properties[pred].push(lit);
+          } else {
+            properties[pred] = [properties[pred], lit];
+          }
+          continue;
+        }
+        // Unknown predicate — preserve under a `__extra` namespace.
+        if (!properties.__extra) properties.__extra = {};
+        properties.__extra[pred] = lit;
+      }
+
+      const ccType = HUB_TO_CC_TYPE[hubType] || "Concept";
+      properties.hubKind = hubType || "unknown";
+      if (aliases.length > 0) properties.aliases = aliases;
+      const name = primaryName || subject; // fallback: id as name
+
+      try {
+        this._addEntity(this._db, {
+          id: subject,
+          name,
+          type: ccType,
+          properties,
+        });
+        entitiesUpserted += 1;
+        this._seenEntities.add(subject);
+        subjectsCreated.add(subject);
+      } catch (err) {
+        const msg = err && err.message ? err.message : String(err);
+        // cc throws "Entity already exists" — treat as success (upsert semantics).
+        if (/already exists/i.test(msg)) {
+          this._seenEntities.add(subject);
+          subjectsCreated.add(subject);
+          continue;
+        }
+        errors.push({ kind: "entity", subject, error: msg });
+        if (this._log) this._log("CcKgSink.addEntity failed", subject, msg);
+      }
+    }
+
+    // ─── Pass 2: add relations ────────────────────────────────────────
+    for (const t of objectTriples) {
+      if (!OBJECT_TRIPLE_PREDICATES.has(t.predicate)) {
+        // Unknown predicate — record for telemetry; cc KG would reject anyway.
+        errors.push({
+          kind: "relation",
+          subject: t.subject,
+          predicate: t.predicate,
+          error: "unknown predicate",
+        });
+        continue;
+      }
+      // cc requires both endpoints already in the KG. Best-effort: if not
+      // in this batch's subjectsCreated AND not in the long-lived _seenEntities,
+      // skip with a warning. (Cross-batch references should be rare in
+      // practice — KG ingest is per-batch from same sync.)
+      if (!this._seenEntities.has(t.subject) || !this._seenEntities.has(t.object)) {
+        errors.push({
+          kind: "relation",
+          subject: t.subject,
+          target: t.object,
+          predicate: t.predicate,
+          error: "endpoint not in KG",
+        });
+        continue;
+      }
+      try {
+        this._addRelation(this._db, {
+          sourceId: t.subject,
+          targetId: t.object,
+          relationType: t.predicate,
+        });
+        relationsAdded += 1;
+      } catch (err) {
+        const msg = err && err.message ? err.message : String(err);
+        // Tolerate "already exists" if cc throws it.
+        if (/already exists|duplicate/i.test(msg)) continue;
+        errors.push({
+          kind: "relation",
+          subject: t.subject,
+          target: t.object,
+          predicate: t.predicate,
+          error: msg,
+        });
+        if (this._log) this._log("CcKgSink.addRelation failed", t.subject, t.object, msg);
+      }
+    }
+
+    return { entitiesUpserted, relationsAdded, errors };
+  }
+}
+
+module.exports = { CcKgSink, HUB_TO_CC_TYPE };

package/lib/bridges/cc-llm-adapter.js

@@ -0,0 +1,169 @@
+/**
+ * CcLLMAdapter — bridges the hub's LLMClient contract to ChainlessChain's
+ * existing llm-manager (or any compatible client).
+ *
+ * Hub package stays decoupled from cc by INJECTING the cc-specific bits:
+ *
+ *   const llmManager = require("desktop-app-vue/src/main/llm/llm-manager");
+ *   const adapter = new CcLLMAdapter({
+ *     chat: (messages, opts) => llmManager.getInstance().chat(messages, opts),
+ *     getActiveProvider: () => llmManager.getInstance().getActiveProvider(),
+ *     getActiveModel: () => llmManager.getInstance().getActiveModel(),
+ *   });
+ *   // adapter satisfies LLMClient — drop into new AnalysisEngine({ vault, llm: adapter }).
+ *
+ * Why injection: the hub is CJS + workspace-portable. cli is ESM, desktop
+ * main is CJS, future iOS bridge is yet another runtime. Each caller knows
+ * how to obtain a `chat(messages, opts)` function in its own module system.
+ * The hub just adapts the response shape.
+ *
+ * Privacy: isLocal is computed from the caller-supplied getActiveProvider().
+ * Providers we consider local (no network egress on chat): ollama, llama-cpp,
+ * vllm-local, lm-studio. Everything else is non-local — AnalysisEngine will
+ * refuse to call unless caller explicitly opts in via acceptNonLocal: true.
+ *
+ * Response normalization: cc's llm-manager returns slightly different
+ * shapes per provider. We coerce to the hub's { text, model, usage }
+ * contract, preferring (in order):
+ *   result.content                       // llm-manager wraps with .content
+ *   result.message.content               // raw provider message
+ *   result.text                          // some clients
+ *   result.choices[0].message.content    // OpenAI-style
+ */
+
+"use strict";
+
+const LOCAL_PROVIDERS = new Set([
+  "ollama",
+  "llama-cpp",
+  "llamacpp",
+  "vllm-local",
+  "lm-studio",
+  "lmstudio",
+]);
+
+function extractText(result) {
+  if (!result || typeof result !== "object") return "";
+  if (typeof result.content === "string") return result.content;
+  if (typeof result.text === "string") return result.text;
+  if (result.message && typeof result.message.content === "string") {
+    return result.message.content;
+  }
+  if (
+    Array.isArray(result.choices) &&
+    result.choices[0] &&
+    result.choices[0].message &&
+    typeof result.choices[0].message.content === "string"
+  ) {
+    return result.choices[0].message.content;
+  }
+  return "";
+}
+
+function extractUsage(result) {
+  if (!result || typeof result !== "object" || !result.usage) {
+    return { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
+  }
+  const u = result.usage;
+  // cc llm-manager + OpenAI-compat: prompt_tokens / completion_tokens / total_tokens
+  // hub contract: camelCase
+  return {
+    promptTokens: u.promptTokens ?? u.prompt_tokens ?? u.input_tokens ?? 0,
+    completionTokens: u.completionTokens ?? u.completion_tokens ?? u.output_tokens ?? 0,
+    totalTokens:
+      u.totalTokens ??
+      u.total_tokens ??
+      (u.promptTokens ?? u.prompt_tokens ?? 0) + (u.completionTokens ?? u.completion_tokens ?? 0),
+  };
+}
+
+class CcLLMAdapter {
+  /**
+   * @param {object} deps
+   * @param {(messages: Array, opts?: object) => Promise<object>} deps.chat
+   * @param {() => string} [deps.getActiveProvider]
+   * @param {() => string} [deps.getActiveModel]
+   * @param {Set<string>|string[]} [deps.localProviders]   override the default local-provider whitelist
+   * @param {string} [deps.name]                            override the .name surface
+   */
+  constructor(deps) {
+    if (!deps || typeof deps !== "object") {
+      throw new Error("CcLLMAdapter: deps required");
+    }
+    if (typeof deps.chat !== "function") {
+      throw new Error("CcLLMAdapter: deps.chat(messages, opts) required");
+    }
+    this._chat = deps.chat;
+    this._getActiveProvider = typeof deps.getActiveProvider === "function" ? deps.getActiveProvider : null;
+    this._getActiveModel = typeof deps.getActiveModel === "function" ? deps.getActiveModel : null;
+    this._localProviders =
+      deps.localProviders instanceof Set
+        ? deps.localProviders
+        : Array.isArray(deps.localProviders)
+          ? new Set(deps.localProviders)
+          : LOCAL_PROVIDERS;
+    this._name = deps.name || null;
+  }
+
+  get name() {
+    if (this._name) return this._name;
+    const model = this._getActiveModel ? this._tryCall(this._getActiveModel, "model") : null;
+    const provider = this._getActiveProvider
+      ? this._tryCall(this._getActiveProvider, "provider")
+      : null;
+    if (provider && model) return `${provider}:${model}`;
+    if (model) return model;
+    if (provider) return provider;
+    return "cc-llm";
+  }
+
+  /**
+   * Privacy invariant: report whether this LLM keeps data on-device. The
+   * AnalysisEngine consults this BEFORE calling chat; non-local clients
+   * are refused unless the caller explicitly passes acceptNonLocal: true.
+   */
+  get isLocal() {
+    if (!this._getActiveProvider) {
+      // No provider info → conservative: assume non-local. Caller must
+      // explicitly mark via constructor opts.localProviders if needed.
+      return false;
+    }
+    const provider = this._tryCall(this._getActiveProvider, "provider");
+    if (!provider) return false;
+    return this._localProviders.has(String(provider).toLowerCase());
+  }
+
+  async chat(messages, opts = {}) {
+    if (!Array.isArray(messages)) {
+      throw new Error("CcLLMAdapter.chat: messages must be an array");
+    }
+    let result;
+    try {
+      result = await this._chat(messages, opts);
+    } catch (err) {
+      const wrapped = new Error(
+        `CcLLMAdapter.chat: underlying client failed — ${err && err.message ? err.message : err}`
+      );
+      wrapped.cause = err;
+      throw wrapped;
+    }
+    return {
+      text: extractText(result),
+      model: this._getActiveModel ? this._tryCall(this._getActiveModel, "model") : result && result.model,
+      usage: extractUsage(result),
+      raw: result,
+    };
+  }
+
+  _tryCall(fn, label) {
+    try {
+      return fn();
+    } catch (err) {
+      // Don't let getActiveProvider/getActiveModel side-effects abort isLocal
+      // computation or name lookup. Fall through to default.
+      return null;
+    }
+  }
+}
+
+module.exports = { CcLLMAdapter, LOCAL_PROVIDERS };

package/lib/bridges/cc-rag-sink.js

@@ -0,0 +1,118 @@
+/**
+ * CcRagSink — feeds hub RagDocs into ChainlessChain's existing BM25 + (later)
+ * Qdrant vector store.
+ *
+ * Hub RagDoc shape: { id, type, text, metadata: { ... } }
+ * cc BM25.addDocument(doc) expects: { id, title?, content? } — concatenates
+ * title + " " + content for tokenization.
+ *
+ * We map:
+ *   doc.id       → doc.id
+ *   doc.metadata.title || doc.type → doc.title (short, BM25-prioritized via tokenization)
+ *   doc.text     → doc.content
+ *
+ * Metadata is also serialized into a `meta` property the BM25 originalDoc
+ * field preserves verbatim — that's how the downstream Q&A flow filters
+ * hits by adapter / time-window / subtype.
+ *
+ * Vector store wiring (Qdrant) is intentionally LEFT OUT of this v0 sink:
+ * the existing cc embed/qdrant integration is async and IPC-shaped; adding
+ * it here would couple us to its lifecycle. v1 sink: BM25 only. The hub
+ * caller can add a second sink in parallel for vector indexing once that
+ * surface stabilizes.
+ *
+ * Like the other bridges this is dependency-injected — caller passes
+ * the BM25 instance (or any object with .addDocument(doc)).
+ */
+
+"use strict";
+
+class CcRagSink {
+  /**
+   * @param {object} deps
+   * @param {{ addDocument: (doc: object) => void }} deps.bm25  cc BM25Search instance
+   * @param {{ index: (docs: Array) => Promise<void> }} [deps.vector]   future Qdrant adapter (not yet used)
+   * @param {(label: string, ...args: any[]) => void} [deps.logger]
+   * @param {(doc: object) => object} [deps.transformDoc]       optional pre-write hook
+   */
+  constructor(deps) {
+    if (!deps || typeof deps !== "object") {
+      throw new Error("CcRagSink: deps required");
+    }
+    if (!deps.bm25 || typeof deps.bm25.addDocument !== "function") {
+      throw new Error("CcRagSink: deps.bm25 with .addDocument(doc) required");
+    }
+    this._bm25 = deps.bm25;
+    this._vector = deps.vector && typeof deps.vector.index === "function" ? deps.vector : null;
+    this._log = typeof deps.logger === "function" ? deps.logger : null;
+    this._transform = typeof deps.transformDoc === "function" ? deps.transformDoc : null;
+    this._writtenIds = new Set();
+  }
+
+  /**
+   * Bound to .write(docs) for use as the registry's ragSink callback.
+   */
+  async write(docs) {
+    if (!Array.isArray(docs) || docs.length === 0) {
+      return { indexed: 0, skipped: 0, errors: [] };
+    }
+    let indexed = 0;
+    let skipped = 0;
+    const errors = [];
+    const forVector = [];
+
+    for (const d of docs) {
+      if (!d || !d.id || typeof d.text !== "string" || d.text.length === 0) {
+        skipped += 1;
+        continue;
+      }
+      // De-dup within process lifetime — BM25.addDocument doesn't dedup
+      // internally; a re-ingest would otherwise double-count term frequencies.
+      if (this._writtenIds.has(d.id)) {
+        skipped += 1;
+        continue;
+      }
+
+      const doc = this._transform ? this._transform(d) : this._toBm25Doc(d);
+      try {
+        this._bm25.addDocument(doc);
+        this._writtenIds.add(d.id);
+        indexed += 1;
+        if (this._vector) forVector.push(d);
+      } catch (err) {
+        const msg = err && err.message ? err.message : String(err);
+        errors.push({ id: d.id, error: msg });
+        if (this._log) this._log("CcRagSink.addDocument failed", d.id, msg);
+      }
+    }
+
+    if (this._vector && forVector.length > 0) {
+      try {
+        await this._vector.index(forVector);
+      } catch (err) {
+        const msg = err && err.message ? err.message : String(err);
+        errors.push({ phase: "vector", error: msg });
+        if (this._log) this._log("CcRagSink.vector.index failed", msg);
+      }
+    }
+
+    return { indexed, skipped, errors };
+  }
+
+  _toBm25Doc(d) {
+    const title =
+      (d.metadata && (d.metadata.title || d.metadata.subtype)) ||
+      d.type ||
+      "";
+    return {
+      id: d.id,
+      title: String(title),
+      content: d.text,
+      // BM25 preserves the original doc in `originalDoc`; metadata lives there.
+      meta: d.metadata || {},
+      hubType: d.type,
+    };
+  }
+}
+
+module.exports = { CcRagSink };