@hasna/experts 0.0.5 → 0.0.7

package/dist/server/index.js

@@ -28,6 +28,206 @@ function expertText(e) {
   return [e.title, e.headline, e.bio].filter(Boolean).join(". ");
 }
 
+// src/score.ts
+var DEFAULT_WEIGHTS = {
+  rating: 0.3,
+  reviews: 0.2,
+  followers: 0.25,
+  featured: 0.1,
+  verified: 0.05,
+  recency: 0.1
+};
+var clamp01 = (n) => Math.max(0, Math.min(1, n));
+var logNorm = (x, cap) => clamp01(Math.log10(1 + Math.max(0, x)) / Math.log10(1 + cap));
+function authorityScore(e, inputs = {}, weights = DEFAULT_WEIGHTS) {
+  const rating = clamp01((e.rating || 0) / 5);
+  const reviews = logNorm(e.ratingCount || 0, 1000);
+  const followers = logNorm(inputs.followers ?? 0, 1e6);
+  const featured = e.featured ? 1 : 0;
+  const verified = e.verified ? 1 : 0;
+  const recency = inputs.daysSinceLastTweet == null ? 0 : clamp01(1 - inputs.daysSinceLastTweet / 30);
+  const raw = weights.rating * rating + weights.reviews * reviews + weights.followers * followers + weights.featured * featured + weights.verified * verified + weights.recency * recency;
+  return Math.round(raw * 1000) / 10;
+}
+
+// src/embed.ts
+var STOPWORDS = new Set([
+  "the",
+  "a",
+  "an",
+  "and",
+  "or",
+  "of",
+  "to",
+  "in",
+  "for",
+  "on",
+  "at",
+  "is",
+  "are",
+  "with",
+  "by",
+  "as",
+  "be",
+  "this",
+  "that",
+  "it",
+  "from",
+  "i",
+  "you",
+  "we",
+  "they"
+]);
+function tokenize(text) {
+  return (text || "").toLowerCase().replace(/https?:\/\/\S+/g, " ").split(/[^a-z0-9]+/).filter((t) => t.length >= 2 && !STOPWORDS.has(t));
+}
+function fnv1a(s) {
+  let h = 2166136261;
+  for (let i = 0;i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
+  }
+  return h >>> 0;
+}
+
+class HashingEmbedder {
+  id = "hash-v1";
+  dim;
+  constructor(dim = 512) {
+    this.dim = dim;
+  }
+  one(text) {
+    const v = new Array(this.dim).fill(0);
+    const toks = tokenize(text);
+    for (let i = 0;i < toks.length; i++) {
+      const uni = toks[i];
+      v[fnv1a(uni) % this.dim] += 1;
+      if (i + 1 < toks.length) {
+        const bi = uni + "_" + toks[i + 1];
+        v[fnv1a(bi) % this.dim] += 0.5;
+      }
+    }
+    const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0)) || 1;
+    return v.map((x) => x / norm);
+  }
+  async embed(texts) {
+    return texts.map((t) => this.one(t));
+  }
+}
+
+class OpenAIEmbedder {
+  id;
+  dim = 1536;
+  apiKey;
+  model;
+  fetchFn;
+  constructor(opts = {}) {
+    this.apiKey = opts.apiKey ?? process.env.OPENAI_API_KEY ?? "";
+    this.model = opts.model ?? "text-embedding-3-small";
+    this.fetchFn = opts.fetchFn ?? fetch;
+    this.id = `openai:${this.model}`;
+  }
+  async embed(texts) {
+    const res = await this.fetchFn("https://api.openai.com/v1/embeddings", {
+      method: "POST",
+      headers: { "content-type": "application/json", authorization: `Bearer ${this.apiKey}` },
+      body: JSON.stringify({ model: this.model, input: texts })
+    });
+    if (!res.ok)
+      throw new Error(`OpenAI embeddings ${res.status}: ${(await res.text()).slice(0, 200)}`);
+    const data = await res.json();
+    return data.data.map((d) => d.embedding);
+  }
+}
+function getEmbedder() {
+  if (process.env.EXPERTS_EMBEDDER === "openai" && process.env.OPENAI_API_KEY) {
+    return new OpenAIEmbedder;
+  }
+  return new HashingEmbedder;
+}
+function cosine(a, b) {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  const n = Math.min(a.length, b.length);
+  for (let i = 0;i < n; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  const d = Math.sqrt(na) * Math.sqrt(nb);
+  return d === 0 ? 0 : dot / d;
+}
+function packVector(v) {
+  const f = new Float32Array(v);
+  return new Uint8Array(f.buffer);
+}
+function unpackVector(buf) {
+  const ab = buf instanceof Uint8Array ? buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) : buf;
+  return Array.from(new Float32Array(ab));
+}
+function expertEmbedText(e) {
+  return [e.fullName, e.title, e.headline, e.bio, e.topics.join(" "), e.tags.join(" ")].filter(Boolean).join(". ");
+}
+
+// src/identity.ts
+function socialHandles(e) {
+  const out = new Set;
+  for (const [platform, url] of Object.entries(e.socials || {})) {
+    if (!url)
+      continue;
+    const m = String(url).match(/(?:[a-z]+\.[a-z]+\/@?)([A-Za-z0-9_.-]+)/i);
+    const handle = (m ? m[1] : String(url)).toLowerCase().replace(/\/+$/, "");
+    if (handle)
+      out.add(`${platform}:${handle}`);
+  }
+  return out;
+}
+var key = (e) => `${e.source}:${e.sourceId}`;
+function clusterPersons(experts) {
+  const parent = new Map;
+  const find = (x) => {
+    let r = x;
+    while (parent.get(r) !== r)
+      r = parent.get(r);
+    let c = x;
+    while (parent.get(c) !== r) {
+      const n = parent.get(c);
+      parent.set(c, r);
+      c = n;
+    }
+    return r;
+  };
+  const union = (a, b) => {
+    const ra = find(a);
+    const rb = find(b);
+    if (ra === rb)
+      return;
+    if (ra < rb)
+      parent.set(rb, ra);
+    else
+      parent.set(ra, rb);
+  };
+  for (const e of experts)
+    parent.set(key(e), key(e));
+  const byHandle = new Map;
+  for (const e of experts) {
+    for (const h of socialHandles(e)) {
+      if (!byHandle.has(h))
+        byHandle.set(h, []);
+      byHandle.get(h).push(key(e));
+    }
+  }
+  for (const members of byHandle.values()) {
+    for (let i = 1;i < members.length; i++)
+      union(members[0], members[i]);
+  }
+  const out = new Map;
+  for (const e of experts)
+    out.set(key(e), find(key(e)));
+  return out;
+}
+
 // src/db.ts
 function defaultDbPath() {
   return process.env.OPEN_EXPERTS_DB || join(homedir(), ".hasna", "experts", "experts.db");
@@ -111,8 +311,59 @@ class ExpertsDB {
         PRIMARY KEY (source, tweet_id)
       );
       CREATE INDEX IF NOT EXISTS idx_tweets_expert ON tweets(source, source_id, created_at DESC);
+
+      -- Enrichment: recent YouTube videos per expert.
+      CREATE TABLE IF NOT EXISTS videos (
+        source TEXT NOT NULL, source_id TEXT NOT NULL,
+        video_id TEXT NOT NULL, title TEXT, description TEXT,
+        published_at TEXT, url TEXT, thumbnail TEXT, view_count INTEGER,
+        PRIMARY KEY (source, video_id)
+      );
+      CREATE INDEX IF NOT EXISTS idx_videos_expert ON videos(source, source_id, published_at DESC);
+
+      -- Generic external enrichment (linkedin, site/newsletter, \u2026) as JSON.
+      CREATE TABLE IF NOT EXISTS ext_profiles (
+        source TEXT NOT NULL, source_id TEXT NOT NULL, kind TEXT NOT NULL,
+        data TEXT, enriched_at TEXT,
+        PRIMARY KEY (source, source_id, kind)
+      );
+
+      -- Enrichment: discovered contact methods (multiple email/phone per expert).
+      CREATE TABLE IF NOT EXISTS contacts (
+        source TEXT NOT NULL, source_id TEXT NOT NULL,
+        type TEXT NOT NULL, value TEXT NOT NULL,
+        label TEXT, provider TEXT, confidence REAL,
+        status TEXT DEFAULT 'unverified', verified_at TEXT, created_at TEXT,
+        PRIMARY KEY (source, source_id, type, value)
+      );
+      CREATE INDEX IF NOT EXISTS idx_contacts_expert ON contacts(source, source_id);
+      CREATE INDEX IF NOT EXISTS idx_contacts_status ON contacts(status);
+
+      -- Semantic search: one embedding vector per expert.
+      CREATE TABLE IF NOT EXISTS vectors (
+        source TEXT NOT NULL, source_id TEXT NOT NULL,
+        embedder TEXT NOT NULL, dim INTEGER, vec BLOB,
+        PRIMARY KEY (source, source_id)
+      );
+
+      -- Identity resolution: maps each expert record to a canonical person.
+      CREATE TABLE IF NOT EXISTS persons (
+        source TEXT NOT NULL, source_id TEXT NOT NULL, person_id TEXT NOT NULL,
+        PRIMARY KEY (source, source_id)
+      );
+      CREATE INDEX IF NOT EXISTS idx_persons_person ON persons(person_id);
+
+      -- Change detection: a log of what changed between crawls.
+      CREATE TABLE IF NOT EXISTS changes (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        source TEXT NOT NULL, source_id TEXT NOT NULL,
+        kind TEXT NOT NULL, field TEXT, old_value TEXT, new_value TEXT,
+        detected_at TEXT
+      );
+      CREATE INDEX IF NOT EXISTS idx_changes_time ON changes(detected_at DESC);
     `);
     this.addColumnIfMissing("experts", "avatar_local", "TEXT");
+    this.addColumnIfMissing("experts", "authority", "REAL DEFAULT 0");
   }
   addColumnIfMissing(table, column, type) {
     const cols = this.db.query(`PRAGMA table_info(${table})`).all();
@@ -198,6 +449,7 @@ class ExpertsDB {
       socials: JSON.parse(r.socials || "{}"),
       extra: JSON.parse(r.extra || "{}"),
       avatarLocal: r.avatar_local || undefined,
+      authority: r.authority ?? 0,
       crawledAt: r.crawled_at
     };
   }
@@ -245,7 +497,7 @@ class ExpertsDB {
       where.push("rating >= ?");
       params.push(filters.minRating);
     }
-    const sortCol = filters.sort === "price" ? "price" : filters.sort === "name" ? "full_name" : filters.sort === "reviews" ? "rating_count" : "rating";
+    const sortCol = filters.sort === "price" ? "price" : filters.sort === "name" ? "full_name" : filters.sort === "reviews" ? "rating_count" : filters.sort === "authority" ? "authority" : "rating";
     const defaultAsc = filters.sort === "name";
     const dir = filters.desc ?? !defaultAsc ? "DESC" : "ASC";
     let sql = "SELECT * FROM experts";
@@ -321,11 +573,11 @@ class ExpertsDB {
     sql += " ORDER BY name";
     return this.db.query(sql).all(...params);
   }
-  setMeta(key, value) {
-    this.db.query("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(key, value);
+  setMeta(key2, value) {
+    this.db.query("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(key2, value);
   }
-  getMeta(key) {
-    const row = this.db.query("SELECT value FROM meta WHERE key = ?").get(key);
+  getMeta(key2) {
+    const row = this.db.query("SELECT value FROM meta WHERE key = ?").get(key2);
     return row ? row.value : null;
   }
   stats(source) {
@@ -349,12 +601,12 @@ class ExpertsDB {
       const nodeIds = new Map;
       const insertNode = this.db.query("INSERT INTO kg_nodes (type, key, label) VALUES (?, ?, ?) ON CONFLICT(type, key) DO UPDATE SET label=excluded.label RETURNING id");
       const insertEdge = this.db.query("INSERT OR REPLACE INTO kg_edges (src, dst, rel, weight) VALUES (?, ?, ?, ?)");
-      const node = (type, key, label) => {
-        const ck = `${type}\x00${key.toLowerCase()}`;
+      const node = (type, key2, label) => {
+        const ck = `${type}\x00${key2.toLowerCase()}`;
         const cached = nodeIds.get(ck);
         if (cached != null)
           return cached;
-        const id = insertNode.get(type, key.toLowerCase(), label).id;
+        const id = insertNode.get(type, key2.toLowerCase(), label).id;
         nodeIds.set(ck, id);
         return id;
       };
@@ -364,7 +616,8 @@ class ExpertsDB {
         for (const topic of e.topics) {
           insertEdge.run(eId, node("topic", topic, topic), "IN_TOPIC", 1);
         }
-        const tags = inferTags(expertText(e), vocabulary);
+        const tweetText = this.recentTweets(e.source, e.sourceId, 30).map((t) => t.text).join(". ");
+        const tags = inferTags(expertText(e) + ". " + tweetText, vocabulary);
         for (const tag of tags) {
           insertEdge.run(eId, node("tag", tag, tag), "HAS_TAG", 1);
         }
@@ -377,11 +630,51 @@ class ExpertsDB {
     this.setMeta("graph_built", new Date().toISOString());
     return { nodes, edges };
   }
-  expertFromNodeKey(key) {
-    const idx = key.indexOf(":");
+  rescore(source) {
+    const experts = this.list({ source });
+    const upd = this.db.query("UPDATE experts SET authority = ? WHERE source = ? AND source_id = ?");
+    const followerStmt = this.db.query("SELECT followers FROM x_profiles WHERE source = ? AND source_id = ?");
+    const lastTweetStmt = this.db.query("SELECT MAX(created_at) AS t FROM tweets WHERE source = ? AND source_id = ?");
+    const tx = this.db.transaction((rows) => {
+      for (const e of rows) {
+        const fr = followerStmt.get(e.source, e.sourceId);
+        const lt = lastTweetStmt.get(e.source, e.sourceId);
+        let daysSince;
+        if (lt?.t) {
+          const ms = Date.now() - Date.parse(lt.t);
+          if (!Number.isNaN(ms))
+            daysSince = ms / 86400000;
+        }
+        const score = authorityScore(e, { followers: fr?.followers ?? 0, daysSinceLastTweet: daysSince });
+        upd.run(score, e.source, e.sourceId);
+      }
+    });
+    tx(experts);
+    this.setMeta("rescored_at", new Date().toISOString());
+    return experts.length;
+  }
+  stalest(opts = {}) {
+    const where = opts.source ? "WHERE e.source = ?" : "";
+    const params = opts.source ? [opts.source] : [];
+    const sql = `
+      SELECT e.*, COALESCE(
+        (SELECT MAX(enriched_at) FROM x_profiles xp WHERE xp.source=e.source AND xp.source_id=e.source_id),
+        e.crawled_at
+      ) AS last_seen
+      FROM experts e ${where}
+      ORDER BY last_seen ASC
+      LIMIT ?`;
+    params.push(opts.limit ?? 25);
+    return this.db.query(sql).all(...params).map((r) => ({
+      expert: this.rowToExpert(r),
+      lastSeen: r.last_seen || ""
+    }));
+  }
+  expertFromNodeKey(key2) {
+    const idx = key2.indexOf(":");
     if (idx < 0)
       return null;
-    return this.get(key.slice(idx + 1), key.slice(0, idx));
+    return this.get(key2.slice(idx + 1), key2.slice(0, idx));
   }
   findByNeeds(needs, opts = {}) {
     const cleaned = needs.map((n) => n.trim().toLowerCase()).filter(Boolean);
@@ -570,6 +863,270 @@ class ExpertsDB {
     const avatars = this.db.query(`SELECT COUNT(*) AS n FROM experts WHERE avatar_local IS NOT NULL${source ? " AND source = ?" : ""}`).get(...args).n;
     return { withHandle, enriched, tweets, avatars };
   }
+  recordChanges(source, incoming) {
+    const existing = new Map(this.list({ source }).map((e) => [e.sourceId, e]));
+    const now = new Date().toISOString();
+    const watched = ["price", "title", "headline", "bio", "slug"];
+    const stmt = this.db.query("INSERT INTO changes (source, source_id, kind, field, old_value, new_value, detected_at) VALUES (?, ?, ?, ?, ?, ?, ?)");
+    let count = 0;
+    const tx = this.db.transaction((rows) => {
+      for (const e of rows) {
+        const prev = existing.get(e.sourceId);
+        if (!prev) {
+          stmt.run(source, e.sourceId, "added", null, null, e.fullName || e.slug, now);
+          count++;
+          continue;
+        }
+        for (const f of watched) {
+          const a = String(prev[f] ?? "");
+          const b = String(e[f] ?? "");
+          if (a !== b) {
+            stmt.run(source, e.sourceId, "updated", f, a, b, now);
+            count++;
+          }
+        }
+      }
+    });
+    tx(incoming);
+    return count;
+  }
+  changes(opts = {}) {
+    const where = opts.source ? "WHERE source = ?" : "";
+    const params = opts.source ? [opts.source] : [];
+    params.push(opts.limit ?? 50);
+    return this.db.query(`SELECT * FROM changes ${where} ORDER BY detected_at DESC, id DESC LIMIT ?`).all(...params);
+  }
+  rebuildPersons() {
+    const experts = this.list();
+    const mapping = clusterPersons(experts);
+    const tx = this.db.transaction(() => {
+      this.db.exec("DELETE FROM persons");
+      const stmt = this.db.query("INSERT OR REPLACE INTO persons (source, source_id, person_id) VALUES (?, ?, ?)");
+      for (const [k, pid] of mapping) {
+        const idx = k.indexOf(":");
+        stmt.run(k.slice(0, idx), k.slice(idx + 1), pid);
+      }
+    });
+    tx();
+    const persons = new Set(mapping.values()).size;
+    this.setMeta("persons_built", new Date().toISOString());
+    return { experts: experts.length, persons };
+  }
+  personIdOf(source, sourceId) {
+    const r = this.db.query("SELECT person_id FROM persons WHERE source = ? AND source_id = ?").get(source, sourceId);
+    return r ? r.person_id : `${source}:${sourceId}`;
+  }
+  expertsForPerson(personId) {
+    const rows = this.db.query("SELECT e.* FROM persons p JOIN experts e ON e.source=p.source AND e.source_id=p.source_id WHERE p.person_id = ?").all(personId);
+    return rows.map((r) => this.rowToExpert(r));
+  }
+  personStats() {
+    const experts = this.count();
+    const row = this.db.query("SELECT COUNT(DISTINCT person_id) n FROM persons").get();
+    const persons = row?.n ?? 0;
+    const dupes = this.db.query("SELECT person_id, COUNT(*) c FROM persons GROUP BY person_id HAVING c > 1 ORDER BY c DESC").all();
+    return { experts, persons: persons || experts, duplicated: dupes.length };
+  }
+  async buildEmbeddings(embedder, opts = {}) {
+    const log = opts.onLog ?? (() => {});
+    const experts = this.list({ source: opts.source });
+    const batch = opts.batch ?? 64;
+    const stmt = this.db.query("INSERT OR REPLACE INTO vectors (source, source_id, embedder, dim, vec) VALUES (?, ?, ?, ?, ?)");
+    let done = 0;
+    for (let i = 0;i < experts.length; i += batch) {
+      const slice = experts.slice(i, i + batch);
+      const vecs = await embedder.embed(slice.map((e) => expertEmbedText(e)));
+      const tx = this.db.transaction(() => {
+        slice.forEach((e, j) => stmt.run(e.source, e.sourceId, embedder.id, embedder.dim, packVector(vecs[j])));
+      });
+      tx();
+      done += slice.length;
+      log(`  embedded ${done}/${experts.length}`);
+    }
+    this.setMeta("embedder", embedder.id);
+    this.setMeta("embedded_at", new Date().toISOString());
+    return done;
+  }
+  vectorCount() {
+    return this.db.query("SELECT COUNT(*) n FROM vectors").get().n;
+  }
+  semanticSearch(queryVec, opts = {}) {
+    const where = opts.source ? "WHERE v.source = ?" : "";
+    const params = opts.source ? [opts.source] : [];
+    const rows = this.db.query(`SELECT e.*, v.vec AS _vec FROM vectors v JOIN experts e ON e.source=v.source AND e.source_id=v.source_id ${where}`).all(...params);
+    const scored = rows.map((r) => ({ expert: this.rowToExpert(r), score: cosine(queryVec, unpackVector(r._vec)) }));
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, opts.limit ?? 25);
+  }
+  upsertExt(source, sourceId, kind, data) {
+    this.db.query(`
+        INSERT INTO ext_profiles (source, source_id, kind, data, enriched_at)
+        VALUES (?, ?, ?, ?, ?)
+        ON CONFLICT(source, source_id, kind) DO UPDATE SET data=excluded.data, enriched_at=excluded.enriched_at
+      `).run(source, sourceId, kind, JSON.stringify(data), new Date().toISOString());
+  }
+  getExt(source, sourceId, kind) {
+    const r = this.db.query("SELECT data FROM ext_profiles WHERE source=? AND source_id=? AND kind=?").get(source, sourceId, kind);
+    return r ? JSON.parse(r.data || "{}") : null;
+  }
+  allExt(source, sourceId) {
+    const rows = this.db.query("SELECT kind, data FROM ext_profiles WHERE source=? AND source_id=?").all(source, sourceId);
+    return Object.fromEntries(rows.map((r) => [r.kind, JSON.parse(r.data || "{}")]));
+  }
+  expertsNeedingExt(platform, kind, opts = {}) {
+    const where = [`json_extract(socials, '$.${platform}') IS NOT NULL`];
+    const bind = [];
+    if (opts.source) {
+      where.push("source = ?");
+      bind.push(opts.source);
+    }
+    if (!opts.refresh) {
+      where.push("NOT EXISTS (SELECT 1 FROM ext_profiles x WHERE x.source=experts.source AND x.source_id=experts.source_id AND x.kind=?)");
+      bind.push(kind);
+    }
+    let sql = "SELECT * FROM experts WHERE " + where.join(" AND ") + " ORDER BY rating_count DESC";
+    if (opts.limit) {
+      sql += " LIMIT ?";
+      bind.push(opts.limit);
+    }
+    return this.db.query(sql).all(...bind).map((r) => this.rowToExpert(r));
+  }
+  replaceVideos(source, sourceId, videos) {
+    const tx = this.db.transaction((rows) => {
+      this.db.query("DELETE FROM videos WHERE source = ? AND source_id = ?").run(source, sourceId);
+      const stmt = this.db.query("INSERT OR REPLACE INTO videos (source, source_id, video_id, title, description, published_at, url, thumbnail, view_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)");
+      for (const v of rows) {
+        stmt.run(v.source, v.sourceId, v.videoId, v.title, v.description, v.publishedAt, v.url, v.thumbnail, v.viewCount);
+      }
+    });
+    tx(videos);
+  }
+  recentVideos(source, sourceId, limit = 10) {
+    const rows = this.db.query("SELECT * FROM videos WHERE source = ? AND source_id = ? ORDER BY published_at DESC LIMIT ?").all(source, sourceId, limit);
+    return rows.map((r) => ({
+      source: r.source,
+      sourceId: r.source_id,
+      videoId: r.video_id,
+      title: r.title || "",
+      description: r.description || "",
+      publishedAt: r.published_at || "",
+      url: r.url || "",
+      thumbnail: r.thumbnail || "",
+      viewCount: r.view_count ?? 0
+    }));
+  }
+  expertsNeedingVideos(opts = {}) {
+    const where = ["json_extract(socials, '$.youtube') IS NOT NULL"];
+    const params = [];
+    if (opts.source) {
+      where.push("source = ?");
+      params.push(opts.source);
+    }
+    if (!opts.refresh) {
+      where.push("NOT EXISTS (SELECT 1 FROM videos v WHERE v.source=experts.source AND v.source_id=experts.source_id)");
+    }
+    let sql = "SELECT * FROM experts WHERE " + where.join(" AND ") + " ORDER BY rating_count DESC";
+    if (opts.limit) {
+      sql += " LIMIT ?";
+      params.push(opts.limit);
+    }
+    return this.db.query(sql).all(...params).map((r) => this.rowToExpert(r));
+  }
+  upsertContact(c) {
+    this.db.query(`
+        INSERT INTO contacts (source, source_id, type, value, label, provider, confidence, status, verified_at, created_at)
+        VALUES ($source, $source_id, $type, $value, $label, $provider, $confidence, $status, $verified_at, $created_at)
+        ON CONFLICT(source, source_id, type, value) DO UPDATE SET
+          label=excluded.label, provider=excluded.provider, confidence=excluded.confidence,
+          status=CASE WHEN excluded.status != 'unverified' THEN excluded.status ELSE contacts.status END,
+          verified_at=COALESCE(excluded.verified_at, contacts.verified_at)
+      `).run({
+      $source: c.source,
+      $source_id: c.sourceId,
+      $type: c.type,
+      $value: c.value,
+      $label: c.label,
+      $provider: c.provider,
+      $confidence: c.confidence,
+      $status: c.status,
+      $verified_at: c.verifiedAt || null,
+      $created_at: c.createdAt || new Date().toISOString()
+    });
+  }
+  setContactStatus(source, sourceId, type, value, status) {
+    this.db.query("UPDATE contacts SET status = ?, verified_at = ? WHERE source = ? AND source_id = ? AND type = ? AND value = ?").run(status, new Date().toISOString(), source, sourceId, type, value);
+  }
+  contacts(source, sourceId) {
+    const rows = this.db.query("SELECT * FROM contacts WHERE source = ? AND source_id = ? ORDER BY type, confidence DESC").all(source, sourceId);
+    return rows.map((r) => ({
+      source: r.source,
+      sourceId: r.source_id,
+      type: r.type,
+      value: r.value,
+      label: r.label || "",
+      provider: r.provider || "",
+      confidence: r.confidence ?? 0,
+      status: r.status || "unverified",
+      verifiedAt: r.verified_at || "",
+      createdAt: r.created_at || ""
+    }));
+  }
+  contactsToVerify(opts = {}) {
+    const where = ["status = 'unverified'"];
+    const params = [];
+    if (opts.source) {
+      where.push("source = ?");
+      params.push(opts.source);
+    }
+    let sql = "SELECT * FROM contacts WHERE " + where.join(" AND ") + " ORDER BY confidence DESC";
+    if (opts.limit) {
+      sql += " LIMIT ?";
+      params.push(opts.limit);
+    }
+    return this.db.query(sql).all(...params).map((r) => ({
+      source: r.source,
+      sourceId: r.source_id,
+      type: r.type,
+      value: r.value,
+      label: r.label || "",
+      provider: r.provider || "",
+      confidence: r.confidence ?? 0,
+      status: r.status || "unverified",
+      verifiedAt: r.verified_at || "",
+      createdAt: r.created_at || ""
+    }));
+  }
+  expertsNeedingContacts(opts = {}) {
+    const where = [];
+    const params = [];
+    if (opts.source) {
+      where.push("source = ?");
+      params.push(opts.source);
+    }
+    if (!opts.refresh) {
+      where.push("NOT EXISTS (SELECT 1 FROM contacts c WHERE c.source = experts.source AND c.source_id = experts.source_id)");
+    }
+    let sql = "SELECT * FROM experts";
+    if (where.length)
+      sql += " WHERE " + where.join(" AND ");
+    sql += " ORDER BY rating_count DESC";
+    if (opts.limit) {
+      sql += " LIMIT ?";
+      params.push(opts.limit);
+    }
+    return this.db.query(sql).all(...params).map((r) => this.rowToExpert(r));
+  }
+  contactStats(source) {
+    const filt = source ? " WHERE source = ?" : "";
+    const args = source ? [source] : [];
+    const total = this.db.query(`SELECT COUNT(*) n FROM contacts${filt}`).get(...args).n;
+    const valid = this.db.query(`SELECT COUNT(*) n FROM contacts WHERE status='valid'${source ? " AND source = ?" : ""}`).get(...args).n;
+    const invalid = this.db.query(`SELECT COUNT(*) n FROM contacts WHERE status='invalid'${source ? " AND source = ?" : ""}`).get(...args).n;
+    const emails = this.db.query(`SELECT COUNT(*) n FROM contacts WHERE type='email'${source ? " AND source = ?" : ""}`).get(...args).n;
+    const phones = this.db.query(`SELECT COUNT(*) n FROM contacts WHERE type='phone'${source ? " AND source = ?" : ""}`).get(...args).n;
+    const expertsWith = this.db.query(`SELECT COUNT(DISTINCT source||source_id) n FROM contacts${filt}`).get(...args).n;
+    return { total, valid, invalid, emails, phones, expertsWith };
+  }
   close() {
     this.db.close();
   }
@@ -788,6 +1345,329 @@ class IntroSource {
   }
 }
 
+// src/sources/common.ts
+function makeExpert(p) {
+  const first = p.firstName ?? "";
+  const last = p.lastName ?? "";
+  return {
+    source: p.source,
+    sourceId: p.sourceId,
+    slug: p.slug ?? "",
+    url: p.url ?? "",
+    fullName: p.fullName ?? [first, last].filter(Boolean).join(" "),
+    firstName: first,
+    lastName: last,
+    title: p.title ?? "",
+    headline: p.headline ?? "",
+    bio: p.bio ?? "",
+    avatar: p.avatar ?? "",
+    price: p.price ?? 0,
+    priceCurrency: p.priceCurrency ?? "USD",
+    priceUnit: p.priceUnit ?? "",
+    rating: p.rating ?? 0,
+    ratingCount: p.ratingCount ?? 0,
+    verified: p.verified ?? false,
+    featured: p.featured ?? false,
+    topics: p.topics ? [...new Set(p.topics)].sort() : [],
+    tags: p.tags ?? [],
+    socials: p.socials ?? {},
+    extra: p.extra ?? {},
+    crawledAt: p.crawledAt ?? new Date().toISOString()
+  };
+}
+function slugify(s) {
+  return (s || "").toLowerCase().normalize("NFKD").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
+}
+async function fetchJson(url, fetchFn, init = {}) {
+  try {
+    const res = await fetchFn(url, {
+      ...init,
+      headers: {
+        "User-Agent": "open-experts (+https://github.com/hasna/experts)",
+        Accept: "application/json",
+        ...init.headers || {}
+      }
+    });
+    if (!res.ok)
+      return null;
+    return await res.json();
+  } catch {
+    return null;
+  }
+}
+
+// src/sources/mentorcruise.ts
+function normalizeMentor(m, crawledAt) {
+  const slug = m.slug || slugify(m.name || String(m.id ?? ""));
+  const socials = {};
+  if (m.twitter)
+    socials.twitter = m.twitter.startsWith("http") ? m.twitter : `https://x.com/${m.twitter}`;
+  if (m.linkedin)
+    socials.linkedin = m.linkedin;
+  return makeExpert({
+    source: "mentorcruise",
+    sourceId: String(m.id ?? slug),
+    slug,
+    url: `https://mentorcruise.com/mentor/${slug}/`,
+    fullName: m.name ?? [m.first_name, m.last_name].filter(Boolean).join(" "),
+    firstName: m.first_name ?? "",
+    lastName: m.last_name ?? "",
+    title: m.job_title ?? "",
+    bio: m.bio ?? "",
+    avatar: m.avatar ?? m.photo ?? "",
+    price: m.price ?? 0,
+    priceCurrency: m.currency ?? "USD",
+    priceUnit: m.price ? "per month" : "",
+    rating: m.rating ?? 0,
+    ratingCount: m.reviews_count ?? 0,
+    verified: Boolean(m.verified),
+    featured: Boolean(m.is_top_mentor),
+    topics: m.categories ?? [],
+    tags: m.skills ?? [],
+    socials,
+    crawledAt
+  });
+}
+
+class MentorCruiseSource {
+  name = "mentorcruise";
+  description = "MentorCruise \u2014 long-term mentorship from vetted mentors";
+  website = "https://mentorcruise.com";
+  fetchFn;
+  apiBase;
+  pageSize;
+  constructor(opts = {}) {
+    this.fetchFn = opts.fetchFn ?? fetch;
+    this.apiBase = opts.apiBase ?? process.env.MENTORCRUISE_API_BASE ?? "https://mentorcruise.com/api";
+    this.pageSize = opts.pageSize ?? 50;
+  }
+  async crawl(opts = {}) {
+    const log = opts.onLog ?? (() => {});
+    const crawledAt = new Date().toISOString();
+    const experts = [];
+    const tags = new Set;
+    let offset = 0;
+    for (;; ) {
+      const data = await fetchJson(`${this.apiBase}/mentors/?limit=${this.pageSize}&offset=${offset}`, this.fetchFn);
+      const items = data?.results ?? data?.data ?? (Array.isArray(data) ? data : []);
+      if (!items.length)
+        break;
+      for (const m of items) {
+        const e = normalizeMentor(m, crawledAt);
+        experts.push(e);
+        for (const t of e.tags)
+          tags.add(t);
+      }
+      offset += items.length;
+      log(`  mentorcruise: ${experts.length}`);
+      if (opts.max && experts.length >= opts.max)
+        break;
+      if (items.length < this.pageSize)
+        break;
+    }
+    if (experts.length === 0) {
+      log("mentorcruise: no public listing reachable (set MENTORCRUISE_API_BASE or provide a fetchFn).");
+    }
+    const topics = [];
+    return {
+      experts: opts.max ? experts.slice(0, opts.max) : experts,
+      topics,
+      tags: [...tags].map((name) => ({ name, topic: "" })),
+      total: experts.length
+    };
+  }
+}
+
+// src/sources/adplist.ts
+function normalizeAdpMentor(m, crawledAt) {
+  const slug = m.username || slugify(m.name || m.full_name || String(m.id ?? ""));
+  const socials = {};
+  if (m.twitter)
+    socials.twitter = m.twitter.startsWith("http") ? m.twitter : `https://x.com/${m.twitter}`;
+  if (m.linkedin)
+    socials.linkedin = m.linkedin;
+  return makeExpert({
+    source: "adplist",
+    sourceId: String(m.id ?? slug),
+    slug,
+    url: `https://adplist.org/mentors/${slug}`,
+    fullName: m.name ?? m.full_name ?? "",
+    title: m.headline ?? m.tagline ?? "",
+    headline: m.tagline ?? "",
+    bio: m.bio ?? m.about ?? "",
+    avatar: m.profile_photo ?? m.avatar ?? "",
+    price: 0,
+    priceUnit: "free session",
+    rating: m.rating ?? 0,
+    ratingCount: m.total_reviews ?? 0,
+    verified: Boolean(m.verified),
+    featured: Boolean(m.is_featured),
+    tags: m.expertise ?? m.skills ?? [],
+    socials,
+    crawledAt
+  });
+}
+
+class ADPListSource {
+  name = "adplist";
+  description = "ADPList \u2014 free mentorship across design, product & engineering";
+  website = "https://adplist.org";
+  fetchFn;
+  apiBase;
+  pageSize;
+  constructor(opts = {}) {
+    this.fetchFn = opts.fetchFn ?? fetch;
+    this.apiBase = opts.apiBase ?? process.env.ADPLIST_API_BASE ?? "https://api.adplist.org/api";
+    this.pageSize = opts.pageSize ?? 50;
+  }
+  async crawl(opts = {}) {
+    const log = opts.onLog ?? (() => {});
+    const crawledAt = new Date().toISOString();
+    const experts = [];
+    const tags = new Set;
+    let page = 1;
+    for (;; ) {
+      const data = await fetchJson(`${this.apiBase}/mentors/?page=${page}&page_size=${this.pageSize}`, this.fetchFn);
+      const items = data?.results ?? data?.data ?? (Array.isArray(data) ? data : []);
+      if (!items.length)
+        break;
+      for (const m of items) {
+        const e = normalizeAdpMentor(m, crawledAt);
+        experts.push(e);
+        for (const t of e.tags)
+          tags.add(t);
+      }
+      log(`  adplist: ${experts.length}`);
+      page++;
+      if (opts.max && experts.length >= opts.max)
+        break;
+      if (items.length < this.pageSize)
+        break;
+    }
+    if (experts.length === 0)
+      log("adplist: no public listing reachable (set ADPLIST_API_BASE or provide a fetchFn).");
+    return {
+      experts: opts.max ? experts.slice(0, opts.max) : experts,
+      topics: [],
+      tags: [...tags].map((name) => ({ name, topic: "" })),
+      total: experts.length
+    };
+  }
+}
+
+// src/sources/clarity.ts
+function normalizeClarityExpert(c, crawledAt) {
+  const slug = c.username || slugify(c.name || String(c.id ?? ""));
+  return makeExpert({
+    source: "clarity",
+    sourceId: String(c.id ?? slug),
+    slug,
+    url: `https://clarity.fm/${slug}`,
+    fullName: c.name ?? "",
+    title: c.title ?? "",
+    bio: c.bio ?? "",
+    avatar: c.image ?? "",
+    price: c.rate_per_minute ?? 0,
+    priceCurrency: "USD",
+    priceUnit: c.rate_per_minute ? "per minute" : "",
+    rating: c.rating ?? 0,
+    ratingCount: c.reviews ?? 0,
+    topics: c.categories ?? [],
+    tags: c.expertise ?? [],
+    crawledAt
+  });
+}
+
+class ClaritySource {
+  name = "clarity";
+  description = "Clarity.fm \u2014 on-demand expert calls billed per minute";
+  website = "https://clarity.fm";
+  fetchFn;
+  apiBase;
+  constructor(opts = {}) {
+    this.fetchFn = opts.fetchFn ?? fetch;
+    this.apiBase = opts.apiBase ?? process.env.CLARITY_API_BASE;
+  }
+  async crawl(opts = {}) {
+    const log = opts.onLog ?? (() => {});
+    if (!this.apiBase) {
+      log("clarity: no public listing API; set CLARITY_API_BASE or inject a fetchFn to crawl.");
+      return { experts: [], topics: [], tags: [], total: 0 };
+    }
+    const crawledAt = new Date().toISOString();
+    const data = await fetchJson(`${this.apiBase}/experts`, this.fetchFn);
+    const items = data?.results ?? data?.data ?? (Array.isArray(data) ? data : []);
+    const experts = items.map((c) => normalizeClarityExpert(c, crawledAt));
+    const tags = new Set;
+    for (const e of experts)
+      for (const t of e.tags)
+        tags.add(t);
+    return {
+      experts: opts.max ? experts.slice(0, opts.max) : experts,
+      topics: [],
+      tags: [...tags].map((name) => ({ name, topic: "" })),
+      total: experts.length
+    };
+  }
+}
+
+// src/sources/glg.ts
+function normalizeGlgExpert(g, crawledAt) {
+  const slug = slugify(g.name || String(g.id ?? ""));
+  return makeExpert({
+    source: "glg",
+    sourceId: String(g.id ?? slug),
+    slug,
+    url: "https://glginsights.com",
+    fullName: g.name ?? "",
+    title: g.title ?? "",
+    bio: g.biography ?? "",
+    price: g.hourly_rate ?? 0,
+    priceCurrency: g.currency ?? "USD",
+    priceUnit: g.hourly_rate ? "per hour" : "",
+    topics: g.industries ?? [],
+    tags: g.expertise_areas ?? [],
+    crawledAt
+  });
+}
+
+class GLGSource {
+  name = "glg";
+  description = "GLG \u2014 enterprise expert network (requires partner API access)";
+  website = "https://glginsights.com";
+  fetchFn;
+  apiBase;
+  apiKey;
+  constructor(opts = {}) {
+    this.fetchFn = opts.fetchFn ?? fetch;
+    this.apiBase = opts.apiBase ?? process.env.GLG_API_BASE;
+    this.apiKey = opts.apiKey ?? process.env.GLG_API_KEY;
+  }
+  async crawl(opts = {}) {
+    const log = opts.onLog ?? (() => {});
+    if (!this.apiBase) {
+      log("glg: enterprise-gated; no public directory. Set GLG_API_BASE + GLG_API_KEY (partner access) to crawl.");
+      return { experts: [], topics: [], tags: [], total: 0 };
+    }
+    const crawledAt = new Date().toISOString();
+    const data = await fetchJson(`${this.apiBase}/experts`, this.fetchFn, {
+      headers: this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}
+    });
+    const items = data?.results ?? data?.data ?? (Array.isArray(data) ? data : []);
+    const experts = items.map((g) => normalizeGlgExpert(g, crawledAt));
+    const tags = new Set;
+    for (const e of experts)
+      for (const t of e.tags)
+        tags.add(t);
+    return {
+      experts: opts.max ? experts.slice(0, opts.max) : experts,
+      topics: [],
+      tags: [...tags].map((name) => ({ name, topic: "" })),
+      total: experts.length
+    };
+  }
+}
+
 // src/sources/index.ts
 var registry = new Map;
 function registerSource(source) {
@@ -800,6 +1680,10 @@ function listSources() {
   return [...registry.values()];
 }
 registerSource(new IntroSource);
+registerSource(new MentorCruiseSource);
+registerSource(new ADPListSource);
+registerSource(new ClaritySource);
+registerSource(new GLGSource);
 
 // src/crawl.ts
 async function crawlSource(db, sourceName, opts = {}) {
@@ -808,6 +1692,7 @@ async function crawlSource(db, sourceName, opts = {}) {
     throw new Error(`Unknown source "${sourceName}". Run \`experts sources\` to list options.`);
   }
   const data = await source.crawl(opts);
+  const changes = db.recordChanges(source.name, data.experts);
   db.upsertExperts(data.experts);
   if (data.topics.length)
     db.setTopics(source.name, data.topics);
@@ -816,13 +1701,16 @@ async function crawlSource(db, sourceName, opts = {}) {
   db.setMeta(`catalog_total:${source.name}`, String(data.total));
   opts.onLog?.("building knowledge graph\u2026");
   const graph = db.rebuildGraph();
+  db.rescore(source.name);
+  db.rebuildPersons();
   return {
     source: source.name,
     experts: data.experts.length,
     topics: data.topics.map((t) => ({ name: t.name, count: t.expertCount })),
     tags: new Set(data.tags.map((t) => t.name)).size,
     total: data.total,
-    graph
+    graph,
+    changes
   };
 }
 
@@ -880,6 +1768,11 @@ function handle(db, req) {
     return json(db.enrichmentStats(q.get("source") || undefined));
   if (path === "/graph")
     return json(db.graphStats());
+  if (path === "/persons")
+    return json(db.personStats());
+  if (path === "/changes") {
+    return json(db.changes({ source: q.get("source") || undefined, limit: num(q.get("limit")) }));
+  }
   if (path === "/find") {
     const needs = (q.get("needs") || "").split(",").map((s) => s.trim()).filter(Boolean);
     return json(db.findByNeeds(needs, {
@@ -922,6 +1815,8 @@ function handle(db, req) {
     };
     return json(db.list(filters));
   }
+  if (path === "/contacts")
+    return json(db.contactStats(q.get("source") || undefined));
   const tw = path.match(/^\/experts\/([^/]+)\/(.+)\/tweets$/);
   if (tw) {
     const e = db.get(decodeURIComponent(tw[2]), decodeURIComponent(tw[1]));
@@ -929,6 +1824,20 @@ function handle(db, req) {
       return json({ error: "not found" }, 404);
     return json(db.recentTweets(e.source, e.sourceId, num(q.get("limit")) ?? 25));
   }
+  const ct = path.match(/^\/experts\/([^/]+)\/(.+)\/contacts$/);
+  if (ct) {
+    const e = db.get(decodeURIComponent(ct[2]), decodeURIComponent(ct[1]));
+    if (!e)
+      return json({ error: "not found" }, 404);
+    return json(db.contacts(e.source, e.sourceId));
+  }
+  const vd = path.match(/^\/experts\/([^/]+)\/(.+)\/videos$/);
+  if (vd) {
+    const e = db.get(decodeURIComponent(vd[2]), decodeURIComponent(vd[1]));
+    if (!e)
+      return json({ error: "not found" }, 404);
+    return json(db.recentVideos(e.source, e.sourceId, num(q.get("limit")) ?? 25));
+  }
   const m = path.match(/^\/experts\/([^/]+)\/(.+)$/);
   if (m) {
     const e = db.get(decodeURIComponent(m[2]), decodeURIComponent(m[1]));
@@ -937,13 +1846,50 @@ function handle(db, req) {
     return json({
       ...e,
       xProfile: db.getXProfile(e.source, e.sourceId),
-      tweets: db.recentTweets(e.source, e.sourceId, 10)
+      tweets: db.recentTweets(e.source, e.sourceId, 10),
+      contacts: db.contacts(e.source, e.sourceId),
+      videos: db.recentVideos(e.source, e.sourceId, 10)
     });
   }
   return json({ error: "not found", path }, 404);
 }
 async function handleAsync(db, req) {
   const url = new URL(req.url);
+  if (url.pathname.replace(/\/+$/, "") === "/ask") {
+    const q = url.searchParams.get("q") || "";
+    if (!q)
+      return json({ error: "missing q" }, 400);
+    if (db.vectorCount() === 0)
+      return json({ error: "no semantic index; run `experts embed`" }, 409);
+    const [qv] = await getEmbedder().embed([q]);
+    return json(db.semanticSearch(qv, {
+      source: url.searchParams.get("source") || undefined,
+      limit: num(url.searchParams.get("limit"))
+    }));
+  }
+  if (url.pathname.replace(/\/+$/, "") === "/brief") {
+    const q = url.searchParams.get("q") || "";
+    if (!q)
+      return json({ error: "missing q" }, 400);
+    if (db.vectorCount() === 0)
+      return json({ error: "no semantic index; run `experts embed`" }, 409);
+    const limit = num(url.searchParams.get("limit")) ?? 10;
+    const [qv] = await getEmbedder().embed([q]);
+    const raw = db.semanticSearch(qv, { source: url.searchParams.get("source") || undefined, limit: (limit + 5) * 4 });
+    const seen = new Set;
+    const briefLc = q.toLowerCase();
+    const out = [];
+    for (const r of raw) {
+      const pid = db.personIdOf(r.expert.source, r.expert.sourceId);
+      if (seen.has(pid))
+        continue;
+      seen.add(pid);
+      out.push({ ...r, why: r.expert.tags.filter((t) => briefLc.includes(t.toLowerCase())).slice(0, 4) });
+      if (out.length >= limit)
+        break;
+    }
+    return json(out);
+  }
   if (req.method === "POST") {
     const m = url.pathname.match(/^\/crawl\/([^/]+)\/?$/);
     if (m) {