@odla-ai/kg 0.5.0 → 0.6.0

package/dist/index.js

@@ -650,6 +650,59 @@ function httpsUrl(value) {
   }
 }
 
+// src/enrich/grade.ts
+var DEFAULT_STANDARD = {
+  minimumSources: 2,
+  minimumFacts: 3,
+  anchors: ["first-party", "authoritative"],
+  anchorRequiredForMedium: false
+};
+function independentHosts(sources) {
+  const hosts = /* @__PURE__ */ new Set();
+  for (const source of sources) {
+    try {
+      hosts.add(new URL(source.sourceUrl).hostname.toLowerCase().replace(/^www\./, ""));
+    } catch {
+    }
+  }
+  return [...hosts];
+}
+function gradeSources(sources, standard = DEFAULT_STANDARD) {
+  const supporting = sources.filter((s) => s.corroboration === "confirmed" || s.corroboration === "name-only");
+  const contradicting = sources.filter((s) => s.corroboration === "absent");
+  const unreadable = sources.filter((s) => s.corroboration === "unreachable" || s.corroboration === "unchecked");
+  const facts = [...new Set(supporting.flatMap((s) => s.facts))];
+  const hosts = independentHosts(supporting);
+  const hasAnchor = supporting.some((s) => standard.anchors.includes(s.sourceClass));
+  const base = {
+    independentSources: hosts.length,
+    factsCorroborated: facts,
+    hasAnchor,
+    contradictions: contradicting.map((s) => s.sourceUrl)
+  };
+  if (supporting.length === 0) {
+    return unreadable.length > 0 && contradicting.length === 0 ? { ...base, grade: "unverifiable", reason: `no source could be read (${unreadable.length} unreachable)` } : { ...base, grade: "unresolved", reason: "no source corroborated the subject" };
+  }
+  if (base.contradictions.length > 0) {
+    return { ...base, grade: "unresolved", reason: `${base.contradictions.length} source(s) named someone else` };
+  }
+  if (hosts.length >= standard.minimumSources && hasAnchor && facts.length >= standard.minimumFacts) {
+    return { ...base, grade: "high", reason: `${hosts.length} independent sources, anchored, agreeing on ${facts.length} facts` };
+  }
+  if (hosts.length >= standard.minimumSources && (hasAnchor || !standard.anchorRequiredForMedium)) {
+    return {
+      ...base,
+      grade: "medium",
+      reason: hasAnchor ? `${hosts.length} sources but only ${facts.length} corroborated facts` : `${hosts.length} sources, none first-party or authoritative`
+    };
+  }
+  return { ...base, grade: "unresolved", reason: hosts.length < standard.minimumSources ? "only one independent source" : "no anchoring source" };
+}
+var ACTIONABLE = ["high", "medium"];
+function isActionable(value) {
+  return ACTIONABLE.includes(value);
+}
+
 // src/enrich/run.ts
 async function enrichCompany(ai, models, company) {
   const where = `${company.city ? `, ${company.city}` : ""}${company.region ? ` (${company.region})` : ""}`;
@@ -711,6 +764,409 @@ async function resolveCounties(ai, model, rows, gazetteer) {
   return out;
 }
 
+// src/enrich/match.ts
+function fold(value) {
+  return value.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
+}
+function mentionsName(haystack, name) {
+  const tokens = fold(name).split(" ").filter((t) => t.length > 2);
+  if (tokens.length < 2) return false;
+  const words = haystack.split(" ");
+  const positions = /* @__PURE__ */ new Map();
+  for (const [i, word] of words.entries()) {
+    if (tokens.includes(word)) positions.set(word, [...positions.get(word) ?? [], i]);
+  }
+  if (tokens.some((t) => !positions.has(t))) return false;
+  const span = tokens.length;
+  const anchors = positions.get(tokens[0]) ?? [];
+  return anchors.some(
+    (anchor) => tokens.every((token) => (positions.get(token) ?? []).some((at) => Math.abs(at - anchor) <= span))
+  );
+}
+function snippetAround(text, name, width = 240) {
+  const hay = fold(text);
+  const first = fold(name).split(" ")[0] ?? "";
+  const at = hay.indexOf(first);
+  if (at < 0) return "";
+  const start = Math.max(0, at - width / 2);
+  return text.slice(start, start + width).replace(/\s+/g, " ").trim();
+}
+function stripMarkup(html) {
+  return html.replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ").replace(/&nbsp;/gi, " ").replace(/&amp;/gi, "&").replace(/\s+/g, " ").slice(0, 2e5);
+}
+
+// src/enrich/strategy.ts
+async function investigatePerson(context, strategies, standard = DEFAULT_STANDARD) {
+  const skipped = [];
+  const runnable = [];
+  for (const strategy of strategies) {
+    if (strategy.applicable(context)) runnable.push(strategy);
+    else skipped.push(strategy.name);
+  }
+  const findings = await Promise.all(
+    runnable.map(async (strategy) => {
+      try {
+        return await strategy.run(context);
+      } catch (err) {
+        return {
+          strategy: strategy.name,
+          sources: [],
+          links: [],
+          notes: "",
+          error: err instanceof Error ? err.message : String(err)
+        };
+      }
+    })
+  );
+  const sources = dedupeBy(findings.flatMap((f) => f.sources), (s) => s.sourceUrl);
+  const links = dedupeBy(findings.flatMap((f) => f.links), (l) => l.url);
+  const report = gradeSources(sources, standard);
+  return { person: context.person, grade: report.grade, report, sources, links, findings, skipped };
+}
+function dedupeBy(items, key) {
+  const seen = /* @__PURE__ */ new Map();
+  for (const item of items) {
+    const at = key(item);
+    if (!seen.has(at)) seen.set(at, item);
+  }
+  return [...seen.values()];
+}
+
+// src/enrich/strategies/org-site.ts
+var TEAM_PATHS = [
+  "/team",
+  "/about",
+  "/about-us",
+  "/our-team",
+  "/leadership",
+  "/people",
+  "/staff",
+  "/who-we-are",
+  "/company/team",
+  "/press",
+  "/news"
+];
+var PEOPLE_WORDS = [
+  "team",
+  "about",
+  "leadership",
+  "staff",
+  "people",
+  "who we are",
+  "our story",
+  "founders",
+  "management",
+  "board",
+  "press",
+  "news",
+  "advisors",
+  "bios"
+];
+function rootOf(website) {
+  try {
+    return new URL(website.startsWith("http") ? website : `https://${website}`).origin;
+  } catch {
+    return null;
+  }
+}
+function candidatePages(website, limit = 4) {
+  const origin = rootOf(website);
+  if (!origin) return [];
+  return TEAM_PATHS.map((p) => `${origin}${p}`).slice(0, limit);
+}
+function peopleLinks(html, origin, limit = 5) {
+  const found = /* @__PURE__ */ new Map();
+  const anchor = /<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]{0,200}?)<\/a>/gi;
+  for (const match of html.matchAll(anchor)) {
+    let url;
+    try {
+      url = new URL(match[1] ?? "", origin);
+    } catch {
+      continue;
+    }
+    if (url.origin !== origin) continue;
+    const text = (match[2] ?? "").replace(/<[^>]+>/g, " ").toLowerCase();
+    const rank = PEOPLE_WORDS.findIndex((w) => `${text} ${url.pathname.toLowerCase()}`.includes(w));
+    if (rank < 0) continue;
+    const clean = `${url.origin}${url.pathname}`.replace(/\/$/, "");
+    if (clean === origin) continue;
+    const prior = found.get(clean);
+    if (prior === void 0 || rank < prior) found.set(clean, rank);
+  }
+  return [...found.entries()].sort((a, b) => a[1] - b[1]).slice(0, limit).map(([url]) => url);
+}
+function judgeOrgPage(name, url, text) {
+  const named = mentionsName(fold(text), name);
+  return { url, named, snippet: named ? snippetAround(text, name) : "" };
+}
+var orgSiteStrategy = {
+  name: "org-site",
+  applicable(context) {
+    return Boolean(context.readPage && rootOf(context.person.companyWebsite ?? ""));
+  },
+  async run(context) {
+    const { person, readPage, retrievedAt } = context;
+    const origin = rootOf(person.companyWebsite ?? "");
+    if (!origin || !readPage) {
+      return { strategy: "org-site", sources: [], links: [], notes: "no employer website to read" };
+    }
+    const hits = [];
+    let read = 0;
+    const visit = async (url) => {
+      const html = await readPage(url);
+      if (html === null) return null;
+      read += 1;
+      const hit = judgeOrgPage(person.name, url, stripMarkup(html));
+      if (hit.named) hits.push(hit);
+      return html;
+    };
+    const home = await visit(origin);
+    const followable = home && hits.length === 0 ? peopleLinks(home, origin) : [];
+    const fallback = read <= 1 ? candidatePages(person.companyWebsite ?? "") : [];
+    for (const url of [...followable, ...fallback]) {
+      if (hits.length > 0) break;
+      await visit(url);
+    }
+    const sources = hits.map((hit) => ({
+      sourceUrl: hit.url,
+      sourceClass: "first-party",
+      facts: ["employer", "affiliation"],
+      verifiedOn: retrievedAt,
+      corroboration: "confirmed"
+    }));
+    return {
+      strategy: "org-site",
+      sources,
+      links: [],
+      notes: hits[0] ? `named on ${hits[0].url}
+
+${hits[0].snippet}` : read > 0 ? `read ${read} page(s) on the employer's site; none named them` : "no page on the employer's site could be read"
+    };
+  }
+};
+
+// src/enrich/strategies/research.ts
+var SELF_ASSERTED_HOSTS = ["linkedin.com", "x.com", "twitter.com", "bsky.app", "github.com", "substack.com"];
+function hostOf(url) {
+  try {
+    return new URL(url).hostname.toLowerCase().replace(/^www\./, "");
+  } catch {
+    return "";
+  }
+}
+function classifySource(url, companyWebsite) {
+  const host = hostOf(url);
+  if (!host) return "third-party";
+  const company = companyWebsite ? hostOf(companyWebsite.startsWith("http") ? companyWebsite : `https://${companyWebsite}`) : "";
+  if (company && (host === company || host.endsWith(`.${company}`))) return "first-party";
+  if (SELF_ASSERTED_HOSTS.some((h) => host === h || host.endsWith(`.${h}`))) return "self-asserted";
+  return "third-party";
+}
+function isAttributed(citation) {
+  return Boolean(citation.citedText || citation.startIndex !== void 0 || citation.endIndex !== void 0);
+}
+var researchStrategy = {
+  name: "research",
+  applicable(context) {
+    return Boolean(context.ai && context.models?.search && context.models?.extract);
+  },
+  async run(context) {
+    const { ai, models, person, retrievedAt } = context;
+    if (!ai || !models) return { strategy: "research", sources: [], links: [], notes: "no models available" };
+    const result = await enrichPerson(ai, models, person, retrievedAt);
+    const sources = result.citations.map((citation) => {
+      const corroboration = isAttributed(citation) ? "name-only" : "unchecked";
+      return {
+        sourceUrl: citation.url,
+        sourceClass: classifySource(citation.url, person.companyWebsite),
+        facts: ["identity"],
+        verifiedOn: retrievedAt,
+        corroboration
+      };
+    });
+    return {
+      strategy: "research",
+      sources,
+      links: result.links,
+      notes: `[${result.verification.verdict}] ${result.notes}`
+    };
+  }
+};
+
+// src/enrich/strategies/targeted-profile.ts
+var PROFILE_HOSTS = {
+  "linkedin.com": /\/in\/[^/?#]+/i,
+  "github.com": /^\/[^/?#]+\/?$/i,
+  "bsky.app": /\/profile\/[^/?#]+/i,
+  "x.com": /^\/[^/?#]+\/?$/i,
+  "twitter.com": /^\/[^/?#]+\/?$/i
+};
+function kindOf(host) {
+  if (host.startsWith("linkedin")) return "linkedin";
+  if (host.startsWith("github")) return "github";
+  if (host.startsWith("bsky")) return "bluesky";
+  if (host.startsWith("x.") || host.startsWith("twitter")) return "x";
+  return "other";
+}
+function targetedQuery(person, platform = "linkedin") {
+  const company = person.companyName ? ` "${person.companyName}"` : "";
+  return `"${person.name}"${company} ${platform}`;
+}
+function candidatesFrom(citations) {
+  const seen = /* @__PURE__ */ new Map();
+  for (const citation of citations) {
+    let url;
+    try {
+      url = new URL(citation.url);
+    } catch {
+      continue;
+    }
+    const host = url.hostname.toLowerCase().replace(/^www\./, "").replace(/^[a-z]{2}\./, "");
+    const match = PROFILE_HOSTS[host]?.exec(url.pathname);
+    if (!match) continue;
+    const handle = match[0].replace(/\/+$/, "").toLowerCase();
+    if (!seen.has(`${host}${handle}`)) seen.set(`${host}${handle}`, { url: citation.url, host, handle });
+  }
+  return [...seen.values()];
+}
+var targetedProfileStrategy = {
+  name: "targeted-profile",
+  applicable(context) {
+    return Boolean(context.ai && context.models?.search);
+  },
+  async run(context) {
+    const { ai, models, person, retrievedAt } = context;
+    if (!ai || !models) return { strategy: "targeted-profile", sources: [], links: [], notes: "no search model" };
+    const query = targetedQuery(person);
+    const { citations } = await ai.search({
+      model: models.search,
+      system: "Search for the exact query given and report what you find. Do not guess a profile URL that did not appear in the results.",
+      user: query,
+      maxTokens: 600
+    });
+    const candidates = candidatesFrom(citations);
+    if (candidates.length !== 1) {
+      return {
+        strategy: "targeted-profile",
+        sources: [],
+        links: [],
+        notes: candidates.length === 0 ? `${query}
+
+no profile matched` : `${query}
+
+${candidates.length} distinct profiles matched \u2014 the name is ambiguous`
+      };
+    }
+    const only = candidates[0];
+    const link = { kind: kindOf(only.host), url: only.url };
+    const source = {
+      sourceUrl: only.url,
+      sourceClass: "self-asserted",
+      facts: ["profile"],
+      verifiedOn: retrievedAt,
+      corroboration: "confirmed"
+    };
+    return {
+      strategy: "targeted-profile",
+      sources: [source],
+      links: [link],
+      notes: `${query}
+
+exactly one distinct profile matched the name and company`
+    };
+  }
+};
+
+// src/identity/person.ts
+var DURABLE_KINDS = ["linkedin", "github", "bluesky", "x"];
+function settled(config) {
+  return {
+    aliasType: config.aliasType,
+    aliasKeyAttr: config.aliasKeyAttr ?? "alias",
+    personKeyAttr: config.personKeyAttr ?? "personKey",
+    personType: config.personType ?? "person",
+    edgeLabel: config.edgeLabel ?? "of_person"
+  };
+}
+function mintPersonKey(random = () => crypto.randomUUID()) {
+  return `p_${random()}`;
+}
+function isMintedKey(key) {
+  return key.startsWith("p_");
+}
+function profileAlias(kind, url) {
+  try {
+    const parsed = new URL(url);
+    const handle = parsed.pathname.replace(/\/+$/, "").toLowerCase();
+    return handle && handle !== "/" ? `${kind}:${handle}` : null;
+  } catch {
+    return null;
+  }
+}
+function nameAlias(name, companyName) {
+  const person = identityKey(name);
+  const company = companyName ? identityKey(companyName) : "";
+  return person && company ? `name:${person}~${company}` : null;
+}
+function aliasesFor(person, links = []) {
+  const out = [];
+  for (const kind of DURABLE_KINDS) {
+    for (const link of links.filter((l) => l.kind === kind)) {
+      const alias = profileAlias(kind, link.url);
+      if (alias && !out.includes(alias)) out.push(alias);
+    }
+  }
+  const byName = nameAlias(person.name, person.companyName);
+  if (byName) out.push(byName);
+  return out;
+}
+async function resolvePersonKey(db, config, aliases) {
+  const { aliasType, aliasKeyAttr, personKeyAttr } = settled(config);
+  if (aliases.length === 0) return null;
+  const result = await db.query({
+    [aliasType]: { $: { where: { [aliasKeyAttr]: { $in: [...aliases] } } } }
+  });
+  const found = /* @__PURE__ */ new Map();
+  for (const row of result[aliasType] ?? []) {
+    const alias = row[aliasKeyAttr];
+    const personKey = row[personKeyAttr];
+    if (typeof alias === "string" && typeof personKey === "string" && personKey) found.set(alias, personKey);
+  }
+  for (const alias of aliases) {
+    const hit = found.get(alias);
+    if (hit) return hit;
+  }
+  return null;
+}
+function applyPersonIdentity(fragment, from, to, aliases, config) {
+  const { aliasType, aliasKeyAttr, personKeyAttr, personType, edgeLabel } = settled(config);
+  const rekey = (type, key) => type === personType && key === from ? to : key;
+  const nodes = fragment.nodes.map((n) => ({ ...n, key: rekey(n.type, n.key) }));
+  const edges = fragment.edges.map((e) => ({
+    from: { type: e.from.type, key: rekey(e.from.type, e.from.key) },
+    to: { type: e.to.type, key: rekey(e.to.type, e.to.key) },
+    label: e.label
+  }));
+  for (const alias of [...new Set(aliases)]) {
+    nodes.push({ type: aliasType, key: alias, attrs: { [personKeyAttr]: to } });
+    edges.push({ from: { type: aliasType, key: alias }, to: { type: personType, key: to }, label: edgeLabel });
+  }
+  return { nodes, edges };
+}
+
+// src/identity/identify.ts
+async function identifyPerson(db, config, person, links, fragment) {
+  const aliases = aliasesFor(person, links);
+  const existing = await resolvePersonKey(db, config, aliases);
+  const personKey = existing ?? mintPersonKey();
+  return {
+    personKey,
+    minted: existing === null,
+    aliases,
+    fragment: applyPersonIdentity(fragment, person.slug, personKey, aliases, config)
+  };
+}
+
 // src/llm/provider.ts
 import { init, extractText } from "@odla-ai/ai";
 var MODEL_TIERS = {
@@ -1390,7 +1846,9 @@ async function ingestFresh(step, deps, fresh, kind, resolve = false) {
   );
 }
 export {
+  ACTIONABLE,
   COMMIT_STEP_CONFIG,
+  DEFAULT_STANDARD,
   FACTS_EXTRACT_SYSTEM,
   FACTS_EXTRACT_TOOL,
   FACTS_SEARCH_SYSTEM,
@@ -1403,11 +1861,17 @@ export {
   PERSON_EXTRACT_TOOL,
   PERSON_SEARCH_SYSTEM,
   Store,
+  TEAM_PATHS,
   WRITABLE_VERDICTS,
   addUsage,
+  aliasesFor,
   allJobs,
+  applyPersonIdentity,
   asStepLike,
+  candidatePages,
+  candidatesFrom,
   canonicalizeFragment,
+  classifySource,
   connectorIds,
   countyKey,
   createProvider,
@@ -1422,34 +1886,56 @@ export {
   fetchText,
   fetchUrl,
   findConnectors,
+  fold,
   fragmentToOps,
   getConnector,
   getProvider,
+  gradeSources,
   htmlToText,
   humanAttrsOf,
+  identifyPerson,
   identityKey,
+  independentHosts,
   ingestFresh,
+  investigatePerson,
+  isActionable,
+  isAttributed,
   isHttpUrl,
   isLikelyPersonName,
+  isMintedKey,
   isVerdict,
   isWritable,
   jobsForHour,
+  judgeOrgPage,
   locatePrompt,
   locationFragment,
   makeContext,
+  mentionsName,
   mergeAttrs,
   mergeRulesOf,
   mergeScalar,
+  mintPersonKey,
+  nameAlias,
   naturalKeysOf,
   normalizeUrl,
   normalizeVerification,
+  orgSiteStrategy,
+  peopleLinks,
   personPrompt,
   personToFragment,
+  profileAlias,
+  researchStrategy,
   resilientStep,
   resolveCounties,
   resolveName,
+  resolvePersonKey,
+  rootOf,
   searchEntities,
   slugify,
+  snippetAround,
+  stripMarkup,
+  targetedProfileStrategy,
+  targetedQuery,
   toSerializedSchema,
   urlHash,
   validateLinks,