@odla-ai/kg 0.3.2 → 0.4.1

package/dist/index.cjs

@@ -21,8 +21,13 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var index_exports = {};
 __export(index_exports, {
   COMMIT_STEP_CONFIG: () => COMMIT_STEP_CONFIG,
+  FACTS_EXTRACT_SYSTEM: () => FACTS_EXTRACT_SYSTEM,
+  FACTS_EXTRACT_TOOL: () => FACTS_EXTRACT_TOOL,
+  FACTS_SEARCH_SYSTEM: () => FACTS_SEARCH_SYSTEM,
   IO_STEP_CONFIG: () => IO_STEP_CONFIG,
   LLM_STEP_CONFIG: () => LLM_STEP_CONFIG,
+  LOCATE_SYSTEM: () => LOCATE_SYSTEM,
+  LOCATE_TOOL: () => LOCATE_TOOL,
   MODEL_TIERS: () => MODEL_TIERS,
   Store: () => Store,
   addUsage: () => addUsage,
@@ -30,14 +35,18 @@ __export(index_exports, {
   asStepLike: () => asStepLike,
   canonicalizeFragment: () => canonicalizeFragment,
   connectorIds: () => connectorIds,
+  countyKey: () => countyKey,
   createProvider: () => createProvider,
   emptyUsage: () => emptyUsage,
   enabledSources: () => enabledSources,
+  enrichCompany: () => enrichCompany,
   exportFragment: () => exportFragment,
   extractJson: () => extractJson,
   extractJsonArray: () => extractJsonArray,
+  factsToFragment: () => factsToFragment,
   fetchText: () => fetchText,
   fetchUrl: () => fetchUrl,
+  findConnectors: () => findConnectors,
   fragmentToOps: () => fragmentToOps,
   getConnector: () => getConnector,
   getProvider: () => getProvider,
@@ -46,6 +55,8 @@ __export(index_exports, {
   ingestFresh: () => ingestFresh,
   isLikelyPersonName: () => isLikelyPersonName,
   jobsForHour: () => jobsForHour,
+  locatePrompt: () => locatePrompt,
+  locationFragment: () => locationFragment,
   makeContext: () => makeContext,
   mergeAttrs: () => mergeAttrs,
   mergeRulesOf: () => mergeRulesOf,
@@ -53,6 +64,7 @@ __export(index_exports, {
   naturalKeysOf: () => naturalKeysOf,
   normalizeUrl: () => normalizeUrl,
   resilientStep: () => resilientStep,
+  resolveCounties: () => resolveCounties,
   resolveName: () => resolveName,
   searchEntities: () => searchEntities,
   slugify: () => slugify,
@@ -366,6 +378,404 @@ async function canonicalizeFragment(db, fragment, ontology) {
   };
 }
 
+// src/temporal/prng.ts
+function mulberry32(seed) {
+  let a = seed >>> 0;
+  return () => {
+    a = a + 1831565813 >>> 0;
+    let t = a;
+    t = Math.imul(t ^ t >>> 15, t | 1);
+    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+function shuffle(items, rand) {
+  const out = items.slice();
+  for (let i = out.length - 1; i > 0; i--) {
+    const j = Math.floor(rand() * (i + 1));
+    const tmp = out[i];
+    out[i] = out[j];
+    out[j] = tmp;
+  }
+  return out;
+}
+
+// src/temporal/communities.ts
+function labelPropagation(nodes, edges, opts = {}) {
+  const { seed = 1, maxRounds = 20 } = opts;
+  const sorted = [...nodes].sort();
+  const labels = new Map(sorted.map((id) => [id, id]));
+  const neighbors = /* @__PURE__ */ new Map();
+  for (const e of edges) {
+    if (!labels.has(e.a) || !labels.has(e.b) || e.a === e.b) continue;
+    const w = e.weight ?? 1;
+    let listA = neighbors.get(e.a);
+    if (!listA) neighbors.set(e.a, listA = []);
+    listA.push({ id: e.b, weight: w });
+    let listB = neighbors.get(e.b);
+    if (!listB) neighbors.set(e.b, listB = []);
+    listB.push({ id: e.a, weight: w });
+  }
+  const rand = mulberry32(seed);
+  for (let round = 0; round < maxRounds; round++) {
+    const order = shuffle(sorted, rand);
+    let changed = false;
+    for (const id of order) {
+      const incident = neighbors.get(id);
+      if (!incident || incident.length === 0) continue;
+      const votes = /* @__PURE__ */ new Map();
+      for (const n of incident) {
+        const label = labels.get(n.id);
+        votes.set(label, (votes.get(label) ?? 0) + n.weight);
+      }
+      let best = [];
+      let bestWeight = -Infinity;
+      for (const [label, weight] of votes) {
+        if (weight > bestWeight) {
+          bestWeight = weight;
+          best = [label];
+        } else if (weight === bestWeight) {
+          best.push(label);
+        }
+      }
+      const current = labels.get(id);
+      if (best.includes(current)) continue;
+      best.sort();
+      const pick = best.length === 1 ? best[0] : best[Math.floor(rand() * best.length)];
+      labels.set(id, pick);
+      changed = true;
+    }
+    if (!changed) break;
+  }
+  const members = /* @__PURE__ */ new Map();
+  for (const [id, label] of labels) {
+    const list = members.get(label);
+    list ? list.push(id) : members.set(label, [id]);
+  }
+  const canonical = /* @__PURE__ */ new Map();
+  for (const [label, ids] of members) canonical.set(label, ids[0]);
+  return new Map(sorted.map((id) => [id, canonical.get(labels.get(id))]));
+}
+
+// src/temporal/lenses.ts
+function reifiedBridgeTargets(fragment, opts) {
+  const bridgesOf = /* @__PURE__ */ new Map();
+  const targetsOf = /* @__PURE__ */ new Map();
+  for (const e of fragment.edges) {
+    if (e.from.type !== opts.reified) continue;
+    const to = `${e.to.type}:${e.to.key}`;
+    if (e.label === opts.bridgeLabel) (bridgesOf.get(e.from.key) ?? bridgesOf.set(e.from.key, []).get(e.from.key)).push(to);
+    if (e.label === opts.targetLabel) (targetsOf.get(e.from.key) ?? targetsOf.set(e.from.key, []).get(e.from.key)).push(to);
+  }
+  const out = /* @__PURE__ */ new Map();
+  for (const [reifiedKey, bridges] of bridgesOf) {
+    for (const bridge of bridges) {
+      for (const target of targetsOf.get(reifiedKey) ?? []) {
+        (out.get(bridge) ?? out.set(bridge, /* @__PURE__ */ new Set()).get(bridge)).add(target);
+      }
+    }
+  }
+  return out;
+}
+function projectBridges(bridges, opts = {}) {
+  const { maxPerBridge = 12, viaCap = 4 } = opts;
+  const acc = /* @__PURE__ */ new Map();
+  for (const [bridge, targetSet] of [...bridges.entries()].sort((x, y) => x[0] < y[0] ? -1 : 1)) {
+    const targets = [...targetSet].sort();
+    if (targets.length < 2 || targets.length > maxPerBridge) continue;
+    for (let i = 0; i < targets.length; i++) {
+      for (let j = i + 1; j < targets.length; j++) {
+        const key = `${targets[i]}|${targets[j]}`;
+        const hit = acc.get(key) ?? { a: targets[i], b: targets[j], w: 0, via: [] };
+        hit.w += 1;
+        if (hit.via.length < viaCap) hit.via.push(bridge);
+        acc.set(key, hit);
+      }
+    }
+  }
+  return [...acc.keys()].sort().map((k) => acc.get(k));
+}
+function blendPairs(sets) {
+  const acc = /* @__PURE__ */ new Map();
+  for (const { pairs, weight } of sets) {
+    if (weight <= 0 || pairs.length === 0) continue;
+    const max = Math.max(...pairs.map((p) => p.w));
+    for (const p of pairs) {
+      const key = `${p.a}|${p.b}`;
+      const hit = acc.get(key) ?? { a: p.a, b: p.b, w: 0, via: [] };
+      hit.w += p.w / max * weight;
+      for (const v of p.via) if (hit.via.length < 6 && !hit.via.includes(v)) hit.via.push(v);
+      acc.set(key, hit);
+    }
+  }
+  return [...acc.keys()].sort().map((key) => acc.get(key));
+}
+
+// src/temporal/bridges.ts
+function bridgingScores(edges, labels) {
+  const cross = /* @__PURE__ */ new Map();
+  const foreign = /* @__PURE__ */ new Map();
+  const touch = (id, otherLabel, w) => {
+    cross.set(id, (cross.get(id) ?? 0) + w);
+    (foreign.get(id) ?? foreign.set(id, /* @__PURE__ */ new Set()).get(id)).add(otherLabel);
+  };
+  for (const e of edges) {
+    const la = labels.get(e.a);
+    const lb = labels.get(e.b);
+    if (la === void 0 || lb === void 0 || la === lb) continue;
+    const w = e.weight ?? 1;
+    touch(e.a, lb, w);
+    touch(e.b, la, w);
+  }
+  let max = 0;
+  const raw = /* @__PURE__ */ new Map();
+  for (const [id, w] of cross) {
+    const r = w * foreign.get(id).size;
+    raw.set(id, r);
+    if (r > max) max = r;
+  }
+  const out = /* @__PURE__ */ new Map();
+  for (const [id] of labels) {
+    const r = raw.get(id) ?? 0;
+    const own = labels.get(id);
+    const groups = r > 0 ? [own, ...[...foreign.get(id)].sort()] : [own];
+    out.set(id, { score: max > 0 ? r / max : 0, groups });
+  }
+  return out;
+}
+function topBridges(bridging, n = 5) {
+  return [...bridging.entries()].filter(([, b]) => b.score > 0).sort((x, y) => y[1].score - x[1].score || (x[0] < y[0] ? -1 : 1)).slice(0, n).map(([id, b]) => ({ id, score: Math.round(b.score * 100) / 100, groups: b.groups }));
+}
+
+// src/graph/connectors.ts
+function findConnectors(fragment, config) {
+  const nameOf = /* @__PURE__ */ new Map();
+  for (const n of fragment.nodes) {
+    const label = n.attrs.name ?? n.attrs.title;
+    if (typeof label === "string" && label) nameOf.set(`${n.type}:${n.key}`, label);
+  }
+  const sets = config.lenses.map((l) => ({
+    pairs: projectBridges(reifiedBridgeTargets(fragment, { reified: l.reified, bridgeLabel: l.bridge, targetLabel: l.of }), {
+      maxPerBridge: config.maxPerBridge ?? 12
+    }),
+    weight: l.weight ?? 1
+  }));
+  const pairs = blendPairs([...sets, ...config.extraPairs ?? []]);
+  if (pairs.length === 0) return { connectors: [], communities: 0, nodes: 0 };
+  const nodes = [...new Set(pairs.flatMap((p) => [p.a, p.b]))].sort();
+  const edges = pairs.map((p) => ({ a: p.a, b: p.b, weight: p.w }));
+  const labels = labelPropagation(nodes, edges, { seed: 1 });
+  const groupName = /* @__PURE__ */ new Map();
+  for (const id of nodes) {
+    const g = labels.get(id);
+    if (!groupName.has(g)) groupName.set(g, nameOf.get(id) ?? id);
+  }
+  const bridging = bridgingScores(edges, labels);
+  const viaOf = (id) => {
+    const names = /* @__PURE__ */ new Set();
+    for (const p of pairs) {
+      if (p.a !== id && p.b !== id) continue;
+      for (const v of p.via) names.add(nameOf.get(v) ?? v.split(":").pop());
+    }
+    return [...names].slice(0, 6);
+  };
+  const connectors = topBridges(bridging, config.top ?? 12).map((b) => ({
+    ref: b.id,
+    name: nameOf.get(b.id) ?? b.id,
+    score: b.score,
+    groups: b.groups.map((g) => groupName.get(g)),
+    // every label has an anchor name
+    via: viaOf(b.id)
+  }));
+  return { connectors, communities: new Set(labels.values()).size, nodes: nodes.length };
+}
+
+// src/util/identity.ts
+function slugify(input) {
+  return input.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60).replace(/-+$/g, "");
+}
+function normalizeUrl(raw) {
+  try {
+    const u = new URL(raw);
+    u.hash = "";
+    const drop = [/^utm_/i, /^ref$/i, /^ref_/i, /^fbclid$/i, /^gclid$/i, /^mc_/i];
+    for (const key of [...u.searchParams.keys()]) {
+      if (drop.some((re) => re.test(key))) u.searchParams.delete(key);
+    }
+    let s = u.toString();
+    s = s.replace(/\/$/, "");
+    return s;
+  } catch {
+    return raw.trim().replace(/#.*$/, "").replace(/\/$/, "");
+  }
+}
+function urlHash(raw) {
+  const s = normalizeUrl(raw);
+  let h = 2166136261;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 16777619);
+  }
+  return (h >>> 0).toString(36);
+}
+function isLikelyPersonName(name) {
+  const n = name.trim();
+  if (!n || /\d/.test(n)) return false;
+  const parts = n.split(/\s+/);
+  if (parts.length < 2 || parts.length > 4) return false;
+  return parts.every((p) => /^[A-Z][a-zA-Z'’.-]*$/.test(p));
+}
+
+// src/enrich/facts.ts
+var LOCATE_SYSTEM = "You resolve US business locations to counties. For each company you are given a city and/or a free-text region. Name the US county containing that city (for independent cities, use the city's own county-equivalent name) plus its 2-letter state. Use nulls when the location is ambiguous, outside the US, or unknown.";
+var LOCATE_TOOL = {
+  name: "resolve_counties",
+  description: "County name + state per company",
+  parameters: {
+    type: "object",
+    properties: {
+      items: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            slug: { type: "string" },
+            county: { type: ["string", "null"], description: 'county name WITHOUT the word "County", e.g. "Santa Clara"' },
+            state: { type: ["string", "null"], description: "2-letter USPS state" }
+          },
+          required: ["slug", "county", "state"]
+        }
+      }
+    },
+    required: ["items"]
+  }
+};
+function locatePrompt(batch) {
+  return "Companies:\n" + batch.map((c) => `- slug ${c.slug}: city=${c.city ?? "?"} region=${c.region ?? "?"}`).join("\n");
+}
+var FACTS_SEARCH_SYSTEM = "You are a research assistant with web search. Find concrete facts about the company: founding year/date, headquarters city and state, founders and key executives (names, roles), and funding (rounds, dates, amounts, investor names). Prefer official/reputable sources. Report dense factual notes; say 'not found' for anything you cannot establish. Do not speculate.";
+var FACTS_EXTRACT_SYSTEM = "Extract structured facts from the research notes. Use null for anything the notes do not establish.";
+var FACTS_EXTRACT_TOOL = {
+  name: "company_facts",
+  description: "Structured company facts from research notes",
+  parameters: {
+    type: "object",
+    properties: {
+      founded: { type: ["string", "null"], description: "YYYY or YYYY-MM or YYYY-MM-DD" },
+      city: { type: ["string", "null"] },
+      state: { type: ["string", "null"], description: "2-letter USPS" },
+      people: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: { name: { type: "string" }, role: { type: "string" } },
+          required: ["name", "role"]
+        }
+      },
+      rounds: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            round: { type: ["string", "null"] },
+            date: { type: ["string", "null"] },
+            amount: { type: ["string", "null"] },
+            investors: { type: "array", items: { type: "string" } }
+          },
+          required: ["investors"]
+        }
+      }
+    },
+    required: ["people", "rounds"]
+  }
+};
+var DATE_RE = /^\d{4}(-\d{2}){0,2}$/;
+var isoDate = (v) => !DATE_RE.test(v) ? null : v.length === 4 ? `${v}-01-01` : v.length === 7 ? `${v}-01` : v;
+var countyKey = (name, st) => `${String(name).toLowerCase().replace(/\bcounty\b|\bparish\b|\bborough\b/g, "").replace(/[^a-z]/g, "")}|${String(st).toUpperCase()}`;
+function factsToFragment(company, draft) {
+  const nodes = [];
+  const edges = [];
+  const attrs = {};
+  const founded = draft.founded ? isoDate(draft.founded) : null;
+  if (founded) attrs.founded = founded;
+  if (draft.city && !company.city) attrs.city = String(draft.city).slice(0, 60);
+  if (Object.keys(attrs).length > 0) {
+    nodes.push({ type: "company", key: company.slug, attrs: { name: company.name, ...attrs } });
+  }
+  const anchorDate = attrs.founded ?? company.founded ?? company.first_seen ?? "";
+  for (const p of (draft.people ?? []).slice(0, 5)) {
+    const personSlug = slugify(p.name);
+    if (!personSlug || personSlug === company.slug) continue;
+    nodes.push({ type: "person", key: personSlug, attrs: { name: p.name } });
+    const key = `${personSlug}~${company.slug}`;
+    nodes.push({ type: "affiliation", key, attrs: { date: anchorDate, role: String(p.role).slice(0, 60) } });
+    edges.push({ from: { type: "affiliation", key }, to: { type: "person", key: personSlug }, label: "of_person" });
+    edges.push({ from: { type: "affiliation", key }, to: { type: "company", key: company.slug }, label: "of_company" });
+  }
+  (draft.rounds ?? []).slice(0, 4).forEach((r, i) => {
+    const investors = (r.investors ?? []).map(slugify).filter(Boolean).slice(0, 6);
+    const date = r.date ? isoDate(r.date) : null;
+    if (investors.length === 0 && !date) return;
+    const key = `${company.slug}~e${i}`;
+    const rAttrs = { date: date ?? anchorDate };
+    if (r.round) rAttrs.round = String(r.round).slice(0, 40);
+    if (r.amount) rAttrs.amount = String(r.amount).slice(0, 40);
+    nodes.push({ type: "funding_round", key, attrs: rAttrs });
+    edges.push({ from: { type: "funding_round", key }, to: { type: "company", key: company.slug }, label: "round_of" });
+    for (const [j, invSlug] of investors.entries()) {
+      nodes.push({ type: "investor", key: invSlug, attrs: { name: (r.investors ?? [])[j] } });
+      edges.push({ from: { type: "funding_round", key }, to: { type: "investor", key: invSlug }, label: "raised_from" });
+    }
+  });
+  return { nodes, edges };
+}
+function locationFragment(slug, name, fips) {
+  return {
+    nodes: [{ type: "company", key: slug, attrs: { name, fips } }],
+    edges: [{ from: { type: "company", key: slug }, to: { type: "county", key: fips }, label: "located_in" }]
+  };
+}
+
+// src/enrich/run.ts
+async function enrichCompany(ai, models, company) {
+  const where = `${company.city ? `, ${company.city}` : ""}${company.region ? ` (${company.region})` : ""}`;
+  const { text } = await ai.search({
+    model: models.search,
+    system: FACTS_SEARCH_SYSTEM,
+    user: `Company: ${company.name}${where}. Research its founding, headquarters, key people, and funding.`,
+    maxTokens: 900
+  });
+  const extractOnce = () => ai.extract({
+    model: models.extract,
+    system: FACTS_EXTRACT_SYSTEM,
+    user: `Company: ${company.name}
+
+Notes:
+${text}`,
+    tool: FACTS_EXTRACT_TOOL,
+    maxTokens: 4e3
+    // reasoning headroom before the forced tool call
+  });
+  const { value } = await extractOnce().catch(extractOnce);
+  return factsToFragment(company, value);
+}
+async function resolveCounties(ai, model, rows, gazetteer) {
+  const { value } = await ai.extract({
+    model,
+    system: LOCATE_SYSTEM,
+    user: locatePrompt(rows),
+    tool: LOCATE_TOOL,
+    maxTokens: 6e3
+    // reasoning models spend tokens before the tool call
+  });
+  const out = [];
+  for (const item of value.items ?? []) {
+    const fips = item.county && item.state ? gazetteer.get(countyKey(item.county, item.state)) : void 0;
+    if (fips) out.push({ slug: item.slug, fips });
+  }
+  return out;
+}
+
 // src/llm/provider.ts
 var import_ai = require("@odla-ai/ai");
 var MODEL_TIERS = {
@@ -607,42 +1017,6 @@ var gdeltConnector = {
   }
 };
 
-// src/util/identity.ts
-function slugify(input) {
-  return input.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60).replace(/-+$/g, "");
-}
-function normalizeUrl(raw) {
-  try {
-    const u = new URL(raw);
-    u.hash = "";
-    const drop = [/^utm_/i, /^ref$/i, /^ref_/i, /^fbclid$/i, /^gclid$/i, /^mc_/i];
-    for (const key of [...u.searchParams.keys()]) {
-      if (drop.some((re) => re.test(key))) u.searchParams.delete(key);
-    }
-    let s = u.toString();
-    s = s.replace(/\/$/, "");
-    return s;
-  } catch {
-    return raw.trim().replace(/#.*$/, "").replace(/\/$/, "");
-  }
-}
-function urlHash(raw) {
-  const s = normalizeUrl(raw);
-  let h = 2166136261;
-  for (let i = 0; i < s.length; i++) {
-    h ^= s.charCodeAt(i);
-    h = Math.imul(h, 16777619);
-  }
-  return (h >>> 0).toString(36);
-}
-function isLikelyPersonName(name) {
-  const n = name.trim();
-  if (!n || /\d/.test(n)) return false;
-  const parts = n.split(/\s+/);
-  if (parts.length < 2 || parts.length > 4) return false;
-  return parts.every((p) => /^[A-Z][a-zA-Z'’.-]*$/.test(p));
-}
-
 // src/connectors/fred.ts
 async function fetchSeries(seriesId, apiKey, start) {
   const url = `https://api.stlouisfed.org/fred/series/observations?series_id=${encodeURIComponent(seriesId)}&api_key=${encodeURIComponent(apiKey)}&file_type=json&observation_start=${encodeURIComponent(start)}`;