@odla-ai/kg 0.1.0

package/dist/index.js

@@ -0,0 +1,924 @@
+// src/types.ts
+function emptyUsage() {
+  return { calls: 0, inputTokens: 0, outputTokens: 0 };
+}
+function addUsage(into, more) {
+  into.calls += more.calls;
+  into.inputTokens += more.inputTokens;
+  into.outputTokens += more.outputTokens;
+}
+
+// src/ontology.ts
+function naturalKeysOf(o) {
+  const m = {};
+  for (const [type, e] of Object.entries(o.entities)) m[type] = e.naturalKey;
+  return m;
+}
+function mergeRulesOf(o) {
+  const out = {};
+  for (const [type, e] of Object.entries(o.entities)) {
+    out[type] = {};
+    for (const [attr, spec] of Object.entries(e.attrs)) out[type][attr] = spec.merge ?? "replace";
+  }
+  return out;
+}
+function humanAttrsOf(o) {
+  const out = {};
+  for (const [type, e] of Object.entries(o.entities)) {
+    out[type] = new Set(Object.entries(e.attrs).filter(([, a]) => a.human).map(([n]) => n));
+  }
+  return out;
+}
+function toSerializedSchema(o) {
+  const entities = {};
+  for (const [type, e] of Object.entries(o.entities)) {
+    const attrs = {};
+    for (const [name, a] of Object.entries(e.attrs)) {
+      attrs[name] = {
+        type: a.type,
+        unique: !!a.unique,
+        indexed: !!a.indexed,
+        optional: !!a.optional
+      };
+    }
+    entities[type] = { attrs };
+  }
+  const links = {};
+  for (const [name, ed] of Object.entries(o.edges)) {
+    links[name] = {
+      forward: { on: ed.from, has: ed.forwardHas ?? "many", label: ed.forwardLabel },
+      reverse: { on: ed.to, has: ed.reverseHas ?? "many", label: ed.reverseLabel }
+    };
+  }
+  return { entities, links };
+}
+
+// src/graph/merge.ts
+function isEmpty(v) {
+  return v === void 0 || v === null || v === "";
+}
+function mergeScalar(rule, existing, incoming) {
+  if (isEmpty(incoming)) return existing;
+  if (isEmpty(existing)) return incoming;
+  switch (rule) {
+    case "prefer":
+      return existing;
+    // first non-empty value sticks
+    case "earliest":
+      return String(incoming) < String(existing) ? incoming : existing;
+    case "latest":
+      return String(incoming) > String(existing) ? incoming : existing;
+    case "replace":
+    default:
+      return incoming;
+  }
+}
+function mergeAttrs(rules, existing, incoming) {
+  const out = {};
+  for (const [attr, val] of Object.entries(incoming)) {
+    const merged = mergeScalar(rules[attr] ?? "replace", existing[attr], val);
+    if (merged !== void 0) out[attr] = merged;
+  }
+  return out;
+}
+
+// src/graph/writer.ts
+function fragmentToOps(fragment, keys) {
+  const ref = (type, key) => {
+    const attr = keys[type];
+    if (!attr) throw new Error(`No natural key configured for node type "${type}"`);
+    return { ns: type, attr, value: key };
+  };
+  const ops = [];
+  for (const n of fragment.nodes) {
+    const attr = keys[n.type];
+    if (!attr) throw new Error(`No natural key configured for node type "${n.type}"`);
+    ops.push({ t: "update", ns: n.type, id: { ns: n.type, attr, value: n.key }, attrs: { ...n.attrs, [attr]: n.key } });
+  }
+  for (const e of fragment.edges) {
+    ops.push({ t: "link", ns: e.from.type, id: ref(e.from.type, e.from.key), label: e.label, target: ref(e.to.type, e.to.key) });
+  }
+  return ops;
+}
+async function writeFragment(db, fragment, ontology, mutationId) {
+  const keys = naturalKeysOf(ontology);
+  const rules = mergeRulesOf(ontology);
+  const humans = humanAttrsOf(ontology);
+  const byType = /* @__PURE__ */ new Map();
+  for (const n of fragment.nodes) {
+    if (!keys[n.type]) throw new Error(`No natural key configured for node type "${n.type}"`);
+    const human = humans[n.type] ?? /* @__PURE__ */ new Set();
+    const clean = {};
+    for (const [k, v] of Object.entries(n.attrs)) if (!human.has(k)) clean[k] = v;
+    const typeMap = byType.get(n.type) ?? /* @__PURE__ */ new Map();
+    const prev = typeMap.get(n.key);
+    typeMap.set(n.key, prev ? mergeAttrs(rules[n.type], prev, clean) : clean);
+    byType.set(n.type, typeMap);
+  }
+  const query = {};
+  for (const [type, m] of byType) {
+    query[type] = { $: { where: { [keys[type]]: { $in: [...m.keys()] } } } };
+  }
+  const existing = Object.keys(query).length ? await db.query(query) : {};
+  const priorOf = (type, key) => {
+    const attr = keys[type];
+    for (const row of existing[type] ?? []) {
+      if (row[attr] === key) return row;
+    }
+    return {};
+  };
+  const ops = [];
+  for (const [type, m] of byType) {
+    const attr = keys[type];
+    for (const [key, incoming] of m) {
+      const merged = mergeAttrs(rules[type], priorOf(type, key), incoming);
+      ops.push({ t: "update", ns: type, id: { ns: type, attr, value: key }, attrs: { ...merged, [attr]: key } });
+    }
+  }
+  const ref = (type, key) => ({ ns: type, attr: keys[type], value: key });
+  for (const e of fragment.edges) {
+    ops.push({ t: "link", ns: e.from.type, id: ref(e.from.type, e.from.key), label: e.label, target: ref(e.to.type, e.to.key) });
+  }
+  if (ops.length === 0) return { txId: 0, ops: 0 };
+  const txId = await db.transact(ops, { mutationId });
+  return { txId, ops: ops.length };
+}
+
+// src/graph/resolve.ts
+async function searchEntities(db, type, q, limit = 5) {
+  if (!q.trim()) return [];
+  const res = await db.query({ [type]: { $: { search: q, limit } } });
+  return res[type] ?? [];
+}
+var norm = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, "");
+async function resolveName(db, type, keyAttr, name) {
+  const target = norm(name);
+  if (!target) return null;
+  for (const h of await searchEntities(db, type, name, 3)) {
+    const key = h[keyAttr];
+    const hname = h["name"];
+    if (typeof key !== "string") continue;
+    if (typeof hname === "string" && norm(hname) === target || norm(key) === target) return key;
+  }
+  return null;
+}
+async function canonicalizeFragment(db, fragment, ontology) {
+  const remap = /* @__PURE__ */ new Map();
+  for (const n of fragment.nodes) {
+    const spec = ontology.entities[n.type];
+    if (!spec?.searchable?.length) continue;
+    const name = typeof n.attrs.name === "string" ? n.attrs.name : "";
+    if (!name) continue;
+    const canon = await resolveName(db, n.type, spec.naturalKey, name);
+    if (canon && canon !== n.key) remap.set(`${n.type}:${n.key}`, canon);
+  }
+  if (remap.size === 0) return fragment;
+  const rk = (type, key) => remap.get(`${type}:${key}`) ?? key;
+  return {
+    nodes: fragment.nodes.map((n) => ({ ...n, key: rk(n.type, n.key) })),
+    edges: fragment.edges.map((e) => ({
+      from: { type: e.from.type, key: rk(e.from.type, e.from.key) },
+      to: { type: e.to.type, key: rk(e.to.type, e.to.key) },
+      label: e.label
+    }))
+  };
+}
+
+// src/llm/provider.ts
+import { init, extractText } from "@odla-ai/ai";
+var MODEL_TIERS = {
+  claude: { cheap: "claude-haiku-4-5", extract: "claude-sonnet-5", search: "claude-sonnet-5" },
+  openai: { cheap: "gpt-5-nano", extract: "gpt-5-mini", search: "gpt-5-search-api" }
+};
+function toUsage(u) {
+  return { calls: 1, inputTokens: u.inputTokens, outputTokens: u.outputTokens };
+}
+function createProvider(cfg) {
+  const tiers = MODEL_TIERS[cfg.provider];
+  const ai = cfg.ai ?? init({ keys: { anthropic: cfg.anthropicKey, openai: cfg.openaiKey } });
+  return {
+    id: cfg.provider,
+    ai,
+    async extract(c) {
+      const { value, usage } = await ai.extract({
+        model: c.model ?? tiers.extract,
+        system: c.system,
+        user: c.user,
+        tool: c.tool,
+        maxTokens: c.maxTokens
+      });
+      return { value, usage: toUsage(usage) };
+    },
+    async search(c) {
+      const { text, usage } = await ai.search({
+        model: c.model ?? tiers.search,
+        system: c.system,
+        user: c.user,
+        maxTokens: c.maxTokens
+      });
+      return { text, usage: toUsage(usage) };
+    },
+    async write(c) {
+      const res = await ai.chat({
+        model: c.model ?? tiers.extract,
+        system: c.system,
+        messages: [{ role: "user", content: c.user }],
+        maxTokens: c.maxTokens ?? 2048
+      });
+      return { text: extractText(res.content), usage: toUsage(res.usage) };
+    }
+  };
+}
+function getProvider(env) {
+  const which = (env.LLM_PROVIDER || "claude").toLowerCase() === "openai" ? "openai" : "claude";
+  if (which === "openai" && !env.OPENAI_API_KEY) throw new Error("LLM_PROVIDER=openai but OPENAI_API_KEY is unset");
+  if (which === "claude" && !env.ANTHROPIC_API_KEY) throw new Error("LLM_PROVIDER=claude but ANTHROPIC_API_KEY is unset");
+  return createProvider({ provider: which, anthropicKey: env.ANTHROPIC_API_KEY, openaiKey: env.OPENAI_API_KEY });
+}
+
+// src/llm/json.ts
+function extractJson(text) {
+  const m = text.match(/\{[\s\S]*\}/);
+  if (!m) return null;
+  try {
+    return JSON.parse(m[0]);
+  } catch {
+    return null;
+  }
+}
+function extractJsonArray(text) {
+  const m = text.match(/\[[\s\S]*\]/);
+  if (!m) return [];
+  try {
+    const v = JSON.parse(m[0]);
+    return Array.isArray(v) ? v : [];
+  } catch {
+    return [];
+  }
+}
+
+// src/util/content.ts
+var MAX_CHARS = 8e3;
+function htmlToText(html, maxChars = MAX_CHARS) {
+  let t = html;
+  t = t.replace(/<script[\s\S]*?<\/script>/gi, " ");
+  t = t.replace(/<style[\s\S]*?<\/style>/gi, " ");
+  t = t.replace(/<!--[\s\S]*?-->/g, " ");
+  t = t.replace(/<\/(p|div|section|article|li|h[1-6]|br)>/gi, "\n");
+  t = t.replace(/<[^>]+>/g, " ");
+  t = decodeEntities(t);
+  t = t.replace(/[ \t\f\r]+/g, " ").replace(/\n\s*\n\s*/g, "\n\n").trim();
+  return t.slice(0, maxChars);
+}
+function decodeEntities(s) {
+  return s.replace(/&nbsp;/gi, " ").replace(/&amp;/gi, "&").replace(/&lt;/gi, "<").replace(/&gt;/gi, ">").replace(/&quot;/gi, '"').replace(/&#39;|&apos;/gi, "'").replace(/&#(\d+);/g, (_, d) => String.fromCharCode(Number(d))).replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16)));
+}
+async function fetchText(url, maxChars = MAX_CHARS) {
+  try {
+    const res = await fetch(url, {
+      headers: {
+        "user-agent": "odla-kg/0.1 (+https://github.com/cory/odla-kg)",
+        accept: "text/html,application/xhtml+xml"
+      }
+    });
+    if (!res.ok) return "";
+    const ct = res.headers.get("content-type") || "";
+    const body = await res.text();
+    if (ct.includes("html") || /<\w+[\s>]/.test(body.slice(0, 200))) {
+      return htmlToText(body, maxChars);
+    }
+    return body.slice(0, maxChars);
+  } catch {
+    return "";
+  }
+}
+
+// src/connectors/rss.ts
+var UA = "Mozilla/5.0 (compatible; odla-kg/0.1)";
+function extractTag(xml, tag) {
+  const cdata = xml.match(new RegExp(`<${tag}[^>]*>\\s*<!\\[CDATA\\[([\\s\\S]*?)\\]\\]>\\s*</${tag}>`, "i"));
+  if (cdata) return cdata[1].trim();
+  const m = xml.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}>`, "i"));
+  return m ? m[1].trim() : "";
+}
+function extractAttr(xml, tag, attr) {
+  const m = xml.match(new RegExp(`<${tag}[^>]*\\s${attr}=["']([^"']*)["']`, "i"));
+  return m ? m[1].trim() : "";
+}
+function splitItems(xml) {
+  return xml.match(/<item[\s>][\s\S]*?<\/item>/gi) || xml.match(/<entry[\s>][\s\S]*?<\/entry>/gi) || [];
+}
+var rssConnector = {
+  id: "rss",
+  capabilities: { fetch: true, structured: false },
+  parseConfig(raw) {
+    const r = raw;
+    if (typeof r.url !== "string" || !r.url) throw new Error("rss connector requires a `url`");
+    return { url: r.url, name: String(r.name ?? r.url), watch: r.watch ?? null };
+  },
+  async fetch(cfg, ctx) {
+    let xml;
+    try {
+      const res = await fetch(cfg.url, { headers: { "User-Agent": UA } });
+      if (!res.ok) {
+        if (res.status === 429 || res.status === 403 || res.status >= 500) return [];
+        throw new Error(`feed ${cfg.url} \u2192 ${res.status}`);
+      }
+      xml = await res.text();
+    } catch (e) {
+      console.log(`rss fetch failed (non-blocking): ${cfg.url}: ${e}`);
+      return [];
+    }
+    const out = [];
+    for (const raw of splitItems(xml)) {
+      const title = extractTag(raw, "title");
+      let url = extractTag(raw, "link") || extractAttr(raw, "link", "href");
+      if (!title || !url) continue;
+      url = url.trim();
+      const date = extractTag(raw, "pubDate") || extractTag(raw, "published") || extractTag(raw, "updated") || void 0;
+      const content = extractTag(raw, "content:encoded") || extractTag(raw, "content") || extractTag(raw, "description") || extractTag(raw, "summary") || "";
+      let text = htmlToText(content);
+      if (text.length < 200) {
+        const page = await ctx.fetchText(url);
+        if (page.length > text.length) text = page;
+      }
+      out.push({ url, title, date, sourceName: cfg.name, sourceKind: "rss", watch: cfg.watch, text });
+    }
+    return out;
+  }
+};
+
+// src/connectors/url.ts
+async function fetchUrl(url) {
+  const text = await fetchText(url);
+  const firstLine = text.split("\n").find((l) => l.trim().length > 0) ?? "";
+  let host = url;
+  try {
+    host = new URL(url).hostname;
+  } catch {
+  }
+  return { url, title: (firstLine || url).slice(0, 200).trim(), sourceName: host, sourceKind: "url", text };
+}
+var urlConnector = {
+  id: "url",
+  capabilities: { fetch: true, structured: false },
+  parseConfig(raw) {
+    const r = raw;
+    if (typeof r.url !== "string" || !r.url) throw new Error("url connector requires a `url`");
+    return { url: r.url, watch: r.watch ?? null };
+  },
+  async fetch(cfg) {
+    const item = await fetchUrl(cfg.url);
+    if (cfg.watch !== void 0) item.watch = cfg.watch;
+    return [item];
+  }
+};
+
+// src/connectors/gdelt.ts
+var UA2 = "Mozilla/5.0 (compatible; odla-kg/0.1)";
+var MAX_ITEMS = 15;
+function isoFromSeendate(s) {
+  const m = s.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z?$/);
+  return m ? `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z` : void 0;
+}
+var gdeltConnector = {
+  id: "gdelt",
+  capabilities: { fetch: true, structured: false },
+  parseConfig(raw) {
+    const r = raw;
+    if (typeof r.query !== "string" || !r.query) throw new Error("gdelt connector requires a `query`");
+    return { query: r.query, name: String(r.name ?? `GDELT: ${r.query}`), watch: r.watch ?? null };
+  },
+  async fetch(cfg, ctx) {
+    const api = `https://api.gdeltproject.org/api/v2/doc/doc?query=${encodeURIComponent(cfg.query)}&mode=artlist&format=json&maxrecords=40&timespan=2d`;
+    let articles = [];
+    try {
+      const res = await fetch(api, { headers: { "User-Agent": UA2 } });
+      if (!res.ok) return [];
+      const text = await res.text();
+      try {
+        articles = JSON.parse(text).articles ?? [];
+      } catch {
+        return [];
+      }
+    } catch (e) {
+      console.log(`gdelt fetch failed (non-blocking): ${e}`);
+      return [];
+    }
+    const seen = /* @__PURE__ */ new Set();
+    const picked = articles.filter((a) => a?.url && a?.title && a.language === "English" && !seen.has(a.url) && (seen.add(a.url), true)).slice(0, MAX_ITEMS);
+    const out = [];
+    for (const a of picked) {
+      const body = await ctx.fetchText(a.url);
+      if (body.length < 200) continue;
+      out.push({
+        url: a.url,
+        title: a.title,
+        date: isoFromSeendate(a.seendate ?? ""),
+        sourceName: cfg.name,
+        sourceKind: "gdelt",
+        watch: cfg.watch,
+        text: body
+      });
+    }
+    return out;
+  }
+};
+
+// src/util/identity.ts
+function slugify(input) {
+  return input.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60).replace(/-+$/g, "");
+}
+function normalizeUrl(raw) {
+  try {
+    const u = new URL(raw);
+    u.hash = "";
+    const drop = [/^utm_/i, /^ref$/i, /^ref_/i, /^fbclid$/i, /^gclid$/i, /^mc_/i];
+    for (const key of [...u.searchParams.keys()]) {
+      if (drop.some((re) => re.test(key))) u.searchParams.delete(key);
+    }
+    let s = u.toString();
+    s = s.replace(/\/$/, "");
+    return s;
+  } catch {
+    return raw.trim().replace(/#.*$/, "").replace(/\/$/, "");
+  }
+}
+function urlHash(raw) {
+  const s = normalizeUrl(raw);
+  let h = 2166136261;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 16777619);
+  }
+  return (h >>> 0).toString(36);
+}
+function isLikelyPersonName(name) {
+  const n = name.trim();
+  if (!n || /\d/.test(n)) return false;
+  const parts = n.split(/\s+/);
+  if (parts.length < 2 || parts.length > 4) return false;
+  return parts.every((p) => /^[A-Z][a-zA-Z'’.-]*$/.test(p));
+}
+
+// src/connectors/fred.ts
+async function fetchSeries(seriesId, apiKey, start) {
+  const url = `https://api.stlouisfed.org/fred/series/observations?series_id=${encodeURIComponent(seriesId)}&api_key=${encodeURIComponent(apiKey)}&file_type=json&observation_start=${encodeURIComponent(start)}`;
+  try {
+    const res = await fetch(url);
+    if (!res.ok) return [];
+    const data = await res.json();
+    const out = [];
+    for (const o of data.observations ?? []) {
+      if (!o?.date || o.value === void 0 || o.value === ".") continue;
+      const v = Number(o.value);
+      if (Number.isFinite(v)) out.push({ date: o.date, value: v });
+    }
+    return out;
+  } catch (e) {
+    console.log(`fred fetch failed for ${seriesId} (non-blocking): ${e}`);
+    return [];
+  }
+}
+var fredConnector = {
+  id: "fred",
+  capabilities: { fetch: true, structured: true },
+  parseConfig(raw) {
+    const r = raw;
+    const series = Array.isArray(r.series) ? r.series : [];
+    if (!series.length) throw new Error("fred connector requires a non-empty `series`");
+    return { series, region: r.region, regionName: r.regionName };
+  },
+  async fetch(cfg, ctx) {
+    const apiKey = ctx.secret("FRED_API_KEY");
+    if (!apiKey) {
+      console.log("fred: FRED_API_KEY unset \u2014 skipping");
+      return [];
+    }
+    const start = new Date(ctx.now.getTime() - 400 * 24 * 3600 * 1e3).toISOString().slice(0, 10);
+    const out = [];
+    for (const s of cfg.series) {
+      const obs = await fetchSeries(s.id, apiKey, start);
+      if (!obs.length) continue;
+      const latest = obs[obs.length - 1];
+      const prior = obs.length > 1 ? obs[obs.length - 2] : void 0;
+      const attrs = { label: s.label, latest: String(latest.value), as_of: latest.date };
+      if (prior) attrs.prior = String(prior.value);
+      if (s.unit) attrs.unit = s.unit;
+      const fragment = { nodes: [{ type: "indicator", key: s.id, attrs }], edges: [] };
+      if (cfg.region) {
+        const rslug = slugify(cfg.region);
+        if (rslug) {
+          fragment.nodes.push({ type: "region", key: rslug, attrs: { name: cfg.regionName ?? cfg.region } });
+          fragment.edges.push({ from: { type: "region", key: rslug }, to: { type: "indicator", key: s.id }, label: "has_indicator" });
+        }
+      }
+      out.push({
+        url: `https://fred.stlouisfed.org/series/${s.id}#${latest.date}`,
+        title: `${s.label}: ${latest.value} (${latest.date})`,
+        date: latest.date,
+        sourceName: "FRED",
+        sourceKind: "fred",
+        fragment
+      });
+    }
+    return out;
+  }
+};
+
+// src/connectors/edgar.ts
+var SEARCH_BASE = "https://efts.sec.gov/LATEST/search-index";
+var MAX_FILINGS = 10;
+var cikTrim = (cik) => cik.replace(/^0+/, "") || cik;
+var indexUrl = (cik, adsh) => `https://www.sec.gov/Archives/edgar/data/${cikTrim(cik)}/${adsh.replace(/-/g, "")}/${adsh}-index.htm`;
+function tagText(scope, tag) {
+  const m = scope.match(new RegExp(`<${tag}[^>]*>\\s*([\\s\\S]*?)\\s*</${tag}>`, "i"));
+  return m ? m[1].trim() : "";
+}
+var edgarConnector = {
+  id: "edgar",
+  capabilities: { fetch: true, structured: true },
+  parseConfig(raw) {
+    const r = raw;
+    const codes = Array.isArray(r.locationCodes) ? r.locationCodes : [];
+    if (!codes.length) throw new Error('edgar connector requires `locationCodes` (e.g. ["UT"])');
+    return {
+      locationCodes: codes,
+      cityFilter: Array.isArray(r.cityFilter) ? r.cityFilter : void 0,
+      days: typeof r.days === "number" ? r.days : 7
+    };
+  },
+  async fetch(cfg, ctx) {
+    const ua = ctx.secret("EDGAR_USER_AGENT");
+    if (!ua) {
+      console.log("edgar: EDGAR_USER_AGENT unset \u2014 skipping");
+      return [];
+    }
+    const enddt = ctx.now.toISOString().slice(0, 10);
+    const startdt = new Date(ctx.now.getTime() - (cfg.days ?? 7) * 24 * 3600 * 1e3).toISOString().slice(0, 10);
+    const searchUrl = `${SEARCH_BASE}?q=&forms=D&locationCodes=${encodeURIComponent(cfg.locationCodes.join(","))}&startdt=${startdt}&enddt=${enddt}`;
+    let hits = [];
+    try {
+      const res = await fetch(searchUrl, { headers: { "User-Agent": ua } });
+      if (!res.ok) return [];
+      hits = (await res.json()).hits?.hits ?? [];
+    } catch (e) {
+      console.log(`edgar search failed (non-blocking): ${e}`);
+      return [];
+    }
+    const out = [];
+    const seen = /* @__PURE__ */ new Set();
+    for (const hit of hits.slice(0, MAX_FILINGS)) {
+      const src = hit._source ?? {};
+      const adsh = src.adsh ?? "";
+      if (!adsh || seen.has(adsh)) continue;
+      seen.add(adsh);
+      const cik = src.ciks?.[0] ?? "";
+      const name = (src.display_names?.[0] ?? "").replace(/\s*\(CIK\s*\d+\)\s*$/i, "").trim();
+      if (!name || !cik) continue;
+      const loc = src.biz_locations?.[0] ?? "";
+      let city = loc.trim();
+      let state = src.biz_states?.[0] ?? "";
+      const comma = loc.lastIndexOf(",");
+      if (comma >= 0) {
+        city = loc.slice(0, comma).trim();
+        state = loc.slice(comma + 1).trim() || state;
+      }
+      if (cfg.cityFilter?.length && !cfg.cityFilter.some((c) => c.toLowerCase() === city.toLowerCase())) continue;
+      const url = indexUrl(cik, adsh);
+      const fileDate = src.file_date ?? "";
+      let industry = "";
+      let offering = "";
+      const persons = [];
+      try {
+        const detailUrl = `https://www.sec.gov/Archives/edgar/data/${cikTrim(cik)}/${adsh.replace(/-/g, "")}/primary_doc.xml`;
+        const dres = await fetch(detailUrl, { headers: { "User-Agent": ua } });
+        if (dres.ok) {
+          const xml = await dres.text();
+          industry = tagText(xml, "industryGroupType");
+          offering = tagText(xml, "totalOfferingAmount");
+          for (const block of xml.match(/<relatedPersonInfo>[\s\S]*?<\/relatedPersonInfo>/gi) ?? []) {
+            const pname = [tagText(block, "firstName"), tagText(block, "lastName")].filter((p) => p && p !== "-").join(" ").trim();
+            if (!pname) continue;
+            const roles = [];
+            const relRe = /<relationship>\s*([\s\S]*?)\s*<\/relationship>/gi;
+            let m;
+            while ((m = relRe.exec(block)) !== null) {
+              const r = m[1].trim();
+              if (r && !roles.includes(r)) roles.push(r);
+            }
+            persons.push({ name: pname, roles });
+          }
+        }
+      } catch {
+      }
+      const cslug = slugify(name);
+      if (!cslug) continue;
+      const f = { nodes: [], edges: [] };
+      const cattrs = { name };
+      if (city) cattrs.city = city;
+      if (state) cattrs.region = state;
+      if (industry) cattrs.sector = industry;
+      if (fileDate) {
+        cattrs.first_seen = fileDate;
+        cattrs.last_seen = fileDate;
+      }
+      f.nodes.push({ type: "company", key: cslug, attrs: cattrs });
+      if (state) {
+        const rslug = slugify(state);
+        if (rslug) {
+          f.nodes.push({ type: "region", key: rslug, attrs: { name: state } });
+          f.edges.push({ from: { type: "company", key: cslug }, to: { type: "region", key: rslug }, label: "located_in" });
+        }
+      }
+      const rkey = `${cslug}:form-d:${fileDate}`;
+      const rattrs = { round: "Form D", source_url: url };
+      if (offering) rattrs.amount = offering;
+      if (fileDate) rattrs.date = fileDate;
+      f.nodes.push({ type: "funding_round", key: rkey, attrs: rattrs });
+      f.edges.push({ from: { type: "funding_round", key: rkey }, to: { type: "company", key: cslug }, label: "round_of" });
+      for (const p of persons) {
+        if (!isLikelyPersonName(p.name)) continue;
+        const pslug = slugify(p.name);
+        if (!pslug) continue;
+        const pattrs = { name: p.name };
+        if (p.roles.length) pattrs.role = p.roles.join(", ");
+        f.nodes.push({ type: "person", key: pslug, attrs: pattrs });
+        f.edges.push({ from: { type: "person", key: pslug }, to: { type: "company", key: cslug }, label: "works_at" });
+      }
+      out.push({
+        url,
+        title: `Form D: ${name}${offering ? ` \u2014 ${offering}` : ""}`,
+        date: fileDate || void 0,
+        sourceName: "SEC EDGAR",
+        sourceKind: "edgar",
+        fragment: f
+      });
+    }
+    return out;
+  }
+};
+
+// src/connectors/govdata.ts
+var govDataConnector = {
+  id: "gov-data",
+  capabilities: { fetch: true, structured: true },
+  parseConfig(raw) {
+    const r = raw;
+    return { dataset: String(r.dataset ?? ""), watch: r.watch ?? null };
+  },
+  async fetch(cfg, _ctx) {
+    console.log(`gov-data: template connector for dataset "${cfg.dataset}" \u2014 implement fetch() to emit RawItems`);
+    return [];
+  }
+};
+
+// src/connectors/web-search.ts
+var SWEEP_SYSTEM = "You are a research scout building a knowledge graph of entrepreneurs, startups, and business trends. Search the web and report concrete, real companies and the people leading them, with specifics (what they do, where, who founded them). Tie every claim to something you actually found \u2014 never invent names, numbers, or companies. If a search turns up nothing solid, say so plainly.";
+var webSearchConnector = {
+  id: "web-search",
+  capabilities: { fetch: false, discover: true, structured: false },
+  parseConfig(raw) {
+    const r = raw;
+    const queries = Array.isArray(r.queries) ? r.queries.filter((q) => typeof q === "string") : typeof r.query === "string" ? [r.query] : [];
+    if (!queries.length) throw new Error("web-search connector requires `queries` (or a `query`)");
+    return { queries, name: String(r.name ?? "web-search"), watch: r.watch ?? null };
+  },
+  async fetch() {
+    return [];
+  },
+  async discover(cfg, ctx) {
+    const provider = ctx.provider();
+    const today = ctx.now.toISOString().slice(0, 10);
+    const out = [];
+    for (const q of cfg.queries) {
+      const { text } = await provider.search({
+        system: SWEEP_SYSTEM,
+        user: `Search task: ${q}
+Today's date: ${today}
+
+Report what you find, naming specific companies and the real people leading them.`
+      });
+      if (text.trim().length < 80) continue;
+      out.push({
+        url: `search://web-search/${urlHash(q)}#${today}`,
+        // synthetic; changes daily so re-sweeps aren't deduped away
+        title: q,
+        date: today,
+        sourceName: cfg.name,
+        sourceKind: "web-search",
+        watch: cfg.watch,
+        text
+      });
+    }
+    return out;
+  }
+};
+
+// src/connectors/registry.ts
+var REGISTRY = /* @__PURE__ */ new Map([
+  [rssConnector.id, rssConnector],
+  [urlConnector.id, urlConnector],
+  [gdeltConnector.id, gdeltConnector],
+  [fredConnector.id, fredConnector],
+  // structured (RawItem.fragment)
+  [edgarConnector.id, edgarConnector],
+  // structured (RawItem.fragment)
+  [webSearchConnector.id, webSearchConnector],
+  // discovery (web search)
+  [govDataConnector.id, govDataConnector]
+  // template — zero core change to add
+]);
+function getConnector(id) {
+  return REGISTRY.get(id);
+}
+function connectorIds() {
+  return [...REGISTRY.keys()];
+}
+function makeContext(cfg) {
+  let provider;
+  return {
+    now: cfg.now ?? /* @__PURE__ */ new Date(),
+    fetchText: (u) => fetchText(u),
+    secret: (name) => cfg.secrets?.[name],
+    ontology: cfg.ontology,
+    provider: () => provider ??= cfg.provider()
+  };
+}
+
+// src/sources.ts
+function enabledSources(sources) {
+  return sources.filter((s) => s.enabled !== false);
+}
+function toJob(s) {
+  return { kind: s.kind, name: s.name, watch: s.watch ?? null, cfg: s };
+}
+function jobsForHour(sources, hour) {
+  return enabledSources(sources).filter((s) => hour % Math.max(1, Math.floor(s.cadenceHours ?? 1)) === 0).map(toJob);
+}
+function allJobs(sources) {
+  return enabledSources(sources).map(toJob);
+}
+
+// src/state/kv.ts
+var SEEN_TTL_S = 120 * 24 * 60 * 60;
+var ENRICHED_TTL_S = 30 * 24 * 60 * 60;
+var Store = class {
+  constructor(kv) {
+    this.kv = kv;
+  }
+  kv;
+  seenKey(url) {
+    return `seen:${urlHash(url)}`;
+  }
+  async hasSeen(url) {
+    return await this.kv.get(this.seenKey(url)) !== null;
+  }
+  async markSeen(url) {
+    await this.kv.put(this.seenKey(url), "1", { expirationTtl: SEEN_TTL_S });
+  }
+  /** Drop items whose URL we've already processed (checked BEFORE any LLM work). */
+  async filterNew(items) {
+    const flags = await Promise.all(items.map((it) => this.hasSeen(it.url)));
+    return items.filter((_, i) => !flags[i]);
+  }
+  async setLastRun(kind, at = (/* @__PURE__ */ new Date()).toISOString()) {
+    await this.kv.put(`lastrun:${kind}`, at);
+  }
+  async getLastRun(kind) {
+    return this.kv.get(`lastrun:${kind}`);
+  }
+  /** Per-entity enrichment cooldown so we don't re-search the same node constantly. */
+  async isEnrichCooling(key) {
+    return await this.kv.get(`enriched:${key}`) !== null;
+  }
+  async markEnriched(key) {
+    await this.kv.put(`enriched:${key}`, (/* @__PURE__ */ new Date()).toISOString(), { expirationTtl: ENRICHED_TTL_S });
+  }
+  /** Runtime source config (edit without redeploy). null = fall back to the static SOURCES. */
+  async getConfigSources() {
+    const raw = await this.kv.get("config:sources");
+    if (!raw) return null;
+    try {
+      const v = JSON.parse(raw);
+      return Array.isArray(v) ? v : null;
+    } catch {
+      return null;
+    }
+  }
+  async setConfigSources(list) {
+    await this.kv.put("config:sources", JSON.stringify(list));
+  }
+};
+
+// src/workflow/helpers.ts
+async function resilientStep(stepName, fn) {
+  try {
+    return await fn();
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    const stack = err instanceof Error ? err.stack : void 0;
+    console.error(`[STEP_ERROR] ${stepName}: ${msg}`);
+    if (stack) console.error(`[STEP_STACK] ${stepName}: ${stack}`);
+    throw err;
+  }
+}
+var IO_STEP_CONFIG = {
+  retries: { limit: 3, delay: "5 seconds", backoff: "exponential" },
+  timeout: "2 minutes"
+};
+var COMMIT_STEP_CONFIG = {
+  retries: { limit: 3, delay: "3 seconds", backoff: "exponential" },
+  timeout: "90 seconds"
+};
+var LLM_STEP_CONFIG = {
+  retries: { limit: 2, delay: "5 seconds", backoff: "exponential" },
+  timeout: "3 minutes"
+};
+function asStepLike(step) {
+  return step;
+}
+
+// src/pipeline/run-items.ts
+async function ingestFresh(step, deps, fresh, kind, resolve = false) {
+  const { db, provider, ontology, extract, store } = deps;
+  if (fresh.length === 0) {
+    await store.setLastRun(kind);
+    return;
+  }
+  for (const item of fresh) {
+    const id = urlHash(item.url);
+    await step.do(
+      `item-${id}`,
+      LLM_STEP_CONFIG,
+      () => resilientStep(`item-${id}`, async () => {
+        try {
+          let fragment = item.fragment;
+          if (!fragment) {
+            const r = await extract(provider, item);
+            if (!r) {
+              await store.markSeen(item.url);
+              return { skipped: true };
+            }
+            fragment = r.fragment;
+          }
+          if (resolve) fragment = await canonicalizeFragment(db, fragment, ontology);
+          const { txId, ops } = await writeFragment(db, fragment, ontology, `ingest:${id}:v1`);
+          await store.markSeen(item.url);
+          return { txId, ops };
+        } catch (e) {
+          console.log(`ingest item failed (non-blocking): ${item.url}: ${e instanceof Error ? e.message : e}`);
+          return { error: true };
+        }
+      })
+    );
+  }
+  await step.do(
+    `stamp-${kind}`,
+    () => resilientStep(`stamp-${kind}`, async () => {
+      await store.setLastRun(kind);
+      return { ok: true };
+    })
+  );
+}
+export {
+  COMMIT_STEP_CONFIG,
+  IO_STEP_CONFIG,
+  LLM_STEP_CONFIG,
+  MODEL_TIERS,
+  Store,
+  addUsage,
+  allJobs,
+  asStepLike,
+  canonicalizeFragment,
+  connectorIds,
+  createProvider,
+  emptyUsage,
+  enabledSources,
+  extractJson,
+  extractJsonArray,
+  fetchText,
+  fetchUrl,
+  fragmentToOps,
+  getConnector,
+  getProvider,
+  htmlToText,
+  humanAttrsOf,
+  ingestFresh,
+  isLikelyPersonName,
+  jobsForHour,
+  makeContext,
+  mergeAttrs,
+  mergeRulesOf,
+  mergeScalar,
+  naturalKeysOf,
+  normalizeUrl,
+  resilientStep,
+  resolveName,
+  searchEntities,
+  slugify,
+  toSerializedSchema,
+  urlHash,
+  writeFragment
+};
+//# sourceMappingURL=index.js.map