@100xprompt/chitta 0.1.0

package/src/embedded/extract.ts

@@ -0,0 +1,12 @@
+// Knowledge extraction - turns raw text into entity nodes + relationship edges so
+// the store becomes a real knowledge graph, not one opaque record. Deterministic
+// (no LLM, no deps) so it runs offline; swap in an LLM extractor later behind the
+// same `Extraction` shape for higher recall.
+//
+// This file is now a thin facade: the implementations live in ./extractors/* and
+// are re-exported here so existing imports (`import { slugify } from "./extract"`)
+// keep resolving unchanged. Public API is preserved exactly.
+
+export type { ExtractedEntity, ExtractedRelation, QuestionIntent, Extraction, KnowledgeExtractor } from "./extractors/types"
+export { slugify, entityId, ENTITY_PREFIX, cleanLine, isBoilerplate, stripBoilerplate } from "./extractors/text-hygiene"
+export { DeterministicExtractor, extractKnowledge } from "./extractors/deterministic"

package/src/embedded/extractors/code.ts

@@ -0,0 +1,308 @@
+// Code → graph extractor (the Graphify capability, ported TS-native). Parses source
+// with tree-sitter (WASM grammars, no Python, no servers) into the SAME entity/edge
+// shape every other extractor produces - so the moment code nodes land, all the rest
+// (ACL, vector recall, bi-temporal edges, context_relate / path / impact / central)
+// works on them for free. This is what makes us a STRICT SUPERSET of Graphify: code
+// graph + NL graph + permissions + vectors + temporal, in one embedded store.
+//
+// ALL 36 tree-sitter grammars are supported. Rather than hand-list node types per
+// grammar (brittle across 36), we CLASSIFY nodes generically: tree-sitter follows
+// strong conventions (`*_declaration`/`*_definition`/`*_item` for defs,
+// `call_expression`/`*_invocation`/`command` for calls, `import|use|include|...` for
+// imports), with a small OUTLIERS table for the few grammars that don't (Ruby,
+// Elixir, Lua, Elm, …). Static AST is the RIGHT tool here (unlike prose): code has a
+// formal grammar, so extraction is exact. Grammars are an optionalDependency; if they
+// fail to load we degrade to an empty extraction (never crash).
+// Re-exported from code-extractor.ts to preserve original import paths.
+
+import { fileURLToPath } from "node:url"
+import type { Extraction, ExtractedEntity, ExtractedRelation, KnowledgeExtractor } from "./types"
+import { slugify } from "./text-hygiene"
+
+// NB: this module lives one directory deeper than the original code-extractor.ts, so
+// the relative path to node_modules gains one extra `../` to resolve identically.
+const WASM_DIR = fileURLToPath(new URL("../../../node_modules/tree-sitter-wasms/out/", import.meta.url))
+
+// All 36 grammars shipped by tree-sitter-wasms (grammar name → wasm file stem).
+const GRAMMARS = [
+  "bash", "c", "c_sharp", "cpp", "css", "dart", "elisp", "elixir", "elm", "embedded_template",
+  "go", "html", "java", "javascript", "json", "kotlin", "lua", "objc", "ocaml", "php",
+  "python", "ql", "rescript", "ruby", "rust", "scala", "solidity", "swift", "systemrdl",
+  "tlaplus", "toml", "tsx", "typescript", "vue", "yaml", "zig",
+] as const
+type Lang = (typeof GRAMMARS)[number]
+const GRAMMAR_SET = new Set<string>(GRAMMARS)
+
+// File extension → grammar. Covers every supported language.
+const EXT_TO_LANG: Record<string, Lang> = {
+  sh: "bash", bash: "bash", zsh: "bash",
+  c: "c", h: "c",
+  cs: "c_sharp",
+  cpp: "cpp", cc: "cpp", cxx: "cpp", hpp: "cpp", hxx: "cpp", hh: "cpp",
+  css: "css", scss: "css",
+  dart: "dart",
+  el: "elisp", emacs: "elisp",
+  ex: "elixir", exs: "elixir",
+  elm: "elm",
+  erb: "embedded_template", ejs: "embedded_template",
+  go: "go",
+  html: "html", htm: "html",
+  java: "java",
+  js: "javascript", mjs: "javascript", cjs: "javascript", jsx: "javascript",
+  json: "json",
+  kt: "kotlin", kts: "kotlin",
+  lua: "lua",
+  m: "objc", mm: "objc",
+  ml: "ocaml", mli: "ocaml",
+  php: "php",
+  py: "python", pyi: "python",
+  ql: "ql",
+  res: "rescript",
+  rb: "ruby",
+  rs: "rust",
+  scala: "scala", sc: "scala",
+  sol: "solidity",
+  swift: "swift",
+  rdl: "systemrdl",
+  tla: "tlaplus",
+  toml: "toml",
+  tsx: "tsx",
+  ts: "typescript", mts: "typescript", cts: "typescript",
+  vue: "vue",
+  yaml: "yaml", yml: "yaml",
+  zig: "zig",
+}
+
+type Kind = "func" | "class" | "call" | "import"
+
+// Generic, convention-based classification of a tree-sitter node type. Tree-sitter
+// grammars vary (`function_declaration`, `function_item`, `function_definition_statement`,
+// …) so we match a KEYWORD plus a DEF-SUFFIX rather than enumerate every combination.
+const IMPORT_RE = /^(import|include|use|using|require|package|open|with|load)(_|$)/
+const CALL_RE = /(^|_)(call|invocation|command)(_expression|_statement)?$/
+const CLASS_KEY = /(^|_)(class|struct|interface|trait|enum|contract|object|protocol|module|impl|type|record|union|actor|mixin|component)(_|$)/
+const FUNC_KEY = /(^|_)(function|method|func|fn|constructor|subroutine|procedure|def|getter|setter)(_|$)/
+const DEF_SUFFIX = /(declaration|definition|specifier|item|signature|spec|statement|binding)/
+
+// Outliers whose node names carry no DEF-SUFFIX keyword. NB: bare "module" is Ruby's
+// `module M` namespace - but it's also Python's FILE ROOT node type, so we must never
+// classify the root (handled by starting the walk at the root's children).
+const EXTRA_CLASS = new Set(["class", "module", "singleton_class", "object_declaration", "protocol_declaration"])
+const EXTRA_FUNC = new Set(["method", "singleton_method", "let_binding", "value_declaration", "function_declaration_left", "defun", "macro_definition"])
+const EXTRA_CALL = new Set(["function_call", "member_call_expression", "scoped_call_expression", "command_call", "method_call"])
+
+function classify(t: string): Kind | null {
+  if (t === "preproc_include") return "import"
+  if (IMPORT_RE.test(t)) return "import"
+  if (CALL_RE.test(t) || EXTRA_CALL.has(t)) return "call"
+  if ((CLASS_KEY.test(t) && DEF_SUFFIX.test(t)) || EXTRA_CLASS.has(t)) return "class"
+  if ((FUNC_KEY.test(t) && DEF_SUFFIX.test(t)) || EXTRA_FUNC.has(t)) return "func"
+  return null
+}
+
+// Elixir (and similar Lisp-y grammars) express definitions AS macro calls - `def`,
+// `defmodule`, etc. are `call` nodes whose head identifier is the macro name.
+const DEF_MACROS = new Set(["def", "defp", "defmacro", "defmacrop", "defmodule", "defprotocol", "defimpl", "defstruct", "defn", "defun", "defmethod", "defclass"])
+const MODULE_MACROS = new Set(["defmodule", "defprotocol", "defimpl", "defclass"])
+
+const C_EXTRACTED = 1.0 // call resolves to a symbol we defined here
+const C_INFERRED = 0.7 // call to an unknown/external symbol
+
+let parserInit: Promise<void> | null = null
+const langCache = new Map<string, unknown>()
+
+async function loadLang(lang: string): Promise<unknown | null> {
+  try {
+    const TS = (await import("web-tree-sitter")).default as any
+    if (!parserInit) parserInit = TS.init()
+    await parserInit
+    if (!langCache.has(lang)) {
+      if (!GRAMMAR_SET.has(lang)) return null
+      langCache.set(lang, await TS.Language.load(WASM_DIR + `tree-sitter-${lang}.wasm`))
+    }
+    return langCache.get(lang)
+  } catch {
+    return null // grammars unavailable (e.g. compiled binary) → graceful no-op
+  }
+}
+
+export class CodeExtractor implements KnowledgeExtractor {
+  /** All languages this extractor can parse. */
+  static languages(): readonly string[] {
+    return GRAMMARS
+  }
+
+  /** Map a filename to a supported language, or null. */
+  static detectLanguage(name?: string): string | null {
+    if (!name) return null
+    const ext = name.toLowerCase().split(".").pop() ?? ""
+    return EXT_TO_LANG[ext] ?? null
+  }
+
+  /** Implements the generic extractor interface. Uses `meta.language` if given, else
+   *  detects from `meta.name` (the filename). Returns empty for non-code / unknown. */
+  async extract(text: string, meta?: { name?: string; language?: string }): Promise<Extraction> {
+    const lang = meta?.language ?? CodeExtractor.detectLanguage(meta?.name)
+    if (!lang) return { entities: [], relations: [] }
+    return this.extractCode(text, lang, meta?.name ?? "source")
+  }
+
+  /** Parse one source file into code entities + relations. Returns empty (never
+   *  throws) if the grammar can't load or its ABI is incompatible with the runtime. */
+  async extractCode(code: string, lang: string, fileName = "source"): Promise<Extraction> {
+    const Language = await loadLang(lang)
+    if (!Language) return { entities: [], relations: [] }
+    let tree: any
+    try {
+      const TS = (await import("web-tree-sitter")).default as any
+      const parser = new TS()
+      parser.setLanguage(Language) // throws on ABI mismatch → caught → graceful empty
+      tree = parser.parse(code)
+    } catch {
+      return { entities: [], relations: [] }
+    }
+
+    const entities = new Map<string, ExtractedEntity>()
+    const relations = new Map<string, ExtractedRelation>()
+    const defined = new Set<string>() // symbol ids we actually define here
+
+    const fileId = `file:${slugify(fileName)}`
+    entities.set(fileId, { id: fileId, label: fileName, type: "FILE" })
+
+    const symId = (name: string) => `sym:${slugify(name)}`
+    const addEnt = (id: string, label: string, type: string) => {
+      if (!entities.has(id)) entities.set(id, { id, label, type })
+    }
+    const addRel = (from: string, to: string, type: string, confidence: number) => {
+      const key = `${from}|${to}|${type}`
+      if (!relations.has(key)) relations.set(key, { from, to, type, confidence })
+    }
+    // Find a definition's name: prefer the `name` field, else a shallow DFS for the
+    // first identifier-like token (skipping parameter lists), so it works across grammars.
+    const IDENT_RE = /(identifier|name|word|constant|type_identifier|field_identifier)/
+    const nameOf = (node: any): string | null => {
+      const n = node.childForFieldName?.("name")
+      if (n?.text) return n.text
+      const stack: any[] = [node]
+      let depth = 0
+      while (stack.length && depth < 400) {
+        const cur = stack.shift()
+        depth++
+        for (let i = 0; i < cur.namedChildCount; i++) {
+          const c = cur.namedChild(i)
+          if (/parameter|argument|body|block/.test(c.type)) continue
+          if (IDENT_RE.test(c.type)) return c.text
+          stack.push(c)
+        }
+      }
+      return null
+    }
+    const lastSegment = (text: string): string | null => {
+      const parts = text.split(/[.:>-]+/).filter(Boolean)
+      return parts[parts.length - 1] || null
+    }
+    // The first identifier-like token in preorder, skipping argument subtrees - for a
+    // call node that's the callee; for an Elixir def-macro that's the macro name.
+    const firstIdent = (node: any): string | null => {
+      for (let i = 0; i < node.namedChildCount; i++) {
+        const c = node.namedChild(i)
+        if (/argument/.test(c.type)) continue
+        if (IDENT_RE.test(c.type)) return c.text
+        const deep = firstIdent(c)
+        if (deep) return deep
+      }
+      return null
+    }
+    const calleeOf = (node: any): string | null => {
+      const f = node.childForFieldName?.("function") || node.childForFieldName?.("name") || node.childForFieldName?.("method")
+      if (f?.text) return lastSegment(f.text)
+      const id = firstIdent(node)
+      return id ? lastSegment(id) : null
+    }
+    // Elixir: name defined by a def-macro call - in its arguments, the first module
+    // alias or the head identifier of the inner call (e.g. `def greet(n)` → greet).
+    const elixirDefName = (node: any): string | null => {
+      let scope = node
+      for (let i = 0; i < node.namedChildCount; i++) if (node.namedChild(i).type === "arguments") scope = node.namedChild(i)
+      for (let i = 0; i < scope.namedChildCount; i++) {
+        const c = scope.namedChild(i)
+        if (c.type === "alias") return c.text
+        if (c.type === "call") return firstIdent(c)
+        if (IDENT_RE.test(c.type)) return c.text
+      }
+      return null
+    }
+
+    const walk = (node: any, enclosing: string, enclosingClass: string | null) => {
+      let nextEnclosing = enclosing
+      let nextClass = enclosingClass
+      const kind = classify(node.type)
+
+      if (kind === "class") {
+        const nm = nameOf(node)
+        if (nm) {
+          const id = symId(nm)
+          addEnt(id, nm, "CLASS")
+          defined.add(id)
+          addRel(fileId, id, "defines", C_EXTRACTED)
+          nextEnclosing = id
+          nextClass = id
+        }
+      } else if (kind === "func") {
+        const nm = nameOf(node)
+        if (nm) {
+          const id = symId(nm)
+          const type = enclosingClass ? "METHOD" : "FUNCTION"
+          addEnt(id, nm, type)
+          defined.add(id)
+          if (enclosingClass) addRel(enclosingClass, id, "has_method", C_EXTRACTED)
+          else addRel(fileId, id, "defines", C_EXTRACTED)
+          nextEnclosing = id
+        }
+      } else if (kind === "import") {
+        const mod = node.text.replace(/^[^A-Za-z0-9_./@]*(import|from|use|using|require|include|package|open|with|load)\s+/i, "").split(/[\s;(){}]/)[0]?.replace(/["'`<>]/g, "")
+        if (mod && mod.length > 1) {
+          const id = `module:${slugify(mod)}`
+          addEnt(id, mod, "MODULE")
+          addRel(fileId, id, "imports", C_EXTRACTED)
+        }
+      } else if (kind === "call") {
+        // Elixir-style: a call to a def-macro is actually a DEFINITION.
+        const head = firstIdent(node)
+        if (head && DEF_MACROS.has(head)) {
+          const nm = elixirDefName(node)
+          if (nm && nm.length > 1) {
+            const id = symId(nm)
+            const isMod = MODULE_MACROS.has(head)
+            addEnt(id, nm, isMod ? "CLASS" : enclosingClass ? "METHOD" : "FUNCTION")
+            defined.add(id)
+            if (isMod) {
+              addRel(fileId, id, "defines", C_EXTRACTED)
+              nextClass = id
+            } else if (enclosingClass) addRel(enclosingClass, id, "has_method", C_EXTRACTED)
+            else addRel(fileId, id, "defines", C_EXTRACTED)
+            nextEnclosing = id
+          }
+        } else {
+          const callee = calleeOf(node)
+          if (callee && callee.length > 1) {
+            const id = symId(callee)
+            addEnt(id, callee, "FUNCTION")
+            addRel(enclosing, id, "calls", defined.has(id) ? C_EXTRACTED : C_INFERRED)
+          }
+        }
+      }
+
+      for (let i = 0; i < node.namedChildCount; i++) walk(node.namedChild(i), nextEnclosing, nextClass)
+    }
+    // Start at the root's CHILDREN - never classify the root itself (e.g. Python's
+    // root node type is "module", which would otherwise look like a class).
+    for (let i = 0; i < tree.rootNode.namedChildCount; i++) walk(tree.rootNode.namedChild(i), fileId, null)
+
+    // Second pass: a call to a symbol that turned out to be locally defined (possibly
+    // seen before its definition) is promoted to EXTRACTED confidence.
+    for (const r of relations.values()) if (r.type === "calls" && defined.has(r.to)) r.confidence = C_EXTRACTED
+
+    return { entities: [...entities.values()], relations: [...relations.values()] }
+  }
+}

package/src/embedded/extractors/deterministic.ts

@@ -0,0 +1,63 @@
+// Deterministic (no-LLM, no-deps) knowledge extractor. Pulls capitalized phrases
+// and acronyms as entities and relates entities that co-occur in a sentence-ish
+// unit. Runs fully offline. Re-exported from extract.ts to preserve import paths.
+
+import type { Extraction, ExtractedEntity, ExtractedRelation, KnowledgeExtractor } from "./types"
+
+export class DeterministicExtractor implements KnowledgeExtractor {
+  async extract(text: string): Promise<Extraction> {
+    return extractKnowledge(text)
+  }
+}
+
+const STOP = new Set([
+  "The", "A", "An", "This", "That", "These", "Those", "It", "Our", "Your", "Their", "We", "You",
+  "I", "He", "She", "They", "And", "Or", "But", "For", "With", "From", "To", "Of", "In", "On",
+  "At", "By", "As", "Is", "Are", "Was", "Were", "Be", "Will", "Each", "All", "Layer", "Step",
+  "Both", "Also", "Use", "Used", "Using", "Here", "There", "When", "Then", "Now", "Both",
+])
+
+function slug(s: string): string {
+  return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "")
+}
+
+// Capitalized phrases (e.g. "100X Prompt Pro", "Fine-Tuned") and acronyms ("SOC-II", "FP8").
+const PHRASE = /\b(?:[0-9]+[A-Za-z]+|[A-Z][a-z0-9]+|[A-Z]{2,}(?:-[A-Z0-9]+)?)(?:[ -](?:[A-Z][A-Za-z0-9]+|[A-Z]{2,}(?:-[A-Z0-9]+)?|[0-9]+))*\b/g
+
+function candidates(line: string): ExtractedEntity[] {
+  const out: ExtractedEntity[] = []
+  const seen = new Set<string>()
+  for (const m of line.matchAll(PHRASE)) {
+    const label = m[0].trim()
+    if (label.length < 2) continue
+    if (STOP.has(label)) continue
+    // single short common capitalized word at sentence start → skip noise
+    if (!label.includes(" ") && !label.includes("-") && label.length < 3) continue
+    const id = slug(label)
+    if (!id || seen.has(id)) continue
+    seen.add(id)
+    out.push({ id, label, type: /^[A-Z0-9-]+$/.test(label) && label === label.toUpperCase() ? "ACRONYM" : "CONCEPT" })
+  }
+  return out
+}
+
+export function extractKnowledge(text: string): Extraction {
+  const entities = new Map<string, ExtractedEntity>()
+  const relations = new Map<string, ExtractedRelation>()
+
+  // Split into lines / sentence-ish units; entities co-occurring in a unit relate.
+  const units = text.split(/[\n.;]+/).map((u) => u.trim()).filter(Boolean)
+  for (const unit of units) {
+    const ents = candidates(unit)
+    for (const e of ents) if (!entities.has(e.id)) entities.set(e.id, e)
+    // pairwise co-occurrence within the unit (cap to avoid explosion)
+    for (let i = 0; i < ents.length; i++) {
+      for (let j = i + 1; j < Math.min(ents.length, i + 6); j++) {
+        const [a, b] = [ents[i].id, ents[j].id].sort()
+        if (a === b) continue
+        relations.set(`${a}|${b}`, { from: a, to: b, type: "relates_to" })
+      }
+    }
+  }
+  return { entities: [...entities.values()], relations: [...relations.values()] }
+}

package/src/embedded/extractors/llm.ts

@@ -0,0 +1,151 @@
+// LLM-backed knowledge extraction. Calls an OpenAI-compatible chat endpoint -
+// point it at a LOCAL/sovereign model (vLLM/SGLang/Ollama) so nothing leaves the
+// building. Unlike the deterministic extractor, it handles casual lowercase text
+// ("i love lavanya" → Lavanya[PERSON], user -loves→ Lavanya).
+// Re-exported from llm-extractor.ts to preserve original import paths.
+
+import type { Extraction, ExtractedEntity, ExtractedRelation, KnowledgeExtractor, QuestionIntent } from "./types"
+import { slugify } from "./text-hygiene"
+
+export interface LlmExtractorConfig {
+  endpoint: string // OpenAI-compatible base, e.g. http://localhost:8000
+  model: string
+  apiKey?: string
+  fetchImpl?: typeof fetch
+}
+
+// Typed-triple extraction: the "code-like" structure - subject/object carry types,
+// the predicate is a real verb, plus a confidence. (Per research: enhancing the LLM
+// extractor beats REBEL/GLiNER/etc. for our TS stack - typed, no deps, no big models.)
+const SYSTEM = [
+  "You extract knowledge-graph TRIPLES from the user's text.",
+  "Return ONLY a JSON object, no prose:",
+  '{"triples":[{"subject":"<entity>","subjectType":"PERSON|ORG|PLACE|PRODUCT|CONCEPT|OTHER",',
+  '"predicate":"<short verb>","object":"<entity>","objectType":"PERSON|ORG|PLACE|PRODUCT|CONCEPT|OTHER","confidence":0.0-1.0}]}',
+  "Cover people, orgs, places, products, key concepts - INCLUDING casual lowercase text.",
+  'Example: "i love lavanya" → {"subject":"user","subjectType":"PERSON","predicate":"loves","object":"Lavanya","objectType":"PERSON","confidence":0.95}.',
+  "Be concise; only meaningful triples.",
+].join(" ")
+
+const INTENT_SYSTEM = [
+  "Parse the user's QUESTION into a structured intent. Return ONLY JSON:",
+  '{"type":"entity_lookup|relation_query|binary_relation","subject":"<noun or null>","predicate":"<verb or null>","object":"<noun or null>"}',
+  'Examples: "who do I love?" → {"type":"relation_query","subject":"user","predicate":"loves","object":null};',
+  '"does Lavanya work at Acme?" → {"type":"binary_relation","subject":"Lavanya","predicate":"works_at","object":"Acme"};',
+  '"what is Pro?" → {"type":"entity_lookup","subject":null,"predicate":null,"object":"Pro"}.',
+  'Map first-person ("I","me","my") to subject "user".',
+].join(" ")
+
+function parseJsonBlock(s: string): any {
+  try {
+    return JSON.parse(s)
+  } catch {
+    const m = s.match(/\{[\s\S]*\}/) // tolerate code fences / stray text
+    if (m) return JSON.parse(m[0])
+    throw new Error("LLM did not return JSON")
+  }
+}
+
+export class LlmExtractor implements KnowledgeExtractor {
+  private readonly fetch: typeof fetch
+  constructor(private readonly cfg: LlmExtractorConfig) {
+    this.fetch = cfg.fetchImpl ?? fetch
+  }
+
+  private async chat(system: string, user: string): Promise<string> {
+    const res = await this.fetch(`${this.cfg.endpoint.replace(/\/$/, "")}/v1/chat/completions`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        ...(this.cfg.apiKey ? { authorization: `Bearer ${this.cfg.apiKey}` } : {}),
+      },
+      body: JSON.stringify({
+        model: this.cfg.model,
+        temperature: 0,
+        messages: [
+          { role: "system", content: system },
+          { role: "user", content: user },
+        ],
+      }),
+    })
+    const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }
+    return body.choices?.[0]?.message?.content ?? ""
+  }
+
+  async extract(text: string): Promise<Extraction> {
+    const content = await this.chat(SYSTEM, text)
+    if (!content) return { entities: [], relations: [] }
+
+    const parsed = parseJsonBlock(content) as {
+      triples?: Array<{
+        subject?: string
+        subjectType?: string
+        predicate?: string
+        object?: string
+        objectType?: string
+        confidence?: number
+      }>
+    }
+
+    const entities: ExtractedEntity[] = []
+    const seen = new Set<string>()
+    const addEntity = (label?: string, type?: string) => {
+      const l = (label ?? "").trim()
+      if (!l) return
+      const id = slugify(l)
+      if (!id || seen.has(id)) return
+      seen.add(id)
+      entities.push({ id, label: l, type: (type ?? "CONCEPT").toUpperCase() })
+    }
+    for (const t of parsed.triples ?? []) {
+      addEntity(t.subject, t.subjectType)
+      addEntity(t.object, t.objectType)
+    }
+    const ids = new Set(entities.map((e) => e.id))
+    const relations: ExtractedRelation[] = []
+    for (const t of parsed.triples ?? []) {
+      const from = slugify(t.subject ?? "")
+      const to = slugify(t.object ?? "")
+      if (from && to && from !== to && ids.has(from) && ids.has(to)) {
+        relations.push({ from, to, type: (t.predicate ?? "relates_to").toLowerCase().replace(/\s+/g, "_"), confidence: t.confidence })
+      }
+    }
+    return { entities, relations }
+  }
+
+  /** Parse a natural-language QUESTION into a graph-query intent (for KGQA). */
+  async parseQuestionIntent(question: string): Promise<QuestionIntent | null> {
+    const content = await this.chat(INTENT_SYSTEM, question)
+    if (!content) return null
+    try {
+      const p = parseJsonBlock(content) as QuestionIntent
+      if (!p?.type) return null
+      return p
+    } catch {
+      return null
+    }
+  }
+}
+
+/** Run two extractors and merge (dedupe by entity id / relation pair). Used to
+ *  combine deterministic + LLM for best recall. */
+export class HybridExtractor implements KnowledgeExtractor {
+  constructor(
+    private readonly a: KnowledgeExtractor,
+    private readonly b: KnowledgeExtractor,
+  ) {}
+  async extract(text: string): Promise<Extraction> {
+    const results = await Promise.allSettled([this.a.extract(text), this.b.extract(text)])
+    const entities = new Map<string, ExtractedEntity>()
+    const relations = new Map<string, ExtractedRelation>()
+    for (const r of results) {
+      if (r.status !== "fulfilled") continue
+      for (const e of r.value.entities) if (!entities.has(e.id)) entities.set(e.id, e)
+      for (const rel of r.value.relations) {
+        const key = [rel.from, rel.to].sort().join("|")
+        if (!relations.has(key)) relations.set(key, rel)
+      }
+    }
+    return { entities: [...entities.values()], relations: [...relations.values()] }
+  }
+}

package/src/embedded/extractors/text-hygiene.ts

@@ -0,0 +1,54 @@
+// Text-hygiene utilities: slug/clean-line normalization and web-boilerplate
+// stripping. Used by the extractors and (via re-export from extract.ts) across
+// the codebase. Re-exported from extract.ts to preserve original import paths.
+
+export function slugify(s: string): string {
+  return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "")
+}
+
+// Knowledge-graph entities share the `nodes` table with principals (users/orgs/groups)
+// and records. Their ids are slugs of free text, so without isolation an ingested
+// document that merely MENTIONS a word matching a principal id would overwrite that
+// principal's node (INSERT OR REPLACE) and silently corrupt the ACL graph. Entity ids
+// live in their own namespace so the collision is impossible - and every writer AND
+// resolver must agree on the scheme, so it is defined ONCE here.
+export const ENTITY_PREFIX = "entity:"
+export const entityId = (slug: string): string => ENTITY_PREFIX + slug
+
+// Strip markdown emphasis / heading / blockquote / bullet noise from a line of
+// (often scraped) text, so stored + returned snippets are clean prose - no `**`,
+// `#`, `>` or leftover bullet markers leaking into answers.
+export function cleanLine(s: string): string {
+  return s
+    .replace(/\*\*|__|`+/g, "") // bold / italic / code markers
+    .replace(/^\s*#{1,6}\s+/, "") // heading hashes
+    .replace(/^\s*>+\s*/, "") // blockquote
+    .replace(/^[-*•▪◦\d.)\s]+/, "") // leading bullets / numbering
+    .replace(/\s+/g, " ")
+    .trim()
+}
+
+// Web boilerplate (cookie banners, nav menus, subscribe CTAs) that pollutes scraped
+// pages and would otherwise become junk entities / noisy chunks. Strong multi-word
+// phrases always drop; short EXACT nav tokens drop; real sentences are preserved.
+const STRONG_BOILER =
+  /(manage cookie consent|cookie consent|consent banner|cookie policy|privacy policy|gdpr.?compliant|opt.?out|skip to content|view preferences|subscribe now|consenting to these technologies|withdrawing consent|adversely affect certain|join our community|all our premium content|delivered straight to your inbox|post a press release|reach our audience|editorial opportunities)/i
+const NAV_TOKENS =
+  /^(accept|deny|subscribe|search|search\.\.\.|view preferences|view all|view all latest|see all|click here|sign in|log in|menu|home|contact us|newsletter|categories|events|resources|more|explore|explore all|explore more|applications|industries|news|search\b)$/i
+
+export function isBoilerplate(line: string): boolean {
+  const t = line.trim()
+  if (!t) return true
+  if (STRONG_BOILER.test(t)) return true
+  if (NAV_TOKENS.test(t)) return true
+  return false
+}
+
+/** Drop boilerplate lines from a block of (scraped) text before chunking/extraction. */
+export function stripBoilerplate(text: string): string {
+  return text
+    .split(/\r?\n/)
+    .filter((l) => !isBoilerplate(l))
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n")
+}

package/src/embedded/extractors/types.ts

@@ -0,0 +1,34 @@
+// Shared knowledge-extraction types. Re-exported from extract.ts to preserve the
+// original import paths (`import { ExtractedEntity } from "../extract"`).
+
+export interface ExtractedEntity {
+  id: string // normalized slug (stable across docs → same concept merges)
+  label: string // surface form as first seen
+  type: string // ACRONYM | CONCEPT | PERSON | ORG | PLACE | PRODUCT | …
+}
+export interface ExtractedRelation {
+  from: string // entity id
+  to: string // entity id
+  type: string // relates_to, or a verb from the LLM (loves, builds, …)
+  confidence?: number // 0..1 (LLM-provided); deterministic omits it
+}
+
+/** Parsed intent of a question, for graph QA. */
+export interface QuestionIntent {
+  type: "entity_lookup" | "relation_query" | "binary_relation"
+  subject?: string
+  predicate?: string
+  object?: string
+}
+export interface Extraction {
+  entities: ExtractedEntity[]
+  relations: ExtractedRelation[]
+}
+
+/** Pluggable extractor - deterministic (offline), LLM-backed (casual prose), or
+ *  tree-sitter (code). Same Extraction shape, so ingest/rebuild don't care which.
+ *  `meta` is an optional hint (filename / language) used by the code extractor;
+ *  text extractors ignore it. */
+export interface KnowledgeExtractor {
+  extract(text: string, meta?: { name?: string; language?: string }): Promise<Extraction>
+}