@lucern/contracts 0.1.0 → 0.1.1-alpha.1

package/src/text-matching.contract.ts

@@ -0,0 +1,347 @@
+/**
+ * Shared lexical matching primitives used across MCP handlers and graph utilities.
+ *
+ * The goal is not to replace downstream LLM scoring. It provides a fast,
+ * deterministic substrate for candidate generation, reranking, and light
+ * classification across belief/question/evidence/entity surfaces.
+ */
+
+export type LexicalStrategy = "tokenOverlap" | "bigramJaccard" | "wordOverlap";
+
+export type PreparedLexicalQuery = {
+  raw: string;
+  tokens: string[];
+  words: string[];
+  bigrams: Set<string>;
+};
+
+export type LexicalSignal = {
+  strategy?: LexicalStrategy;
+  text: string | null | undefined;
+  weight: number;
+};
+
+export type LexicalRerankOptions = {
+  lexicalWeight?: number;
+  rankWeight?: number;
+};
+
+const TOKEN_SPLIT_REGEX = /[^a-z0-9]+/;
+const NON_ALPHANUMERIC_REGEX = /[^a-z0-9]/g;
+
+/** Stop words that add noise to scoring. */
+const STOP_WORDS = new Set([
+  "the",
+  "a",
+  "an",
+  "and",
+  "or",
+  "but",
+  "in",
+  "on",
+  "at",
+  "to",
+  "for",
+  "of",
+  "with",
+  "by",
+  "from",
+  "is",
+  "it",
+  "as",
+  "be",
+  "was",
+  "are",
+  "this",
+  "that",
+  "has",
+  "had",
+  "have",
+  "not",
+  "all",
+  "can",
+  "do",
+  "its",
+  "may",
+  "will",
+  "how",
+  "what",
+  "which",
+  "who",
+  "when",
+  "where",
+  "than",
+  "then",
+  "each",
+  "into",
+  "such",
+  "any",
+  "been",
+  "if",
+  "would",
+  "about",
+  "should",
+  "these",
+  "those",
+  "their",
+  "we",
+  "our",
+  "so",
+]);
+
+/** Tokenize a string into lowercase words, removing stop words. */
+export function tokenizeSearchText(text: string): string[] {
+  return text
+    .toLowerCase()
+    .split(TOKEN_SPLIT_REGEX)
+    .filter((token) => token.length >= 2 && !STOP_WORDS.has(token));
+}
+
+/** Simple stemmer: strip common English suffixes for fuzzy matching. */
+export function stemToken(word: string): string {
+  if (word.length <= 4) {
+    return word;
+  }
+  if (word.endsWith("ation")) {
+    return word.slice(0, -5);
+  }
+  if (word.endsWith("ment")) {
+    return word.slice(0, -4);
+  }
+  if (word.endsWith("ness")) {
+    return word.slice(0, -4);
+  }
+  if (word.endsWith("ical")) {
+    return word.slice(0, -4);
+  }
+  if (word.endsWith("tion")) {
+    return word.slice(0, -4);
+  }
+  if (word.endsWith("sion")) {
+    return word.slice(0, -4);
+  }
+  if (word.endsWith("ing")) {
+    return word.slice(0, -3);
+  }
+  if (word.endsWith("ous")) {
+    return word.slice(0, -3);
+  }
+  if (word.endsWith("ive")) {
+    return word.slice(0, -3);
+  }
+  if (word.endsWith("ity")) {
+    return word.slice(0, -3);
+  }
+  if (word.endsWith("ics")) {
+    return word.slice(0, -3);
+  }
+  if (word.endsWith("ly")) {
+    return word.slice(0, -2);
+  }
+  if (word.endsWith("ed")) {
+    return word.slice(0, -2);
+  }
+  if (word.endsWith("er")) {
+    return word.slice(0, -2);
+  }
+  if (word.endsWith("es")) {
+    return word.slice(0, -2);
+  }
+  if (word.endsWith("al")) {
+    return word.slice(0, -2);
+  }
+  if (word.endsWith("ic")) {
+    return word.slice(0, -2);
+  }
+  if (word.endsWith("s") && !word.endsWith("ss")) {
+    return word.slice(0, -1);
+  }
+  return word;
+}
+
+/** Compute token overlap score between query tokens and text tokens. */
+export function tokenOverlapScore(
+  queryTokens: string[],
+  textTokens: string[]
+): number {
+  if (queryTokens.length === 0 || textTokens.length === 0) {
+    return 0;
+  }
+
+  const stemmedText = new Set(textTokens.map(stemToken));
+  let matchCount = 0;
+
+  for (const queryToken of queryTokens) {
+    const stemmedQuery = stemToken(queryToken);
+
+    if (stemmedText.has(stemmedQuery)) {
+      matchCount += 1;
+      continue;
+    }
+
+    for (const textToken of stemmedText) {
+      if (
+        textToken.startsWith(stemmedQuery) ||
+        stemmedQuery.startsWith(textToken)
+      ) {
+        matchCount += 0.5;
+        break;
+      }
+    }
+  }
+
+  return matchCount / queryTokens.length;
+}
+
+/**
+ * Extract character bigrams from text. Normalizes to lowercase, removes
+ * non-alphanumeric characters, and generates overlapping pairs.
+ */
+export function bigramTokenize(text: string): Set<string> {
+  const normalized = text.toLowerCase().replace(NON_ALPHANUMERIC_REGEX, "");
+  const bigrams = new Set<string>();
+  for (let i = 0; i < normalized.length - 1; i++) {
+    bigrams.add(normalized.slice(i, i + 2));
+  }
+  return bigrams;
+}
+
+/**
+ * Extract word-level tokens from text (for coarser matching).
+ * Normalizes to lowercase, splits on non-alphanumeric.
+ */
+export function wordTokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .split(TOKEN_SPLIT_REGEX)
+    .filter((token) => token.length > 1);
+}
+
+/** Jaccard similarity between two sets: |A ∩ B| / |A ∪ B|. */
+export function jaccardSimilarity(
+  setA: Set<string>,
+  setB: Set<string>
+): number {
+  if (setA.size === 0 && setB.size === 0) {
+    return 0;
+  }
+
+  let intersectionSize = 0;
+  const smaller = setA.size <= setB.size ? setA : setB;
+  const larger = setA.size <= setB.size ? setB : setA;
+
+  for (const item of smaller) {
+    if (larger.has(item)) {
+      intersectionSize++;
+    }
+  }
+
+  const unionSize = setA.size + setB.size - intersectionSize;
+  return unionSize === 0 ? 0 : intersectionSize / unionSize;
+}
+
+/** Exact word overlap score: fraction of type words found in input text. */
+export function wordOverlapScore(
+  inputWords: string[],
+  typeWords: string[]
+): number {
+  if (typeWords.length === 0) {
+    return 0;
+  }
+  let matches = 0;
+  for (const word of typeWords) {
+    if (inputWords.includes(word)) {
+      matches++;
+    }
+  }
+  return matches / typeWords.length;
+}
+
+/** Pre-compute reusable lexical structures for a query. */
+export function prepareLexicalQuery(query: string): PreparedLexicalQuery {
+  return {
+    raw: query,
+    tokens: tokenizeSearchText(query),
+    words: wordTokenize(query),
+    bigrams: bigramTokenize(query),
+  };
+}
+
+/** Score a single lexical signal against a prepared query. */
+export function scoreLexicalSignal(
+  query: PreparedLexicalQuery,
+  signal: LexicalSignal
+): number {
+  const text = signal.text?.trim();
+  if (!text) {
+    return 0;
+  }
+
+  switch (signal.strategy ?? "tokenOverlap") {
+    case "bigramJaccard":
+      return jaccardSimilarity(query.bigrams, bigramTokenize(text));
+    case "wordOverlap":
+      return wordOverlapScore(query.words, wordTokenize(text));
+    default:
+      return tokenOverlapScore(query.tokens, tokenizeSearchText(text));
+  }
+}
+
+/** Weighted lexical score across multiple textual signals. */
+export function scoreLexicalSignals(
+  query: PreparedLexicalQuery,
+  signals: LexicalSignal[]
+): number {
+  let weightedScore = 0;
+  let totalWeight = 0;
+
+  for (const signal of signals) {
+    if (!signal.text?.trim() || signal.weight <= 0) {
+      continue;
+    }
+    weightedScore += scoreLexicalSignal(query, signal) * signal.weight;
+    totalWeight += signal.weight;
+  }
+
+  return totalWeight === 0 ? 0 : weightedScore / totalWeight;
+}
+
+/** Map a candidate's original rank position into a 0..1 prior. */
+export function rankWindowScore(index: number, total: number): number {
+  if (total <= 1) {
+    return 1;
+  }
+  const clampedIndex = Math.max(0, Math.min(index, total - 1));
+  return 1 - clampedIndex / (total - 1);
+}
+
+/** Rerank a candidate window by lexical overlap while preserving original-rank prior. */
+export function rerankLexicalWindow<T>(
+  query: string,
+  items: T[],
+  getText: (item: T) => string | null | undefined,
+  options?: LexicalRerankOptions
+): T[] {
+  const preparedQuery = prepareLexicalQuery(query);
+  if (preparedQuery.tokens.length === 0 || items.length <= 1) {
+    return items;
+  }
+
+  const lexicalWeight = options?.lexicalWeight ?? 0.65;
+  const rankWeight = options?.rankWeight ?? 0.35;
+
+  return items
+    .map((item, index) => {
+      const lexicalScore = scoreLexicalSignals(preparedQuery, [
+        { text: getText(item) ?? "", weight: 1, strategy: "tokenOverlap" },
+      ]);
+      const rankScore = rankWindowScore(index, items.length);
+
+      return {
+        item,
+        combinedScore: lexicalScore * lexicalWeight + rankScore * rankWeight,
+      };
+    })
+    .sort((left, right) => right.combinedScore - left.combinedScore)
+    .map(({ item }) => item);
+}

package/src/topic-scope.contract.ts

@@ -0,0 +1,9 @@
+/**
+ * @lucern/contracts — topic-scope compat shim
+ *
+ * This file moved to ./v1/topics/v1.ts during EK-16 T1 PR 2.
+ * Retained here until the Lucern 1.0.0 barrel-sunset cut (D12).
+ * New code should import from "@lucern/contracts/v1/topics" directly.
+ */
+
+export * from "./v1/topics/v1";

package/src/topics/v1.ts

@@ -0,0 +1,8 @@
+/**
+ * Topics API Contract v1
+ *
+ * Defines the public surface for topic operations:
+ * create, update, tree traversal, coverage analysis.
+ * Migrates from front-end/lucern/contracts/ during EK-16.
+ */
+export {};

package/src/v1/agents/v1.ts

@@ -0,0 +1,8 @@
+/**
+ * @lucern/contracts — Agents v1 namespace
+ *
+ * Placeholder. Resource contracts land in EK-16 T1 PR 2 (two-taxonomy migration).
+ * Do not add symbols here — they must move from the flat contract files
+ * during the PR 2 migration.
+ */
+export {};

package/src/v1/beliefs/v1.ts

@@ -0,0 +1,8 @@
+/**
+ * @lucern/contracts — Beliefs v1 namespace
+ *
+ * Placeholder. Resource contracts land in EK-16 T1 PR 2 (two-taxonomy migration).
+ * Do not add symbols here — they must move from the flat contract files
+ * during the PR 2 migration.
+ */
+export {};

package/src/v1/evidence/v1.ts

@@ -0,0 +1,8 @@
+/**
+ * @lucern/contracts — Evidence v1 namespace
+ *
+ * Placeholder. Resource contracts land in EK-16 T1 PR 2 (two-taxonomy migration).
+ * Do not add symbols here — they must move from the flat contract files
+ * during the PR 2 migration.
+ */
+export {};

package/src/v1/graph/v1.ts

@@ -0,0 +1,8 @@
+/**
+ * @lucern/contracts — Graph v1 namespace
+ *
+ * Placeholder. Resource contracts land in EK-16 T1 PR 2 (two-taxonomy migration).
+ * Do not add symbols here — they must move from the flat contract files
+ * during the PR 2 migration.
+ */
+export {};

package/src/v1/ontologies/v1.ts

@@ -0,0 +1,276 @@
+/**
+ * @lucern/contracts — OntologiesV1 namespace (resource contracts)
+ *
+ * Ontology Matching Engine — L0 entity type classification and similarity scoring.
+ * Provides bigram-based text similarity for matching free text against
+ * ontology entity types. Domain-agnostic: works identically for companies,
+ * molecules, code modules, or any tenant-defined entity vocabulary.
+ *
+ * Moved from src/ontology-matching.contract.ts in EK-16 T1 PR 2.
+ * Compat shim remains at the old path until the Lucern 1.0.0 cut.
+ */
+
+import {
+  bigramTokenize,
+  jaccardSimilarity,
+  prepareLexicalQuery,
+  scoreLexicalSignals,
+  wordOverlapScore,
+  wordTokenize,
+} from "../../text-matching.contract";
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+/** An entity type definition from a resolved ontology version. */
+export type OntologyEntityType = {
+  value: string;
+  label: string;
+  description?: string;
+  subtypes?: Array<{ value: string; label: string; description?: string }>;
+};
+
+/** A scored match between input text and an entity type. */
+export type EntityTypeMatch = {
+  entityType: string;
+  label: string;
+  score: number;
+  reason: string;
+};
+
+/** A candidate entity node that can be matched against a target node. */
+export type EntityMatchCandidate = {
+  nodeId: string;
+  entityType: string;
+  title: string;
+  canonicalText: string;
+  connectedBeliefCount: number;
+  connectedEvidenceCount: number;
+};
+
+/** A scored entity match with suggested bridge edge type. */
+export type EntityConnectionMatch = {
+  entityNodeId: string;
+  entityType: string;
+  title: string;
+  score: number;
+  suggestedEdgeType: string;
+  reason: string;
+};
+
+// =============================================================================
+// ENTITY TYPE SCORING
+// =============================================================================
+
+/** Weights for combining scoring signals. */
+const MATCH_WEIGHTS = {
+  tokenOverlap: 0.35,
+  bigramSimilarity: 0.25,
+  wordOverlap: 0.2,
+  descriptionBonus: 0.2,
+} as const;
+
+/**
+ * Score how well input text matches a single entity type definition.
+ * Combines bigram Jaccard similarity, word overlap, and description matching.
+ */
+export function scoreEntityTypeMatch(
+  inputText: string,
+  entityType: OntologyEntityType
+): EntityTypeMatch {
+  const preparedQuery = prepareLexicalQuery(inputText);
+  const labelText = `${entityType.label} ${entityType.value}`;
+
+  const tokenScore = scoreLexicalSignals(preparedQuery, [
+    { text: labelText, weight: 1, strategy: "tokenOverlap" },
+  ]);
+
+  // Score against the type label (primary signal)
+  const labelBigrams = bigramTokenize(entityType.label);
+  const bigramScore = jaccardSimilarity(preparedQuery.bigrams, labelBigrams);
+
+  // Word overlap with label + value
+  const labelWords = wordTokenize(labelText);
+  const wordScore = wordOverlapScore(preparedQuery.words, labelWords);
+
+  // Description matching bonus (if description exists)
+  let descScore = 0;
+  if (entityType.description) {
+    descScore = scoreLexicalSignals(preparedQuery, [
+      { text: entityType.description, weight: 1, strategy: "tokenOverlap" },
+    ]);
+  }
+
+  // Check subtypes for additional matches
+  let subtypeBonus = 0;
+  if (entityType.subtypes && entityType.subtypes.length > 0) {
+    for (const subtype of entityType.subtypes) {
+      const subtypeScore = scoreLexicalSignals(preparedQuery, [
+        {
+          text: `${subtype.label} ${subtype.value} ${subtype.description || ""}`,
+          weight: 1,
+          strategy: "tokenOverlap",
+        },
+      ]);
+      subtypeBonus = Math.max(subtypeBonus, subtypeScore * 0.3); // up to 30% subtype bonus
+    }
+  }
+
+  const score = Math.min(
+    1.0,
+    tokenScore * MATCH_WEIGHTS.tokenOverlap +
+      bigramScore * MATCH_WEIGHTS.bigramSimilarity +
+      wordScore * MATCH_WEIGHTS.wordOverlap +
+      descScore * MATCH_WEIGHTS.descriptionBonus +
+      subtypeBonus
+  );
+
+  // Generate human-readable reason
+  const reasons: string[] = [];
+  if (tokenScore > 0.3) {
+    reasons.push(`stem match: ${(tokenScore * 100).toFixed(0)}%`);
+  }
+  if (bigramScore > 0.3) {
+    reasons.push(`text similarity: ${(bigramScore * 100).toFixed(0)}%`);
+  }
+  if (wordScore > 0.3) {
+    reasons.push(`word match: ${(wordScore * 100).toFixed(0)}%`);
+  }
+  if (descScore > 0.2) {
+    reasons.push("description match");
+  }
+  if (subtypeBonus > 0.05) {
+    reasons.push("subtype match");
+  }
+  const reason = reasons.length > 0 ? reasons.join(", ") : "low similarity";
+
+  return {
+    entityType: entityType.value,
+    label: entityType.label,
+    score,
+    reason,
+  };
+}
+
+/**
+ * Rank all entity types in an ontology against input text.
+ * Returns matches sorted by score (descending), filtered to score > minScore.
+ */
+export function rankEntityTypeMatches(
+  inputText: string,
+  entityTypes: OntologyEntityType[],
+  options?: { minScore?: number; limit?: number }
+): EntityTypeMatch[] {
+  const minScore = options?.minScore ?? 0.05;
+  const limit = options?.limit ?? 10;
+
+  const matches = entityTypes
+    .map((et) => scoreEntityTypeMatch(inputText, et))
+    .filter((m) => m.score >= minScore)
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit);
+
+  return matches;
+}
+
+// =============================================================================
+// ENTITY DISCOVERY SCORING
+// =============================================================================
+
+/**
+ * Score how well a node's text matches an entity candidate.
+ * Used by discover_entity_connections to suggest missing bridge edges.
+ */
+export function scoreEntityConnection(
+  nodeText: string,
+  candidate: EntityMatchCandidate,
+  options?: { connectivityWeight?: number }
+): EntityConnectionMatch {
+  const preparedQuery = prepareLexicalQuery(nodeText);
+  const connectivityWeight = options?.connectivityWeight ?? 0.3;
+  const textWeight = 1.0 - connectivityWeight;
+  const candidateText = `${candidate.title} ${candidate.canonicalText}`;
+  const tokenScore = scoreLexicalSignals(preparedQuery, [
+    { text: candidateText, weight: 1, strategy: "tokenOverlap" },
+  ]);
+  const textScore = scoreLexicalSignals(preparedQuery, [
+    { text: candidateText, weight: 1, strategy: "bigramJaccard" },
+  ]);
+  const wordScore = scoreLexicalSignals(preparedQuery, [
+    { text: candidateText, weight: 1, strategy: "wordOverlap" },
+  ]);
+
+  // Connectivity signal (normalized externally by caller)
+  const maxConnections = Math.max(
+    1,
+    candidate.connectedBeliefCount + candidate.connectedEvidenceCount
+  );
+  const connectivityScore = Math.min(1.0, maxConnections / 10); // soft-cap at 10
+
+  const combinedTextScore =
+    tokenScore * 0.45 + textScore * 0.35 + wordScore * 0.2;
+  const score =
+    combinedTextScore * textWeight + connectivityScore * connectivityWeight;
+
+  // Suggest edge type based on entity type
+  const suggestedEdgeType = suggestEdgeType(candidate.entityType);
+
+  const reason =
+    tokenScore > 0.3
+      ? `stem match: ${(tokenScore * 100).toFixed(0)}%`
+      : textScore > 0.2
+        ? `name similarity: ${(textScore * 100).toFixed(0)}%`
+        : wordScore > 0.2
+          ? `keyword match: ${(wordScore * 100).toFixed(0)}%`
+          : `connectivity: ${candidate.connectedBeliefCount} beliefs`;
+
+  return {
+    entityNodeId: candidate.nodeId,
+    entityType: candidate.entityType,
+    title: candidate.title,
+    score,
+    suggestedEdgeType,
+    reason,
+  };
+}
+
+/**
+ * Suggest the most appropriate bridge edge type for an entity type.
+ */
+function suggestEdgeType(entityType: string): string {
+  switch (entityType) {
+    case "company":
+    case "person":
+    case "investor":
+      return "contains";
+    case "function":
+    case "value_chain":
+      return "impacts";
+    default:
+      return "contains";
+  }
+}
+
+/**
+ * Rank entity candidates against a node's text.
+ * Returns sorted matches above the minimum score threshold.
+ */
+export function rankEntityConnections(
+  nodeText: string,
+  candidates: EntityMatchCandidate[],
+  options?: { minScore?: number; limit?: number; connectivityWeight?: number }
+): EntityConnectionMatch[] {
+  const minScore = options?.minScore ?? 0.05;
+  const limit = options?.limit ?? 10;
+
+  return candidates
+    .map((c) =>
+      scoreEntityConnection(nodeText, c, {
+        connectivityWeight: options?.connectivityWeight,
+      })
+    )
+    .filter((m) => m.score >= minScore)
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit);
+}

package/src/v1/questions/v1.ts

@@ -0,0 +1,8 @@
+/**
+ * @lucern/contracts — Questions v1 namespace
+ *
+ * Placeholder. Resource contracts land in EK-16 T1 PR 2 (two-taxonomy migration).
+ * Do not add symbols here — they must move from the flat contract files
+ * during the PR 2 migration.
+ */
+export {};