@startsimpli/integrations 0.1.5 → 0.2.1

package/src/corpus/index.ts

@@ -9,6 +9,19 @@ export type {
   AnswerDraft,
   AnswerRequest,
   AskMode,
+  Commitment,
+  CommitmentApi,
+  CommitmentDraft,
+  CommitmentMention,
+  CommitmentQuery,
+  CommitmentStatus,
+  CommitmentSubtype,
+  CommitmentTrackerApi,
+  CorpusExtractorLlm,
+  ExtractionDraft,
+  ExtractionRequest,
+  ExtractionResult,
+  ResolutionClaim,
   AskRequest,
   AskResult,
   AskStatus,
@@ -32,8 +45,13 @@ export type {
 } from './types';
 
 export {
+  DEFAULT_PARTICIPANT_EDGES,
   DEFAULT_TYPE_KEYS,
+  indexParticipants,
+  commitmentToEntity,
+  mergeParticipants,
   moneyEventAsDocument,
+  toCommitment,
   scopeOf,
   toMoneyEvent,
   toParty,
@@ -41,7 +59,7 @@ export {
   toSections,
   toSourceDocument,
 } from './entities';
-export type { CorpusTypeKeys, TenantEntity } from './entities';
+export type { CorpusTypeKeys, TenantEntity, TenantRelationship } from './entities';
 
 export {
   enforceScope,
@@ -63,17 +81,55 @@ export {
 } from './ask';
 export type { AskCorpusOptions } from './ask';
 
+// Extraction — conversations in, trackable commitments out (bd startsim-768w.38.25).
+export {
+  EXTRACTOR_VERSION,
+  extractCommitments,
+  extractSystemPrompt,
+  renderExtractPrompt,
+  rulesExtract,
+} from './extract';
+export type { ExtractCommitmentsOptions, ExtractDeps, ExtractRequest } from './extract';
+
+export {
+  applyResolutions,
+  commitmentIdentity,
+  isSameCommitment,
+  mergeDrafts,
+  reconcile,
+} from './commitments';
+export type {
+  CommitmentIdentity,
+  MergedDraft,
+  ReconcileContext,
+  ReconcileResult,
+  ResolutionResult,
+} from './commitments';
+
+export { jaccard, outcomeSections, splitSections, stem, words } from './text';
+export type { TextSection } from './text';
+
 export { createCorpusApi } from './client';
 export type { CorpusHttp, CreateCorpusApiOptions } from './client';
 
 export {
   createFixtureCorpusApi,
+  createFixtureCorpusHttp,
+  createFixtureExtractor,
   createFixtureLlm,
   FIXTURE_DOCUMENTS,
   FIXTURE_MONEY_EVENTS,
   FIXTURE_ORG,
   FIXTURE_OTHER_ORG,
+  FIXTURE_PARTICIPANTS,
   FIXTURE_PARTIES,
+  FIXTURE_SOURCE_LABELS,
   FIXTURE_SOURCES,
+  FIXTURE_TENANT_DOCUMENTS,
+  FIXTURE_TENANT_MONEY_EVENTS,
+  FIXTURE_TENANT_PARTIES,
+  FIXTURE_TENANT_RELATIONSHIPS,
+  FIXTURE_TENANT_TRACKED,
+  FIXTURE_TRACKED_DOCUMENTS,
 } from './fixtures';
 export type { FixtureCorpusOptions } from './fixtures';

package/src/corpus/llm.ts

@@ -18,7 +18,16 @@
 import { extractJson, LLMService } from '@startsimpli/llm';
 
 import { answerSystemPrompt, renderAnswerPrompt } from './ask';
-import type { AnswerDraft, AnswerRequest, CorpusLlm, LlmDescription } from './types';
+import { extractSystemPrompt, renderExtractPrompt } from './extract';
+import type {
+  AnswerDraft,
+  AnswerRequest,
+  CorpusExtractorLlm,
+  CorpusLlm,
+  ExtractionDraft,
+  ExtractionRequest,
+  LlmDescription,
+} from './types';
 
 export interface CorpusLlmOptions {
   /** Reuse an already-configured service; otherwise one is constructed. */
@@ -27,6 +36,26 @@ export interface CorpusLlmOptions {
   model?: string;
   temperature?: number;
   maxTokens?: number;
+  /** Injectable environment, so the configured-model lookup is testable. */
+  env?: Record<string, string | undefined>;
+}
+
+/**
+ * The model this deployment CONFIGURED for a provider, from its own environment.
+ *
+ * Only used until a real response reports its own model — see `describe()`.
+ * Provider-specific first (`OPENAI_MODEL`), then the provider-agnostic
+ * `LLM_MODEL`, because a tenant that sets both means the specific one.
+ */
+export function configuredModel(
+  provider: string,
+  env: Record<string, string | undefined> = typeof process !== 'undefined' ? process.env : {},
+): string {
+  const specific: Record<string, string | undefined> = {
+    openai: env.OPENAI_MODEL,
+    anthropic: env.ANTHROPIC_MODEL,
+  };
+  return (specific[provider] || env.LLM_MODEL || '').trim();
 }
 
 /**
@@ -139,13 +168,19 @@ export function createCorpusLlm(options: CorpusLlmOptions = {}): CorpusLlm {
     });
 
   let lastModel = options.model ?? '';
+  const env = options.env ?? (typeof process !== 'undefined' ? process.env : {});
 
   const describe = (): LlmDescription => {
     const available = service.getAvailableProviders();
     const provider = available[0] ?? 'none';
+    // WHAT ANSWERED beats WHAT WAS CONFIGURED, always — but before the first
+    // call there is no "what answered", and `(provider default)` is a string
+    // dressed up as a fact. The tenant's own OPENAI_MODEL is the honest answer
+    // to "which model will run", so it is named until a response overrides it
+    // (bd startsim-768w.38.21).
     return {
       provider,
-      model: lastModel || options.model || '(provider default)',
+      model: lastModel || options.model || configuredModel(provider, env) || '',
       live: provider !== 'none' && provider !== 'mock',
     };
   };
@@ -195,3 +230,100 @@ export function createCorpusLlm(options: CorpusLlmOptions = {}): CorpusLlm {
     },
   };
 }
+
+// ---------------------------------------------------------------------------
+//  extraction (bd startsim-768w.38.25)
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse + shape-check the extractor's JSON. Anything unusable returns null so
+ * the caller can retry once, and after that the run keeps whatever the
+ * deterministic rules pass produced rather than failing outright.
+ */
+export function parseExtractionDraft(raw: string): ExtractionDraft | null {
+  const extraction = extractJson(raw);
+  if (!extraction.success || !extraction.json || typeof extraction.json !== 'object') return null;
+  const json = extraction.json as Record<string, unknown>;
+  const items = Array.isArray(json.items) ? json.items : null;
+  if (!items) return null;
+  // Shape-checking proper happens in `extract.ts`, against the set of document
+  // ids the run actually supplied — this only rejects entries that are not
+  // objects at all, so a single malformed row cannot discard the whole list.
+  const objects = (values: unknown[]) => values.filter((item) => Boolean(item) && typeof item === 'object');
+  return {
+    items: objects(items) as ExtractionDraft['items'],
+    resolutions: objects(Array.isArray(json.resolutions) ? json.resolutions : []) as ExtractionDraft['resolutions'],
+  };
+}
+
+/**
+ * A `CorpusExtractorLlm` backed by `@startsimpli/llm`.
+ *
+ * ONE CALL, N DOCUMENTS, exactly like `createCorpusLlm` — the batching rule is
+ * structural, and the only second request that can ever happen is a JSON-repair
+ * retry of the same request.
+ *
+ * TEMPERATURE 0. This is transcription of things people said they would do, not
+ * writing. A model that improvises here invents a commitment, which is the one
+ * failure the tracking surface cannot render its way out of: an item nobody
+ * agreed to, sitting on someone's list with a quote that does not appear in the
+ * record. `maxTokens` is large because the output is a list whose length is set
+ * by the corpus, and a truncated list is a silently short one.
+ */
+export function createCorpusExtractor(options: CorpusLlmOptions = {}): CorpusExtractorLlm {
+  const service =
+    options.service ??
+    new LLMService<unknown>({
+      preferredProvider: options.preferredProvider ?? 'anthropic',
+      defaultModel: options.model ?? '',
+      defaultTemperature: options.temperature ?? 0,
+      defaultMaxTokens: options.maxTokens ?? 8192,
+    });
+
+  let lastModel = options.model ?? '';
+  const env = options.env ?? (typeof process !== 'undefined' ? process.env : {});
+
+  return {
+    describe(): LlmDescription {
+      const available = service.getAvailableProviders();
+      const provider = available[0] ?? 'none';
+      return {
+        provider,
+        model: lastModel || options.model || configuredModel(provider, env) || '',
+        live: provider !== 'none' && provider !== 'mock',
+      };
+    },
+
+    async extract(input: ExtractionRequest): Promise<ExtractionDraft> {
+      if (!service.isAvailable()) {
+        // Named, not swallowed. The caller catches this and still writes the
+        // rules-derived commitments, so the run degrades to the deterministic
+        // half rather than to nothing.
+        throw new Error(
+          'No LLM provider is reachable. Configure ANTHROPIC_API_KEY for this tenant.',
+        );
+      }
+
+      const system = extractSystemPrompt();
+      const user = renderExtractPrompt(input);
+
+      const first = await service.chat(user, system);
+      lastModel = first.model || lastModel;
+      const parsed = parseExtractionDraft(first.content);
+      if (parsed) return parsed;
+
+      const repair = await service.chat(
+        [
+          'Your previous response was not valid JSON in the required shape.',
+          'Return ONLY the JSON object, with keys "items" and "resolutions".',
+          '',
+          'Your previous response was:',
+          first.content,
+        ].join('\n'),
+        system,
+      );
+      lastModel = repair.model || lastModel;
+      return parseExtractionDraft(repair.content) ?? { items: [], resolutions: [] };
+    },
+  };
+}

package/src/corpus/text.ts

@@ -0,0 +1,283 @@
+/**
+ * Text normalisation under the extraction pipeline (bd startsim-768w.38.25).
+ *
+ * TWO JOBS, both of them boring on purpose:
+ *
+ *   1. SECTIONS. The documents this runs over are markdown — the tenant's
+ *      `source_document.summary` arrives with `###` headings, one per topic, and
+ *      on every one of the sixteen real records on the live tandem tenant the
+ *      last of those headings is "Next Steps". Splitting on the author's own
+ *      headings is not a heuristic; it is reading the structure the vendor
+ *      already produced, which is the cheapest extraction there is.
+ *
+ *   2. STEMS. The identity of a commitment cannot be a string the model emitted,
+ *      because the model will emit a different string tomorrow. It has to be a
+ *      NORMALISED bag of stems, so that "Set a deadline for 2040-01" and
+ *      "Deadline needs setting on matter 2040-01" collapse to the same key. That
+ *      is the whole reason a stemmer is in this package.
+ *
+ * Nothing here knows what a commitment is, and nothing here calls a model.
+ */
+
+/**
+ * Terms too common to carry meaning. Deliberately small: an aggressive stoplist
+ * silently deletes the distinguishing words. Domain nouns all stay in, because
+ * they are exactly what separates one commitment from another.
+ */
+const STOPWORDS = new Set([
+  'a', 'about', 'all', 'am', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'been',
+  'but', 'by', 'can', 'did', 'do', 'does', 'for', 'from', 'had', 'has', 'have',
+  'he', 'her', 'his', 'how', 'i', 'if', 'in', 'is', 'it', 'its', 'me', 'my', 'of',
+  'on', 'or', 'our', 'she', 'so', 'that', 'the', 'their', 'them', 'then', 'there',
+  'these', 'they', 'this', 'to', 'us', 'was', 'we', 'were', 'what', 'when',
+  'where', 'which', 'who', 'why', 'will', 'with', 'you', 'your',
+]);
+
+/** Raw lowercase word tokens, before stemming. */
+export function words(text: string): string[] {
+  return ((text ?? '').toLowerCase().match(/[a-z0-9][a-z0-9'&.\-/]*/g) ?? [])
+    .map((token) => token.replace(/^[.'\-/]+|[.'\-/]+$/g, ''))
+    .filter((token) => token.length >= 2 && !STOPWORDS.has(token));
+}
+
+/** Lowercase STEMS, stopwords removed. The unit identity is computed on. */
+export function tokenize(text: string): string[] {
+  return words(text).map(stem);
+}
+
+/** Overlap of two token sets, 0..1. The near-match half of commitment identity. */
+export function jaccard(a: Iterable<string>, b: Iterable<string>): number {
+  const left = new Set(a);
+  const right = new Set(b);
+  if (left.size === 0 || right.size === 0) return 0;
+  let shared = 0;
+  for (const token of left) if (right.has(token)) shared += 1;
+  return shared / (left.size + right.size - shared);
+}
+
+// ---------------------------------------------------------------------------
+//  stemming
+// ---------------------------------------------------------------------------
+//
+// Porter, with ONE deliberate departure: `-ment` / `-ement` is stripped at m>0
+// rather than m>1. Vanilla Porter leaves `agreement` as `agreem` while `agree`
+// becomes `agre`, so two mentions of the same commitment — one phrased "we
+// agreed", one "per the agreement" — would land as two items. The measure test
+// stays on every other suffix, which is what keeps `sign`/`signal` and
+// `rent`/`rental` apart; those are different words and collapsing them would
+// merge two commitments that have nothing to do with each other, which is a
+// worse failure than listing one twice.
+//
+// Anything containing a digit is returned untouched: `2040-01` is a matter
+// number, and a matter number has no morphology. That matters more than it
+// sounds — matter numbers are the strongest identity signal in this corpus.
+
+const VOWELS = new Set(['a', 'e', 'i', 'o', 'u']);
+
+function isConsonant(word: string, index: number): boolean {
+  const letter = word[index];
+  if (VOWELS.has(letter)) return false;
+  if (letter !== 'y') return true;
+  return index === 0 ? true : !isConsonant(word, index - 1);
+}
+
+/** Porter's m: how many vowel→consonant transitions the word contains. */
+function measure(word: string): number {
+  let count = 0;
+  let index = 0;
+  while (index < word.length && isConsonant(word, index)) index += 1;
+  while (index < word.length) {
+    while (index < word.length && !isConsonant(word, index)) index += 1;
+    if (index >= word.length) break;
+    count += 1;
+    while (index < word.length && isConsonant(word, index)) index += 1;
+  }
+  return count;
+}
+
+const hasVowel = (word: string) => [...word].some((_, index) => !isConsonant(word, index));
+
+const endsDoubleConsonant = (word: string) =>
+  word.length >= 2 &&
+  word[word.length - 1] === word[word.length - 2] &&
+  isConsonant(word, word.length - 1);
+
+/** Porter's *o: ends consonant-vowel-consonant where the last is not w, x or y. */
+function endsCvc(word: string): boolean {
+  const size = word.length;
+  if (size < 3) return false;
+  return (
+    isConsonant(word, size - 1) &&
+    !isConsonant(word, size - 2) &&
+    isConsonant(word, size - 3) &&
+    !'wxy'.includes(word[size - 1])
+  );
+}
+
+function replaceIf(word: string, suffix: string, replacement: string, minMeasure: number): string | null {
+  if (!word.endsWith(suffix)) return null;
+  const base = word.slice(0, word.length - suffix.length);
+  return measure(base) > minMeasure ? base + replacement : null;
+}
+
+const STEP2: Array<[string, string]> = [
+  ['ational', 'ate'], ['tional', 'tion'], ['enci', 'ence'], ['anci', 'ance'],
+  ['izer', 'ize'], ['abli', 'able'], ['alli', 'al'], ['entli', 'ent'], ['eli', 'e'],
+  ['ousli', 'ous'], ['ization', 'ize'], ['ation', 'ate'], ['ator', 'ate'],
+  ['alism', 'al'], ['iveness', 'ive'], ['fulness', 'ful'], ['ousness', 'ous'],
+  ['aliti', 'al'], ['iviti', 'ive'], ['biliti', 'ble'],
+];
+
+const STEP3: Array<[string, string]> = [
+  ['icate', 'ic'], ['ative', ''], ['alize', 'al'], ['iciti', 'ic'], ['ical', 'ic'],
+  ['ful', ''], ['ness', ''],
+];
+
+const STEP4 = [
+  'al', 'ance', 'ence', 'er', 'ic', 'able', 'ible', 'ant', 'ou', 'ism', 'ate',
+  'iti', 'ous', 'ive', 'ize',
+];
+
+/**
+ * Reduce a word to the form it shares with its inflections.
+ *
+ * NOT a dictionary and not a lemmatizer: the output is frequently not a word
+ * (`invoice` and `invoicing` both become `invoic`). That is fine — it is only
+ * ever compared against another stem, and it is never shown to a user. Every
+ * string on screen is the original text.
+ */
+export function stem(input: string): string {
+  let word = (input ?? '').toLowerCase();
+  if (word.length <= 2 || /\d/.test(word)) return word;
+
+  // Step 1a — plurals.
+  if (word.endsWith('sses')) word = word.slice(0, -2);
+  else if (word.endsWith('ies')) word = `${word.slice(0, -3)}i`;
+  else if (word.endsWith('ss')) { /* keep */ }
+  else if (word.endsWith('s')) word = word.slice(0, -1);
+
+  // Step 1b — past tense and gerund.
+  let step1bFired = false;
+  if (word.endsWith('eed')) {
+    if (measure(word.slice(0, -3)) > 0) word = word.slice(0, -1);
+  } else if (word.endsWith('ed') && hasVowel(word.slice(0, -2))) {
+    word = word.slice(0, -2);
+    step1bFired = true;
+  } else if (word.endsWith('ing') && hasVowel(word.slice(0, -3))) {
+    word = word.slice(0, -3);
+    step1bFired = true;
+  }
+  if (step1bFired) {
+    if (word.endsWith('at') || word.endsWith('bl') || word.endsWith('iz')) word += 'e';
+    else if (endsDoubleConsonant(word) && !'lsz'.includes(word[word.length - 1])) word = word.slice(0, -1);
+    else if (measure(word) === 1 && endsCvc(word)) word += 'e';
+  }
+
+  // Step 1c — terminal y.
+  if (word.endsWith('y') && hasVowel(word.slice(0, -1))) word = `${word.slice(0, -1)}i`;
+
+  // Steps 2 and 3 — derivational suffixes.
+  for (const [suffix, replacement] of STEP2) {
+    const next = replaceIf(word, suffix, replacement, 0);
+    if (next !== null) { word = next; break; }
+  }
+  for (const [suffix, replacement] of STEP3) {
+    const next = replaceIf(word, suffix, replacement, 0);
+    if (next !== null) { word = next; break; }
+  }
+
+  // Step 4 — strip a final derivational suffix on a long enough stem.
+  // `-ment` and NOT `-ement`, deliberately. Stripping `ement` from `agreement`
+  // leaves `agre`, which step 5 then trims again to `agr` — while `agree`
+  // itself only reaches `agre`, so the two would never meet. Cutting at `ment`
+  // leaves `agree` and lets the ordinary trailing-e rule do the rest.
+  const mentStripped = replaceIf(word, 'ment', '', 0);
+  if (mentStripped !== null) {
+    word = mentStripped;
+  } else {
+    let stripped = false;
+    for (const suffix of STEP4) {
+      const next = replaceIf(word, suffix, '', 1);
+      if (next !== null) { word = next; stripped = true; break; }
+    }
+    if (!stripped) {
+      const ent = replaceIf(word, 'ent', '', 1);
+      if (ent !== null) { word = ent; stripped = true; }
+    }
+    if (!stripped && word.endsWith('ion')) {
+      const base = word.slice(0, -3);
+      if (measure(base) > 1 && (base.endsWith('s') || base.endsWith('t'))) word = base;
+    }
+  }
+
+  // Step 5 — the trailing e, and a doubled l.
+  if (word.endsWith('e')) {
+    const base = word.slice(0, -1);
+    const m = measure(base);
+    if (m > 1 || (m === 1 && !endsCvc(base))) word = base;
+  }
+  if (measure(word) > 1 && endsDoubleConsonant(word) && word.endsWith('l')) word = word.slice(0, -1);
+
+  return word;
+}
+
+// ---------------------------------------------------------------------------
+//  markdown sections
+// ---------------------------------------------------------------------------
+
+export interface TextSection {
+  /** The heading text, without the leading hashes. Empty for a preamble. */
+  heading: string;
+  /** Everything under that heading, up to the next one. */
+  body: string;
+}
+
+const HEADING = /^\s{0,3}(#{1,6})\s+(\S.*)$/;
+
+/**
+ * Split markdown into its author's own sections.
+ *
+ * Used for one thing that matters: finding the block the vendor already labelled
+ * as the outcome of the meeting. On the live tandem tenant every single one of
+ * the sixteen records ends with a `## Next Steps` block of bolded imperatives
+ * with owners in parentheses — an action list the summariser already wrote. A
+ * pipeline that sends the whole document to a model and asks it to find the
+ * action items is redoing that work, less reliably, for money.
+ */
+export function splitSections(text: string): TextSection[] {
+  const body = (text ?? '').trim();
+  if (!body) return [];
+
+  const sections: TextSection[] = [];
+  let heading = '';
+  let buffer: string[] = [];
+
+  const flush = () => {
+    const joined = buffer.join('\n').trim();
+    if (heading || joined) sections.push({ heading, body: joined });
+    buffer = [];
+  };
+
+  for (const line of body.split('\n')) {
+    const match = HEADING.exec(line);
+    if (match) {
+      flush();
+      heading = match[2].trim();
+    } else {
+      buffer.push(line);
+    }
+  }
+  flush();
+  return sections;
+}
+
+/**
+ * The section whose heading names the outcome of the meeting, if the source
+ * wrote one. Matched on the WORDS, not on an exact string, because summarisers
+ * write "Next Steps", "Action Items" and "Follow-ups" interchangeably.
+ */
+const OUTCOME_HEADING = /\b(next\s*steps?|action\s*items?|follow[\s-]?ups?|to[\s-]?dos?)\b/i;
+
+export function outcomeSections(text: string): TextSection[] {
+  return splitSections(text).filter((section) => OUTCOME_HEADING.test(section.heading));
+}