@startsimpli/integrations 0.1.3 → 0.1.5

package/src/corpus/llm.ts

@@ -0,0 +1,197 @@
+/**
+ * The one place a real model is called (bd startsim-768w.38.7).
+ *
+ * SERVER-ONLY. This module is reachable at `@startsimpli/integrations/corpus-llm`
+ * and is deliberately NOT re-exported from the package index, so importing the
+ * composers can never drag a provider client — or a provider key — into a
+ * browser bundle.
+ *
+ * IT ADDS NO PROVIDER CLIENT. `@startsimpli/llm` already owns provider
+ * selection, retries, JSON extraction and the OpenAI/Anthropic HTTP shapes.
+ * Everything below is an adapter from `LLMService` to the `CorpusLlm` seam.
+ *
+ * ONE CALL, N DOCUMENTS. `CorpusLlm.answer` takes the whole retrieved set and
+ * makes a single request with echoed ids (feedback_batch_llm_calls). The only
+ * second request that can ever happen is a JSON-repair retry of that same call,
+ * which is a correction, not a per-document loop.
+ */
+import { extractJson, LLMService } from '@startsimpli/llm';
+
+import { answerSystemPrompt, renderAnswerPrompt } from './ask';
+import type { AnswerDraft, AnswerRequest, CorpusLlm, LlmDescription } from './types';
+
+export interface CorpusLlmOptions {
+  /** Reuse an already-configured service; otherwise one is constructed. */
+  service?: Pick<LLMService<unknown>, 'chat' | 'isAvailable' | 'getAvailableProviders'>;
+  preferredProvider?: 'openai' | 'anthropic' | 'mock';
+  model?: string;
+  temperature?: number;
+  maxTokens?: number;
+}
+
+/**
+ * What a provider key looks like, so a misfiled one is reported instead of
+ * failing at request time with an opaque 401.
+ *
+ * This exists because of a real incident (reference_prod_anthropic_key_is_openai):
+ * the value deployed as `ANTHROPIC_API_KEY` in prod is an OpenAI `sk-proj` key.
+ * Anything built on "ANTHROPIC_API_KEY is set, so Anthropic is reachable" is
+ * wrong there, so the assumption is checked rather than made.
+ */
+export interface ProviderDiagnosis {
+  provider: 'openai' | 'anthropic';
+  envVar: string;
+  present: boolean;
+  /** True when the key's own prefix contradicts the variable it is filed under. */
+  looksMisfiled: boolean;
+  detail: string;
+}
+
+export function diagnoseProviders(
+  env: Record<string, string | undefined> = typeof process !== 'undefined' ? process.env : {},
+): ProviderDiagnosis[] {
+  const openaiKey = env.OPENAI_API_KEY ?? '';
+  const anthropicKey = env.ANTHROPIC_API_KEY ?? '';
+
+  const anthropicLooksOpenAI = anthropicKey.startsWith('sk-proj') || anthropicKey.startsWith('sk-None');
+  const openaiLooksAnthropic = openaiKey.startsWith('sk-ant');
+
+  return [
+    {
+      provider: 'openai',
+      envVar: 'OPENAI_API_KEY',
+      present: Boolean(openaiKey),
+      looksMisfiled: openaiLooksAnthropic,
+      detail: !openaiKey
+        ? 'not set'
+        : openaiLooksAnthropic
+          ? 'value starts with sk-ant — this is an Anthropic key filed under OPENAI_API_KEY'
+          : 'set',
+    },
+    {
+      provider: 'anthropic',
+      envVar: 'ANTHROPIC_API_KEY',
+      present: Boolean(anthropicKey),
+      looksMisfiled: anthropicLooksOpenAI,
+      detail: !anthropicKey
+        ? 'not set'
+        : anthropicLooksOpenAI
+          ? 'value starts with sk-proj — this is an OpenAI key filed under ANTHROPIC_API_KEY; Anthropic calls will 401'
+          : 'set',
+    },
+  ];
+}
+
+/**
+ * Which provider will ACTUALLY answer, given what is configured and what the
+ * keys really are. A key whose prefix contradicts its variable is treated as
+ * absent for that provider — pretending otherwise just moves the failure to
+ * request time.
+ */
+export function reachableProviders(
+  env: Record<string, string | undefined> = typeof process !== 'undefined' ? process.env : {},
+): string[] {
+  if (env.LLM_PROVIDER === 'mock') return ['mock'];
+  return diagnoseProviders(env)
+    .filter((diagnosis) => diagnosis.present && !diagnosis.looksMisfiled)
+    .map((diagnosis) => diagnosis.provider);
+}
+
+/** Parse + shape-check the model's JSON. Anything unusable returns null. */
+export function parseAnswerDraft(raw: string): AnswerDraft | null {
+  const extraction = extractJson(raw);
+  if (!extraction.success || !extraction.json || typeof extraction.json !== 'object') return null;
+  const json = extraction.json as Record<string, unknown>;
+  const answer = typeof json.answer === 'string' ? json.answer : '';
+  if (!answer && json.unanswerable !== true) return null;
+  const citations = Array.isArray(json.citations)
+    ? json.citations
+        .map((entry) => {
+          if (!entry || typeof entry !== 'object') return null;
+          const record = entry as Record<string, unknown>;
+          const documentId = String(record.documentId ?? record.document_id ?? '').trim();
+          if (!documentId) return null;
+          return { documentId, quote: typeof record.quote === 'string' ? record.quote : '' };
+        })
+        .filter((entry): entry is { documentId: string; quote: string } => entry !== null)
+    : [];
+  return { answer, citations, unanswerable: json.unanswerable === true };
+}
+
+/**
+ * A `CorpusLlm` backed by `@startsimpli/llm`.
+ *
+ * `describe()` reports what really answered rather than what was requested, so
+ * the surface can state the model by name and can say `live: false` when the run
+ * came from the mock provider. Claiming a model that did not answer is exactly
+ * the kind of small lie that makes the citations unbelievable too.
+ */
+export function createCorpusLlm(options: CorpusLlmOptions = {}): CorpusLlm {
+  const service =
+    options.service ??
+    new LLMService<unknown>({
+      preferredProvider: options.preferredProvider ?? 'openai',
+      defaultModel: options.model ?? '',
+      // Low temperature: this is extraction over the user's own records, not
+      // creative writing. A model that improvises here invents a contract term.
+      defaultTemperature: options.temperature ?? 0.2,
+      defaultMaxTokens: options.maxTokens ?? 4096,
+    });
+
+  let lastModel = options.model ?? '';
+
+  const describe = (): LlmDescription => {
+    const available = service.getAvailableProviders();
+    const provider = available[0] ?? 'none';
+    return {
+      provider,
+      model: lastModel || options.model || '(provider default)',
+      live: provider !== 'none' && provider !== 'mock',
+    };
+  };
+
+  return {
+    describe,
+    async answer(input: AnswerRequest): Promise<AnswerDraft> {
+      if (!service.isAvailable()) {
+        // Say what is wrong rather than returning an empty answer that reads
+        // like "your corpus has nothing about that".
+        throw new Error(
+          'No LLM provider is reachable. Configure OPENAI_API_KEY or ANTHROPIC_API_KEY for this tenant.',
+        );
+      }
+
+      const system = answerSystemPrompt(input.mode);
+      const user = renderAnswerPrompt(input);
+
+      const first = await service.chat(user, system);
+      lastModel = first.model || lastModel;
+      const parsed = parseAnswerDraft(first.content);
+      if (parsed) return parsed;
+
+      // ONE repair attempt on the SAME call. Not a per-document loop — the
+      // documents were all in the first request and are all in this one.
+      const repair = await service.chat(
+        [
+          'Your previous response was not valid JSON in the required shape.',
+          'Return ONLY the JSON object, with keys "answer", "citations" and "unanswerable".',
+          '',
+          'Your previous response was:',
+          first.content,
+        ].join('\n'),
+        system,
+      );
+      lastModel = repair.model || lastModel;
+      const repaired = parseAnswerDraft(repair.content);
+      if (repaired) return repaired;
+
+      // A model that will not produce a citable shape has produced nothing that
+      // can be presented as sourced. Refuse rather than degrade to uncited prose.
+      return {
+        answer: '',
+        citations: [],
+        unanswerable: true,
+      };
+    },
+  };
+}

package/src/corpus/retrieval.ts

@@ -0,0 +1,237 @@
+/**
+ * Scope enforcement and retrieval for the Ask surface (bd startsim-768w.38.7).
+ *
+ * WHAT WAS CHOSEN, AND WHY, since the bead demands this be written down:
+ *
+ *   SCOPED KEYWORD + RECENCY over a few hundred documents. No embeddings, no
+ *   vector store, no index to keep warm. The corpus is one tenant's own
+ *   documents; the questions are about named companies, named people and named
+ *   deals; and every term the user types appears verbatim in the records they
+ *   mean. A BM25-ish term score with a recency prior answers those honestly, is
+ *   deterministic, is explainable ("it matched these terms"), and needs no
+ *   pipeline to be running before an answer is possible.
+ *
+ *   WHAT BREAKS IT AT 10x: (a) synonyms and paraphrase — "what did we promise
+ *   about pricing" will miss a note that says "rate card"; (b) recall over
+ *   ~5k documents, where the page of candidates the server returns stops
+ *   containing the right record at all; (c) long documents, where a single
+ *   term hit in a 40-minute transcript outranks a whole-document match. The
+ *   upgrade — embeddings + chunking, retrieved server-side — is filed rather
+ *   than pre-built, because pre-building it before there is a corpus to measure
+ *   is guessing.
+ *
+ * THE SCOPE GUARD IS NOT A CONVENIENCE. The tenant data API scopes reads by
+ * `org_id` and by readable type, and Postgres RLS enforces the same boundary at
+ * the DB. This is a THIRD check, immediately before the prompt is built, for one
+ * reason: an AI that retrieves across a scope boundary leaks data in the most
+ * convincing possible format. If any of the layers above ever hands back a row
+ * it should not have, the answer must still not contain it — and the count of
+ * what was dropped is surfaced rather than swallowed, because a non-zero drop is
+ * a bug somewhere upstream that someone needs to see.
+ */
+import type { CorpusScope, RetrievedDocument, SourceDocument } from './types';
+
+/**
+ * Terms too common to carry meaning. Deliberately small: an aggressive stoplist
+ * silently deletes the user's question. `contract`, `invoice` and every domain
+ * word stay in, because they are exactly what distinguishes one record here.
+ */
+const STOPWORDS = new Set([
+  'a', 'about', 'all', 'am', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'been',
+  'but', 'by', 'can', 'did', 'do', 'does', 'for', 'from', 'had', 'has', 'have',
+  'he', 'her', 'his', 'how', 'i', 'if', 'in', 'is', 'it', 'its', 'me', 'my', 'of',
+  'on', 'or', 'our', 'she', 'so', 'that', 'the', 'their', 'them', 'then', 'there',
+  'these', 'they', 'this', 'to', 'us', 'was', 'we', 'were', 'what', 'when',
+  'where', 'which', 'who', 'why', 'will', 'with', 'you', 'your',
+]);
+
+/** Lowercase word tokens, 2+ chars, stopwords removed. */
+export function tokenize(text: string): string[] {
+  return (text.toLowerCase().match(/[a-z0-9][a-z0-9'&.-]*/g) ?? [])
+    .map((token) => token.replace(/^[.'-]+|[.'-]+$/g, ''))
+    .filter((token) => token.length >= 2 && !STOPWORDS.has(token));
+}
+
+/**
+ * Is this record inside the asking principal's scope?
+ *
+ * `orgId` must match EXACTLY — it is the customer-workspace boundary (bd
+ * startsim-768w.34.4) and there is no hierarchy to walk here; a foundry
+ * founder wanting to read across customers does so by acting AS that org, which
+ * changes the claim and therefore changes this comparison.
+ *
+ * `ownerSub` is only enforced when the caller asked for it. That mirrors what
+ * the tenant API actually does today (org + readable-type, not per-owner), so
+ * this guard never claims a narrower posture than the server enforces — and
+ * when a caller does want per-owner, it is one field, not a new code path.
+ */
+export function isInScope(record: { scope: CorpusScope }, scope: CorpusScope): boolean {
+  if (!scope.orgId) return false; // an empty scope grants nothing, ever
+  if (record.scope?.orgId !== scope.orgId) return false;
+  if (scope.ownerSub && record.scope?.ownerSub !== scope.ownerSub) return false;
+  return true;
+}
+
+export interface ScopeFilterResult<T> {
+  kept: T[];
+  dropped: T[];
+}
+
+/** Partition a candidate set by the scope guard. Nothing is dropped silently. */
+export function enforceScope<T extends { scope: CorpusScope }>(
+  records: T[],
+  scope: CorpusScope,
+): ScopeFilterResult<T> {
+  const kept: T[] = [];
+  const dropped: T[] = [];
+  for (const record of records) {
+    (isInScope(record, scope) ? kept : dropped).push(record);
+  }
+  return { kept, dropped };
+}
+
+// ---------------------------------------------------------------------------
+//  ranking
+// ---------------------------------------------------------------------------
+
+/** Half-life of the recency prior, in days. */
+const RECENCY_HALF_LIFE_DAYS = 120;
+
+/** Fields are weighted because a title hit means more than a transcript hit. */
+const FIELD_WEIGHTS = { title: 3, participants: 2, body: 1, sections: 0.6 } as const;
+
+function recencyWeight(occurredAt: string | null | undefined, now: number): number {
+  if (!occurredAt) return 0.5; // undated: neither fresh nor stale, just less certain
+  const at = Date.parse(occurredAt);
+  if (Number.isNaN(at)) return 0.5;
+  const ageDays = Math.max(0, (now - at) / 86_400_000);
+  return 2 ** (-ageDays / RECENCY_HALF_LIFE_DAYS);
+}
+
+function countHits(tokens: string[], terms: Set<string>): { hits: number; matched: string[] } {
+  let hits = 0;
+  const matched = new Set<string>();
+  for (const token of tokens) {
+    if (terms.has(token)) {
+      hits += 1;
+      matched.add(token);
+    }
+  }
+  return { hits, matched: [...matched] };
+}
+
+/**
+ * A passage around the first matching term, so the citation card shows the user
+ * WHY this record came back rather than making them open it to find out.
+ */
+export function snippetFor(document: SourceDocument, terms: Set<string>, length = 220): string {
+  const haystacks = [document.body, ...document.sections.map((section) => section.text)];
+  for (const text of haystacks) {
+    if (!text) continue;
+    const lower = text.toLowerCase();
+    for (const term of terms) {
+      const at = lower.indexOf(term);
+      if (at === -1) continue;
+      const start = Math.max(0, at - Math.floor(length / 3));
+      const slice = text.slice(start, start + length).trim();
+      return `${start > 0 ? '…' : ''}${slice}${start + length < text.length ? '…' : ''}`;
+    }
+  }
+  const fallback = document.body || document.sections[0]?.text || '';
+  return fallback.length > length ? `${fallback.slice(0, length).trim()}…` : fallback;
+}
+
+export interface RetrieveOptions {
+  /** How many records the model is allowed to see. */
+  limit?: number;
+  /** Injectable clock, so recency is testable. */
+  now?: number;
+  /**
+   * Score floor. A record that matched nothing is never included: padding the
+   * prompt with irrelevant documents is how a model ends up citing one.
+   */
+  minScore?: number;
+}
+
+/**
+ * Rank an ALREADY SCOPE-FILTERED candidate set against a question.
+ *
+ * Takes scope-filtered input by construction rather than filtering internally,
+ * so a caller cannot accidentally reach the ranker with unfiltered records —
+ * `retrieveForQuestion` below is the entry point that does both in the right
+ * order.
+ */
+export function rankDocuments(
+  documents: SourceDocument[],
+  question: string,
+  options: RetrieveOptions = {},
+): RetrievedDocument[] {
+  const { limit = 8, now = Date.now(), minScore = 0.0001 } = options;
+  const terms = new Set(tokenize(question));
+  if (terms.size === 0) {
+    // No usable terms — fall back to pure recency rather than returning
+    // nothing. "What happened lately" is a real question.
+    return [...documents]
+      .sort((a, b) => recencyWeight(b.occurredAt, now) - recencyWeight(a.occurredAt, now))
+      .slice(0, limit)
+      .map((document) => ({
+        document,
+        score: recencyWeight(document.occurredAt, now),
+        matchedTerms: [],
+        snippet: snippetFor(document, terms),
+      }));
+  }
+
+  const scored: RetrievedDocument[] = [];
+  for (const document of documents) {
+    const title = countHits(tokenize(document.title), terms);
+    const participants = countHits(
+      tokenize(document.participants.map((party) => `${party.name} ${party.email ?? ''}`).join(' ')),
+      terms,
+    );
+    const body = countHits(tokenize(document.body), terms);
+    const sections = countHits(
+      tokenize(document.sections.map((section) => section.text).join(' ')),
+      terms,
+    );
+
+    // Saturating term counts (log1p) so one long transcript repeating a word
+    // cannot outrank a document that is actually about it.
+    const termScore =
+      FIELD_WEIGHTS.title * Math.log1p(title.hits) +
+      FIELD_WEIGHTS.participants * Math.log1p(participants.hits) +
+      FIELD_WEIGHTS.body * Math.log1p(body.hits) +
+      FIELD_WEIGHTS.sections * Math.log1p(sections.hits);
+
+    if (termScore <= 0) continue;
+
+    const matched = [
+      ...new Set([...title.matched, ...participants.matched, ...body.matched, ...sections.matched]),
+    ];
+    // Coverage rewards matching MORE OF the question rather than one term often.
+    const coverage = matched.length / terms.size;
+    const score = termScore * (0.5 + 0.5 * coverage) * (0.6 + 0.4 * recencyWeight(document.occurredAt, now));
+
+    if (score < minScore) continue;
+    scored.push({ document, score, matchedTerms: matched, snippet: snippetFor(document, terms) });
+  }
+
+  return scored
+    .sort((a, b) => b.score - a.score || (b.document.occurredAt ?? '').localeCompare(a.document.occurredAt ?? ''))
+    .slice(0, limit);
+}
+
+/**
+ * The entry point: guard, THEN rank. Returns the drop count so the caller can
+ * surface it — see the note at the top of this file about why a non-zero drop
+ * is worth someone's attention rather than a silent success.
+ */
+export function retrieveForQuestion(
+  candidates: SourceDocument[],
+  question: string,
+  scope: CorpusScope,
+  options: RetrieveOptions = {},
+): { retrieved: RetrievedDocument[]; droppedOutOfScope: number } {
+  const { kept, dropped } = enforceScope(candidates, scope);
+  return { retrieved: rankDocuments(kept, question, options), droppedOutOfScope: dropped.length };
+}