@startsimpli/integrations 0.1.4 → 0.1.6

package/src/corpus/ask.ts

@@ -0,0 +1,321 @@
+/**
+ * Ask & Generate over the corpus (bd startsim-768w.38.7).
+ *
+ * This is the whole pipeline, server-side, in one function:
+ *
+ *   1. pull CANDIDATES from the tenant data API, already org-scoped by the
+ *      server and narrowed by the same filters the Library uses;
+ *   2. run the SCOPE GUARD over them — a third check, immediately before the
+ *      prompt is built (see ./retrieval for why);
+ *   3. RANK them, keyword + recency;
+ *   4. ONE LLM call carrying all N documents with ids the model must echo
+ *      (feedback_batch_llm_calls — never a loop of N);
+ *   5. RESOLVE every citation back to a record retrieval actually supplied,
+ *      drop the ones that do not resolve, and report that it happened;
+ *   6. refuse to present an UNCITED answer as an answer.
+ *
+ * STEP 5 AND 6 ARE THE POINT. An answer the user cannot trace is an answer they
+ * cannot trust with a contract. A model that cites a document id it was never
+ * given has invented a source, and the only safe rendering of that is to say so.
+ */
+import { moneyEventAsDocument } from './entities';
+import { retrieveForQuestion } from './retrieval';
+import type {
+  AnswerRequest,
+  AskRequest,
+  AskResult,
+  Citation,
+  CorpusApi,
+  CorpusLlm,
+  PromptDocument,
+  RetrievedDocument,
+  SourceDocument,
+  SourceFreshness,
+} from './types';
+
+export interface AskCorpusOptions {
+  /** How many candidate records to pull from the API before ranking. */
+  candidateLimit?: number;
+  /** How many ranked records the model is allowed to see. */
+  contextLimit?: number;
+  /** Characters of each document's text included in the prompt. */
+  maxCharsPerDocument?: number;
+  /** Injectable clock for the recency prior. */
+  now?: number;
+}
+
+const DEFAULTS: Required<Omit<AskCorpusOptions, 'now'>> = {
+  candidateLimit: 200,
+  contextLimit: 8,
+  maxCharsPerDocument: 6000,
+};
+
+/**
+ * The system prompt. It is short on purpose: every rule in it is enforced in
+ * code below as well, because a prompt is a request and a check is a guarantee.
+ */
+function systemPrompt(mode: AskRequest['mode']): string {
+  const shared = [
+    'You answer only from the DOCUMENTS supplied in this message.',
+    'Every claim you make must be supported by at least one document, and you must cite it by its exact id.',
+    'Never cite an id that is not in the list. If the documents do not support an answer, set "unanswerable": true and say what is missing.',
+    'Respond with JSON only: {"answer": string, "citations": [{"documentId": string, "quote": string}], "unanswerable": boolean}.',
+    'Each "quote" must be a short verbatim span copied from the cited document.',
+  ];
+  if (mode === 'generate') {
+    return [
+      'You draft business documents for a company using only that company\'s own records.',
+      ...shared,
+      'Put the finished draft in "answer", ready to send or sign. Do not add commentary around it.',
+      'Do not invent names, dates, amounts or terms that are not in the documents. Leave an explicit [TO CONFIRM: ...] placeholder instead.',
+    ].join('\n');
+  }
+  return [
+    'You answer questions about a company using only that company\'s own records.',
+    ...shared,
+    'Be specific and short. Prefer naming the record and the date over hedging.',
+  ].join('\n');
+}
+
+/** ONE payload carrying every retrieved document, each tagged with the id to echo. */
+export function buildAnswerRequest(
+  request: AskRequest,
+  retrieved: RetrievedDocument[],
+  maxChars: number,
+): AnswerRequest {
+  const documents: PromptDocument[] = retrieved.map(({ document }) => ({
+    id: document.id,
+    title: document.title,
+    // The source travels INTO the prompt because "we learned this from a
+    // meeting" and "we learned this from a contract in Drive" are different
+    // claims, and the model should be able to say which it used.
+    source: document.sourceLabel || document.source,
+    occurredAt: document.occurredAt,
+    participants: document.participants.map((party) => party.name),
+    text: documentText(document, maxChars),
+  }));
+  return {
+    question: request.question,
+    mode: request.mode,
+    instructions: request.instructions,
+    documents,
+  };
+}
+
+/**
+ * A document's text for the prompt: body first, then each named section, cut at
+ * a budget. Sections are labelled with the source's own words, so the model sees
+ * "Transcript" or "Extracted text" without this package knowing what either is.
+ */
+export function documentText(document: SourceDocument, maxChars: number): string {
+  const parts = [document.body, ...document.sections.map((s) => `## ${s.label}\n${s.text}`)];
+  const joined = parts.filter(Boolean).join('\n\n');
+  return joined.length > maxChars ? `${joined.slice(0, maxChars)}\n…[truncated]` : joined;
+}
+
+/** Render the batched request as the single user message. */
+export function renderAnswerPrompt(input: AnswerRequest): string {
+  const blocks = input.documents.map((document) =>
+    [
+      `<document id="${document.id}">`,
+      `title: ${document.title}`,
+      `source: ${document.source}`,
+      `occurred_at: ${document.occurredAt ?? 'unknown'}`,
+      document.participants.length ? `participants: ${document.participants.join(', ')}` : '',
+      '',
+      document.text,
+      '</document>',
+    ]
+      .filter((line) => line !== '')
+      .join('\n'),
+  );
+  const task =
+    input.mode === 'generate'
+      ? `DRAFT REQUEST:\n${input.question}`
+      : `QUESTION:\n${input.question}`;
+  return [
+    `DOCUMENTS (${input.documents.length}):`,
+    blocks.join('\n\n'),
+    '',
+    task,
+    input.instructions ? `\nADDITIONAL INSTRUCTIONS:\n${input.instructions}` : '',
+  ]
+    .filter(Boolean)
+    .join('\n');
+}
+
+export { systemPrompt as answerSystemPrompt };
+
+/**
+ * Resolve the model's citations against what retrieval actually supplied.
+ *
+ * An id the model invented resolves to nothing and lands in `unresolved`. It is
+ * NOT quietly discarded: `AskResult.unresolvedCitations` carries it to the
+ * screen, because a model inventing sources is a fact about the answer.
+ */
+export function resolveCitations(
+  draft: { citations: Array<{ documentId: string; quote?: string }> },
+  retrieved: RetrievedDocument[],
+): { citations: Citation[]; unresolved: string[] } {
+  const byId = new Map(retrieved.map((item) => [item.document.id, item.document]));
+  const citations: Citation[] = [];
+  const unresolved: string[] = [];
+  const seen = new Set<string>();
+
+  for (const raw of draft.citations ?? []) {
+    // Both spellings. A model asked for `documentId` returns `document_id` often
+    // enough, and the cost of being strict here is EVERY citation silently
+    // dropped — which the surface would then correctly, uselessly, report as an
+    // uncited answer. The tolerance belongs at the seam, not only in the one
+    // adapter that happens to parse the JSON.
+    const record = (raw ?? {}) as { documentId?: unknown; document_id?: unknown; quote?: unknown };
+    const id = String(record.documentId ?? record.document_id ?? '').trim();
+    if (!id) continue;
+    const document = byId.get(id);
+    if (!document) {
+      if (!unresolved.includes(id)) unresolved.push(id);
+      continue;
+    }
+    if (seen.has(id)) continue;
+    seen.add(id);
+    citations.push({
+      documentId: id,
+      document,
+      quote: String(record.quote ?? '').trim() || snippetOf(document),
+    });
+  }
+  return { citations, unresolved };
+}
+
+function snippetOf(document: SourceDocument): string {
+  const text = document.body || document.sections[0]?.text || '';
+  return text.length > 200 ? `${text.slice(0, 200).trim()}…` : text;
+}
+
+/**
+ * Pull the candidate set. Two reads, in PARALLEL, not one read per document —
+ * the server pages and filters, this only decides how wide a net to cast.
+ *
+ * Money events are folded in as documents so one ranking pass spans meetings and
+ * revenue; "what did we agree with this company" is a question about both.
+ */
+async function loadCandidates(
+  corpus: CorpusApi,
+  request: AskRequest,
+  candidateLimit: number,
+): Promise<SourceDocument[]> {
+  const documentQuery = {
+    search: request.question,
+    ...request.filters,
+    page: 1,
+    pageSize: candidateLimit,
+  };
+
+  const [documents, money] = await Promise.all([
+    corpus.listDocuments(documentQuery),
+    corpus.listMoneyEvents
+      ? corpus
+          .listMoneyEvents({
+            search: request.question,
+            partyId: request.filters?.partyId,
+            page: 1,
+            pageSize: Math.ceil(candidateLimit / 4),
+          })
+          // A tenant with no payment connector, or a money endpoint that is down,
+          // must not take the whole answer with it — the meetings still answer.
+          .catch(() => ({ count: 0, next: null, previous: null, results: [] }))
+      : Promise.resolve({ count: 0, next: null, previous: null, results: [] }),
+  ]);
+
+  return [...documents.results, ...money.results.map(moneyEventAsDocument)];
+}
+
+/**
+ * Ask (or generate) over the corpus.
+ *
+ * Runs SERVER-SIDE — it holds a `CorpusLlm`, which holds a provider key. The
+ * browser composer talks to a route that calls this, never to a provider.
+ */
+export async function askCorpus(
+  request: AskRequest,
+  deps: { corpus: CorpusApi; llm: CorpusLlm },
+  options: AskCorpusOptions = {},
+): Promise<AskResult> {
+  const { candidateLimit, contextLimit, maxCharsPerDocument } = { ...DEFAULTS, ...options };
+  const { corpus, llm } = deps;
+
+  // An empty scope grants nothing (see `isInScope`). Refusing here rather than
+  // relying on the guard downstream keeps a missing claim from ever reaching a
+  // prompt builder in the first place.
+  if (!request.scope?.orgId) {
+    return emptyResult(request, llm, 'This session has no tenant scope, so there is nothing to search.');
+  }
+
+  const candidates = await loadCandidates(corpus, request, candidateLimit);
+  const { retrieved, droppedOutOfScope } = retrieveForQuestion(
+    candidates,
+    request.question,
+    request.scope,
+    { limit: contextLimit, now: options.now },
+  );
+
+  const freshness = await corpus.listSources().catch(() => [] as SourceFreshness[]);
+
+  if (retrieved.length === 0) {
+    return {
+      status: 'no_evidence',
+      answer:
+        'Nothing in your corpus matches that. Either it has not been pulled in yet, or it belongs to a different workspace.',
+      citations: [],
+      considered: [],
+      unresolvedCitations: [],
+      droppedOutOfScope,
+      provider: llm.describe(),
+      freshness,
+    };
+  }
+
+  const draft = await llm.answer(
+    buildAnswerRequest(request, retrieved, maxCharsPerDocument),
+  );
+  const { citations, unresolved } = resolveCitations(draft, retrieved);
+
+  const answer = String(draft.answer ?? '').trim();
+  const status = deriveStatus(draft.unanswerable === true, answer, citations.length);
+
+  return {
+    status,
+    answer,
+    citations,
+    considered: retrieved,
+    unresolvedCitations: unresolved,
+    droppedOutOfScope,
+    provider: llm.describe(),
+    freshness,
+  };
+}
+
+/**
+ * `uncited` is a real outcome, not a failure to report. The model produced text
+ * and nothing traceable; the surface renders that as a warning over the text
+ * rather than as an answer, which is the difference between a demo that impresses
+ * on Monday and one that survives Tuesday.
+ */
+function deriveStatus(unanswerable: boolean, answer: string, citationCount: number): AskResult['status'] {
+  if (unanswerable && citationCount === 0) return 'no_evidence';
+  if (!answer) return 'no_evidence';
+  return citationCount > 0 ? 'answered' : 'uncited';
+}
+
+function emptyResult(request: AskRequest, llm: CorpusLlm, answer: string): AskResult {
+  void request;
+  return {
+    status: 'no_evidence',
+    answer,
+    citations: [],
+    considered: [],
+    unresolvedCitations: [],
+    droppedOutOfScope: 0,
+    provider: llm.describe(),
+  };
+}

package/src/corpus/client.ts

@@ -0,0 +1,303 @@
+/**
+ * A `CorpusApi` over the tenant data plane's generic `/api/v1/*` endpoints.
+ *
+ * This is the ONLY place that knows the corpus is stored as generic `Entity`
+ * rows behind `?type=`. Swap this adapter and every other file in the package —
+ * both composers, retrieval, the ask pipeline, all the tests — is unaffected.
+ *
+ * SERVER-SIDE FILTERING AND PAGING, ALWAYS (CLAUDE.md rule 8). Every filter goes
+ * out as a query parameter and the DRF page envelope comes back untouched. There
+ * is no client-side `.filter()` on a page of results anywhere in this file, and
+ * there must not be: it produces a list whose count disagrees with its rows and
+ * that stops working at exactly the corpus size where it starts to matter.
+ *
+ * WHAT THE BACKEND HONOURS TODAY: `tenant-starter`'s `EntityViewSet` filters on
+ * `?type=` and nothing else — no `search`, no `source`, no date window (see
+ * `apps/api/views.py`). So `supportedFilters` defaults to `[]` and the composer
+ * says so on screen rather than pretending. Pass the real list once the tenant
+ * API gains them; the request already sends every parameter.
+ *
+ * PEOPLE ARE A SECOND READ, NOT A SECOND COPY (bd startsim-768w.38.22). The
+ * corpus delivery writes participants as typed `authored_by` / `involves` edges
+ * and leaves only a `participant_count` on the document, so this adapter reads
+ * `/api/v1/relationships/` and joins. The alternative — denormalizing names back
+ * onto `data.participants` at delivery time — is one fewer round trip and a
+ * second copy of the graph that will drift; the edge is the declared contract,
+ * so the edge is what is read. The join is cached, bounded and fails soft.
+ */
+import {
+  DEFAULT_PARTICIPANT_EDGES,
+  DEFAULT_TYPE_KEYS,
+  indexParticipants,
+  toMoneyEvent,
+  toParty,
+  toSourceDocument,
+} from './entities';
+import type { CorpusTypeKeys, TenantEntity, TenantRelationship } from './entities';
+import type {
+  CorpusApi,
+  CorpusPage,
+  CorpusScope,
+  DocumentQuery,
+  MoneyEvent,
+  Party,
+  PartyRef,
+  SourceFreshness,
+} from './types';
+
+/** The `@startsimpli/api` client surface this adapter needs, and no more. */
+export interface CorpusHttp {
+  get<T>(path: string, options?: { params?: Record<string, unknown> }): Promise<T>;
+}
+
+export interface CreateCorpusApiOptions {
+  http: CorpusHttp;
+  /** The asking principal's scope, used as the fallback when a row omits its own. */
+  scope: CorpusScope;
+  /** Override when a tenant modelled its types under different keys. */
+  typeKeys?: Partial<CorpusTypeKeys>;
+  /** Human names per source key, from the connector catalog. Data, not a mapping table. */
+  sourceLabels?: Record<string, string>;
+  /**
+   * Which `DocumentQuery` keys this deployment's API really applies. Anything
+   * not listed is reported to the user as unfiltered rather than assumed.
+   */
+  supportedFilters?: Array<keyof DocumentQuery>;
+  /** Endpoint bases, without leading/trailing slashes (the client convention). */
+  paths?: { entities?: string; connections?: string; relationships?: string };
+  /** How old a source's last sync may be before the Library calls it stale. */
+  staleAfterHours?: number;
+  /**
+   * Resolve people from the tenant's typed EDGES into `participants`.
+   *
+   * On by default because it is what the shipped pipeline writes: the corpus
+   * delivery lands `source_document --involves--> party` edges and a bare
+   * `participant_count` on the document (bd startsim-768w.38.18). A deployment
+   * whose connectors denormalize participants inline can turn this off and save
+   * the round trip; the inline array is read either way.
+   */
+  resolveParticipants?: boolean;
+  /** Which edge types mean "this party is on this record". Data, not an enum. */
+  participantEdges?: readonly string[];
+  /**
+   * Bound on the graph read, in PAGES.
+   *
+   * The tenant relationships endpoint has no `?source=` filter today
+   * (`tenant-starter/apps/api/views.py` — see also bd startsim-768w.38.12 for
+   * the same gap on entities), so the edges for a page of documents cannot be
+   * requested directly and the graph is read in bulk and cached. This cap is
+   * what keeps that bounded rather than unbounded: past it, participants simply
+   * stop resolving for the older tail instead of the page hanging.
+   */
+  maxGraphPages?: number;
+  /** How long a loaded graph is reused before it is read again. */
+  graphTtlMs?: number;
+}
+
+interface DrfPage<T> {
+  count: number;
+  next: string | null;
+  previous: string | null;
+  results: T[];
+}
+
+const EMPTY_PAGE = { count: 0, next: null, previous: null, results: [] };
+
+function params(query: DocumentQuery, type: string) {
+  const out: Record<string, unknown> = { type, page: query.page ?? 1 };
+  if (query.pageSize) out.page_size = query.pageSize;
+  if (query.search) out.search = query.search;
+  if (query.source) out.source = query.source;
+  if (query.partyId) out.party = query.partyId;
+  if (query.occurredAfter) out.occurred_after = query.occurredAfter;
+  if (query.occurredBefore) out.occurred_before = query.occurredBefore;
+  return out;
+}
+
+/** Which of the filters the caller asked for are actually honoured server-side. */
+function appliedFilters(
+  query: DocumentQuery,
+  supported: Array<keyof DocumentQuery>,
+): string[] {
+  const requested = (['search', 'source', 'partyId', 'occurredAfter', 'occurredBefore'] as const).filter(
+    (key) => Boolean(query[key]),
+  );
+  return requested.filter((key) => supported.includes(key));
+}
+
+export function createCorpusApi(options: CreateCorpusApiOptions): CorpusApi {
+  const {
+    http,
+    scope,
+    sourceLabels = {},
+    supportedFilters = [],
+    staleAfterHours = 26,
+    resolveParticipants = true,
+    participantEdges = DEFAULT_PARTICIPANT_EDGES,
+    maxGraphPages = 20,
+    graphTtlMs = 30_000,
+  } = options;
+  const types: CorpusTypeKeys = { ...DEFAULT_TYPE_KEYS, ...options.typeKeys };
+  const entities = options.paths?.entities ?? 'api/v1/entities';
+  const connections = options.paths?.connections ?? 'api/v1/connections';
+  const relationships = options.paths?.relationships ?? 'api/v1/relationships';
+
+  async function page<T>(
+    type: string,
+    query: DocumentQuery,
+    map: (entity: TenantEntity) => T,
+  ): Promise<CorpusPage<T>> {
+    const response = await http.get<DrfPage<TenantEntity>>(entities, {
+      params: params(query, type),
+    });
+    return {
+      count: response.count ?? response.results?.length ?? 0,
+      next: response.next ?? null,
+      previous: response.previous ?? null,
+      results: (response.results ?? []).map(map),
+      appliedFilters: appliedFilters(query, supportedFilters),
+    };
+  }
+
+  /** Every page of a DRF list, up to the cap. Stops on the first failure. */
+  async function readAll<T>(path: string, extra: Record<string, unknown>): Promise<T[]> {
+    const rows: T[] = [];
+    for (let index = 1; index <= maxGraphPages; index += 1) {
+      const response = await http.get<DrfPage<T>>(path, { params: { ...extra, page: index } });
+      const results = response?.results ?? [];
+      rows.push(...results);
+      if (!response?.next || results.length === 0) break;
+    }
+    return rows;
+  }
+
+  // ---------------------------------------------------------------------
+  //  the participant graph
+  // ---------------------------------------------------------------------
+  //
+  // ONE read for a page of documents, not one per document — and cached, so a
+  // screen that lists, opens a record and then asks a question spends it once.
+  // It FAILS SOFT: a tenant whose relationships endpoint is missing or refused
+  // renders exactly what it rendered before this existed, rather than an error
+  // where a Library used to be.
+  let graph: { at: number; value: Promise<Map<string, PartyRef[]>> } | null = null;
+
+  async function loadGraph(): Promise<Map<string, PartyRef[]>> {
+    const empty = new Map<string, PartyRef[]>();
+    if (!resolveParticipants) return empty;
+    try {
+      const partyRows = await readAll<TenantEntity>(entities, { type: types.party });
+      // No parties means no edge can resolve to a name, so the second read is
+      // pure cost — which is the common case on a freshly connected workspace.
+      if (partyRows.length === 0) return empty;
+      const parties = partyRows.map((entity) => toParty(entity, scope));
+      const edges = await readAll<TenantRelationship>(relationships, {});
+      return indexParticipants(edges, parties, participantEdges);
+    } catch {
+      return empty;
+    }
+  }
+
+  function participantGraph(): Promise<Map<string, PartyRef[]>> {
+    const now = Date.now();
+    if (!graph || now - graph.at > graphTtlMs) {
+      graph = { at: now, value: loadGraph() };
+    }
+    return graph.value;
+  }
+
+  return {
+    async listDocuments(query) {
+      const documents = await page<TenantEntity>(types.document, query, (entity) => entity);
+      // Skip the graph entirely for an empty page: an empty corpus should cost
+      // exactly the one request it costs today.
+      const byRecord = documents.results.length ? await participantGraph() : new Map();
+      return {
+        ...documents,
+        results: documents.results.map((entity) =>
+          toSourceDocument(entity, scope, sourceLabels, byRecord.get(String(entity.id)) ?? []),
+        ),
+      };
+    },
+
+    async getDocument(id) {
+      const [entity, byRecord] = await Promise.all([
+        http.get<TenantEntity>(`${entities}/${encodeURIComponent(id)}`),
+        participantGraph(),
+      ]);
+      return toSourceDocument(entity, scope, sourceLabels, byRecord.get(String(entity.id)) ?? []);
+    },
+
+    listParties(query = {}) {
+      return page<Party>(types.party, query, (entity) => toParty(entity, scope));
+    },
+
+    async listMoneyEvents(query = {}) {
+      const events = await page<TenantEntity>(types.moneyEvent, query, (entity) => entity);
+      const byRecord = events.results.length ? await participantGraph() : new Map();
+      return {
+        ...events,
+        results: events.results.map((entity) =>
+          toMoneyEvent(entity, scope, sourceLabels, byRecord.get(String(entity.id)) ?? []),
+        ),
+      };
+    },
+
+    /**
+     * Freshness per SOURCE, assembled from the connections the tenant has.
+     *
+     * It reads the connections API rather than the documents, because "Dropbox
+     * has not synced since Tuesday" is a true and useful statement even when
+     * Dropbox has contributed zero documents — and a count derived from the
+     * corpus alone can never say it.
+     */
+    async listSources(): Promise<SourceFreshness[]> {
+      const [connectionPage, documentPage] = await Promise.all([
+        http
+          .get<DrfPage<Record<string, unknown>>>(connections)
+          .catch(() => EMPTY_PAGE as DrfPage<Record<string, unknown>>),
+        http
+          .get<DrfPage<TenantEntity>>(entities, { params: { type: types.document, page_size: 1 } })
+          .catch(() => EMPTY_PAGE as DrfPage<TenantEntity>),
+      ]);
+
+      void documentPage; // total count is per-source below; kept for the empty-corpus case
+
+      const now = Date.now();
+      return (connectionPage.results ?? []).map((connection) => {
+        const key = String(connection.connector_key ?? connection.connectorKey ?? 'unknown');
+        const lastSyncedAt =
+          (connection.last_synced_at as string) ??
+          (connection.lastSyncedAt as string) ??
+          (connection.last_checked_at as string) ??
+          (connection.lastCheckedAt as string) ??
+          null;
+        const documentCount = Number(connection.document_count ?? connection.documentCount ?? 0);
+        const error = String(connection.check_error ?? connection.checkError ?? '');
+        const status = connection.status ?? '';
+
+        let health: SourceFreshness['status'] = 'ok';
+        let detail = '';
+        if (error || status === 'error' || status === 'needs_reauth') {
+          health = 'error';
+          detail = error || `Connection status is ${String(status)}.`;
+        } else if (!lastSyncedAt) {
+          health = 'never';
+          detail = 'Connected, but nothing has arrived yet.';
+        } else if (now - Date.parse(lastSyncedAt) > staleAfterHours * 3_600_000) {
+          health = 'stale';
+          detail = `Last sync was more than ${staleAfterHours} hours ago.`;
+        }
+
+        return {
+          source: key,
+          label: String(connection.connector_name ?? connection.connectorName ?? sourceLabels[key] ?? key),
+          documentCount,
+          lastSyncedAt,
+          status: health,
+          ...(detail ? { detail } : {}),
+        };
+      });
+    },
+  };
+}