@startsimpli/integrations 0.2.13 → 0.2.16

package/src/corpus/text.ts

@@ -281,3 +281,79 @@ const OUTCOME_HEADING = /\b(next\s*steps?|action\s*items?|follow[\s-]?ups?|to[\s
 export function outcomeSections(text: string): TextSection[] {
   return splitSections(text).filter((section) => OUTCOME_HEADING.test(section.heading));
 }
+
+// ---------------------------------------------------------------------------
+//  assembling a document's text
+// ---------------------------------------------------------------------------
+
+/** The text-bearing shape, so this file still knows nothing about the corpus. */
+interface TextBearing {
+  body: string;
+  summary?: string;
+  sections: Array<{ label: string; text: string }>;
+}
+
+/**
+ * Every distinct piece of text a document carries, summary FIRST.
+ *
+ * DE-DUPLICATED, because `body` deliberately falls back to `summary` on a corpus
+ * that has never seen a full text — and sending the same 5,000 characters to a
+ * model twice is a bill, not a feature.
+ *
+ * SUMMARY FIRST is not cosmetic. It is the piece a person already curated, the
+ * piece the deterministic pass reads, and the piece that must survive when the
+ * rest has to be cut.
+ */
+export function documentParts(document: TextBearing): string[] {
+  const summary = (document.summary ?? '').trim();
+  const body = (document.body ?? '').trim();
+  const parts = [summary, body === summary ? '' : body];
+  for (const section of document.sections ?? []) {
+    const text = (section.text ?? '').trim();
+    if (!text || parts.includes(text)) continue;
+    parts.push(`## ${section.label}\n${text}`);
+  }
+  return parts.filter(Boolean);
+}
+
+/**
+ * Trim to a budget while KEEPING THE END.
+ *
+ * A prefix `slice(0, maxChars)` is the wrong cut for a conversation. People
+ * agree things in the last five minutes of a meeting — "I'll send it Friday",
+ * "put that on Megan" — so a head-first truncation deletes precisely the half
+ * the extractor exists to read, and does it silently. Keeping a quarter from the
+ * front preserves who is in the room and what it is about; the rest of the
+ * budget goes to the end.
+ */
+export function trimKeepingTheEnd(text: string, maxChars: number): string {
+  if (maxChars <= 0) return '';
+  if (text.length <= maxChars) return text;
+  const head = Math.floor(maxChars * 0.25);
+  const tail = maxChars - head;
+  return [
+    text.slice(0, head),
+    `…[truncated ${text.length - maxChars} characters]…`,
+    text.slice(text.length - tail),
+  ].join('\n');
+}
+
+/**
+ * A document as one string for a prompt, inside a character budget.
+ *
+ * THE SUMMARY IS PAID FOR FIRST and only trimmed if it alone does not fit. Once
+ * a transcript lands, `body` is ten times the size of the summary; a naive join
+ * plus a prefix cut spends the entire budget on the opening minutes of the
+ * meeting and never reaches the outcome block at all — on the Ask surface, whose
+ * per-document budget is 6,000 characters, the summary would simply stop
+ * reaching the model.
+ */
+export function documentText(document: TextBearing, maxChars: number): string {
+  const [first = '', ...rest] = documentParts(document);
+  const summary = trimKeepingTheEnd(first, maxChars);
+  const verbatim = rest.join('\n\n');
+  if (!verbatim) return summary;
+  const remaining = maxChars - summary.length - 2;
+  const kept = trimKeepingTheEnd(verbatim, remaining);
+  return [summary, kept].filter(Boolean).join('\n\n');
+}

package/src/corpus/types.ts

@@ -92,8 +92,26 @@ export interface DocumentSection {
 export interface SourceDocument {
   id: string;
   title: string;
-  /** The primary readable text — a summary, a description, a body. */
+  /**
+   * The FULL text — a transcript, a PDF's extracted text, a document's contents.
+   *
+   * Falls back to `summary` when the source has never delivered a full text, so
+   * that every consumer of `body` keeps working on a corpus of summaries. When
+   * both exist they are DIFFERENT strings and both survive; see `summary`.
+   */
   body: string;
+  /**
+   * The short form somebody or something already WROTE — Granola's AI summary, a
+   * file's description. Two fields and not one, exactly as the tenant's canonical
+   * schema declares them, because the day a transcript arrives is the day a
+   * single field would silently lose the summary.
+   *
+   * THAT IS NOT HYPOTHETICAL. This mapper collapsed them for the whole of the
+   * epic so far, and the collapse would have deleted the deterministic rules pass
+   * — the `## Next Steps` block lives in the summary and nowhere else — on the
+   * exact deploy that first switched transcripts on.
+   */
+  summary?: string;
   /** Additional long-form the source provided. See `DocumentSection`. */
   sections: DocumentSection[];
   /** When the thing HAPPENED, which is not when it was ingested. */
@@ -109,6 +127,19 @@ export interface SourceDocument {
   scope: CorpusScope;
   /** When the connector last wrote this row. Drives the freshness claim. */
   ingestedAt?: string | null;
+  /**
+   * Attribute names the SERVER withheld from this reader
+   * (bd startsim-768w.38.33). The tenant's per-field read gate strips them from
+   * `data` and names them back in `redacted_fields`; this carries that fact all
+   * the way to the screen and into the prompt.
+   *
+   * IT HAS TO TRAVEL, and that is the whole reason it is on the document rather
+   * than swallowed at the mapper. A record whose transcript was withheld is NOT
+   * a record with no transcript. Anything that cannot tell the two apart — a
+   * reader, or a model asked "was this discussed?" — answers "no" with total
+   * confidence about text it was simply never shown.
+   */
+  withheldFields?: string[];
 }
 
 /**
@@ -281,6 +312,14 @@ export interface AskResult {
    * returned something it should not have, and is worth seeing.
    */
   droppedOutOfScope: number;
+  /**
+   * Attribute names the server withheld from this reader on at least one record
+   * that was considered (bd startsim-768w.38.33). Reported for exactly the
+   * reason `droppedOutOfScope` is: an answer built from a partial record has to
+   * say so, or "your corpus has nothing about that" and "you were not shown the
+   * part that does" become the same sentence.
+   */
+  withheldFields?: string[];
   /** Which provider/model actually answered, and whether it was a real one. */
   provider: LlmDescription;
   /** Freshness at answer time — an answer over a stale corpus says so. */
@@ -305,6 +344,13 @@ export interface PromptDocument {
   occurredAt: string | null;
   participants: string[];
   text: string;
+  /**
+   * Fields the reader was not allowed to see (bd startsim-768w.38.33). Rendered
+   * into the document block so the model states a limit instead of inventing a
+   * certainty: without it, a summary-only view of a meeting whose transcript was
+   * withheld reads as proof the thing was never discussed.
+   */
+  withheldFields?: string[];
 }
 
 export interface AnswerRequest {
@@ -515,9 +561,56 @@ export interface CommitmentApi {
  */
 export interface CorpusExtractorLlm {
   extract(input: ExtractionRequest): Promise<ExtractionDraft>;
+  /**
+   * CONFIRM the model's own candidates against the records they came from.
+   *
+   * OPTIONAL, and the run reports `verified: false` when it is missing rather
+   * than pretending a check happened. Present on the shipped adapter.
+   *
+   * WHY IT EXISTS. More text raises recall and lowers precision, and transcripts
+   * are roughly ten times the text of a summary. Measured on the summaries
+   * alone, adding the model pass took recall 56.1% → 65.9% and precision 95.8% →
+   * 67.5%; a corpus of verbatim speech makes that trade worse, not better,
+   * because a transcript is full of sentences that SOUND like commitments and
+   * are hedges, jokes, or somebody reading an agenda aloud.
+   *
+   * ONE BATCHED CALL over every candidate with echoed ids
+   * (feedback_batch_llm_calls) — never a call per item.
+   */
+  verify?(input: VerificationRequest): Promise<VerificationDraft>;
   describe(): LlmDescription;
 }
 
+/**
+ * What the verifier is asked. The candidates AND the records they claim to come
+ * from, so the check is against the source rather than against the candidate's
+ * own plausibility.
+ */
+export interface VerificationRequest {
+  documents: Array<{ id: string; title: string; text: string }>;
+  candidates: Array<{
+    /** Echoed back on the verdict. Stable only within this call. */
+    id: string;
+    documentId: string;
+    subtype: string;
+    title: string;
+    owner?: string;
+    /** The span the extractor said this came from. */
+    quote: string;
+  }>;
+}
+
+export interface VerificationDraft {
+  verdicts: Array<{
+    id: string;
+    confirmed: boolean;
+    /** The supporting span, which the verifier may correct. */
+    quote?: string;
+    /** Why not, in the verifier's words. Surfaced, never silently swallowed. */
+    reason?: string;
+  }>;
+}
+
 export interface ExtractionRequest {
   documents: Array<{
     id: string;
@@ -591,8 +684,21 @@ export interface ExtractionResult {
   nextOffset: number | null;
   /** Items the vendor's own structure produced, with no model involved. */
   fromRules: number;
-  /** Items only the model found. */
+  /** Items only the model found, AFTER verification. */
   fromModel: number;
+  /**
+   * Model candidates thrown away because the span they quoted does not appear in
+   * the record. Free to check, and it is the signature of an invented item.
+   */
+  ungrounded: number;
+  /** Grounded candidates the verification pass declined to confirm. */
+  unconfirmed: number;
+  /**
+   * Whether the model verification pass actually ran. False means the surviving
+   * model items passed the deterministic grounding check and nothing else — the
+   * number is still reported, it is just not called verified.
+   */
+  verified: boolean;
   created: number;
   /** Existing items a new mention was attached to — the dedupe, counted. */
   merged: number;