@startsimpli/integrations 0.2.2 → 0.2.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@startsimpli/integrations",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "description": "Provider-agnostic types, API clients, hooks, and UI primitives for email + calendar integrations across StartSimpli apps.",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/__tests__/corpus-extract.test.ts

@@ -431,6 +431,28 @@ describe('extractCommitments', () => {
     expect(result.created).toBeGreaterThan(0);
   });
 
+  it('slices a sweep so one call fits the gateway, and says where to resume', async () => {
+    // A deployed tenant closes a request at 60s and one call over the whole
+    // corpus does not finish inside it (measured on tandem — 504). The sweep is
+    // therefore BATCHED INTO CALLS THAT FIT, never one call per document.
+    const corpus = createFixtureCorpusApi({ scope: TANDEM, documents: FIXTURE_TRACKED_DOCUMENTS });
+    const extractor = createFixtureExtractor();
+    const spy = vi.spyOn(extractor, 'extract');
+
+    const first = await extractCommitments({ scope: TANDEM, limit: 2 }, { corpus, extractor });
+    expect(first.documentsProcessed).toBe(2);
+    expect(first.totalDocuments).toBe(FIXTURE_TRACKED_DOCUMENTS.length);
+    expect(first.nextOffset).toBe(2);
+    expect(spy.mock.calls[0][0].documents).toHaveLength(2); // still N per call
+
+    const second = await extractCommitments(
+      { scope: TANDEM, offset: first.nextOffset!, limit: 2 },
+      { corpus, extractor },
+    );
+    expect(second.documentsProcessed).toBe(1);
+    expect(second.nextOffset).toBeNull();
+  });
+
   it('refuses outright when the session carries no workspace claim', async () => {
     const corpus = createFixtureCorpusApi({ leakScope: true, documents: FIXTURE_TRACKED_DOCUMENTS });
     const listDocuments = vi.spyOn(corpus, 'listDocuments');

package/src/__tests__/corpus-tracker.test.tsx

@@ -196,6 +196,8 @@ describe('TrackerPage — what is outstanding', () => {
 describe('TrackerPage — running the extraction', () => {
   const RESULT = {
     documentsProcessed: 16,
+    totalDocuments: 16,
+    nextOffset: null,
     fromRules: 24,
     fromModel: 11,
     created: 30,

package/src/corpus/extract.ts

@@ -235,6 +235,17 @@ export interface ExtractRequest {
   scope: CorpusScope;
   /** Narrow the run to specific records. Omit to sweep the corpus. */
   documentIds?: string[];
+  /**
+   * Where this call starts in the in-scope corpus, and how many documents it
+   * may take. Omit both to do the whole thing in one model call.
+   *
+   * This exists for a deployment fact, not a design preference: the tandem ALB
+   * closes a request at 60 seconds, and one batched call over all sixteen real
+   * documents does not finish inside it (measured — 504). Slicing keeps every
+   * call batched over N documents while letting the caller walk the corpus.
+   */
+  offset?: number;
+  limit?: number;
 }
 
 export interface ExtractDeps {
@@ -258,12 +269,24 @@ export async function extractCommitments(
 
   const { documents: candidates, matched } = await readDocuments(corpus, documentLimit, maxPages);
   const { kept, dropped } = enforceScope(candidates, request.scope);
-  const documents = request.documentIds?.length
+  const inScope = request.documentIds?.length
     ? kept.filter((document) => request.documentIds!.includes(document.id))
     : kept;
 
+  // The slice this call handles. `listDocuments` orders stably (the tenant's
+  // (org_id, entity_type, -created_at) index), so an offset means the same thing
+  // between calls of one sweep.
+  const offset = Math.max(0, request.offset ?? 0);
+  const documents = request.limit ? inScope.slice(offset, offset + request.limit) : inScope.slice(offset);
+  const consumed = offset + documents.length;
+  const nextOffset = consumed < inScope.length ? consumed : null;
+
   if (documents.length === 0) {
-    return { ...empty(extractor, 'nothing in scope'), droppedOutOfScope: dropped.length };
+    return {
+      ...empty(extractor, 'nothing in scope'),
+      totalDocuments: inScope.length,
+      droppedOutOfScope: dropped.length,
+    };
   }
 
   // 3. rules first, so the model can be told what not to repeat.
@@ -317,6 +340,8 @@ export async function extractCommitments(
 
   return {
     documentsProcessed: documents.length,
+    totalDocuments: inScope.length,
+    nextOffset,
     fromRules: ruleDrafts.length,
     fromModel: modelDrafts.length,
     created: reconciled.created,
@@ -325,8 +350,8 @@ export async function extractCommitments(
     needsReview: resolutions.needsReview,
     droppedOutOfScope: dropped.length,
     provider: extractor.describe(),
-    ...(matched > candidates.length ? { } : {}),
   };
+  void matched;
 }
 
 /**
@@ -424,6 +449,8 @@ function empty(extractor: CorpusExtractorLlm, _why: string): ExtractionResult {
   void _why;
   return {
     documentsProcessed: 0,
+    totalDocuments: 0,
+    nextOffset: null,
     fromRules: 0,
     fromModel: 0,
     created: 0,

package/src/corpus/types.ts

@@ -565,12 +565,30 @@ export interface CommitmentTrackerApi extends CommitmentApi {
    * Run extraction over documents that have not been processed. Optional: a
    * read-only deployment omits it and the control does not render.
    */
-  extract?(input: { scope: CorpusScope; documentIds?: string[] }): Promise<ExtractionResult>;
+  extract?(input: {
+    scope: CorpusScope;
+    documentIds?: string[];
+    /** Resume point for a batched sweep. See `ExtractionResult.nextOffset`. */
+    offset?: number;
+    limit?: number;
+  }): Promise<ExtractionResult>;
 }
 
 export interface ExtractionResult {
-  /** Documents read and sent to the extractor. */
+  /** Documents read and sent to the extractor IN THIS CALL. */
   documentsProcessed: number;
+  /** Documents in scope altogether, so a partial sweep can say what it is of. */
+  totalDocuments: number;
+  /**
+   * Where a resumed sweep should start, or `null` when the corpus is done.
+   *
+   * A run is RESUMABLE because it has to be: a deployed tenant sits behind a
+   * gateway with a request timeout (60s on the tandem ALB, measured — a single
+   * call over sixteen documents 504s), and one model call per document is
+   * forbidden. So the sweep is batched into calls that fit, and the caller walks
+   * the offsets. Still N documents per model call; just not always all of them.
+   */
+  nextOffset: number | null;
   /** Items the vendor's own structure produced, with no model involved. */
   fromRules: number;
   /** Items only the model found. */