@startsimpli/integrations 0.2.32 → 0.2.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@startsimpli/integrations",
3
- "version": "0.2.32",
3
+ "version": "0.2.33",
4
4
  "description": "Provider-agnostic types, API clients, hooks, and UI primitives for email + calendar integrations across StartSimpli apps.",
5
5
  "main": "./src/index.ts",
6
6
  "types": "./src/index.ts",
@@ -0,0 +1,337 @@
1
+ /**
2
+ * THE CLIENT ASKS THE SERVER FOR A NARROWER ANSWER, AND CHECKS THAT IT GOT ONE.
3
+ *
4
+ * Two bugs, one root cause (bd startsim-o0o6o.2, bd startsim-o0o6o.3). The
5
+ * corpus adapter was written against a tenant data plane that filtered on
6
+ * `?type=` and nothing else, and several of its comments still assert that as
7
+ * fact. It has not been true since bd startsim-768w.38.12 and bd startsim-te3r:
8
+ * `RelationshipQuery` applies `entity__in` bounded at `MAX_ID_LIST` ids, and
9
+ * `TenantPageNumberPagination` honours `page_size` up to 200.
10
+ *
11
+ * WHY THE ANSWER IS CHECKED AND NOT ASSUMED. The tenant's response to a filter
12
+ * it does not recognise is an UNFILTERED PAGE with the parameter named in
13
+ * `ignored_filters` (bd startsim-768w.38.39) — not a 400 and not an empty list.
14
+ * So "send the filter and trust the rows" is the dangerous shape here: the reply
15
+ * to a narrowing request that was ignored looks exactly like a narrow answer
16
+ * with a lot of rows in it. start-simpli#154 set the precedent — read the
17
+ * server's own `applied_filters` and do not use an unconfirmed result — and the
18
+ * fakes below behave the way the wire behaves, ignoring what they do not support
19
+ * and answering as wide as they were asked.
20
+ */
21
+ import { describe, expect, it } from 'vitest';
22
+
23
+ import { createCorpusApi } from '../corpus/client';
24
+ import type { CorpusHttp } from '../corpus/client';
25
+ import { startExtractionRun } from '../corpus/runs';
26
+ import {
27
+ DEFAULT_RUN_KEY,
28
+ DEFAULT_TYPE_KEYS,
29
+ MAX_ID_LIST,
30
+ extractionRunToEntity,
31
+ } from '../corpus/entities';
32
+ import type { TenantEntity, TenantRelationship } from '../corpus/entities';
33
+ import {
34
+ createFixtureCorpusApi,
35
+ createFixtureCorpusHttp,
36
+ createFixtureExtractor,
37
+ FIXTURE_ORG,
38
+ FIXTURE_SOURCE_LABELS,
39
+ } from '../corpus/fixtures';
40
+ import type { ExtractionRun, ExtractionRunStore } from '../corpus/types';
41
+
42
+ const SCOPE = { orgId: FIXTURE_ORG };
43
+
44
+ function apiOver(overrides: Parameters<typeof createFixtureCorpusHttp>[0] = {}) {
45
+ const { http, requests } = createFixtureCorpusHttp(overrides);
46
+ const api = createCorpusApi({ http, scope: SCOPE, sourceLabels: FIXTURE_SOURCE_LABELS });
47
+ return { api, requests };
48
+ }
49
+
50
+ // ---------------------------------------------------------------------------
51
+ // bd startsim-o0o6o.3 — the sweep record, past the fifth row
52
+ // ---------------------------------------------------------------------------
53
+
54
+ function run(id: string): ExtractionRun {
55
+ return {
56
+ runId: id,
57
+ scope: SCOPE,
58
+ status: 'succeeded',
59
+ cursor: 4,
60
+ totalDocuments: 4,
61
+ totals: {
62
+ documentsProcessed: 4, fromRules: 2, fromModel: 1, ungrounded: 0, unconfirmed: 0,
63
+ created: 3, merged: 0, resolved: 0, droppedOutOfScope: 0, modelCalls: 4, modelFailures: 0,
64
+ },
65
+ processedIds: [],
66
+ modelIncompleteIds: [],
67
+ needsReview: [],
68
+ provider: { provider: 'fixture', model: 'fixture-rules', live: false },
69
+ startedAt: '2026-08-01T10:00:00Z',
70
+ updatedAt: '2026-08-01T10:06:00Z',
71
+ finishedAt: '2026-08-01T10:06:00Z',
72
+ leaseOwner: null,
73
+ leaseUntil: null,
74
+ error: null,
75
+ lastModelFailure: null,
76
+ history: [],
77
+ };
78
+ }
79
+
80
+ /** The singleton row, exactly as `writeRun` lands it. */
81
+ function singleton(): TenantEntity {
82
+ return {
83
+ id: 'run-singleton',
84
+ ...extractionRunToEntity(run('sweep-1'), DEFAULT_TYPE_KEYS.extractionRun, DEFAULT_RUN_KEY),
85
+ };
86
+ }
87
+
88
+ /**
89
+ * Rows that are NOT the singleton, and which is which is not a hypothetical.
90
+ *
91
+ * A second workspace key, a renamed `runKey`, a partial migration or a
92
+ * hand-written row all put `extraction_run` rows next to it, and the tenant
93
+ * orders entities `-created_at` — so the singleton, which is the OLDEST because
94
+ * it was written first and is rewritten in place, sorts LAST. Newest first is
95
+ * the order these arrive in.
96
+ */
97
+ function noise(count: number): TenantEntity[] {
98
+ return Array.from({ length: count }, (_, index) => ({
99
+ id: `run-noise-${index}`,
100
+ entity_type: DEFAULT_TYPE_KEYS.extractionRun,
101
+ external_id: `corpus-extraction-sweep-other-${index}`,
102
+ name: `Extraction sweep other-${index}`,
103
+ data: { run: JSON.stringify(run(`other-${index}`)), _origin: 'tenant-extraction' },
104
+ }));
105
+ }
106
+
107
+ describe('readRun tells "nothing has ever run" apart from "I could not tell"', () => {
108
+ it('finds the singleton when it is NOT among the newest five rows', async () => {
109
+ // Six rows, singleton last: the exact corpus at which `page_size: 5` starts
110
+ // answering `null` — which every caller reads as "no sweep has ever run" and
111
+ // `startExtractionRun` acts on by starting a duplicate sweep.
112
+ const { api } = apiOver({ entities: [...noise(5), singleton()], relationships: [] });
113
+
114
+ const read = await api.readRun();
115
+
116
+ expect(read?.runId).toBe('sweep-1');
117
+ });
118
+
119
+ it('THROWS rather than reporting "never run" when the row could be past the cap', async () => {
120
+ // Past the server's own `max_page_size`, one read genuinely cannot see every
121
+ // row — and the honest answer to "has a sweep run here" is then neither the
122
+ // run nor `null`. `null` is a FACT in this contract; a truncated page is not
123
+ // evidence for it.
124
+ const { api } = apiOver({
125
+ entities: [...noise(MAX_ID_LIST + 20), singleton()],
126
+ relationships: [],
127
+ });
128
+
129
+ await expect(api.readRun()).rejects.toThrow(/could not/i);
130
+ });
131
+
132
+ it('still answers null for a workspace where nothing HAS run', async () => {
133
+ // The fact this whole contract exists to preserve: absence is reportable.
134
+ const { api } = apiOver({ entities: [], relationships: [] });
135
+
136
+ expect(await api.readRun()).toBeNull();
137
+ });
138
+
139
+ it('matches the singleton key in EITHER spelling the wire delivers it in', async () => {
140
+ // `@startsimpli/api`'s fetch wrapper transforms every response key
141
+ // snake_case -> camelCase, so a browser-side read of this row carries
142
+ // `externalId` and never `external_id`. This lookup read one spelling, so in
143
+ // the browser it matched nothing at ANY page size and reported "never run"
144
+ // over a workspace with a sweep in it — the same seam, and the same lie, as
145
+ // reading one spelling of `applied_filters`.
146
+ const { external_id: key, ...rest } = singleton();
147
+ const { api } = apiOver({
148
+ entities: [{ ...rest, externalId: key }],
149
+ relationships: [],
150
+ });
151
+
152
+ expect((await api.readRun())?.runId).toBe('sweep-1');
153
+ });
154
+
155
+ it('asks for the widest page the server will serve, not five rows', async () => {
156
+ const { api, requests } = apiOver({ entities: [singleton()], relationships: [] });
157
+ await api.readRun();
158
+
159
+ const read = requests.find(
160
+ (request) => request.params.type === DEFAULT_TYPE_KEYS.extractionRun,
161
+ );
162
+ expect(read?.params.page_size).toBe(MAX_ID_LIST);
163
+ });
164
+ });
165
+
166
+ // ---------------------------------------------------------------------------
167
+ // bd startsim-o0o6o.2 — the edges of ONE PAGE, not the whole edge table
168
+ // ---------------------------------------------------------------------------
169
+
170
+ const PARTY: TenantEntity = {
171
+ id: 'party_dana', entity_type: 'party', external_id: 'granola:dana@example.com',
172
+ name: 'Dana Reed',
173
+ data: { display_name: 'Dana Reed', email: 'dana@example.com', source: 'granola' },
174
+ };
175
+
176
+ function document(id: string): TenantEntity {
177
+ return {
178
+ id, entity_type: 'source_document', external_id: `granola:${id}`, name: `Record ${id}`,
179
+ data: { source: 'granola', title: `Record ${id}`, occurred_at: '2026-08-01T09:00:00Z' },
180
+ };
181
+ }
182
+
183
+ describe('the participant join reads the edges of a PAGE, not the edge table', () => {
184
+ it('names the page’s ids in entity__in instead of asking for everything', async () => {
185
+ const { api, requests } = apiOver();
186
+ await api.listDocuments({ page: 1, pageSize: 3 });
187
+
188
+ const edgeReads = requests.filter((request) => request.path.includes('relationships'));
189
+ expect(edgeReads.length).toBeGreaterThan(0);
190
+ // THE ASSERTION THIS FILE EXISTS FOR. `readAll(relationships, {})` sent no
191
+ // parameters at all and walked up to twenty pages of every edge in the
192
+ // workspace, on a comment claiming the endpoint had no filters.
193
+ for (const read of edgeReads) {
194
+ const named = String(read.params.entity__in ?? '').split(',').filter(Boolean);
195
+ expect(named.length).toBeGreaterThan(0);
196
+ expect(named.length).toBeLessThanOrEqual(MAX_ID_LIST);
197
+ }
198
+ expect(String(edgeReads[0].params.entity__in).split(',')).toHaveLength(3);
199
+ });
200
+
201
+ it('still resolves the people on the page it narrowed to', async () => {
202
+ const { api } = apiOver({
203
+ entities: [document('doc-1'), PARTY],
204
+ relationships: [{ id: 'e1', rel_type: 'authored_by', source: 'doc-1', target: 'party_dana' }],
205
+ });
206
+
207
+ const page = await api.listDocuments({ page: 1 });
208
+
209
+ expect(page.results[0].participants.map((party) => party.name)).toEqual(['Dana Reed']);
210
+ });
211
+
212
+ it('chunks a page wider than the server’s id bound rather than being refused', async () => {
213
+ // A fork whose `max_page_size` is larger than `MAX_ID_LIST` hands back more
214
+ // ids than one `entity__in` may carry, and `_uuid_list` answers a longer
215
+ // list with a 400 — so the ids are chunked. Truncating them instead would
216
+ // drop the people off the tail of the page silently, which is the failure
217
+ // this whole change is undoing.
218
+ const documents = Array.from({ length: MAX_ID_LIST + 40 }, (_, index) =>
219
+ document(`doc-${index}`),
220
+ );
221
+ const last = documents[documents.length - 1];
222
+ const { api, requests } = apiOver({
223
+ pageSize: documents.length,
224
+ entities: [...documents, PARTY],
225
+ relationships: [
226
+ { id: 'e-last', rel_type: 'involves', source: String(last.id), target: 'party_dana' },
227
+ ],
228
+ });
229
+
230
+ const page = await api.listDocuments({ page: 1 });
231
+
232
+ const edgeReads = requests.filter((request) => request.path.includes('relationships'));
233
+ expect(edgeReads.length).toBeGreaterThan(1);
234
+ for (const read of edgeReads) {
235
+ expect(String(read.params.entity__in).split(',').length).toBeLessThanOrEqual(MAX_ID_LIST);
236
+ }
237
+ const tail = page.results.find((record) => record.id === String(last.id));
238
+ expect(tail?.participants.map((party) => party.name)).toEqual(['Dana Reed']);
239
+ });
240
+ });
241
+
242
+ /**
243
+ * A fork that has NOT shipped `RelationshipQuery`.
244
+ *
245
+ * It does what the tenant documents itself as doing with a parameter it does not
246
+ * know: names it in `ignored_filters` and answers with the page exactly as wide
247
+ * as it was. A fake that returned `[]` here would let the client pass this suite
248
+ * while reading the whole edge table in production, which is the trap this test
249
+ * is written to spring.
250
+ */
251
+ function unfilteredEdgeHttp(edges: TenantRelationship[], rows: TenantEntity[]) {
252
+ const requests: Array<{ path: string; params: Record<string, unknown> }> = [];
253
+ const http: CorpusHttp = {
254
+ async get<T>(path: string, init?: { params?: Record<string, unknown> }): Promise<T> {
255
+ const params = init?.params ?? {};
256
+ requests.push({ path, params });
257
+ if (path.includes('relationships')) {
258
+ return {
259
+ count: edges.length, next: null, previous: null, results: edges,
260
+ applied_filters: [],
261
+ ignored_filters: Object.keys(params).filter(
262
+ (key) => key !== 'page' && key !== 'page_size',
263
+ ),
264
+ } as T;
265
+ }
266
+ if (path.includes('entities')) {
267
+ const type = params.type ? String(params.type) : '';
268
+ const matching = type ? rows.filter((row) => row.entity_type === type) : rows;
269
+ return {
270
+ count: matching.length, next: null, previous: null, results: matching,
271
+ applied_filters: [],
272
+ ignored_filters: [],
273
+ } as T;
274
+ }
275
+ throw Object.assign(new Error(`${path} responded 404`), { status: 404 });
276
+ },
277
+ };
278
+ return { http, requests };
279
+ }
280
+
281
+ describe('an unconfirmed narrowing is not used', () => {
282
+ it('renders no participants rather than joining an unnarrowed page of edges', async () => {
283
+ // start-simpli#154's precedent, on the read that matters most: the server
284
+ // reported it did NOT apply `entity__in`, so the rows it sent are an
285
+ // arbitrary page of the whole edge table and joining them would put the
286
+ // wrong people — or a partial list of the right ones — on the record. The
287
+ // Library degrades to what it showed before the join existed.
288
+ const { http, requests } = unfilteredEdgeHttp(
289
+ [{ id: 'e1', rel_type: 'authored_by', source: 'doc-1', target: 'party_dana' }],
290
+ [document('doc-1'), PARTY],
291
+ );
292
+ const api = createCorpusApi({ http, scope: SCOPE, sourceLabels: FIXTURE_SOURCE_LABELS });
293
+
294
+ const page = await api.listDocuments({ page: 1 });
295
+
296
+ expect(page.results).toHaveLength(1);
297
+ expect(page.results[0].participants).toEqual([]);
298
+ // And it does not go on to read the party roster for a join it will not do.
299
+ expect(requests.filter((request) => request.params.type === 'party')).toHaveLength(0);
300
+ });
301
+ });
302
+
303
+ describe('a store that could not answer never becomes "no sweep has ever run"', () => {
304
+ it('refuses to start a second sweep over a cursor it was unable to read', async () => {
305
+ // THE REASON THE THIRD ANSWER HAD TO REACH THIS CALLER. `startExtractionRun`
306
+ // swallowed every `readRun` failure into `null` and `null` means "start a
307
+ // new sweep" one branch later, so an undetermined read put a SECOND driver
308
+ // on a live cursor — and two model calls over one document produce two
309
+ // differently-worded items dedupe cannot see are the same, which is how a
310
+ // live re-run once turned 82 commitments into 98.
311
+ let writes = 0;
312
+ const runs: ExtractionRunStore = {
313
+ async readRun() {
314
+ throw Object.assign(
315
+ new Error('Could not determine whether a sweep has run in this workspace.'),
316
+ { status: 409 },
317
+ );
318
+ },
319
+ async writeRun(run) {
320
+ writes += 1;
321
+ return run;
322
+ },
323
+ };
324
+
325
+ await expect(
326
+ startExtractionRun(
327
+ { scope: SCOPE },
328
+ {
329
+ corpus: createFixtureCorpusApi({ scope: SCOPE }),
330
+ extractor: createFixtureExtractor(),
331
+ runs,
332
+ },
333
+ ),
334
+ ).rejects.toThrow(/could not determine/i);
335
+ expect(writes).toBe(0);
336
+ });
337
+ });
@@ -191,8 +191,10 @@ describe('both storage shapes, because both are real', () => {
191
191
  });
192
192
  });
193
193
 
194
- describe('the graph read is bounded, cached and fails soft', () => {
195
- it('costs ONE graph read across a list, a detail and a second page', async () => {
194
+ describe('the graph read is narrowed, cached and fails soft', () => {
195
+ it('costs ONE edge read across a list, a detail and a second page', async () => {
196
+ // The records are remembered per row, so re-listing them and then opening
197
+ // one of them asks the server nothing further (bd startsim-o0o6o.2).
196
198
  const { api, requests } = apiOver();
197
199
  await api.listDocuments({ page: 1 });
198
200
  await api.listDocuments({ page: 1 });
@@ -220,17 +222,26 @@ describe('the graph read is bounded, cached and fails soft', () => {
220
222
  expect(page.results.every((document) => document.participants.length === 0)).toBe(true);
221
223
  });
222
224
 
223
- it('walks every page of the graph, since the tenant ignores page_size', async () => {
224
- // Two rows a page: the edges for the last document are only reachable by
225
- // paging, which is exactly what the real PAGE_SIZE=50 does at scale.
225
+ it('narrows the edge read to the ids of the page it is rendering', async () => {
226
+ // THIS TEST USED TO ASSERT THE OPPOSITE, and its title said why: "walks
227
+ // every page of the graph, since the tenant ignores page_size". Neither half
228
+ // survived contact with the deployed server. `page_size` is honoured
229
+ // (`TenantPageNumberPagination`, bd startsim-768w.38.12) and the edge table
230
+ // is filterable (`RelationshipQuery`, bd startsim-te3r) — so a page deep in
231
+ // the corpus no longer reaches its people by reading every edge in the
232
+ // workspace and joining in the browser. It ASKS for the edges of the two
233
+ // rows it is rendering, which is the read that scales with the page instead
234
+ // of with the corpus (bd startsim-o0o6o.2).
226
235
  const { api, requests } = apiOver({ pageSize: 2 });
227
- // `pageSize: 100` is ignored by the server, exactly as the tenant ignores
228
- // it — so the Dropbox row is on page THREE however wide the caller asked.
229
- const page = await api.listDocuments({ page: 3, pageSize: 100 });
236
+ const page = await api.listDocuments({ page: 3 });
230
237
  const dropbox = page.results.find((document) => document.id === 'dbx_9812f0');
231
238
 
232
239
  expect(dropbox?.participants.map((party) => party.name)).toEqual(['Acme Robotics', 'Dana Reed']);
233
- expect(requests.filter((request) => request.path.includes('relationships')).length).toBeGreaterThan(1);
240
+ const edgeReads = requests.filter((request) => request.path.includes('relationships'));
241
+ expect(edgeReads).toHaveLength(1);
242
+ expect(String(edgeReads[0].params.entity__in).split(',')).toEqual(
243
+ page.results.map((document) => document.id),
244
+ );
234
245
  });
235
246
  });
236
247
 
@@ -34,6 +34,7 @@ import {
34
34
  DEFAULT_PARTICIPANT_EDGES,
35
35
  DEFAULT_RUN_KEY,
36
36
  DEFAULT_TYPE_KEYS,
37
+ MAX_ID_LIST,
37
38
  commitmentToEntity,
38
39
  extractionRunToEntity,
39
40
  indexParticipants,
@@ -143,17 +144,20 @@ export interface CreateCorpusApiOptions {
143
144
  /** Which edge types mean "this party is on this record". Data, not an enum. */
144
145
  participantEdges?: readonly string[];
145
146
  /**
146
- * Bound on the graph read, in PAGES.
147
+ * Bound on a walked list, in PAGES.
147
148
  *
148
- * The tenant relationships endpoint has no `?source=` filter today
149
- * (`tenant-starter/apps/api/views.py` — see also bd startsim-768w.38.12 for
150
- * the same gap on entities), so the edges for a page of documents cannot be
151
- * requested directly and the graph is read in bulk and cached. This cap is
152
- * what keeps that bounded rather than unbounded: past it, participants simply
153
- * stop resolving for the older tail instead of the page hanging.
149
+ * IT USED TO BE THE ONLY BOUND THERE WAS. This said the tenant relationships
150
+ * endpoint "has no `?source=` filter today", so the edges for a page of
151
+ * documents could not be requested directly and the whole edge table was read
152
+ * in bulk — twenty pages of it — and joined in the browser. That stopped being
153
+ * true with bd startsim-te3r: `RelationshipQuery` applies `entity__in`, whose
154
+ * own docstring says what it is for ("a comma list of ids is how a client asks
155
+ * for the edges of ONE PAGE of records"). The edge read is now narrowed to the
156
+ * ids on screen (bd startsim-o0o6o.2) and this cap is a backstop on the walk
157
+ * within that narrowing, and on the party roster read that names the far ends.
154
158
  */
155
159
  maxGraphPages?: number;
156
- /** How long a loaded graph is reused before it is read again. */
160
+ /** How long a resolved roster and a record's people are reused before re-reading. */
157
161
  graphTtlMs?: number;
158
162
  /**
159
163
  * The `external_id` the singleton extraction-run row lives at. One per
@@ -393,6 +397,26 @@ function notFound(id: string): Error & { status: number } {
393
397
  return Object.assign(new Error(`No record ${id} in this workspace.`), { status: 404 });
394
398
  }
395
399
 
400
+ /**
401
+ * "I could not tell", which is the third answer and not a variant of the second.
402
+ *
403
+ * A read that could not see every candidate row is not evidence of absence, and
404
+ * `null` in this adapter is a claim of absence that callers ACT on: a sweep
405
+ * reported as never-run is started again, over the same cursor, by a second
406
+ * driver. Thrown rather than returned so the existing `ExtractionRun | null`
407
+ * contract keeps meaning what it says — the two answers a caller may act on are
408
+ * a run and no run, and everything else is a failure to answer.
409
+ */
410
+ function undetermined(what: string): Error & { status: number } {
411
+ return Object.assign(
412
+ new Error(
413
+ `Could not determine ${what}: this workspace has more rows of that type ` +
414
+ `than one page can cover, and no filter on the tenant can name the one row.`,
415
+ ),
416
+ { status: 409 },
417
+ );
418
+ }
419
+
396
420
  export function createCorpusApi(
397
421
  options: CreateCorpusApiOptions,
398
422
  ): CorpusApi & CommitmentApi & ExtractionRunStore {
@@ -487,43 +511,132 @@ export function createCorpusApi(
487
511
  // the participant graph
488
512
  // ---------------------------------------------------------------------
489
513
  //
490
- // ONE read for a page of documents, not one per document — and cached, so a
491
- // screen that lists, opens a record and then asks a question spends it once.
514
+ // THE EDGES OF THE RECORDS ON SCREEN, AND NO OTHERS (bd startsim-o0o6o.2).
515
+ // This used to be `readAll(relationships, {})` — every edge in the workspace,
516
+ // twenty pages of it, on a comment saying the endpoint had no filters. It has
517
+ // them: `entity__in` takes the ids of one page, bounded at `MAX_ID_LIST`,
518
+ // which is the shape `RelationshipQuery`'s own docstring says it exists for.
519
+ //
520
+ // ONE read for a page of records, not one per record — and cached per record,
521
+ // so a screen that lists, opens a row and then asks a question spends it once.
492
522
  // It FAILS SOFT: a tenant whose relationships endpoint is missing or refused
493
523
  // renders exactly what it rendered before this existed, rather than an error
494
524
  // where a Library used to be.
495
- let graph: { at: number; value: Promise<Map<string, PartyRef[]>> } | null = null;
525
+ let roster: { at: number; value: Promise<Party[]> } | null = null;
526
+ const known = new Map<string, { at: number; value: PartyRef[] }>();
496
527
 
497
- async function loadGraph(): Promise<Map<string, PartyRef[]>> {
498
- const empty = new Map<string, PartyRef[]>();
499
- if (!resolveParticipants) return empty;
500
- try {
501
- const partyRows = await readAll<TenantEntity>(entities, { type: types.party });
502
- // No parties means no edge can resolve to a name, so the second read is
503
- // pure cost — which is the common case on a freshly connected workspace.
504
- if (partyRows.length === 0) return empty;
505
- const parties = partyRows.map((entity) => toParty(entity, scope));
506
- const edges = await readAll<TenantRelationship>(relationships, {});
507
- return indexParticipants(edges, parties, participantEdges);
508
- } catch {
509
- return empty;
528
+ /**
529
+ * The people an edge can point AT, read once and reused.
530
+ *
531
+ * Still a walked list, because there is no id filter on `/entities/`: the
532
+ * recognised set is `search`, `source`, `party`, the occurred window,
533
+ * `related_*`, `order` and `attr.*`, and an unrecognised `id__in` would be
534
+ * swept into `ignored_filters` and answered with the whole page. It is at
535
+ * least a TYPED read — `?type=party` is applied server-side — and it is only
536
+ * reached when the narrowed edge read came back with edges to resolve.
537
+ */
538
+ function partyRoster(): Promise<Party[]> {
539
+ const now = Date.now();
540
+ if (!roster || now - roster.at > graphTtlMs) {
541
+ roster = {
542
+ at: now,
543
+ value: readAll<TenantEntity>(entities, { type: types.party })
544
+ .then((rows) => rows.map((entity) => toParty(entity, scope)))
545
+ .catch(() => []),
546
+ };
510
547
  }
548
+ return roster.value;
511
549
  }
512
550
 
513
- function participantGraph(): Promise<Map<string, PartyRef[]>> {
551
+ /**
552
+ * The edges touching these records, or `null` when the server did not confirm
553
+ * it narrowed by them.
554
+ *
555
+ * `null` IS NOT "NO EDGES", and the difference is the whole point. A fork
556
+ * without `RelationshipQuery` accepts `entity__in`, names it in
557
+ * `ignored_filters` and answers with an arbitrary page of the WHOLE edge table
558
+ * (bd startsim-768w.38.39) — rows that look exactly like a narrow answer and
559
+ * would put the wrong people, or a fraction of the right ones, on a record.
560
+ * start-simpli#154 settled what to do with a result the server did not
561
+ * confirm: do not use it. So this reports that it could not ask, and the
562
+ * caller renders what it rendered before the join existed.
563
+ *
564
+ * `rel_type` rides along unchecked on purpose: it only ever makes the answer
565
+ * SMALLER, and `indexParticipants` ranks by edge type anyway. A fork that
566
+ * ignores it pays in PAGES rather than in correctness — a record with many
567
+ * non-participant edges can push its people onto a second page, and the walk
568
+ * below follows `next` until it has them.
569
+ */
570
+ async function edgesFor(recordIds: string[]): Promise<TenantRelationship[] | null> {
571
+ const rows: TenantRelationship[] = [];
572
+ for (let start = 0; start < recordIds.length; start += MAX_ID_LIST) {
573
+ // CHUNKED, because `_uuid_list` answers a longer list with a 400 rather
574
+ // than truncating it. Truncating here instead would drop the people off
575
+ // the tail of a wide page silently, which is the failure being undone.
576
+ const chunk = recordIds.slice(start, start + MAX_ID_LIST);
577
+ const extra: Record<string, unknown> = {
578
+ entity__in: chunk.join(','),
579
+ page_size: MAX_ID_LIST,
580
+ };
581
+ if (participantEdges.length) extra.rel_type = participantEdges.join(',');
582
+ for (let index = 1; index <= maxGraphPages; index += 1) {
583
+ const response = await http.get<DrfPage<TenantRelationship> & ServerFilterReport>(
584
+ relationships,
585
+ { params: { ...extra, page: index } },
586
+ );
587
+ const applied = response.applied_filters ?? response.appliedFilters;
588
+ if (!applied?.includes('entity__in')) return null;
589
+ const results = response.results ?? [];
590
+ rows.push(...results);
591
+ if (!response.next || results.length === 0) break;
592
+ }
593
+ }
594
+ return rows;
595
+ }
596
+
597
+ async function participantsFor(
598
+ records: Array<string | number>,
599
+ ): Promise<Map<string, PartyRef[]>> {
600
+ const out = new Map<string, PartyRef[]>();
601
+ if (!resolveParticipants) return out;
514
602
  const now = Date.now();
515
- if (!graph || now - graph.at > graphTtlMs) {
516
- graph = { at: now, value: loadGraph() };
603
+ const wanted: string[] = [];
604
+ for (const record of records) {
605
+ const id = String(record ?? '').trim();
606
+ if (!id || out.has(id) || wanted.includes(id)) continue;
607
+ const cached = known.get(id);
608
+ if (cached && now - cached.at <= graphTtlMs) out.set(id, cached.value);
609
+ else wanted.push(id);
517
610
  }
518
- return graph.value;
611
+ if (wanted.length === 0) return out;
612
+ try {
613
+ const edges = await edgesFor(wanted);
614
+ // Could not ask. Nothing is cached, so the next page asks again rather
615
+ // than remembering an answer that was never given.
616
+ if (!edges) return out;
617
+ const parties = edges.length ? await partyRoster() : [];
618
+ const indexed = edges.length
619
+ ? indexParticipants(edges, parties, participantEdges)
620
+ : new Map<string, PartyRef[]>();
621
+ for (const id of wanted) {
622
+ // A record the server confirmed has no participant edges is REMEMBERED
623
+ // as having none — otherwise every page re-asks for the same silence.
624
+ const people = indexed.get(id) ?? [];
625
+ known.set(id, { at: now, value: people });
626
+ out.set(id, people);
627
+ }
628
+ } catch {
629
+ // The Library renders without its Parties block rather than not at all.
630
+ }
631
+ return out;
519
632
  }
520
633
 
521
634
  return {
522
635
  async listDocuments(query) {
523
636
  const documents = await page<TenantEntity>(types.document, query, (entity) => entity);
524
- // Skip the graph entirely for an empty page: an empty corpus should cost
525
- // exactly the one request it costs today.
526
- const byRecord = documents.results.length ? await participantGraph() : new Map();
637
+ // Skip the edge read entirely for an empty page: an empty corpus should
638
+ // cost exactly the one request it costs today.
639
+ const byRecord = await participantsFor(documents.results.map((entity) => entity.id));
527
640
  return {
528
641
  ...documents,
529
642
  results: documents.results.map((entity) =>
@@ -557,7 +670,10 @@ export function createCorpusApi(
557
670
  async getDocument(id) {
558
671
  const [entity, byRecord] = await Promise.all([
559
672
  http.get<TenantEntity>(`${entities}/${encodeURIComponent(id)}`),
560
- participantGraph(),
673
+ // The edges of ONE record, which is what this screen needs — it no
674
+ // longer waits on a read of the whole workspace's graph to render a
675
+ // record it already named (bd startsim-o0o6o.2).
676
+ participantsFor([id]),
561
677
  ]);
562
678
  if (!isRecord(entity, id)) throw notFound(id);
563
679
  return toSourceDocument(entity, scope, sourceLabels, byRecord.get(String(entity.id)) ?? []);
@@ -569,7 +685,7 @@ export function createCorpusApi(
569
685
 
570
686
  async listMoneyEvents(query = {}) {
571
687
  const events = await page<TenantEntity>(types.moneyEvent, query, (entity) => entity);
572
- const byRecord = events.results.length ? await participantGraph() : new Map();
688
+ const byRecord = await participantsFor(events.results.map((entity) => entity.id));
573
689
  return {
574
690
  ...events,
575
691
  results: events.results.map((entity) =>
@@ -641,21 +757,38 @@ export function createCorpusApi(
641
757
  /**
642
758
  * The sweep record, read back (bd startsim-768w.38.28).
643
759
  *
644
- * ONE ROW PER WORKSPACE, at a fixed `external_id`, because the tenant entity
645
- * API filters on `?type=` and nothing else — there is no way to ask for "the
646
- * newest run", so a row per sweep would mean scanning a list that grows
647
- * forever on every poll. A singleton is one deterministic read that never
648
- * grows, and the last few sweeps ride along inside it.
760
+ * ONE ROW PER WORKSPACE, at a fixed `external_id`, because a row per sweep
761
+ * would mean scanning a list that grows forever on every poll. A singleton
762
+ * is one read that never grows, and the last few sweeps ride along inside it.
763
+ *
764
+ * THE LOOKUP CANNOT BE PUSHED DOWN, AND SAYING SO IS THE POINT. `external_id`
765
+ * is not among the parameters `/entities/` recognises — `search`, `source`,
766
+ * `party`, the occurred window, `related_*`, `order`, `attr.*` — so the match
767
+ * happens here. `attr.external_id` is NOT the way around that: an attribute
768
+ * nobody declared has no `core_attribute` rows, and `EntityQuery._attribute`
769
+ * reports the filter as APPLIED before any row matches, so the reply to it is
770
+ * an empty page that the envelope swears was narrowed. A filter that cannot
771
+ * be told apart from "no such record" is worse than no filter.
772
+ *
773
+ * SO THE BOUND IS WHAT HAS TO BE HONEST (bd startsim-o0o6o.3). This asked for
774
+ * five rows, because the singleton "is" a singleton — but nothing enforces
775
+ * that, entities come back `-created_at`, and the singleton is the OLDEST row
776
+ * of its type because it is rewritten in place. Six `extraction_run` rows and
777
+ * the read stopped seeing it, returned `null`, and every caller read that as
778
+ * "no sweep has ever run" — which `startExtractionRun` acts on by starting a
779
+ * duplicate sweep over a live cursor. It now asks for the widest page the
780
+ * paginator will serve and, when the row could still be past the end of it,
781
+ * says it could not tell instead of saying no.
649
782
  *
650
783
  * FAIL-SOFT ON ABSENCE, NOT ON FAILURE. No run yet is `null`, which is a
651
- * fact. A 5xx is thrown, because "we could not reach the workspace" and
652
- * "nothing has ever run" are different answers and the screen renders them
653
- * differently.
784
+ * fact. Everything else throws, because "we could not reach the workspace",
785
+ * "we could not see far enough" and "nothing has ever run" are different
786
+ * answers and only the last one is safe to act on.
654
787
  */
655
788
  async readRun() {
656
789
  const response = await http
657
790
  .get<DrfPage<TenantEntity>>(entities, {
658
- params: { type: types.extractionRun, page: 1, page_size: 5 },
791
+ params: { type: types.extractionRun, page: 1, page_size: MAX_ID_LIST },
659
792
  })
660
793
  .catch((err: { status?: number }) => {
661
794
  // An undeclared type is a 403 from the read gate, and it means exactly
@@ -664,9 +797,18 @@ export function createCorpusApi(
664
797
  throw err;
665
798
  });
666
799
  const row = (response.results ?? []).find(
667
- (entity) => String(entity.external_id ?? '') === runKey,
800
+ // BOTH SPELLINGS, for the reason `ServerFilterReport` documents: the
801
+ // `@startsimpli/api` fetch wrapper turns `external_id` into `externalId`
802
+ // on the way in, so a browser-side read matched nothing at ANY page size
803
+ // and reported "never run" over a workspace with a sweep in it. Every
804
+ // other reader in `./entities` already takes both; this one did not.
805
+ (entity) => String(entity.external_id ?? entity.externalId ?? '') === runKey,
668
806
  );
669
- return row ? toExtractionRun(row) : null;
807
+ if (row) return toExtractionRun(row);
808
+ // A page with more behind it did not fail to find the row — it failed to
809
+ // look at every row, and those are not the same answer.
810
+ if (response.next) throw undetermined(`whether a sweep has run in this workspace`);
811
+ return null;
670
812
  },
671
813
 
672
814
  async writeRun(run: ExtractionRun) {
@@ -54,6 +54,47 @@ export interface TenantRelationship {
54
54
  createdAt?: string | null;
55
55
  }
56
56
 
57
+ /**
58
+ * The most ids one `__in` list may carry, and it is the SERVER's number.
59
+ *
60
+ * `apps/api/filters.py::MAX_ID_LIST` is 200 and `_uuid_list` answers a longer
61
+ * list with a 400 rather than truncating it — "bounded so the parameter cannot
62
+ * become an unpaginated read by another name". A client that sends more does not
63
+ * get a wider answer, it gets no answer, so the ids of a page are CHUNKED at
64
+ * this number rather than sent in one request and hoped for.
65
+ */
66
+ export const MAX_ID_LIST = 200;
67
+
68
+ /**
69
+ * Parameters `/api/v1/relationships/` really applies (`RelationshipQuery._FILTERS`).
70
+ *
71
+ * Named here rather than inside either the adapter or the fixture, because both
72
+ * have to agree about it and a second copy is how they stop agreeing. It is a
73
+ * DESCRIPTION of a deployed server, not a contract this package enforces: a fork
74
+ * that has not shipped `RelationshipQuery` ignores every one of these, which is
75
+ * why `applied_filters` is read on the response rather than this list being
76
+ * trusted.
77
+ */
78
+ export const RELATIONSHIP_FILTERS: ReadonlySet<string> = new Set([
79
+ 'rel_type',
80
+ 'source',
81
+ 'target',
82
+ 'entity',
83
+ 'source__in',
84
+ 'target__in',
85
+ 'entity__in',
86
+ 'source_type',
87
+ 'target_type',
88
+ ]);
89
+
90
+ /** Consumed by pagination and negotiation, never a filter (`filters.py::_RESERVED`). */
91
+ export const RESERVED_PARAMS: ReadonlySet<string> = new Set([
92
+ 'page',
93
+ 'page_size',
94
+ 'format',
95
+ 'type',
96
+ ]);
97
+
57
98
  /**
58
99
  * Edges that mean "this party is ON this record", in the order they should be
59
100
  * listed. DATA, not a closed set: a tenant that models its own edge passes its
@@ -42,6 +42,7 @@ import {
42
42
  toParty,
43
43
  toSourceDocument,
44
44
  } from './entities';
45
+ import { MAX_ID_LIST, RELATIONSHIP_FILTERS, RESERVED_PARAMS } from './entities';
45
46
  import type { TenantEntity, TenantRelationship } from './entities';
46
47
  import { splitSections, stem } from './text';
47
48
  import type {
@@ -875,10 +876,22 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
875
876
  * stands in for an OLDER fork — which is a useful thing to have a stand-in
876
877
  * for, since that is the fork `listSources` refuses to print a count for
877
878
  * (bd startsim-jevtu), but it is a lagging stub and not the contract;
878
- * • `page_size` is IGNORED (`PageNumberPagination` with no
879
- * `page_size_query_param`), so a caller asking for 200 gets `pageSize`;
880
- * • `/api/v1/relationships/` returns every edge in the org, unfiltered,
881
- * because the tenant has no `?source=` on it either.
879
+ * • `page_size` IS honoured, capped at 200. It was written here as ignored,
880
+ * citing a `PageNumberPagination` with no `page_size_query_param`, and that
881
+ * stopped being true with `TenantPageNumberPagination` (bd
882
+ * startsim-768w.38.12): "a client asking for one row to read a count got
883
+ * fifty, and a client asking for two hundred to walk a graph got fifty and
884
+ * no indication that its request had been overruled". A fixture that keeps
885
+ * ignoring it manufactures evidence for a cap the wire does not impose;
886
+ * • `/api/v1/relationships/` FILTERS — `rel_type`, `source`, `target`,
887
+ * `entity` and the `__in` forms of the last three, bounded at
888
+ * `MAX_ID_LIST` ids and reported in `applied_filters`. This too was written
889
+ * here as "every edge in the org, unfiltered, because the tenant has no
890
+ * `?source=` on it either", and bd startsim-te3r shipped `RelationshipQuery`
891
+ * for exactly the read that comment was excusing. An UNRECOGNISED parameter
892
+ * is still ignored and the page comes back as wide as it was — that part is
893
+ * real, it is what makes an unconfirmed filter dangerous, and it is what
894
+ * `applied_filters` exists to let a client detect.
882
895
  *
883
896
  * Every request is recorded, so "did the browser filter it" and "how many round
884
897
  * trips did one screen cost" are both assertable.
@@ -886,7 +899,7 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
886
899
  export function createFixtureCorpusHttp(options: {
887
900
  entities?: TenantEntity[];
888
901
  relationships?: TenantRelationship[];
889
- /** Rows per page, matching the tenant's own `PAGE_SIZE`. */
902
+ /** Rows per page when the caller does not ask, matching the tenant's `PAGE_SIZE`. */
890
903
  pageSize?: number;
891
904
  /** Endpoints to fail, so the fail-soft paths are exercised. */
892
905
  missing?: string[];
@@ -906,15 +919,90 @@ export function createFixtureCorpusHttp(options: {
906
919
 
907
920
  function slice<T>(rows: T[], params: Record<string, unknown>) {
908
921
  const page = Number(params.page ?? 1) || 1;
909
- const start = (page - 1) * pageSize;
922
+ // `TenantPageNumberPagination.page_size_query_param` is `page_size` and its
923
+ // `max_page_size` is 200 — an OVERRULED request is served at the cap, not
924
+ // refused, which is the behaviour a client that walks a page has to survive.
925
+ const asked = Number(params.page_size);
926
+ const size = Number.isFinite(asked) && asked > 0 ? Math.min(asked, MAX_ID_LIST) : pageSize;
927
+ const start = (page - 1) * size;
910
928
  return {
911
929
  count: rows.length,
912
- next: start + pageSize < rows.length ? `?page=${page + 1}` : null,
930
+ next: start + size < rows.length ? `?page=${page + 1}` : null,
913
931
  previous: page > 1 ? `?page=${page - 1}` : null,
914
- results: rows.slice(start, start + pageSize),
932
+ results: rows.slice(start, start + size),
915
933
  };
916
934
  }
917
935
 
936
+ /**
937
+ * `/api/v1/relationships/`, as `RelationshipQuery.apply` really answers it.
938
+ *
939
+ * THE SHAPE THAT MATTERS IS THE ONE IT REFUSES TO NARROW. A parameter this
940
+ * endpoint does not recognise is swept into `ignored_filters` and the page
941
+ * comes back exactly as wide as it was — so a fixture that returned `[]` for
942
+ * an unsupported filter would let a client pass here and read the whole edge
943
+ * table in production. Every filter below is one the server really applies,
944
+ * and everything else is really ignored.
945
+ */
946
+ function edgeQuery(rows: TenantRelationship[], params: Record<string, unknown>) {
947
+ const applied: string[] = [];
948
+ const ignored: string[] = [];
949
+ const idList = (key: string): Set<string> => {
950
+ const parts = String(params[key]).split(',').map((part) => part.trim()).filter(Boolean);
951
+ // `_uuid_list` refuses a longer list with a 400 rather than truncating it,
952
+ // which is what keeps `__in` from becoming an unpaginated read by another
953
+ // name — and what a client that chunks its ids has to be tested against.
954
+ if (parts.length > MAX_ID_LIST) {
955
+ throw Object.assign(new Error(`${key}: at most ${MAX_ID_LIST} ids`), { status: 400 });
956
+ }
957
+ return new Set(parts);
958
+ };
959
+ const typeOf = (id: unknown): string =>
960
+ String(entities.find((entity) => String(entity.id) === String(id))?.entity_type ?? '');
961
+ let out = rows;
962
+
963
+ for (const [key, field] of [['source', 'source'], ['target', 'target']] as const) {
964
+ if (key in params) {
965
+ const wanted = String(params[key]);
966
+ out = out.filter((edge) => String(edge[field]) === wanted);
967
+ applied.push(key);
968
+ }
969
+ const listed = `${key}__in`;
970
+ if (listed in params) {
971
+ const wanted = idList(listed);
972
+ out = out.filter((edge) => wanted.has(String(edge[field])));
973
+ applied.push(listed);
974
+ }
975
+ }
976
+ if ('entity' in params) {
977
+ const wanted = String(params.entity);
978
+ out = out.filter((edge) => String(edge.source) === wanted || String(edge.target) === wanted);
979
+ applied.push('entity');
980
+ }
981
+ if ('entity__in' in params) {
982
+ const wanted = idList('entity__in');
983
+ out = out.filter((edge) => wanted.has(String(edge.source)) || wanted.has(String(edge.target)));
984
+ applied.push('entity__in');
985
+ }
986
+ if ('rel_type' in params) {
987
+ const wanted = new Set(
988
+ String(params.rel_type).split(',').map((part) => part.trim()).filter(Boolean),
989
+ );
990
+ out = out.filter((edge) => wanted.has(String(edge.rel_type ?? edge.relType ?? '')));
991
+ applied.push('rel_type');
992
+ }
993
+ for (const [key, end] of [['source_type', 'source'], ['target_type', 'target']] as const) {
994
+ if (key in params) {
995
+ const wanted = String(params[key]);
996
+ out = out.filter((edge) => typeOf(edge[end]) === wanted);
997
+ applied.push(key);
998
+ }
999
+ }
1000
+ for (const key of Object.keys(params)) {
1001
+ if (!RESERVED_PARAMS.has(key) && !RELATIONSHIP_FILTERS.has(key)) ignored.push(key);
1002
+ }
1003
+ return { rows: out, applied, ignored };
1004
+ }
1005
+
918
1006
  const http = {
919
1007
  async get<T>(path: string, init?: { params?: Record<string, unknown> }): Promise<T> {
920
1008
  const params = init?.params ?? {};
@@ -925,7 +1013,12 @@ export function createFixtureCorpusHttp(options: {
925
1013
  throw Object.assign(new Error(`${path} responded 404`), { status: 404 });
926
1014
  }
927
1015
  if (path.includes('relationships')) {
928
- return slice(relationships, params) as T;
1016
+ const query = edgeQuery(relationships, params);
1017
+ return {
1018
+ ...slice(query.rows, params),
1019
+ applied_filters: query.applied.slice().sort(),
1020
+ ignored_filters: query.ignored.slice().sort(),
1021
+ } as T;
929
1022
  }
930
1023
  if (path.includes('entities')) {
931
1024
  const id = /entities\/(.+)$/.exec(path)?.[1];
@@ -303,7 +303,16 @@ export async function startExtractionRun(
303
303
  const at = now();
304
304
  const driver = request.driverId ?? options.driverId ?? defaultDriverId();
305
305
 
306
- const existing = await deps.runs.readRun().catch(() => null);
306
+ // NOT `.catch(() => null)`. A store that could not answer is not a store that
307
+ // said "nothing has ever run", and this is the one place where confusing them
308
+ // is expensive: the branch below reads `null` as "start a new sweep", so a
309
+ // transient failure — or a read that could not see far enough (bd
310
+ // startsim-o0o6o.3) — put a SECOND driver on a live cursor, and two model
311
+ // calls over one document produce two items identity resolution cannot see are
312
+ // the same. `readRun` reports absence as `null` and everything else as a
313
+ // throw, precisely so this caller can tell them apart; swallowing the throw
314
+ // here threw that distinction away one line after it was made.
315
+ const existing = await deps.runs.readRun();
307
316
 
308
317
  if (existing && !COMPLETE.has(existing.status) && !request.restart) {
309
318
  if (isLeaseLive(existing, at) && existing.leaseOwner !== driver) {