@startsimpli/integrations 0.2.32 → 0.2.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/corpus-narrowed-reads.test.ts +337 -0
- package/src/__tests__/corpus-storage.test.ts +20 -9
- package/src/corpus/client.ts +186 -44
- package/src/corpus/entities.ts +41 -0
- package/src/corpus/fixtures.ts +102 -9
- package/src/corpus/runs.ts +10 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@startsimpli/integrations",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.33",
|
|
4
4
|
"description": "Provider-agnostic types, API clients, hooks, and UI primitives for email + calendar integrations across StartSimpli apps.",
|
|
5
5
|
"main": "./src/index.ts",
|
|
6
6
|
"types": "./src/index.ts",
|
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE CLIENT ASKS THE SERVER FOR A NARROWER ANSWER, AND CHECKS THAT IT GOT ONE.
|
|
3
|
+
*
|
|
4
|
+
* Two bugs, one root cause (bd startsim-o0o6o.2, bd startsim-o0o6o.3). The
|
|
5
|
+
* corpus adapter was written against a tenant data plane that filtered on
|
|
6
|
+
* `?type=` and nothing else, and several of its comments still assert that as
|
|
7
|
+
* fact. It has not been true since bd startsim-768w.38.12 and bd startsim-te3r:
|
|
8
|
+
* `RelationshipQuery` applies `entity__in` bounded at `MAX_ID_LIST` ids, and
|
|
9
|
+
* `TenantPageNumberPagination` honours `page_size` up to 200.
|
|
10
|
+
*
|
|
11
|
+
* WHY THE ANSWER IS CHECKED AND NOT ASSUMED. The tenant's response to a filter
|
|
12
|
+
* it does not recognise is an UNFILTERED PAGE with the parameter named in
|
|
13
|
+
* `ignored_filters` (bd startsim-768w.38.39) — not a 400 and not an empty list.
|
|
14
|
+
* So "send the filter and trust the rows" is the dangerous shape here: the reply
|
|
15
|
+
* to a narrowing request that was ignored looks exactly like a narrow answer
|
|
16
|
+
* with a lot of rows in it. start-simpli#154 set the precedent — read the
|
|
17
|
+
* server's own `applied_filters` and do not use an unconfirmed result — and the
|
|
18
|
+
* fakes below behave the way the wire behaves, ignoring what they do not support
|
|
19
|
+
* and answering as wide as they were asked.
|
|
20
|
+
*/
|
|
21
|
+
import { describe, expect, it } from 'vitest';
|
|
22
|
+
|
|
23
|
+
import { createCorpusApi } from '../corpus/client';
|
|
24
|
+
import type { CorpusHttp } from '../corpus/client';
|
|
25
|
+
import { startExtractionRun } from '../corpus/runs';
|
|
26
|
+
import {
|
|
27
|
+
DEFAULT_RUN_KEY,
|
|
28
|
+
DEFAULT_TYPE_KEYS,
|
|
29
|
+
MAX_ID_LIST,
|
|
30
|
+
extractionRunToEntity,
|
|
31
|
+
} from '../corpus/entities';
|
|
32
|
+
import type { TenantEntity, TenantRelationship } from '../corpus/entities';
|
|
33
|
+
import {
|
|
34
|
+
createFixtureCorpusApi,
|
|
35
|
+
createFixtureCorpusHttp,
|
|
36
|
+
createFixtureExtractor,
|
|
37
|
+
FIXTURE_ORG,
|
|
38
|
+
FIXTURE_SOURCE_LABELS,
|
|
39
|
+
} from '../corpus/fixtures';
|
|
40
|
+
import type { ExtractionRun, ExtractionRunStore } from '../corpus/types';
|
|
41
|
+
|
|
42
|
+
const SCOPE = { orgId: FIXTURE_ORG };
|
|
43
|
+
|
|
44
|
+
function apiOver(overrides: Parameters<typeof createFixtureCorpusHttp>[0] = {}) {
|
|
45
|
+
const { http, requests } = createFixtureCorpusHttp(overrides);
|
|
46
|
+
const api = createCorpusApi({ http, scope: SCOPE, sourceLabels: FIXTURE_SOURCE_LABELS });
|
|
47
|
+
return { api, requests };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// ---------------------------------------------------------------------------
|
|
51
|
+
// bd startsim-o0o6o.3 — the sweep record, past the fifth row
|
|
52
|
+
// ---------------------------------------------------------------------------
|
|
53
|
+
|
|
54
|
+
function run(id: string): ExtractionRun {
|
|
55
|
+
return {
|
|
56
|
+
runId: id,
|
|
57
|
+
scope: SCOPE,
|
|
58
|
+
status: 'succeeded',
|
|
59
|
+
cursor: 4,
|
|
60
|
+
totalDocuments: 4,
|
|
61
|
+
totals: {
|
|
62
|
+
documentsProcessed: 4, fromRules: 2, fromModel: 1, ungrounded: 0, unconfirmed: 0,
|
|
63
|
+
created: 3, merged: 0, resolved: 0, droppedOutOfScope: 0, modelCalls: 4, modelFailures: 0,
|
|
64
|
+
},
|
|
65
|
+
processedIds: [],
|
|
66
|
+
modelIncompleteIds: [],
|
|
67
|
+
needsReview: [],
|
|
68
|
+
provider: { provider: 'fixture', model: 'fixture-rules', live: false },
|
|
69
|
+
startedAt: '2026-08-01T10:00:00Z',
|
|
70
|
+
updatedAt: '2026-08-01T10:06:00Z',
|
|
71
|
+
finishedAt: '2026-08-01T10:06:00Z',
|
|
72
|
+
leaseOwner: null,
|
|
73
|
+
leaseUntil: null,
|
|
74
|
+
error: null,
|
|
75
|
+
lastModelFailure: null,
|
|
76
|
+
history: [],
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The singleton row, exactly as `writeRun` lands it. */
|
|
81
|
+
function singleton(): TenantEntity {
|
|
82
|
+
return {
|
|
83
|
+
id: 'run-singleton',
|
|
84
|
+
...extractionRunToEntity(run('sweep-1'), DEFAULT_TYPE_KEYS.extractionRun, DEFAULT_RUN_KEY),
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Rows that are NOT the singleton, and which is which is not a hypothetical.
|
|
90
|
+
*
|
|
91
|
+
* A second workspace key, a renamed `runKey`, a partial migration or a
|
|
92
|
+
* hand-written row all put `extraction_run` rows next to it, and the tenant
|
|
93
|
+
* orders entities `-created_at` — so the singleton, which is the OLDEST because
|
|
94
|
+
* it was written first and is rewritten in place, sorts LAST. Newest first is
|
|
95
|
+
* the order these arrive in.
|
|
96
|
+
*/
|
|
97
|
+
function noise(count: number): TenantEntity[] {
|
|
98
|
+
return Array.from({ length: count }, (_, index) => ({
|
|
99
|
+
id: `run-noise-${index}`,
|
|
100
|
+
entity_type: DEFAULT_TYPE_KEYS.extractionRun,
|
|
101
|
+
external_id: `corpus-extraction-sweep-other-${index}`,
|
|
102
|
+
name: `Extraction sweep other-${index}`,
|
|
103
|
+
data: { run: JSON.stringify(run(`other-${index}`)), _origin: 'tenant-extraction' },
|
|
104
|
+
}));
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
describe('readRun tells "nothing has ever run" apart from "I could not tell"', () => {
|
|
108
|
+
it('finds the singleton when it is NOT among the newest five rows', async () => {
|
|
109
|
+
// Six rows, singleton last: the exact corpus at which `page_size: 5` starts
|
|
110
|
+
// answering `null` — which every caller reads as "no sweep has ever run" and
|
|
111
|
+
// `startExtractionRun` acts on by starting a duplicate sweep.
|
|
112
|
+
const { api } = apiOver({ entities: [...noise(5), singleton()], relationships: [] });
|
|
113
|
+
|
|
114
|
+
const read = await api.readRun();
|
|
115
|
+
|
|
116
|
+
expect(read?.runId).toBe('sweep-1');
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('THROWS rather than reporting "never run" when the row could be past the cap', async () => {
|
|
120
|
+
// Past the server's own `max_page_size`, one read genuinely cannot see every
|
|
121
|
+
// row — and the honest answer to "has a sweep run here" is then neither the
|
|
122
|
+
// run nor `null`. `null` is a FACT in this contract; a truncated page is not
|
|
123
|
+
// evidence for it.
|
|
124
|
+
const { api } = apiOver({
|
|
125
|
+
entities: [...noise(MAX_ID_LIST + 20), singleton()],
|
|
126
|
+
relationships: [],
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
await expect(api.readRun()).rejects.toThrow(/could not/i);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('still answers null for a workspace where nothing HAS run', async () => {
|
|
133
|
+
// The fact this whole contract exists to preserve: absence is reportable.
|
|
134
|
+
const { api } = apiOver({ entities: [], relationships: [] });
|
|
135
|
+
|
|
136
|
+
expect(await api.readRun()).toBeNull();
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it('matches the singleton key in EITHER spelling the wire delivers it in', async () => {
|
|
140
|
+
// `@startsimpli/api`'s fetch wrapper transforms every response key
|
|
141
|
+
// snake_case -> camelCase, so a browser-side read of this row carries
|
|
142
|
+
// `externalId` and never `external_id`. This lookup read one spelling, so in
|
|
143
|
+
// the browser it matched nothing at ANY page size and reported "never run"
|
|
144
|
+
// over a workspace with a sweep in it — the same seam, and the same lie, as
|
|
145
|
+
// reading one spelling of `applied_filters`.
|
|
146
|
+
const { external_id: key, ...rest } = singleton();
|
|
147
|
+
const { api } = apiOver({
|
|
148
|
+
entities: [{ ...rest, externalId: key }],
|
|
149
|
+
relationships: [],
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
expect((await api.readRun())?.runId).toBe('sweep-1');
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it('asks for the widest page the server will serve, not five rows', async () => {
|
|
156
|
+
const { api, requests } = apiOver({ entities: [singleton()], relationships: [] });
|
|
157
|
+
await api.readRun();
|
|
158
|
+
|
|
159
|
+
const read = requests.find(
|
|
160
|
+
(request) => request.params.type === DEFAULT_TYPE_KEYS.extractionRun,
|
|
161
|
+
);
|
|
162
|
+
expect(read?.params.page_size).toBe(MAX_ID_LIST);
|
|
163
|
+
});
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
// ---------------------------------------------------------------------------
|
|
167
|
+
// bd startsim-o0o6o.2 — the edges of ONE PAGE, not the whole edge table
|
|
168
|
+
// ---------------------------------------------------------------------------
|
|
169
|
+
|
|
170
|
+
const PARTY: TenantEntity = {
|
|
171
|
+
id: 'party_dana', entity_type: 'party', external_id: 'granola:dana@example.com',
|
|
172
|
+
name: 'Dana Reed',
|
|
173
|
+
data: { display_name: 'Dana Reed', email: 'dana@example.com', source: 'granola' },
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
function document(id: string): TenantEntity {
|
|
177
|
+
return {
|
|
178
|
+
id, entity_type: 'source_document', external_id: `granola:${id}`, name: `Record ${id}`,
|
|
179
|
+
data: { source: 'granola', title: `Record ${id}`, occurred_at: '2026-08-01T09:00:00Z' },
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
describe('the participant join reads the edges of a PAGE, not the edge table', () => {
|
|
184
|
+
it('names the page’s ids in entity__in instead of asking for everything', async () => {
|
|
185
|
+
const { api, requests } = apiOver();
|
|
186
|
+
await api.listDocuments({ page: 1, pageSize: 3 });
|
|
187
|
+
|
|
188
|
+
const edgeReads = requests.filter((request) => request.path.includes('relationships'));
|
|
189
|
+
expect(edgeReads.length).toBeGreaterThan(0);
|
|
190
|
+
// THE ASSERTION THIS FILE EXISTS FOR. `readAll(relationships, {})` sent no
|
|
191
|
+
// parameters at all and walked up to twenty pages of every edge in the
|
|
192
|
+
// workspace, on a comment claiming the endpoint had no filters.
|
|
193
|
+
for (const read of edgeReads) {
|
|
194
|
+
const named = String(read.params.entity__in ?? '').split(',').filter(Boolean);
|
|
195
|
+
expect(named.length).toBeGreaterThan(0);
|
|
196
|
+
expect(named.length).toBeLessThanOrEqual(MAX_ID_LIST);
|
|
197
|
+
}
|
|
198
|
+
expect(String(edgeReads[0].params.entity__in).split(',')).toHaveLength(3);
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it('still resolves the people on the page it narrowed to', async () => {
|
|
202
|
+
const { api } = apiOver({
|
|
203
|
+
entities: [document('doc-1'), PARTY],
|
|
204
|
+
relationships: [{ id: 'e1', rel_type: 'authored_by', source: 'doc-1', target: 'party_dana' }],
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
const page = await api.listDocuments({ page: 1 });
|
|
208
|
+
|
|
209
|
+
expect(page.results[0].participants.map((party) => party.name)).toEqual(['Dana Reed']);
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
it('chunks a page wider than the server’s id bound rather than being refused', async () => {
|
|
213
|
+
// A fork whose `max_page_size` is larger than `MAX_ID_LIST` hands back more
|
|
214
|
+
// ids than one `entity__in` may carry, and `_uuid_list` answers a longer
|
|
215
|
+
// list with a 400 — so the ids are chunked. Truncating them instead would
|
|
216
|
+
// drop the people off the tail of the page silently, which is the failure
|
|
217
|
+
// this whole change is undoing.
|
|
218
|
+
const documents = Array.from({ length: MAX_ID_LIST + 40 }, (_, index) =>
|
|
219
|
+
document(`doc-${index}`),
|
|
220
|
+
);
|
|
221
|
+
const last = documents[documents.length - 1];
|
|
222
|
+
const { api, requests } = apiOver({
|
|
223
|
+
pageSize: documents.length,
|
|
224
|
+
entities: [...documents, PARTY],
|
|
225
|
+
relationships: [
|
|
226
|
+
{ id: 'e-last', rel_type: 'involves', source: String(last.id), target: 'party_dana' },
|
|
227
|
+
],
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
const page = await api.listDocuments({ page: 1 });
|
|
231
|
+
|
|
232
|
+
const edgeReads = requests.filter((request) => request.path.includes('relationships'));
|
|
233
|
+
expect(edgeReads.length).toBeGreaterThan(1);
|
|
234
|
+
for (const read of edgeReads) {
|
|
235
|
+
expect(String(read.params.entity__in).split(',').length).toBeLessThanOrEqual(MAX_ID_LIST);
|
|
236
|
+
}
|
|
237
|
+
const tail = page.results.find((record) => record.id === String(last.id));
|
|
238
|
+
expect(tail?.participants.map((party) => party.name)).toEqual(['Dana Reed']);
|
|
239
|
+
});
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* A fork that has NOT shipped `RelationshipQuery`.
|
|
244
|
+
*
|
|
245
|
+
* It does what the tenant documents itself as doing with a parameter it does not
|
|
246
|
+
* know: names it in `ignored_filters` and answers with the page exactly as wide
|
|
247
|
+
* as it was. A fake that returned `[]` here would let the client pass this suite
|
|
248
|
+
* while reading the whole edge table in production, which is the trap this test
|
|
249
|
+
* is written to spring.
|
|
250
|
+
*/
|
|
251
|
+
function unfilteredEdgeHttp(edges: TenantRelationship[], rows: TenantEntity[]) {
|
|
252
|
+
const requests: Array<{ path: string; params: Record<string, unknown> }> = [];
|
|
253
|
+
const http: CorpusHttp = {
|
|
254
|
+
async get<T>(path: string, init?: { params?: Record<string, unknown> }): Promise<T> {
|
|
255
|
+
const params = init?.params ?? {};
|
|
256
|
+
requests.push({ path, params });
|
|
257
|
+
if (path.includes('relationships')) {
|
|
258
|
+
return {
|
|
259
|
+
count: edges.length, next: null, previous: null, results: edges,
|
|
260
|
+
applied_filters: [],
|
|
261
|
+
ignored_filters: Object.keys(params).filter(
|
|
262
|
+
(key) => key !== 'page' && key !== 'page_size',
|
|
263
|
+
),
|
|
264
|
+
} as T;
|
|
265
|
+
}
|
|
266
|
+
if (path.includes('entities')) {
|
|
267
|
+
const type = params.type ? String(params.type) : '';
|
|
268
|
+
const matching = type ? rows.filter((row) => row.entity_type === type) : rows;
|
|
269
|
+
return {
|
|
270
|
+
count: matching.length, next: null, previous: null, results: matching,
|
|
271
|
+
applied_filters: [],
|
|
272
|
+
ignored_filters: [],
|
|
273
|
+
} as T;
|
|
274
|
+
}
|
|
275
|
+
throw Object.assign(new Error(`${path} responded 404`), { status: 404 });
|
|
276
|
+
},
|
|
277
|
+
};
|
|
278
|
+
return { http, requests };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
describe('an unconfirmed narrowing is not used', () => {
|
|
282
|
+
it('renders no participants rather than joining an unnarrowed page of edges', async () => {
|
|
283
|
+
// start-simpli#154's precedent, on the read that matters most: the server
|
|
284
|
+
// reported it did NOT apply `entity__in`, so the rows it sent are an
|
|
285
|
+
// arbitrary page of the whole edge table and joining them would put the
|
|
286
|
+
// wrong people — or a partial list of the right ones — on the record. The
|
|
287
|
+
// Library degrades to what it showed before the join existed.
|
|
288
|
+
const { http, requests } = unfilteredEdgeHttp(
|
|
289
|
+
[{ id: 'e1', rel_type: 'authored_by', source: 'doc-1', target: 'party_dana' }],
|
|
290
|
+
[document('doc-1'), PARTY],
|
|
291
|
+
);
|
|
292
|
+
const api = createCorpusApi({ http, scope: SCOPE, sourceLabels: FIXTURE_SOURCE_LABELS });
|
|
293
|
+
|
|
294
|
+
const page = await api.listDocuments({ page: 1 });
|
|
295
|
+
|
|
296
|
+
expect(page.results).toHaveLength(1);
|
|
297
|
+
expect(page.results[0].participants).toEqual([]);
|
|
298
|
+
// And it does not go on to read the party roster for a join it will not do.
|
|
299
|
+
expect(requests.filter((request) => request.params.type === 'party')).toHaveLength(0);
|
|
300
|
+
});
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
describe('a store that could not answer never becomes "no sweep has ever run"', () => {
|
|
304
|
+
it('refuses to start a second sweep over a cursor it was unable to read', async () => {
|
|
305
|
+
// THE REASON THE THIRD ANSWER HAD TO REACH THIS CALLER. `startExtractionRun`
|
|
306
|
+
// swallowed every `readRun` failure into `null` and `null` means "start a
|
|
307
|
+
// new sweep" one branch later, so an undetermined read put a SECOND driver
|
|
308
|
+
// on a live cursor — and two model calls over one document produce two
|
|
309
|
+
// differently-worded items dedupe cannot see are the same, which is how a
|
|
310
|
+
// live re-run once turned 82 commitments into 98.
|
|
311
|
+
let writes = 0;
|
|
312
|
+
const runs: ExtractionRunStore = {
|
|
313
|
+
async readRun() {
|
|
314
|
+
throw Object.assign(
|
|
315
|
+
new Error('Could not determine whether a sweep has run in this workspace.'),
|
|
316
|
+
{ status: 409 },
|
|
317
|
+
);
|
|
318
|
+
},
|
|
319
|
+
async writeRun(run) {
|
|
320
|
+
writes += 1;
|
|
321
|
+
return run;
|
|
322
|
+
},
|
|
323
|
+
};
|
|
324
|
+
|
|
325
|
+
await expect(
|
|
326
|
+
startExtractionRun(
|
|
327
|
+
{ scope: SCOPE },
|
|
328
|
+
{
|
|
329
|
+
corpus: createFixtureCorpusApi({ scope: SCOPE }),
|
|
330
|
+
extractor: createFixtureExtractor(),
|
|
331
|
+
runs,
|
|
332
|
+
},
|
|
333
|
+
),
|
|
334
|
+
).rejects.toThrow(/could not determine/i);
|
|
335
|
+
expect(writes).toBe(0);
|
|
336
|
+
});
|
|
337
|
+
});
|
|
@@ -191,8 +191,10 @@ describe('both storage shapes, because both are real', () => {
|
|
|
191
191
|
});
|
|
192
192
|
});
|
|
193
193
|
|
|
194
|
-
describe('the graph read is
|
|
195
|
-
it('costs ONE
|
|
194
|
+
describe('the graph read is narrowed, cached and fails soft', () => {
|
|
195
|
+
it('costs ONE edge read across a list, a detail and a second page', async () => {
|
|
196
|
+
// The records are remembered per row, so re-listing them and then opening
|
|
197
|
+
// one of them asks the server nothing further (bd startsim-o0o6o.2).
|
|
196
198
|
const { api, requests } = apiOver();
|
|
197
199
|
await api.listDocuments({ page: 1 });
|
|
198
200
|
await api.listDocuments({ page: 1 });
|
|
@@ -220,17 +222,26 @@ describe('the graph read is bounded, cached and fails soft', () => {
|
|
|
220
222
|
expect(page.results.every((document) => document.participants.length === 0)).toBe(true);
|
|
221
223
|
});
|
|
222
224
|
|
|
223
|
-
it('
|
|
224
|
-
//
|
|
225
|
-
//
|
|
225
|
+
it('narrows the edge read to the ids of the page it is rendering', async () => {
|
|
226
|
+
// THIS TEST USED TO ASSERT THE OPPOSITE, and its title said why: "walks
|
|
227
|
+
// every page of the graph, since the tenant ignores page_size". Neither half
|
|
228
|
+
// survived contact with the deployed server. `page_size` is honoured
|
|
229
|
+
// (`TenantPageNumberPagination`, bd startsim-768w.38.12) and the edge table
|
|
230
|
+
// is filterable (`RelationshipQuery`, bd startsim-te3r) — so a page deep in
|
|
231
|
+
// the corpus no longer reaches its people by reading every edge in the
|
|
232
|
+
// workspace and joining in the browser. It ASKS for the edges of the two
|
|
233
|
+
// rows it is rendering, which is the read that scales with the page instead
|
|
234
|
+
// of with the corpus (bd startsim-o0o6o.2).
|
|
226
235
|
const { api, requests } = apiOver({ pageSize: 2 });
|
|
227
|
-
|
|
228
|
-
// it — so the Dropbox row is on page THREE however wide the caller asked.
|
|
229
|
-
const page = await api.listDocuments({ page: 3, pageSize: 100 });
|
|
236
|
+
const page = await api.listDocuments({ page: 3 });
|
|
230
237
|
const dropbox = page.results.find((document) => document.id === 'dbx_9812f0');
|
|
231
238
|
|
|
232
239
|
expect(dropbox?.participants.map((party) => party.name)).toEqual(['Acme Robotics', 'Dana Reed']);
|
|
233
|
-
|
|
240
|
+
const edgeReads = requests.filter((request) => request.path.includes('relationships'));
|
|
241
|
+
expect(edgeReads).toHaveLength(1);
|
|
242
|
+
expect(String(edgeReads[0].params.entity__in).split(',')).toEqual(
|
|
243
|
+
page.results.map((document) => document.id),
|
|
244
|
+
);
|
|
234
245
|
});
|
|
235
246
|
});
|
|
236
247
|
|
package/src/corpus/client.ts
CHANGED
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
DEFAULT_PARTICIPANT_EDGES,
|
|
35
35
|
DEFAULT_RUN_KEY,
|
|
36
36
|
DEFAULT_TYPE_KEYS,
|
|
37
|
+
MAX_ID_LIST,
|
|
37
38
|
commitmentToEntity,
|
|
38
39
|
extractionRunToEntity,
|
|
39
40
|
indexParticipants,
|
|
@@ -143,17 +144,20 @@ export interface CreateCorpusApiOptions {
|
|
|
143
144
|
/** Which edge types mean "this party is on this record". Data, not an enum. */
|
|
144
145
|
participantEdges?: readonly string[];
|
|
145
146
|
/**
|
|
146
|
-
* Bound on
|
|
147
|
+
* Bound on a walked list, in PAGES.
|
|
147
148
|
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
149
|
+
* IT USED TO BE THE ONLY BOUND THERE WAS. This said the tenant relationships
|
|
150
|
+
* endpoint "has no `?source=` filter today", so the edges for a page of
|
|
151
|
+
* documents could not be requested directly and the whole edge table was read
|
|
152
|
+
* in bulk — twenty pages of it — and joined in the browser. That stopped being
|
|
153
|
+
* true with bd startsim-te3r: `RelationshipQuery` applies `entity__in`, whose
|
|
154
|
+
* own docstring says what it is for ("a comma list of ids is how a client asks
|
|
155
|
+
* for the edges of ONE PAGE of records"). The edge read is now narrowed to the
|
|
156
|
+
* ids on screen (bd startsim-o0o6o.2) and this cap is a backstop on the walk
|
|
157
|
+
* within that narrowing, and on the party roster read that names the far ends.
|
|
154
158
|
*/
|
|
155
159
|
maxGraphPages?: number;
|
|
156
|
-
/** How long a
|
|
160
|
+
/** How long a resolved roster and a record's people are reused before re-reading. */
|
|
157
161
|
graphTtlMs?: number;
|
|
158
162
|
/**
|
|
159
163
|
* The `external_id` the singleton extraction-run row lives at. One per
|
|
@@ -393,6 +397,26 @@ function notFound(id: string): Error & { status: number } {
|
|
|
393
397
|
return Object.assign(new Error(`No record ${id} in this workspace.`), { status: 404 });
|
|
394
398
|
}
|
|
395
399
|
|
|
400
|
+
/**
|
|
401
|
+
* "I could not tell", which is the third answer and not a variant of the second.
|
|
402
|
+
*
|
|
403
|
+
* A read that could not see every candidate row is not evidence of absence, and
|
|
404
|
+
* `null` in this adapter is a claim of absence that callers ACT on: a sweep
|
|
405
|
+
* reported as never-run is started again, over the same cursor, by a second
|
|
406
|
+
* driver. Thrown rather than returned so the existing `ExtractionRun | null`
|
|
407
|
+
* contract keeps meaning what it says — the two answers a caller may act on are
|
|
408
|
+
* a run and no run, and everything else is a failure to answer.
|
|
409
|
+
*/
|
|
410
|
+
function undetermined(what: string): Error & { status: number } {
|
|
411
|
+
return Object.assign(
|
|
412
|
+
new Error(
|
|
413
|
+
`Could not determine ${what}: this workspace has more rows of that type ` +
|
|
414
|
+
`than one page can cover, and no filter on the tenant can name the one row.`,
|
|
415
|
+
),
|
|
416
|
+
{ status: 409 },
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
|
|
396
420
|
export function createCorpusApi(
|
|
397
421
|
options: CreateCorpusApiOptions,
|
|
398
422
|
): CorpusApi & CommitmentApi & ExtractionRunStore {
|
|
@@ -487,43 +511,132 @@ export function createCorpusApi(
|
|
|
487
511
|
// the participant graph
|
|
488
512
|
// ---------------------------------------------------------------------
|
|
489
513
|
//
|
|
490
|
-
//
|
|
491
|
-
//
|
|
514
|
+
// THE EDGES OF THE RECORDS ON SCREEN, AND NO OTHERS (bd startsim-o0o6o.2).
|
|
515
|
+
// This used to be `readAll(relationships, {})` — every edge in the workspace,
|
|
516
|
+
// twenty pages of it, on a comment saying the endpoint had no filters. It has
|
|
517
|
+
// them: `entity__in` takes the ids of one page, bounded at `MAX_ID_LIST`,
|
|
518
|
+
// which is the shape `RelationshipQuery`'s own docstring says it exists for.
|
|
519
|
+
//
|
|
520
|
+
// ONE read for a page of records, not one per record — and cached per record,
|
|
521
|
+
// so a screen that lists, opens a row and then asks a question spends it once.
|
|
492
522
|
// It FAILS SOFT: a tenant whose relationships endpoint is missing or refused
|
|
493
523
|
// renders exactly what it rendered before this existed, rather than an error
|
|
494
524
|
// where a Library used to be.
|
|
495
|
-
let
|
|
525
|
+
let roster: { at: number; value: Promise<Party[]> } | null = null;
|
|
526
|
+
const known = new Map<string, { at: number; value: PartyRef[] }>();
|
|
496
527
|
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
528
|
+
/**
|
|
529
|
+
* The people an edge can point AT, read once and reused.
|
|
530
|
+
*
|
|
531
|
+
* Still a walked list, because there is no id filter on `/entities/`: the
|
|
532
|
+
* recognised set is `search`, `source`, `party`, the occurred window,
|
|
533
|
+
* `related_*`, `order` and `attr.*`, and an unrecognised `id__in` would be
|
|
534
|
+
* swept into `ignored_filters` and answered with the whole page. It is at
|
|
535
|
+
* least a TYPED read — `?type=party` is applied server-side — and it is only
|
|
536
|
+
* reached when the narrowed edge read came back with edges to resolve.
|
|
537
|
+
*/
|
|
538
|
+
function partyRoster(): Promise<Party[]> {
|
|
539
|
+
const now = Date.now();
|
|
540
|
+
if (!roster || now - roster.at > graphTtlMs) {
|
|
541
|
+
roster = {
|
|
542
|
+
at: now,
|
|
543
|
+
value: readAll<TenantEntity>(entities, { type: types.party })
|
|
544
|
+
.then((rows) => rows.map((entity) => toParty(entity, scope)))
|
|
545
|
+
.catch(() => []),
|
|
546
|
+
};
|
|
510
547
|
}
|
|
548
|
+
return roster.value;
|
|
511
549
|
}
|
|
512
550
|
|
|
513
|
-
|
|
551
|
+
/**
|
|
552
|
+
* The edges touching these records, or `null` when the server did not confirm
|
|
553
|
+
* it narrowed by them.
|
|
554
|
+
*
|
|
555
|
+
* `null` IS NOT "NO EDGES", and the difference is the whole point. A fork
|
|
556
|
+
* without `RelationshipQuery` accepts `entity__in`, names it in
|
|
557
|
+
* `ignored_filters` and answers with an arbitrary page of the WHOLE edge table
|
|
558
|
+
* (bd startsim-768w.38.39) — rows that look exactly like a narrow answer and
|
|
559
|
+
* would put the wrong people, or a fraction of the right ones, on a record.
|
|
560
|
+
* start-simpli#154 settled what to do with a result the server did not
|
|
561
|
+
* confirm: do not use it. So this reports that it could not ask, and the
|
|
562
|
+
* caller renders what it rendered before the join existed.
|
|
563
|
+
*
|
|
564
|
+
* `rel_type` rides along unchecked on purpose: it only ever makes the answer
|
|
565
|
+
* SMALLER, and `indexParticipants` ranks by edge type anyway. A fork that
|
|
566
|
+
* ignores it pays in PAGES rather than in correctness — a record with many
|
|
567
|
+
* non-participant edges can push its people onto a second page, and the walk
|
|
568
|
+
* below follows `next` until it has them.
|
|
569
|
+
*/
|
|
570
|
+
async function edgesFor(recordIds: string[]): Promise<TenantRelationship[] | null> {
|
|
571
|
+
const rows: TenantRelationship[] = [];
|
|
572
|
+
for (let start = 0; start < recordIds.length; start += MAX_ID_LIST) {
|
|
573
|
+
// CHUNKED, because `_uuid_list` answers a longer list with a 400 rather
|
|
574
|
+
// than truncating it. Truncating here instead would drop the people off
|
|
575
|
+
// the tail of a wide page silently, which is the failure being undone.
|
|
576
|
+
const chunk = recordIds.slice(start, start + MAX_ID_LIST);
|
|
577
|
+
const extra: Record<string, unknown> = {
|
|
578
|
+
entity__in: chunk.join(','),
|
|
579
|
+
page_size: MAX_ID_LIST,
|
|
580
|
+
};
|
|
581
|
+
if (participantEdges.length) extra.rel_type = participantEdges.join(',');
|
|
582
|
+
for (let index = 1; index <= maxGraphPages; index += 1) {
|
|
583
|
+
const response = await http.get<DrfPage<TenantRelationship> & ServerFilterReport>(
|
|
584
|
+
relationships,
|
|
585
|
+
{ params: { ...extra, page: index } },
|
|
586
|
+
);
|
|
587
|
+
const applied = response.applied_filters ?? response.appliedFilters;
|
|
588
|
+
if (!applied?.includes('entity__in')) return null;
|
|
589
|
+
const results = response.results ?? [];
|
|
590
|
+
rows.push(...results);
|
|
591
|
+
if (!response.next || results.length === 0) break;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
return rows;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
async function participantsFor(
|
|
598
|
+
records: Array<string | number>,
|
|
599
|
+
): Promise<Map<string, PartyRef[]>> {
|
|
600
|
+
const out = new Map<string, PartyRef[]>();
|
|
601
|
+
if (!resolveParticipants) return out;
|
|
514
602
|
const now = Date.now();
|
|
515
|
-
|
|
516
|
-
|
|
603
|
+
const wanted: string[] = [];
|
|
604
|
+
for (const record of records) {
|
|
605
|
+
const id = String(record ?? '').trim();
|
|
606
|
+
if (!id || out.has(id) || wanted.includes(id)) continue;
|
|
607
|
+
const cached = known.get(id);
|
|
608
|
+
if (cached && now - cached.at <= graphTtlMs) out.set(id, cached.value);
|
|
609
|
+
else wanted.push(id);
|
|
517
610
|
}
|
|
518
|
-
return
|
|
611
|
+
if (wanted.length === 0) return out;
|
|
612
|
+
try {
|
|
613
|
+
const edges = await edgesFor(wanted);
|
|
614
|
+
// Could not ask. Nothing is cached, so the next page asks again rather
|
|
615
|
+
// than remembering an answer that was never given.
|
|
616
|
+
if (!edges) return out;
|
|
617
|
+
const parties = edges.length ? await partyRoster() : [];
|
|
618
|
+
const indexed = edges.length
|
|
619
|
+
? indexParticipants(edges, parties, participantEdges)
|
|
620
|
+
: new Map<string, PartyRef[]>();
|
|
621
|
+
for (const id of wanted) {
|
|
622
|
+
// A record the server confirmed has no participant edges is REMEMBERED
|
|
623
|
+
// as having none — otherwise every page re-asks for the same silence.
|
|
624
|
+
const people = indexed.get(id) ?? [];
|
|
625
|
+
known.set(id, { at: now, value: people });
|
|
626
|
+
out.set(id, people);
|
|
627
|
+
}
|
|
628
|
+
} catch {
|
|
629
|
+
// The Library renders without its Parties block rather than not at all.
|
|
630
|
+
}
|
|
631
|
+
return out;
|
|
519
632
|
}
|
|
520
633
|
|
|
521
634
|
return {
|
|
522
635
|
async listDocuments(query) {
|
|
523
636
|
const documents = await page<TenantEntity>(types.document, query, (entity) => entity);
|
|
524
|
-
// Skip the
|
|
525
|
-
// exactly the one request it costs today.
|
|
526
|
-
const byRecord = documents.results.
|
|
637
|
+
// Skip the edge read entirely for an empty page: an empty corpus should
|
|
638
|
+
// cost exactly the one request it costs today.
|
|
639
|
+
const byRecord = await participantsFor(documents.results.map((entity) => entity.id));
|
|
527
640
|
return {
|
|
528
641
|
...documents,
|
|
529
642
|
results: documents.results.map((entity) =>
|
|
@@ -557,7 +670,10 @@ export function createCorpusApi(
|
|
|
557
670
|
async getDocument(id) {
|
|
558
671
|
const [entity, byRecord] = await Promise.all([
|
|
559
672
|
http.get<TenantEntity>(`${entities}/${encodeURIComponent(id)}`),
|
|
560
|
-
|
|
673
|
+
// The edges of ONE record, which is what this screen needs — it no
|
|
674
|
+
// longer waits on a read of the whole workspace's graph to render a
|
|
675
|
+
// record it already named (bd startsim-o0o6o.2).
|
|
676
|
+
participantsFor([id]),
|
|
561
677
|
]);
|
|
562
678
|
if (!isRecord(entity, id)) throw notFound(id);
|
|
563
679
|
return toSourceDocument(entity, scope, sourceLabels, byRecord.get(String(entity.id)) ?? []);
|
|
@@ -569,7 +685,7 @@ export function createCorpusApi(
|
|
|
569
685
|
|
|
570
686
|
async listMoneyEvents(query = {}) {
|
|
571
687
|
const events = await page<TenantEntity>(types.moneyEvent, query, (entity) => entity);
|
|
572
|
-
const byRecord = events.results.
|
|
688
|
+
const byRecord = await participantsFor(events.results.map((entity) => entity.id));
|
|
573
689
|
return {
|
|
574
690
|
...events,
|
|
575
691
|
results: events.results.map((entity) =>
|
|
@@ -641,21 +757,38 @@ export function createCorpusApi(
|
|
|
641
757
|
/**
|
|
642
758
|
* The sweep record, read back (bd startsim-768w.38.28).
|
|
643
759
|
*
|
|
644
|
-
* ONE ROW PER WORKSPACE, at a fixed `external_id`, because
|
|
645
|
-
*
|
|
646
|
-
*
|
|
647
|
-
*
|
|
648
|
-
*
|
|
760
|
+
* ONE ROW PER WORKSPACE, at a fixed `external_id`, because a row per sweep
|
|
761
|
+
* would mean scanning a list that grows forever on every poll. A singleton
|
|
762
|
+
* is one read that never grows, and the last few sweeps ride along inside it.
|
|
763
|
+
*
|
|
764
|
+
* THE LOOKUP CANNOT BE PUSHED DOWN, AND SAYING SO IS THE POINT. `external_id`
|
|
765
|
+
* is not among the parameters `/entities/` recognises — `search`, `source`,
|
|
766
|
+
* `party`, the occurred window, `related_*`, `order`, `attr.*` — so the match
|
|
767
|
+
* happens here. `attr.external_id` is NOT the way around that: an attribute
|
|
768
|
+
* nobody declared has no `core_attribute` rows, and `EntityQuery._attribute`
|
|
769
|
+
* reports the filter as APPLIED before any row matches, so the reply to it is
|
|
770
|
+
* an empty page that the envelope swears was narrowed. A filter that cannot
|
|
771
|
+
* be told apart from "no such record" is worse than no filter.
|
|
772
|
+
*
|
|
773
|
+
* SO THE BOUND IS WHAT HAS TO BE HONEST (bd startsim-o0o6o.3). This asked for
|
|
774
|
+
* five rows, because the singleton "is" a singleton — but nothing enforces
|
|
775
|
+
* that, entities come back `-created_at`, and the singleton is the OLDEST row
|
|
776
|
+
* of its type because it is rewritten in place. Six `extraction_run` rows and
|
|
777
|
+
* the read stopped seeing it, returned `null`, and every caller read that as
|
|
778
|
+
* "no sweep has ever run" — which `startExtractionRun` acts on by starting a
|
|
779
|
+
* duplicate sweep over a live cursor. It now asks for the widest page the
|
|
780
|
+
* paginator will serve and, when the row could still be past the end of it,
|
|
781
|
+
* says it could not tell instead of saying no.
|
|
649
782
|
*
|
|
650
783
|
* FAIL-SOFT ON ABSENCE, NOT ON FAILURE. No run yet is `null`, which is a
|
|
651
|
-
* fact.
|
|
652
|
-
* "nothing has ever run" are different
|
|
653
|
-
*
|
|
784
|
+
* fact. Everything else throws, because "we could not reach the workspace",
|
|
785
|
+
* "we could not see far enough" and "nothing has ever run" are different
|
|
786
|
+
* answers and only the last one is safe to act on.
|
|
654
787
|
*/
|
|
655
788
|
async readRun() {
|
|
656
789
|
const response = await http
|
|
657
790
|
.get<DrfPage<TenantEntity>>(entities, {
|
|
658
|
-
params: { type: types.extractionRun, page: 1, page_size:
|
|
791
|
+
params: { type: types.extractionRun, page: 1, page_size: MAX_ID_LIST },
|
|
659
792
|
})
|
|
660
793
|
.catch((err: { status?: number }) => {
|
|
661
794
|
// An undeclared type is a 403 from the read gate, and it means exactly
|
|
@@ -664,9 +797,18 @@ export function createCorpusApi(
|
|
|
664
797
|
throw err;
|
|
665
798
|
});
|
|
666
799
|
const row = (response.results ?? []).find(
|
|
667
|
-
|
|
800
|
+
// BOTH SPELLINGS, for the reason `ServerFilterReport` documents: the
|
|
801
|
+
// `@startsimpli/api` fetch wrapper turns `external_id` into `externalId`
|
|
802
|
+
// on the way in, so a browser-side read matched nothing at ANY page size
|
|
803
|
+
// and reported "never run" over a workspace with a sweep in it. Every
|
|
804
|
+
// other reader in `./entities` already takes both; this one did not.
|
|
805
|
+
(entity) => String(entity.external_id ?? entity.externalId ?? '') === runKey,
|
|
668
806
|
);
|
|
669
|
-
|
|
807
|
+
if (row) return toExtractionRun(row);
|
|
808
|
+
// A page with more behind it did not fail to find the row — it failed to
|
|
809
|
+
// look at every row, and those are not the same answer.
|
|
810
|
+
if (response.next) throw undetermined(`whether a sweep has run in this workspace`);
|
|
811
|
+
return null;
|
|
670
812
|
},
|
|
671
813
|
|
|
672
814
|
async writeRun(run: ExtractionRun) {
|
package/src/corpus/entities.ts
CHANGED
|
@@ -54,6 +54,47 @@ export interface TenantRelationship {
|
|
|
54
54
|
createdAt?: string | null;
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* The most ids one `__in` list may carry, and it is the SERVER's number.
|
|
59
|
+
*
|
|
60
|
+
* `apps/api/filters.py::MAX_ID_LIST` is 200 and `_uuid_list` answers a longer
|
|
61
|
+
* list with a 400 rather than truncating it — "bounded so the parameter cannot
|
|
62
|
+
* become an unpaginated read by another name". A client that sends more does not
|
|
63
|
+
* get a wider answer, it gets no answer, so the ids of a page are CHUNKED at
|
|
64
|
+
* this number rather than sent in one request and hoped for.
|
|
65
|
+
*/
|
|
66
|
+
export const MAX_ID_LIST = 200;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Parameters `/api/v1/relationships/` really applies (`RelationshipQuery._FILTERS`).
|
|
70
|
+
*
|
|
71
|
+
* Named here rather than inside either the adapter or the fixture, because both
|
|
72
|
+
* have to agree about it and a second copy is how they stop agreeing. It is a
|
|
73
|
+
* DESCRIPTION of a deployed server, not a contract this package enforces: a fork
|
|
74
|
+
* that has not shipped `RelationshipQuery` ignores every one of these, which is
|
|
75
|
+
* why `applied_filters` is read on the response rather than this list being
|
|
76
|
+
* trusted.
|
|
77
|
+
*/
|
|
78
|
+
export const RELATIONSHIP_FILTERS: ReadonlySet<string> = new Set([
|
|
79
|
+
'rel_type',
|
|
80
|
+
'source',
|
|
81
|
+
'target',
|
|
82
|
+
'entity',
|
|
83
|
+
'source__in',
|
|
84
|
+
'target__in',
|
|
85
|
+
'entity__in',
|
|
86
|
+
'source_type',
|
|
87
|
+
'target_type',
|
|
88
|
+
]);
|
|
89
|
+
|
|
90
|
+
/** Consumed by pagination and negotiation, never a filter (`filters.py::_RESERVED`). */
|
|
91
|
+
export const RESERVED_PARAMS: ReadonlySet<string> = new Set([
|
|
92
|
+
'page',
|
|
93
|
+
'page_size',
|
|
94
|
+
'format',
|
|
95
|
+
'type',
|
|
96
|
+
]);
|
|
97
|
+
|
|
57
98
|
/**
|
|
58
99
|
* Edges that mean "this party is ON this record", in the order they should be
|
|
59
100
|
* listed. DATA, not a closed set: a tenant that models its own edge passes its
|
package/src/corpus/fixtures.ts
CHANGED
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
toParty,
|
|
43
43
|
toSourceDocument,
|
|
44
44
|
} from './entities';
|
|
45
|
+
import { MAX_ID_LIST, RELATIONSHIP_FILTERS, RESERVED_PARAMS } from './entities';
|
|
45
46
|
import type { TenantEntity, TenantRelationship } from './entities';
|
|
46
47
|
import { splitSections, stem } from './text';
|
|
47
48
|
import type {
|
|
@@ -875,10 +876,22 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
|
|
|
875
876
|
* stands in for an OLDER fork — which is a useful thing to have a stand-in
|
|
876
877
|
* for, since that is the fork `listSources` refuses to print a count for
|
|
877
878
|
* (bd startsim-jevtu), but it is a lagging stub and not the contract;
|
|
878
|
-
* • `page_size`
|
|
879
|
-
*
|
|
880
|
-
*
|
|
881
|
-
*
|
|
879
|
+
* • `page_size` IS honoured, capped at 200. It was written here as ignored,
|
|
880
|
+
* citing a `PageNumberPagination` with no `page_size_query_param`, and that
|
|
881
|
+
* stopped being true with `TenantPageNumberPagination` (bd
|
|
882
|
+
* startsim-768w.38.12): "a client asking for one row to read a count got
|
|
883
|
+
* fifty, and a client asking for two hundred to walk a graph got fifty and
|
|
884
|
+
* no indication that its request had been overruled". A fixture that keeps
|
|
885
|
+
* ignoring it manufactures evidence for a cap the wire does not impose;
|
|
886
|
+
* • `/api/v1/relationships/` FILTERS — `rel_type`, `source`, `target`,
|
|
887
|
+
* `entity` and the `__in` forms of the last three, bounded at
|
|
888
|
+
* `MAX_ID_LIST` ids and reported in `applied_filters`. This too was written
|
|
889
|
+
* here as "every edge in the org, unfiltered, because the tenant has no
|
|
890
|
+
* `?source=` on it either", and bd startsim-te3r shipped `RelationshipQuery`
|
|
891
|
+
* for exactly the read that comment was excusing. An UNRECOGNISED parameter
|
|
892
|
+
* is still ignored and the page comes back as wide as it was — that part is
|
|
893
|
+
* real, it is what makes an unconfirmed filter dangerous, and it is what
|
|
894
|
+
* `applied_filters` exists to let a client detect.
|
|
882
895
|
*
|
|
883
896
|
* Every request is recorded, so "did the browser filter it" and "how many round
|
|
884
897
|
* trips did one screen cost" are both assertable.
|
|
@@ -886,7 +899,7 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
|
|
|
886
899
|
export function createFixtureCorpusHttp(options: {
|
|
887
900
|
entities?: TenantEntity[];
|
|
888
901
|
relationships?: TenantRelationship[];
|
|
889
|
-
/** Rows per page, matching the tenant's
|
|
902
|
+
/** Rows per page when the caller does not ask, matching the tenant's `PAGE_SIZE`. */
|
|
890
903
|
pageSize?: number;
|
|
891
904
|
/** Endpoints to fail, so the fail-soft paths are exercised. */
|
|
892
905
|
missing?: string[];
|
|
@@ -906,15 +919,90 @@ export function createFixtureCorpusHttp(options: {
|
|
|
906
919
|
|
|
907
920
|
function slice<T>(rows: T[], params: Record<string, unknown>) {
|
|
908
921
|
const page = Number(params.page ?? 1) || 1;
|
|
909
|
-
|
|
922
|
+
// `TenantPageNumberPagination.page_size_query_param` is `page_size` and its
|
|
923
|
+
// `max_page_size` is 200 — an OVERRULED request is served at the cap, not
|
|
924
|
+
// refused, which is the behaviour a client that walks a page has to survive.
|
|
925
|
+
const asked = Number(params.page_size);
|
|
926
|
+
const size = Number.isFinite(asked) && asked > 0 ? Math.min(asked, MAX_ID_LIST) : pageSize;
|
|
927
|
+
const start = (page - 1) * size;
|
|
910
928
|
return {
|
|
911
929
|
count: rows.length,
|
|
912
|
-
next: start +
|
|
930
|
+
next: start + size < rows.length ? `?page=${page + 1}` : null,
|
|
913
931
|
previous: page > 1 ? `?page=${page - 1}` : null,
|
|
914
|
-
results: rows.slice(start, start +
|
|
932
|
+
results: rows.slice(start, start + size),
|
|
915
933
|
};
|
|
916
934
|
}
|
|
917
935
|
|
|
936
|
+
/**
|
|
937
|
+
* `/api/v1/relationships/`, as `RelationshipQuery.apply` really answers it.
|
|
938
|
+
*
|
|
939
|
+
* THE SHAPE THAT MATTERS IS THE ONE IT REFUSES TO NARROW. A parameter this
|
|
940
|
+
* endpoint does not recognise is swept into `ignored_filters` and the page
|
|
941
|
+
* comes back exactly as wide as it was — so a fixture that returned `[]` for
|
|
942
|
+
* an unsupported filter would let a client pass here and read the whole edge
|
|
943
|
+
* table in production. Every filter below is one the server really applies,
|
|
944
|
+
* and everything else is really ignored.
|
|
945
|
+
*/
|
|
946
|
+
function edgeQuery(rows: TenantRelationship[], params: Record<string, unknown>) {
|
|
947
|
+
const applied: string[] = [];
|
|
948
|
+
const ignored: string[] = [];
|
|
949
|
+
const idList = (key: string): Set<string> => {
|
|
950
|
+
const parts = String(params[key]).split(',').map((part) => part.trim()).filter(Boolean);
|
|
951
|
+
// `_uuid_list` refuses a longer list with a 400 rather than truncating it,
|
|
952
|
+
// which is what keeps `__in` from becoming an unpaginated read by another
|
|
953
|
+
// name — and what a client that chunks its ids has to be tested against.
|
|
954
|
+
if (parts.length > MAX_ID_LIST) {
|
|
955
|
+
throw Object.assign(new Error(`${key}: at most ${MAX_ID_LIST} ids`), { status: 400 });
|
|
956
|
+
}
|
|
957
|
+
return new Set(parts);
|
|
958
|
+
};
|
|
959
|
+
const typeOf = (id: unknown): string =>
|
|
960
|
+
String(entities.find((entity) => String(entity.id) === String(id))?.entity_type ?? '');
|
|
961
|
+
let out = rows;
|
|
962
|
+
|
|
963
|
+
for (const [key, field] of [['source', 'source'], ['target', 'target']] as const) {
|
|
964
|
+
if (key in params) {
|
|
965
|
+
const wanted = String(params[key]);
|
|
966
|
+
out = out.filter((edge) => String(edge[field]) === wanted);
|
|
967
|
+
applied.push(key);
|
|
968
|
+
}
|
|
969
|
+
const listed = `${key}__in`;
|
|
970
|
+
if (listed in params) {
|
|
971
|
+
const wanted = idList(listed);
|
|
972
|
+
out = out.filter((edge) => wanted.has(String(edge[field])));
|
|
973
|
+
applied.push(listed);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
if ('entity' in params) {
|
|
977
|
+
const wanted = String(params.entity);
|
|
978
|
+
out = out.filter((edge) => String(edge.source) === wanted || String(edge.target) === wanted);
|
|
979
|
+
applied.push('entity');
|
|
980
|
+
}
|
|
981
|
+
if ('entity__in' in params) {
|
|
982
|
+
const wanted = idList('entity__in');
|
|
983
|
+
out = out.filter((edge) => wanted.has(String(edge.source)) || wanted.has(String(edge.target)));
|
|
984
|
+
applied.push('entity__in');
|
|
985
|
+
}
|
|
986
|
+
if ('rel_type' in params) {
|
|
987
|
+
const wanted = new Set(
|
|
988
|
+
String(params.rel_type).split(',').map((part) => part.trim()).filter(Boolean),
|
|
989
|
+
);
|
|
990
|
+
out = out.filter((edge) => wanted.has(String(edge.rel_type ?? edge.relType ?? '')));
|
|
991
|
+
applied.push('rel_type');
|
|
992
|
+
}
|
|
993
|
+
for (const [key, end] of [['source_type', 'source'], ['target_type', 'target']] as const) {
|
|
994
|
+
if (key in params) {
|
|
995
|
+
const wanted = String(params[key]);
|
|
996
|
+
out = out.filter((edge) => typeOf(edge[end]) === wanted);
|
|
997
|
+
applied.push(key);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
for (const key of Object.keys(params)) {
|
|
1001
|
+
if (!RESERVED_PARAMS.has(key) && !RELATIONSHIP_FILTERS.has(key)) ignored.push(key);
|
|
1002
|
+
}
|
|
1003
|
+
return { rows: out, applied, ignored };
|
|
1004
|
+
}
|
|
1005
|
+
|
|
918
1006
|
const http = {
|
|
919
1007
|
async get<T>(path: string, init?: { params?: Record<string, unknown> }): Promise<T> {
|
|
920
1008
|
const params = init?.params ?? {};
|
|
@@ -925,7 +1013,12 @@ export function createFixtureCorpusHttp(options: {
|
|
|
925
1013
|
throw Object.assign(new Error(`${path} responded 404`), { status: 404 });
|
|
926
1014
|
}
|
|
927
1015
|
if (path.includes('relationships')) {
|
|
928
|
-
|
|
1016
|
+
const query = edgeQuery(relationships, params);
|
|
1017
|
+
return {
|
|
1018
|
+
...slice(query.rows, params),
|
|
1019
|
+
applied_filters: query.applied.slice().sort(),
|
|
1020
|
+
ignored_filters: query.ignored.slice().sort(),
|
|
1021
|
+
} as T;
|
|
929
1022
|
}
|
|
930
1023
|
if (path.includes('entities')) {
|
|
931
1024
|
const id = /entities\/(.+)$/.exec(path)?.[1];
|
package/src/corpus/runs.ts
CHANGED
|
@@ -303,7 +303,16 @@ export async function startExtractionRun(
|
|
|
303
303
|
const at = now();
|
|
304
304
|
const driver = request.driverId ?? options.driverId ?? defaultDriverId();
|
|
305
305
|
|
|
306
|
-
|
|
306
|
+
// NOT `.catch(() => null)`. A store that could not answer is not a store that
|
|
307
|
+
// said "nothing has ever run", and this is the one place where confusing them
|
|
308
|
+
// is expensive: the branch below reads `null` as "start a new sweep", so a
|
|
309
|
+
// transient failure — or a read that could not see far enough (bd
|
|
310
|
+
// startsim-o0o6o.3) — put a SECOND driver on a live cursor, and two model
|
|
311
|
+
// calls over one document produce two items identity resolution cannot see are
|
|
312
|
+
// the same. `readRun` reports absence as `null` and everything else as a
|
|
313
|
+
// throw, precisely so this caller can tell them apart; swallowing the throw
|
|
314
|
+
// here threw that distinction away one line after it was made.
|
|
315
|
+
const existing = await deps.runs.readRun();
|
|
307
316
|
|
|
308
317
|
if (existing && !COMPLETE.has(existing.status) && !request.restart) {
|
|
309
318
|
if (isLeaseLive(existing, at) && existing.leaseOwner !== driver) {
|