@startsimpli/integrations 0.2.31 → 0.2.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/corpus-narrowed-reads.test.ts +337 -0
- package/src/__tests__/corpus-source-strip.test.tsx +225 -0
- package/src/__tests__/corpus-storage.test.ts +20 -9
- package/src/components/pages/LibraryPage.tsx +26 -2
- package/src/corpus/client.ts +294 -65
- package/src/corpus/entities.ts +41 -0
- package/src/corpus/fixtures.ts +108 -10
- package/src/corpus/runs.ts +10 -1
- package/src/corpus/types.ts +16 -2
package/src/corpus/client.ts
CHANGED
|
@@ -11,11 +11,16 @@
|
|
|
11
11
|
* there must not be: it produces a list whose count disagrees with its rows and
|
|
12
12
|
* that stops working at exactly the corpus size where it starts to matter.
|
|
13
13
|
*
|
|
14
|
-
* WHAT THE BACKEND HONOURS
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
14
|
+
* WHAT THE BACKEND HONOURS IS THE BACKEND'S TO SAY, NOT THIS FILE'S. It once
|
|
15
|
+
* said here that `tenant-starter`'s `EntityViewSet` filters on `?type=` and
|
|
16
|
+
* nothing else; bd startsim-768w.38.12 shipped `search`, `source`, `party`, the
|
|
17
|
+
* date window and `order`, and the comment stayed behind — which is exactly why
|
|
18
|
+
* a hardcoded list is the wrong mechanism. Every list response carries the
|
|
19
|
+
* server's own `applied_filters`/`ignored_filters`, `/api/v1/capabilities/`
|
|
20
|
+
* describes the deployed build, and both are read in preference to anything
|
|
21
|
+
* declared client-side. `supportedFilters` remains only as the fallback for a
|
|
22
|
+
* fork too old to report, and a deployment that passes a stale one is corrected
|
|
23
|
+
* by the server rather than believed.
|
|
19
24
|
*
|
|
20
25
|
* PEOPLE ARE A SECOND READ, NOT A SECOND COPY (bd startsim-768w.38.22). The
|
|
21
26
|
* corpus delivery writes participants as typed `authored_by` / `involves` edges
|
|
@@ -29,6 +34,7 @@ import {
|
|
|
29
34
|
DEFAULT_PARTICIPANT_EDGES,
|
|
30
35
|
DEFAULT_RUN_KEY,
|
|
31
36
|
DEFAULT_TYPE_KEYS,
|
|
37
|
+
MAX_ID_LIST,
|
|
32
38
|
commitmentToEntity,
|
|
33
39
|
extractionRunToEntity,
|
|
34
40
|
indexParticipants,
|
|
@@ -138,17 +144,20 @@ export interface CreateCorpusApiOptions {
|
|
|
138
144
|
/** Which edge types mean "this party is on this record". Data, not an enum. */
|
|
139
145
|
participantEdges?: readonly string[];
|
|
140
146
|
/**
|
|
141
|
-
* Bound on
|
|
147
|
+
* Bound on a walked list, in PAGES.
|
|
142
148
|
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
+
* IT USED TO BE THE ONLY BOUND THERE WAS. This said the tenant relationships
|
|
150
|
+
* endpoint "has no `?source=` filter today", so the edges for a page of
|
|
151
|
+
* documents could not be requested directly and the whole edge table was read
|
|
152
|
+
* in bulk — twenty pages of it — and joined in the browser. That stopped being
|
|
153
|
+
* true with bd startsim-te3r: `RelationshipQuery` applies `entity__in`, whose
|
|
154
|
+
* own docstring says what it is for ("a comma list of ids is how a client asks
|
|
155
|
+
* for the edges of ONE PAGE of records"). The edge read is now narrowed to the
|
|
156
|
+
* ids on screen (bd startsim-o0o6o.2) and this cap is a backstop on the walk
|
|
157
|
+
* within that narrowing, and on the party roster read that names the far ends.
|
|
149
158
|
*/
|
|
150
159
|
maxGraphPages?: number;
|
|
151
|
-
/** How long a
|
|
160
|
+
/** How long a resolved roster and a record's people are reused before re-reading. */
|
|
152
161
|
graphTtlMs?: number;
|
|
153
162
|
/**
|
|
154
163
|
* The `external_id` the singleton extraction-run row lives at. One per
|
|
@@ -388,6 +397,26 @@ function notFound(id: string): Error & { status: number } {
|
|
|
388
397
|
return Object.assign(new Error(`No record ${id} in this workspace.`), { status: 404 });
|
|
389
398
|
}
|
|
390
399
|
|
|
400
|
+
/**
|
|
401
|
+
* "I could not tell", which is the third answer and not a variant of the second.
|
|
402
|
+
*
|
|
403
|
+
* A read that could not see every candidate row is not evidence of absence, and
|
|
404
|
+
* `null` in this adapter is a claim of absence that callers ACT on: a sweep
|
|
405
|
+
* reported as never-run is started again, over the same cursor, by a second
|
|
406
|
+
* driver. Thrown rather than returned so the existing `ExtractionRun | null`
|
|
407
|
+
* contract keeps meaning what it says — the two answers a caller may act on are
|
|
408
|
+
* a run and no run, and everything else is a failure to answer.
|
|
409
|
+
*/
|
|
410
|
+
function undetermined(what: string): Error & { status: number } {
|
|
411
|
+
return Object.assign(
|
|
412
|
+
new Error(
|
|
413
|
+
`Could not determine ${what}: this workspace has more rows of that type ` +
|
|
414
|
+
`than one page can cover, and no filter on the tenant can name the one row.`,
|
|
415
|
+
),
|
|
416
|
+
{ status: 409 },
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
|
|
391
420
|
export function createCorpusApi(
|
|
392
421
|
options: CreateCorpusApiOptions,
|
|
393
422
|
): CorpusApi & CommitmentApi & ExtractionRunStore {
|
|
@@ -430,6 +459,42 @@ export function createCorpusApi(
|
|
|
430
459
|
};
|
|
431
460
|
}
|
|
432
461
|
|
|
462
|
+
/**
|
|
463
|
+
* How many DOCUMENTS one source has put in the corpus, or `null` when the
|
|
464
|
+
* server did not confirm it narrowed by source.
|
|
465
|
+
*
|
|
466
|
+
* ONE ROW, AND THE ENVELOPE'S OWN `count`. `page_size=1` is not a sample: the
|
|
467
|
+
* DRF paginator counts the filtered queryset in SQL and reports it beside the
|
|
468
|
+
* page, so this is one COUNT with `?source=` pushed down, not eleven pages
|
|
469
|
+
* added up. `types.document` is passed for the same reason the list below the
|
|
470
|
+
* strip passes it — a count over every entity type would sweep in parties,
|
|
471
|
+
* money events and commitments and stop agreeing with the list it sits above.
|
|
472
|
+
*
|
|
473
|
+
* THE GUARD IS THE SERVER'S OWN REPORT, in both spellings for the reason
|
|
474
|
+
* `ServerFilterReport` documents. A fork that predates bd startsim-768w.38.12
|
|
475
|
+
* accepts `?source=` and ignores it, so the honest-looking `count` that comes
|
|
476
|
+
* back is the whole corpus — the same number on every card. `applied_filters`
|
|
477
|
+
* naming `source` is what separates "none from this source" from "I could not
|
|
478
|
+
* ask", and only the first of those is a number worth printing.
|
|
479
|
+
*/
|
|
480
|
+
async function countDocumentsFrom(source: string): Promise<number | null> {
|
|
481
|
+
if (!source || source === 'unknown') return null;
|
|
482
|
+
try {
|
|
483
|
+
const response = await http.get<DrfPage<TenantEntity> & ServerFilterReport>(entities, {
|
|
484
|
+
params: { type: types.document, source, page_size: 1 },
|
|
485
|
+
});
|
|
486
|
+
const applied = response.applied_filters ?? response.appliedFilters;
|
|
487
|
+
if (!applied?.includes('source')) return null;
|
|
488
|
+
const count = Number(response.count);
|
|
489
|
+
return Number.isFinite(count) ? count : null;
|
|
490
|
+
} catch {
|
|
491
|
+
// A count is an ornament on a card whose point is freshness. Losing it
|
|
492
|
+
// costs the reader a number; failing the whole strip would cost the four
|
|
493
|
+
// facts the card exists for.
|
|
494
|
+
return null;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
433
498
|
/** Every page of a DRF list, up to the cap. Stops on the first failure. */
|
|
434
499
|
async function readAll<T>(path: string, extra: Record<string, unknown>): Promise<T[]> {
|
|
435
500
|
const rows: T[] = [];
|
|
@@ -446,43 +511,132 @@ export function createCorpusApi(
|
|
|
446
511
|
// the participant graph
|
|
447
512
|
// ---------------------------------------------------------------------
|
|
448
513
|
//
|
|
449
|
-
//
|
|
450
|
-
//
|
|
514
|
+
// THE EDGES OF THE RECORDS ON SCREEN, AND NO OTHERS (bd startsim-o0o6o.2).
|
|
515
|
+
// This used to be `readAll(relationships, {})` — every edge in the workspace,
|
|
516
|
+
// twenty pages of it, on a comment saying the endpoint had no filters. It has
|
|
517
|
+
// them: `entity__in` takes the ids of one page, bounded at `MAX_ID_LIST`,
|
|
518
|
+
// which is the shape `RelationshipQuery`'s own docstring says it exists for.
|
|
519
|
+
//
|
|
520
|
+
// ONE read for a page of records, not one per record — and cached per record,
|
|
521
|
+
// so a screen that lists, opens a row and then asks a question spends it once.
|
|
451
522
|
// It FAILS SOFT: a tenant whose relationships endpoint is missing or refused
|
|
452
523
|
// renders exactly what it rendered before this existed, rather than an error
|
|
453
524
|
// where a Library used to be.
|
|
454
|
-
let
|
|
525
|
+
let roster: { at: number; value: Promise<Party[]> } | null = null;
|
|
526
|
+
const known = new Map<string, { at: number; value: PartyRef[] }>();
|
|
455
527
|
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
528
|
+
/**
|
|
529
|
+
* The people an edge can point AT, read once and reused.
|
|
530
|
+
*
|
|
531
|
+
* Still a walked list, because there is no id filter on `/entities/`: the
|
|
532
|
+
* recognised set is `search`, `source`, `party`, the occurred window,
|
|
533
|
+
* `related_*`, `order` and `attr.*`, and an unrecognised `id__in` would be
|
|
534
|
+
* swept into `ignored_filters` and answered with the whole page. It is at
|
|
535
|
+
* least a TYPED read — `?type=party` is applied server-side — and it is only
|
|
536
|
+
* reached when the narrowed edge read came back with edges to resolve.
|
|
537
|
+
*/
|
|
538
|
+
function partyRoster(): Promise<Party[]> {
|
|
539
|
+
const now = Date.now();
|
|
540
|
+
if (!roster || now - roster.at > graphTtlMs) {
|
|
541
|
+
roster = {
|
|
542
|
+
at: now,
|
|
543
|
+
value: readAll<TenantEntity>(entities, { type: types.party })
|
|
544
|
+
.then((rows) => rows.map((entity) => toParty(entity, scope)))
|
|
545
|
+
.catch(() => []),
|
|
546
|
+
};
|
|
547
|
+
}
|
|
548
|
+
return roster.value;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* The edges touching these records, or `null` when the server did not confirm
|
|
553
|
+
* it narrowed by them.
|
|
554
|
+
*
|
|
555
|
+
* `null` IS NOT "NO EDGES", and the difference is the whole point. A fork
|
|
556
|
+
* without `RelationshipQuery` accepts `entity__in`, names it in
|
|
557
|
+
* `ignored_filters` and answers with an arbitrary page of the WHOLE edge table
|
|
558
|
+
* (bd startsim-768w.38.39) — rows that look exactly like a narrow answer and
|
|
559
|
+
* would put the wrong people, or a fraction of the right ones, on a record.
|
|
560
|
+
* start-simpli#154 settled what to do with a result the server did not
|
|
561
|
+
* confirm: do not use it. So this reports that it could not ask, and the
|
|
562
|
+
* caller renders what it rendered before the join existed.
|
|
563
|
+
*
|
|
564
|
+
* `rel_type` rides along unchecked on purpose: it only ever makes the answer
|
|
565
|
+
* SMALLER, and `indexParticipants` ranks by edge type anyway. A fork that
|
|
566
|
+
* ignores it pays in PAGES rather than in correctness — a record with many
|
|
567
|
+
* non-participant edges can push its people onto a second page, and the walk
|
|
568
|
+
* below follows `next` until it has them.
|
|
569
|
+
*/
|
|
570
|
+
async function edgesFor(recordIds: string[]): Promise<TenantRelationship[] | null> {
|
|
571
|
+
const rows: TenantRelationship[] = [];
|
|
572
|
+
for (let start = 0; start < recordIds.length; start += MAX_ID_LIST) {
|
|
573
|
+
// CHUNKED, because `_uuid_list` answers a longer list with a 400 rather
|
|
574
|
+
// than truncating it. Truncating here instead would drop the people off
|
|
575
|
+
// the tail of a wide page silently, which is the failure being undone.
|
|
576
|
+
const chunk = recordIds.slice(start, start + MAX_ID_LIST);
|
|
577
|
+
const extra: Record<string, unknown> = {
|
|
578
|
+
entity__in: chunk.join(','),
|
|
579
|
+
page_size: MAX_ID_LIST,
|
|
580
|
+
};
|
|
581
|
+
if (participantEdges.length) extra.rel_type = participantEdges.join(',');
|
|
582
|
+
for (let index = 1; index <= maxGraphPages; index += 1) {
|
|
583
|
+
const response = await http.get<DrfPage<TenantRelationship> & ServerFilterReport>(
|
|
584
|
+
relationships,
|
|
585
|
+
{ params: { ...extra, page: index } },
|
|
586
|
+
);
|
|
587
|
+
const applied = response.applied_filters ?? response.appliedFilters;
|
|
588
|
+
if (!applied?.includes('entity__in')) return null;
|
|
589
|
+
const results = response.results ?? [];
|
|
590
|
+
rows.push(...results);
|
|
591
|
+
if (!response.next || results.length === 0) break;
|
|
592
|
+
}
|
|
469
593
|
}
|
|
594
|
+
return rows;
|
|
470
595
|
}
|
|
471
596
|
|
|
472
|
-
function
|
|
597
|
+
async function participantsFor(
|
|
598
|
+
records: Array<string | number>,
|
|
599
|
+
): Promise<Map<string, PartyRef[]>> {
|
|
600
|
+
const out = new Map<string, PartyRef[]>();
|
|
601
|
+
if (!resolveParticipants) return out;
|
|
473
602
|
const now = Date.now();
|
|
474
|
-
|
|
475
|
-
|
|
603
|
+
const wanted: string[] = [];
|
|
604
|
+
for (const record of records) {
|
|
605
|
+
const id = String(record ?? '').trim();
|
|
606
|
+
if (!id || out.has(id) || wanted.includes(id)) continue;
|
|
607
|
+
const cached = known.get(id);
|
|
608
|
+
if (cached && now - cached.at <= graphTtlMs) out.set(id, cached.value);
|
|
609
|
+
else wanted.push(id);
|
|
476
610
|
}
|
|
477
|
-
return
|
|
611
|
+
if (wanted.length === 0) return out;
|
|
612
|
+
try {
|
|
613
|
+
const edges = await edgesFor(wanted);
|
|
614
|
+
// Could not ask. Nothing is cached, so the next page asks again rather
|
|
615
|
+
// than remembering an answer that was never given.
|
|
616
|
+
if (!edges) return out;
|
|
617
|
+
const parties = edges.length ? await partyRoster() : [];
|
|
618
|
+
const indexed = edges.length
|
|
619
|
+
? indexParticipants(edges, parties, participantEdges)
|
|
620
|
+
: new Map<string, PartyRef[]>();
|
|
621
|
+
for (const id of wanted) {
|
|
622
|
+
// A record the server confirmed has no participant edges is REMEMBERED
|
|
623
|
+
// as having none — otherwise every page re-asks for the same silence.
|
|
624
|
+
const people = indexed.get(id) ?? [];
|
|
625
|
+
known.set(id, { at: now, value: people });
|
|
626
|
+
out.set(id, people);
|
|
627
|
+
}
|
|
628
|
+
} catch {
|
|
629
|
+
// The Library renders without its Parties block rather than not at all.
|
|
630
|
+
}
|
|
631
|
+
return out;
|
|
478
632
|
}
|
|
479
633
|
|
|
480
634
|
return {
|
|
481
635
|
async listDocuments(query) {
|
|
482
636
|
const documents = await page<TenantEntity>(types.document, query, (entity) => entity);
|
|
483
|
-
// Skip the
|
|
484
|
-
// exactly the one request it costs today.
|
|
485
|
-
const byRecord = documents.results.
|
|
637
|
+
// Skip the edge read entirely for an empty page: an empty corpus should
|
|
638
|
+
// cost exactly the one request it costs today.
|
|
639
|
+
const byRecord = await participantsFor(documents.results.map((entity) => entity.id));
|
|
486
640
|
return {
|
|
487
641
|
...documents,
|
|
488
642
|
results: documents.results.map((entity) =>
|
|
@@ -516,7 +670,10 @@ export function createCorpusApi(
|
|
|
516
670
|
async getDocument(id) {
|
|
517
671
|
const [entity, byRecord] = await Promise.all([
|
|
518
672
|
http.get<TenantEntity>(`${entities}/${encodeURIComponent(id)}`),
|
|
519
|
-
|
|
673
|
+
// The edges of ONE record, which is what this screen needs — it no
|
|
674
|
+
// longer waits on a read of the whole workspace's graph to render a
|
|
675
|
+
// record it already named (bd startsim-o0o6o.2).
|
|
676
|
+
participantsFor([id]),
|
|
520
677
|
]);
|
|
521
678
|
if (!isRecord(entity, id)) throw notFound(id);
|
|
522
679
|
return toSourceDocument(entity, scope, sourceLabels, byRecord.get(String(entity.id)) ?? []);
|
|
@@ -528,7 +685,7 @@ export function createCorpusApi(
|
|
|
528
685
|
|
|
529
686
|
async listMoneyEvents(query = {}) {
|
|
530
687
|
const events = await page<TenantEntity>(types.moneyEvent, query, (entity) => entity);
|
|
531
|
-
const byRecord = events.results.
|
|
688
|
+
const byRecord = await participantsFor(events.results.map((entity) => entity.id));
|
|
532
689
|
return {
|
|
533
690
|
...events,
|
|
534
691
|
results: events.results.map((entity) =>
|
|
@@ -600,21 +757,38 @@ export function createCorpusApi(
|
|
|
600
757
|
/**
|
|
601
758
|
* The sweep record, read back (bd startsim-768w.38.28).
|
|
602
759
|
*
|
|
603
|
-
* ONE ROW PER WORKSPACE, at a fixed `external_id`, because
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
*
|
|
760
|
+
* ONE ROW PER WORKSPACE, at a fixed `external_id`, because a row per sweep
|
|
761
|
+
* would mean scanning a list that grows forever on every poll. A singleton
|
|
762
|
+
* is one read that never grows, and the last few sweeps ride along inside it.
|
|
763
|
+
*
|
|
764
|
+
* THE LOOKUP CANNOT BE PUSHED DOWN, AND SAYING SO IS THE POINT. `external_id`
|
|
765
|
+
* is not among the parameters `/entities/` recognises — `search`, `source`,
|
|
766
|
+
* `party`, the occurred window, `related_*`, `order`, `attr.*` — so the match
|
|
767
|
+
* happens here. `attr.external_id` is NOT the way around that: an attribute
|
|
768
|
+
* nobody declared has no `core_attribute` rows, and `EntityQuery._attribute`
|
|
769
|
+
* reports the filter as APPLIED before any row matches, so the reply to it is
|
|
770
|
+
* an empty page that the envelope swears was narrowed. A filter that cannot
|
|
771
|
+
* be told apart from "no such record" is worse than no filter.
|
|
772
|
+
*
|
|
773
|
+
* SO THE BOUND IS WHAT HAS TO BE HONEST (bd startsim-o0o6o.3). This asked for
|
|
774
|
+
* five rows, because the singleton "is" a singleton — but nothing enforces
|
|
775
|
+
* that, entities come back `-created_at`, and the singleton is the OLDEST row
|
|
776
|
+
* of its type because it is rewritten in place. Six `extraction_run` rows and
|
|
777
|
+
* the read stopped seeing it, returned `null`, and every caller read that as
|
|
778
|
+
* "no sweep has ever run" — which `startExtractionRun` acts on by starting a
|
|
779
|
+
* duplicate sweep over a live cursor. It now asks for the widest page the
|
|
780
|
+
* paginator will serve and, when the row could still be past the end of it,
|
|
781
|
+
* says it could not tell instead of saying no.
|
|
608
782
|
*
|
|
609
783
|
* FAIL-SOFT ON ABSENCE, NOT ON FAILURE. No run yet is `null`, which is a
|
|
610
|
-
* fact.
|
|
611
|
-
* "nothing has ever run" are different
|
|
612
|
-
*
|
|
784
|
+
* fact. Everything else throws, because "we could not reach the workspace",
|
|
785
|
+
* "we could not see far enough" and "nothing has ever run" are different
|
|
786
|
+
* answers and only the last one is safe to act on.
|
|
613
787
|
*/
|
|
614
788
|
async readRun() {
|
|
615
789
|
const response = await http
|
|
616
790
|
.get<DrfPage<TenantEntity>>(entities, {
|
|
617
|
-
params: { type: types.extractionRun, page: 1, page_size:
|
|
791
|
+
params: { type: types.extractionRun, page: 1, page_size: MAX_ID_LIST },
|
|
618
792
|
})
|
|
619
793
|
.catch((err: { status?: number }) => {
|
|
620
794
|
// An undeclared type is a 403 from the read gate, and it means exactly
|
|
@@ -623,9 +797,18 @@ export function createCorpusApi(
|
|
|
623
797
|
throw err;
|
|
624
798
|
});
|
|
625
799
|
const row = (response.results ?? []).find(
|
|
626
|
-
|
|
800
|
+
// BOTH SPELLINGS, for the reason `ServerFilterReport` documents: the
|
|
801
|
+
// `@startsimpli/api` fetch wrapper turns `external_id` into `externalId`
|
|
802
|
+
// on the way in, so a browser-side read matched nothing at ANY page size
|
|
803
|
+
// and reported "never run" over a workspace with a sweep in it. Every
|
|
804
|
+
// other reader in `./entities` already takes both; this one did not.
|
|
805
|
+
(entity) => String(entity.external_id ?? entity.externalId ?? '') === runKey,
|
|
627
806
|
);
|
|
628
|
-
|
|
807
|
+
if (row) return toExtractionRun(row);
|
|
808
|
+
// A page with more behind it did not fail to find the row — it failed to
|
|
809
|
+
// look at every row, and those are not the same answer.
|
|
810
|
+
if (response.next) throw undetermined(`whether a sweep has run in this workspace`);
|
|
811
|
+
return null;
|
|
629
812
|
},
|
|
630
813
|
|
|
631
814
|
async writeRun(run: ExtractionRun) {
|
|
@@ -670,26 +853,72 @@ export function createCorpusApi(
|
|
|
670
853
|
* exactly the lie bd startsim-1cow removed. Rows are read from one endpoint,
|
|
671
854
|
* as handed.
|
|
672
855
|
*
|
|
673
|
-
* `documentCount`
|
|
674
|
-
*
|
|
675
|
-
*
|
|
676
|
-
*
|
|
856
|
+
* `documentCount` IS THE ONE CORPUS-SIDE FACT ON THIS CARD, and it is
|
|
857
|
+
* counted here rather than read off the connection (bd startsim-jevtu, first
|
|
858
|
+
* filed as startsim-o0o6o.1). It used to be
|
|
859
|
+
* `connection.document_count ?? connection.documentCount ?? 0` over a
|
|
860
|
+
* payload that has never carried either spelling, so every card on every
|
|
861
|
+
* foundry read "0 records" — on tandem, four of them, directly above a list
|
|
862
|
+
* reading "208 records".
|
|
863
|
+
*
|
|
864
|
+
* CENTRAL CANNOT ANSWER IT. The corpus lives in the fork; central delivers
|
|
865
|
+
* INTO it and keeps no copy, so a count annotated onto the connection would
|
|
866
|
+
* be a count of what central believes it shipped — a different number from
|
|
867
|
+
* the list under the strip as soon as an upsert collapses two records or a
|
|
868
|
+
* delivery half-lands. A strip that disagrees with its own list is the fault
|
|
869
|
+
* being fixed, not a fix for it. And the disagreement is the POINT: every
|
|
870
|
+
* other field on this card comes from central's connection row, so "central
|
|
871
|
+
* says it synced an hour ago, the corpus holds nothing from it" is the
|
|
872
|
+
* delivery-hop failure made visible, which no connection field can say.
|
|
873
|
+
*
|
|
874
|
+
* ONE COUNT PER SOURCE, NOT A SCAN. `page_size=1` and the paginator's own
|
|
875
|
+
* `count` — one SQL COUNT with `?source=` pushed down. That is the idiom the
|
|
876
|
+
* tenant's `/api/v1/capabilities/` declares (`"counts": "page-count"`, and
|
|
877
|
+
* "a focus card's '12 of 40 done' is two `page_size=1` reads … so a client
|
|
878
|
+
* does not go looking for a group_by endpoint that deliberately does not
|
|
879
|
+
* exist"). Paging the corpus per card would be the same wrong answer with
|
|
880
|
+
* worse latency.
|
|
881
|
+
*
|
|
882
|
+
* `null`, NEVER A GUESSED `0`. A fork that predates the `?source=` filter
|
|
883
|
+
* (bd startsim-768w.38.12) narrows nothing and hands back the WHOLE corpus,
|
|
884
|
+
* which would print the same total on every card. The server's own
|
|
885
|
+
* `applied_filters` is the only thing that tells a narrowed count from an
|
|
886
|
+
* unnarrowed one, so a count it did not confirm is not reported at all —
|
|
887
|
+
* and `0` is left free to mean what it says.
|
|
677
888
|
*/
|
|
678
889
|
async listSources(): Promise<SourceFreshness[]> {
|
|
679
|
-
const
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
.catch(() => EMPTY_PAGE as DrfPage<Record<string, unknown>>),
|
|
683
|
-
http
|
|
684
|
-
.get<DrfPage<TenantEntity>>(entities, { params: { type: types.document, page_size: 1 } })
|
|
685
|
-
.catch(() => EMPTY_PAGE as DrfPage<TenantEntity>),
|
|
686
|
-
]);
|
|
890
|
+
const connectionPage = await http
|
|
891
|
+
.get<DrfPage<Record<string, unknown>>>(connections)
|
|
892
|
+
.catch(() => EMPTY_PAGE as DrfPage<Record<string, unknown>>);
|
|
687
893
|
|
|
688
|
-
|
|
894
|
+
const live = (connectionPage.results ?? []).filter(isALiveSource).map((connection) => ({
|
|
895
|
+
connection,
|
|
896
|
+
key: String(connection.connector_key ?? connection.connectorKey ?? 'unknown'),
|
|
897
|
+
}));
|
|
898
|
+
// ONE READ PER DISTINCT SOURCE KEY, in parallel — the cards are
|
|
899
|
+
// independent and a slow source must not hold up the rest of the strip.
|
|
900
|
+
//
|
|
901
|
+
// Distinct, because a card is per CONNECTION and a count is per SOURCE,
|
|
902
|
+
// and those are not the same thing. Central's page unions the foundry's
|
|
903
|
+
// shared connections with the member's own (bd startsim-768w.22.25.11),
|
|
904
|
+
// so a shared Granola and a personal Granola arrive as two rows naming one
|
|
905
|
+
// vendor. Whose account it is belongs on the card; it does not narrow the
|
|
906
|
+
// corpus, so counting it twice would be two identical queries for one
|
|
907
|
+
// answer. Both cards then report the same total, which is the true one:
|
|
908
|
+
// the corpus does not record which of two accounts a document came in
|
|
909
|
+
// through.
|
|
910
|
+
const pending = new Map<string, Promise<number | null>>();
|
|
911
|
+
for (const { key } of live) {
|
|
912
|
+
if (!pending.has(key)) pending.set(key, countDocumentsFrom(key));
|
|
913
|
+
}
|
|
914
|
+
const counts = new Map(
|
|
915
|
+
await Promise.all(
|
|
916
|
+
[...pending].map(async ([key, count]) => [key, await count] as const),
|
|
917
|
+
),
|
|
918
|
+
);
|
|
689
919
|
|
|
690
920
|
const now = Date.now();
|
|
691
|
-
return
|
|
692
|
-
const key = String(connection.connector_key ?? connection.connectorKey ?? 'unknown');
|
|
921
|
+
return live.map(({ connection, key }) => {
|
|
693
922
|
// THREE FACTS, READ SEPARATELY (bd startsim-1cow). This was one `??`
|
|
694
923
|
// chain ending in `last_checked_at`, and central's own view says of that
|
|
695
924
|
// column that it "is a far more optimistic fact than either and is never
|
|
@@ -709,7 +938,7 @@ export function createCorpusApi(
|
|
|
709
938
|
const lastCheckedAt = at(connection.last_checked_at ?? connection.lastCheckedAt);
|
|
710
939
|
const failures =
|
|
711
940
|
Number(connection.consecutive_failures ?? connection.consecutiveFailures ?? 0) || 0;
|
|
712
|
-
const documentCount =
|
|
941
|
+
const documentCount = counts.get(key) ?? null;
|
|
713
942
|
const error = String(connection.check_error ?? connection.checkError ?? '');
|
|
714
943
|
const status = connection.status ?? '';
|
|
715
944
|
|
package/src/corpus/entities.ts
CHANGED
|
@@ -54,6 +54,47 @@ export interface TenantRelationship {
|
|
|
54
54
|
createdAt?: string | null;
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* The most ids one `__in` list may carry, and it is the SERVER's number.
|
|
59
|
+
*
|
|
60
|
+
* `apps/api/filters.py::MAX_ID_LIST` is 200 and `_uuid_list` answers a longer
|
|
61
|
+
* list with a 400 rather than truncating it — "bounded so the parameter cannot
|
|
62
|
+
* become an unpaginated read by another name". A client that sends more does not
|
|
63
|
+
* get a wider answer, it gets no answer, so the ids of a page are CHUNKED at
|
|
64
|
+
* this number rather than sent in one request and hoped for.
|
|
65
|
+
*/
|
|
66
|
+
export const MAX_ID_LIST = 200;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Parameters `/api/v1/relationships/` really applies (`RelationshipQuery._FILTERS`).
|
|
70
|
+
*
|
|
71
|
+
* Named here rather than inside either the adapter or the fixture, because both
|
|
72
|
+
* have to agree about it and a second copy is how they stop agreeing. It is a
|
|
73
|
+
* DESCRIPTION of a deployed server, not a contract this package enforces: a fork
|
|
74
|
+
* that has not shipped `RelationshipQuery` ignores every one of these, which is
|
|
75
|
+
* why `applied_filters` is read on the response rather than this list being
|
|
76
|
+
* trusted.
|
|
77
|
+
*/
|
|
78
|
+
export const RELATIONSHIP_FILTERS: ReadonlySet<string> = new Set([
|
|
79
|
+
'rel_type',
|
|
80
|
+
'source',
|
|
81
|
+
'target',
|
|
82
|
+
'entity',
|
|
83
|
+
'source__in',
|
|
84
|
+
'target__in',
|
|
85
|
+
'entity__in',
|
|
86
|
+
'source_type',
|
|
87
|
+
'target_type',
|
|
88
|
+
]);
|
|
89
|
+
|
|
90
|
+
/** Consumed by pagination and negotiation, never a filter (`filters.py::_RESERVED`). */
|
|
91
|
+
export const RESERVED_PARAMS: ReadonlySet<string> = new Set([
|
|
92
|
+
'page',
|
|
93
|
+
'page_size',
|
|
94
|
+
'format',
|
|
95
|
+
'type',
|
|
96
|
+
]);
|
|
97
|
+
|
|
57
98
|
/**
|
|
58
99
|
* Edges that mean "this party is ON this record", in the order they should be
|
|
59
100
|
* listed. DATA, not a closed set: a tenant that models its own edge passes its
|