@startsimpli/integrations 0.2.32 → 0.2.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/connection-activity.test.tsx +160 -0
- package/src/__tests__/corpus-narrowed-reads.test.ts +337 -0
- package/src/__tests__/corpus-storage.test.ts +20 -9
- package/src/__tests__/library-failure-reason.test.tsx +178 -0
- package/src/components/pages/ConnectionsPage.tsx +174 -68
- package/src/corpus/client.ts +221 -49
- package/src/corpus/entities.ts +41 -0
- package/src/corpus/fixtures.ts +102 -9
- package/src/corpus/runs.ts +10 -1
- package/src/corpus/types.ts +28 -0
package/src/corpus/client.ts
CHANGED
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
DEFAULT_PARTICIPANT_EDGES,
|
|
35
35
|
DEFAULT_RUN_KEY,
|
|
36
36
|
DEFAULT_TYPE_KEYS,
|
|
37
|
+
MAX_ID_LIST,
|
|
37
38
|
commitmentToEntity,
|
|
38
39
|
extractionRunToEntity,
|
|
39
40
|
indexParticipants,
|
|
@@ -143,17 +144,20 @@ export interface CreateCorpusApiOptions {
|
|
|
143
144
|
/** Which edge types mean "this party is on this record". Data, not an enum. */
|
|
144
145
|
participantEdges?: readonly string[];
|
|
145
146
|
/**
|
|
146
|
-
* Bound on
|
|
147
|
+
* Bound on a walked list, in PAGES.
|
|
147
148
|
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
149
|
+
* IT USED TO BE THE ONLY BOUND THERE WAS. This said the tenant relationships
|
|
150
|
+
* endpoint "has no `?source=` filter today", so the edges for a page of
|
|
151
|
+
* documents could not be requested directly and the whole edge table was read
|
|
152
|
+
* in bulk — twenty pages of it — and joined in the browser. That stopped being
|
|
153
|
+
* true with bd startsim-te3r: `RelationshipQuery` applies `entity__in`, whose
|
|
154
|
+
* own docstring says what it is for ("a comma list of ids is how a client asks
|
|
155
|
+
* for the edges of ONE PAGE of records"). The edge read is now narrowed to the
|
|
156
|
+
* ids on screen (bd startsim-o0o6o.2) and this cap is a backstop on the walk
|
|
157
|
+
* within that narrowing, and on the party roster read that names the far ends.
|
|
154
158
|
*/
|
|
155
159
|
maxGraphPages?: number;
|
|
156
|
-
/** How long a
|
|
160
|
+
/** How long a resolved roster and a record's people are reused before re-reading. */
|
|
157
161
|
graphTtlMs?: number;
|
|
158
162
|
/**
|
|
159
163
|
* The `external_id` the singleton extraction-run row lives at. One per
|
|
@@ -393,6 +397,26 @@ function notFound(id: string): Error & { status: number } {
|
|
|
393
397
|
return Object.assign(new Error(`No record ${id} in this workspace.`), { status: 404 });
|
|
394
398
|
}
|
|
395
399
|
|
|
400
|
+
/**
|
|
401
|
+
* "I could not tell", which is the third answer and not a variant of the second.
|
|
402
|
+
*
|
|
403
|
+
* A read that could not see every candidate row is not evidence of absence, and
|
|
404
|
+
* `null` in this adapter is a claim of absence that callers ACT on: a sweep
|
|
405
|
+
* reported as never-run is started again, over the same cursor, by a second
|
|
406
|
+
* driver. Thrown rather than returned so the existing `ExtractionRun | null`
|
|
407
|
+
* contract keeps meaning what it says — the two answers a caller may act on are
|
|
408
|
+
* a run and no run, and everything else is a failure to answer.
|
|
409
|
+
*/
|
|
410
|
+
function undetermined(what: string): Error & { status: number } {
|
|
411
|
+
return Object.assign(
|
|
412
|
+
new Error(
|
|
413
|
+
`Could not determine ${what}: this workspace has more rows of that type ` +
|
|
414
|
+
`than one page can cover, and no filter on the tenant can name the one row.`,
|
|
415
|
+
),
|
|
416
|
+
{ status: 409 },
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
|
|
396
420
|
export function createCorpusApi(
|
|
397
421
|
options: CreateCorpusApiOptions,
|
|
398
422
|
): CorpusApi & CommitmentApi & ExtractionRunStore {
|
|
@@ -487,43 +511,132 @@ export function createCorpusApi(
|
|
|
487
511
|
// the participant graph
|
|
488
512
|
// ---------------------------------------------------------------------
|
|
489
513
|
//
|
|
490
|
-
//
|
|
491
|
-
//
|
|
514
|
+
// THE EDGES OF THE RECORDS ON SCREEN, AND NO OTHERS (bd startsim-o0o6o.2).
|
|
515
|
+
// This used to be `readAll(relationships, {})` — every edge in the workspace,
|
|
516
|
+
// twenty pages of it, on a comment saying the endpoint had no filters. It has
|
|
517
|
+
// them: `entity__in` takes the ids of one page, bounded at `MAX_ID_LIST`,
|
|
518
|
+
// which is the shape `RelationshipQuery`'s own docstring says it exists for.
|
|
519
|
+
//
|
|
520
|
+
// ONE read for a page of records, not one per record — and cached per record,
|
|
521
|
+
// so a screen that lists, opens a row and then asks a question spends it once.
|
|
492
522
|
// It FAILS SOFT: a tenant whose relationships endpoint is missing or refused
|
|
493
523
|
// renders exactly what it rendered before this existed, rather than an error
|
|
494
524
|
// where a Library used to be.
|
|
495
|
-
let
|
|
525
|
+
let roster: { at: number; value: Promise<Party[]> } | null = null;
|
|
526
|
+
const known = new Map<string, { at: number; value: PartyRef[] }>();
|
|
496
527
|
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
528
|
+
/**
|
|
529
|
+
* The people an edge can point AT, read once and reused.
|
|
530
|
+
*
|
|
531
|
+
* Still a walked list, because there is no id filter on `/entities/`: the
|
|
532
|
+
* recognised set is `search`, `source`, `party`, the occurred window,
|
|
533
|
+
* `related_*`, `order` and `attr.*`, and an unrecognised `id__in` would be
|
|
534
|
+
* swept into `ignored_filters` and answered with the whole page. It is at
|
|
535
|
+
* least a TYPED read — `?type=party` is applied server-side — and it is only
|
|
536
|
+
* reached when the narrowed edge read came back with edges to resolve.
|
|
537
|
+
*/
|
|
538
|
+
function partyRoster(): Promise<Party[]> {
|
|
539
|
+
const now = Date.now();
|
|
540
|
+
if (!roster || now - roster.at > graphTtlMs) {
|
|
541
|
+
roster = {
|
|
542
|
+
at: now,
|
|
543
|
+
value: readAll<TenantEntity>(entities, { type: types.party })
|
|
544
|
+
.then((rows) => rows.map((entity) => toParty(entity, scope)))
|
|
545
|
+
.catch(() => []),
|
|
546
|
+
};
|
|
510
547
|
}
|
|
548
|
+
return roster.value;
|
|
511
549
|
}
|
|
512
550
|
|
|
513
|
-
|
|
551
|
+
/**
|
|
552
|
+
* The edges touching these records, or `null` when the server did not confirm
|
|
553
|
+
* it narrowed by them.
|
|
554
|
+
*
|
|
555
|
+
* `null` IS NOT "NO EDGES", and the difference is the whole point. A fork
|
|
556
|
+
* without `RelationshipQuery` accepts `entity__in`, names it in
|
|
557
|
+
* `ignored_filters` and answers with an arbitrary page of the WHOLE edge table
|
|
558
|
+
* (bd startsim-768w.38.39) — rows that look exactly like a narrow answer and
|
|
559
|
+
* would put the wrong people, or a fraction of the right ones, on a record.
|
|
560
|
+
* start-simpli#154 settled what to do with a result the server did not
|
|
561
|
+
* confirm: do not use it. So this reports that it could not ask, and the
|
|
562
|
+
* caller renders what it rendered before the join existed.
|
|
563
|
+
*
|
|
564
|
+
* `rel_type` rides along unchecked on purpose: it only ever makes the answer
|
|
565
|
+
* SMALLER, and `indexParticipants` ranks by edge type anyway. A fork that
|
|
566
|
+
* ignores it pays in PAGES rather than in correctness — a record with many
|
|
567
|
+
* non-participant edges can push its people onto a second page, and the walk
|
|
568
|
+
* below follows `next` until it has them.
|
|
569
|
+
*/
|
|
570
|
+
async function edgesFor(recordIds: string[]): Promise<TenantRelationship[] | null> {
|
|
571
|
+
const rows: TenantRelationship[] = [];
|
|
572
|
+
for (let start = 0; start < recordIds.length; start += MAX_ID_LIST) {
|
|
573
|
+
// CHUNKED, because `_uuid_list` answers a longer list with a 400 rather
|
|
574
|
+
// than truncating it. Truncating here instead would drop the people off
|
|
575
|
+
// the tail of a wide page silently, which is the failure being undone.
|
|
576
|
+
const chunk = recordIds.slice(start, start + MAX_ID_LIST);
|
|
577
|
+
const extra: Record<string, unknown> = {
|
|
578
|
+
entity__in: chunk.join(','),
|
|
579
|
+
page_size: MAX_ID_LIST,
|
|
580
|
+
};
|
|
581
|
+
if (participantEdges.length) extra.rel_type = participantEdges.join(',');
|
|
582
|
+
for (let index = 1; index <= maxGraphPages; index += 1) {
|
|
583
|
+
const response = await http.get<DrfPage<TenantRelationship> & ServerFilterReport>(
|
|
584
|
+
relationships,
|
|
585
|
+
{ params: { ...extra, page: index } },
|
|
586
|
+
);
|
|
587
|
+
const applied = response.applied_filters ?? response.appliedFilters;
|
|
588
|
+
if (!applied?.includes('entity__in')) return null;
|
|
589
|
+
const results = response.results ?? [];
|
|
590
|
+
rows.push(...results);
|
|
591
|
+
if (!response.next || results.length === 0) break;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
return rows;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
async function participantsFor(
|
|
598
|
+
records: Array<string | number>,
|
|
599
|
+
): Promise<Map<string, PartyRef[]>> {
|
|
600
|
+
const out = new Map<string, PartyRef[]>();
|
|
601
|
+
if (!resolveParticipants) return out;
|
|
514
602
|
const now = Date.now();
|
|
515
|
-
|
|
516
|
-
|
|
603
|
+
const wanted: string[] = [];
|
|
604
|
+
for (const record of records) {
|
|
605
|
+
const id = String(record ?? '').trim();
|
|
606
|
+
if (!id || out.has(id) || wanted.includes(id)) continue;
|
|
607
|
+
const cached = known.get(id);
|
|
608
|
+
if (cached && now - cached.at <= graphTtlMs) out.set(id, cached.value);
|
|
609
|
+
else wanted.push(id);
|
|
517
610
|
}
|
|
518
|
-
return
|
|
611
|
+
if (wanted.length === 0) return out;
|
|
612
|
+
try {
|
|
613
|
+
const edges = await edgesFor(wanted);
|
|
614
|
+
// Could not ask. Nothing is cached, so the next page asks again rather
|
|
615
|
+
// than remembering an answer that was never given.
|
|
616
|
+
if (!edges) return out;
|
|
617
|
+
const parties = edges.length ? await partyRoster() : [];
|
|
618
|
+
const indexed = edges.length
|
|
619
|
+
? indexParticipants(edges, parties, participantEdges)
|
|
620
|
+
: new Map<string, PartyRef[]>();
|
|
621
|
+
for (const id of wanted) {
|
|
622
|
+
// A record the server confirmed has no participant edges is REMEMBERED
|
|
623
|
+
// as having none — otherwise every page re-asks for the same silence.
|
|
624
|
+
const people = indexed.get(id) ?? [];
|
|
625
|
+
known.set(id, { at: now, value: people });
|
|
626
|
+
out.set(id, people);
|
|
627
|
+
}
|
|
628
|
+
} catch {
|
|
629
|
+
// The Library renders without its Parties block rather than not at all.
|
|
630
|
+
}
|
|
631
|
+
return out;
|
|
519
632
|
}
|
|
520
633
|
|
|
521
634
|
return {
|
|
522
635
|
async listDocuments(query) {
|
|
523
636
|
const documents = await page<TenantEntity>(types.document, query, (entity) => entity);
|
|
524
|
-
// Skip the
|
|
525
|
-
// exactly the one request it costs today.
|
|
526
|
-
const byRecord = documents.results.
|
|
637
|
+
// Skip the edge read entirely for an empty page: an empty corpus should
|
|
638
|
+
// cost exactly the one request it costs today.
|
|
639
|
+
const byRecord = await participantsFor(documents.results.map((entity) => entity.id));
|
|
527
640
|
return {
|
|
528
641
|
...documents,
|
|
529
642
|
results: documents.results.map((entity) =>
|
|
@@ -557,7 +670,10 @@ export function createCorpusApi(
|
|
|
557
670
|
async getDocument(id) {
|
|
558
671
|
const [entity, byRecord] = await Promise.all([
|
|
559
672
|
http.get<TenantEntity>(`${entities}/${encodeURIComponent(id)}`),
|
|
560
|
-
|
|
673
|
+
// The edges of ONE record, which is what this screen needs — it no
|
|
674
|
+
// longer waits on a read of the whole workspace's graph to render a
|
|
675
|
+
// record it already named (bd startsim-o0o6o.2).
|
|
676
|
+
participantsFor([id]),
|
|
561
677
|
]);
|
|
562
678
|
if (!isRecord(entity, id)) throw notFound(id);
|
|
563
679
|
return toSourceDocument(entity, scope, sourceLabels, byRecord.get(String(entity.id)) ?? []);
|
|
@@ -569,7 +685,7 @@ export function createCorpusApi(
|
|
|
569
685
|
|
|
570
686
|
async listMoneyEvents(query = {}) {
|
|
571
687
|
const events = await page<TenantEntity>(types.moneyEvent, query, (entity) => entity);
|
|
572
|
-
const byRecord = events.results.
|
|
688
|
+
const byRecord = await participantsFor(events.results.map((entity) => entity.id));
|
|
573
689
|
return {
|
|
574
690
|
...events,
|
|
575
691
|
results: events.results.map((entity) =>
|
|
@@ -641,21 +757,38 @@ export function createCorpusApi(
|
|
|
641
757
|
/**
|
|
642
758
|
* The sweep record, read back (bd startsim-768w.38.28).
|
|
643
759
|
*
|
|
644
|
-
* ONE ROW PER WORKSPACE, at a fixed `external_id`, because
|
|
645
|
-
*
|
|
646
|
-
*
|
|
647
|
-
*
|
|
648
|
-
*
|
|
760
|
+
* ONE ROW PER WORKSPACE, at a fixed `external_id`, because a row per sweep
|
|
761
|
+
* would mean scanning a list that grows forever on every poll. A singleton
|
|
762
|
+
* is one read that never grows, and the last few sweeps ride along inside it.
|
|
763
|
+
*
|
|
764
|
+
* THE LOOKUP CANNOT BE PUSHED DOWN, AND SAYING SO IS THE POINT. `external_id`
|
|
765
|
+
* is not among the parameters `/entities/` recognises — `search`, `source`,
|
|
766
|
+
* `party`, the occurred window, `related_*`, `order`, `attr.*` — so the match
|
|
767
|
+
* happens here. `attr.external_id` is NOT the way around that: an attribute
|
|
768
|
+
* nobody declared has no `core_attribute` rows, and `EntityQuery._attribute`
|
|
769
|
+
* reports the filter as APPLIED before any row matches, so the reply to it is
|
|
770
|
+
* an empty page that the envelope swears was narrowed. A filter that cannot
|
|
771
|
+
* be told apart from "no such record" is worse than no filter.
|
|
772
|
+
*
|
|
773
|
+
* SO THE BOUND IS WHAT HAS TO BE HONEST (bd startsim-o0o6o.3). This asked for
|
|
774
|
+
* five rows, because the singleton "is" a singleton — but nothing enforces
|
|
775
|
+
* that, entities come back `-created_at`, and the singleton is the OLDEST row
|
|
776
|
+
* of its type because it is rewritten in place. Six `extraction_run` rows and
|
|
777
|
+
* the read stopped seeing it, returned `null`, and every caller read that as
|
|
778
|
+
* "no sweep has ever run" — which `startExtractionRun` acts on by starting a
|
|
779
|
+
* duplicate sweep over a live cursor. It now asks for the widest page the
|
|
780
|
+
* paginator will serve and, when the row could still be past the end of it,
|
|
781
|
+
* says it could not tell instead of saying no.
|
|
649
782
|
*
|
|
650
783
|
* FAIL-SOFT ON ABSENCE, NOT ON FAILURE. No run yet is `null`, which is a
|
|
651
|
-
* fact.
|
|
652
|
-
* "nothing has ever run" are different
|
|
653
|
-
*
|
|
784
|
+
* fact. Everything else throws, because "we could not reach the workspace",
|
|
785
|
+
* "we could not see far enough" and "nothing has ever run" are different
|
|
786
|
+
* answers and only the last one is safe to act on.
|
|
654
787
|
*/
|
|
655
788
|
async readRun() {
|
|
656
789
|
const response = await http
|
|
657
790
|
.get<DrfPage<TenantEntity>>(entities, {
|
|
658
|
-
params: { type: types.extractionRun, page: 1, page_size:
|
|
791
|
+
params: { type: types.extractionRun, page: 1, page_size: MAX_ID_LIST },
|
|
659
792
|
})
|
|
660
793
|
.catch((err: { status?: number }) => {
|
|
661
794
|
// An undeclared type is a 403 from the read gate, and it means exactly
|
|
@@ -664,9 +797,18 @@ export function createCorpusApi(
|
|
|
664
797
|
throw err;
|
|
665
798
|
});
|
|
666
799
|
const row = (response.results ?? []).find(
|
|
667
|
-
|
|
800
|
+
// BOTH SPELLINGS, for the reason `ServerFilterReport` documents: the
|
|
801
|
+
// `@startsimpli/api` fetch wrapper turns `external_id` into `externalId`
|
|
802
|
+
// on the way in, so a browser-side read matched nothing at ANY page size
|
|
803
|
+
// and reported "never run" over a workspace with a sweep in it. Every
|
|
804
|
+
// other reader in `./entities` already takes both; this one did not.
|
|
805
|
+
(entity) => String(entity.external_id ?? entity.externalId ?? '') === runKey,
|
|
668
806
|
);
|
|
669
|
-
|
|
807
|
+
if (row) return toExtractionRun(row);
|
|
808
|
+
// A page with more behind it did not fail to find the row — it failed to
|
|
809
|
+
// look at every row, and those are not the same answer.
|
|
810
|
+
if (response.next) throw undetermined(`whether a sweep has run in this workspace`);
|
|
811
|
+
return null;
|
|
670
812
|
},
|
|
671
813
|
|
|
672
814
|
async writeRun(run: ExtractionRun) {
|
|
@@ -798,6 +940,11 @@ export function createCorpusApi(
|
|
|
798
940
|
Number(connection.consecutive_failures ?? connection.consecutiveFailures ?? 0) || 0;
|
|
799
941
|
const documentCount = counts.get(key) ?? null;
|
|
800
942
|
const error = String(connection.check_error ?? connection.checkError ?? '');
|
|
943
|
+
// WHY THE LAST POLL FAILED — not why the credential did (bd
|
|
944
|
+
// startsim-ei8gm.2). Both spellings for the same reason the three
|
|
945
|
+
// timestamps above read both: `@startsimpli/api` camelCases response
|
|
946
|
+
// keys in the browser and a plain server-side fetch does not.
|
|
947
|
+
const lastError = String(connection.last_error ?? connection.lastError ?? '');
|
|
801
948
|
const status = connection.status ?? '';
|
|
802
949
|
|
|
803
950
|
let health: SourceFreshness['status'] = 'ok';
|
|
@@ -805,7 +952,7 @@ export function createCorpusApi(
|
|
|
805
952
|
if (error || status === 'error' || status === 'needs_reauth') {
|
|
806
953
|
health = 'error';
|
|
807
954
|
detail = error || `Connection status is ${String(status)}.`;
|
|
808
|
-
} else if (failures > 0 || (lastAttemptAt && !lastSyncedAt)) {
|
|
955
|
+
} else if (failures > 0 || lastError || (lastAttemptAt && !lastSyncedAt)) {
|
|
809
956
|
// IT TRIED AND IT FAILED. Central's view exists partly so that "a card
|
|
810
957
|
// cannot read 'nothing yet' while it is in fact failing on a
|
|
811
958
|
// schedule" — a run that FINISHED while nothing has ever finished well
|
|
@@ -813,12 +960,33 @@ export function createCorpusApi(
|
|
|
813
960
|
// when an older sync did land. Note what does NOT appear in this
|
|
814
961
|
// condition: `lastCheckedAt`. A verified key is not a delivered
|
|
815
962
|
// record, in either direction.
|
|
963
|
+
//
|
|
964
|
+
// `lastError` JOINS THAT CONDITION, because the two facts have
|
|
965
|
+
// different writers: `consecutive_failures` is a counter on the
|
|
966
|
+
// CONNECTION and the reason is derived from the newest RUN. Central
|
|
967
|
+
// sends a reason only while that newest run is the failed one, so a
|
|
968
|
+
// reason with a zero streak behind it is still a source whose last
|
|
969
|
+
// sync failed — and turning only on the counter reported that as
|
|
970
|
+
// `stale`, which reads as "nothing to do yet".
|
|
816
971
|
health = 'error';
|
|
817
|
-
|
|
972
|
+
// THE COUNT SURVIVES; IT IS NO LONGER THE WHOLE SENTENCE. "One blip"
|
|
973
|
+
// and "sixteen days" are different problems, so the streak stays —
|
|
974
|
+
// what was missing was the half a reader can act on. Composed here
|
|
975
|
+
// rather than server-side because the streak is a connection field
|
|
976
|
+
// and the reason is a run field, and central sends them separately
|
|
977
|
+
// on purpose (bd startsim-1cow: conflating two facts is how a
|
|
978
|
+
// freshness strip lies).
|
|
979
|
+
const streak =
|
|
818
980
|
failures > 0
|
|
819
|
-
? `${failures} sync${failures === 1 ? '' : 's'} in a row failed`
|
|
820
|
-
|
|
821
|
-
|
|
981
|
+
? `${failures} sync${failures === 1 ? '' : 's'} in a row failed`
|
|
982
|
+
: lastError
|
|
983
|
+
? 'The last sync failed'
|
|
984
|
+
: 'A sync ran but nothing arrived';
|
|
985
|
+
const nothingYet =
|
|
986
|
+
failures > 0 && !lastSyncedAt ? ' and nothing has arrived yet' : '';
|
|
987
|
+
// An em dash, not a colon: the server's reason usually contains its
|
|
988
|
+
// own colons ("[config_error] ...") and a second one reads as a typo.
|
|
989
|
+
detail = lastError ? `${streak}${nothingYet} — ${lastError}` : `${streak}${nothingYet}.`;
|
|
822
990
|
} else if (!lastSyncedAt) {
|
|
823
991
|
// `never` regardless of `lastCheckedAt`, which is the whole bead.
|
|
824
992
|
health = 'never';
|
|
@@ -840,6 +1008,10 @@ export function createCorpusApi(
|
|
|
840
1008
|
consecutiveFailures: failures,
|
|
841
1009
|
status: health,
|
|
842
1010
|
...(detail ? { detail } : {}),
|
|
1011
|
+
// Carried separately as well as folded into `detail`, so a surface
|
|
1012
|
+
// that wants the reason on its own — a tooltip, a copy button, the
|
|
1013
|
+
// control plane's card — does not have to unpick a composed sentence.
|
|
1014
|
+
...(lastError ? { lastError } : {}),
|
|
843
1015
|
};
|
|
844
1016
|
});
|
|
845
1017
|
},
|
package/src/corpus/entities.ts
CHANGED
|
@@ -54,6 +54,47 @@ export interface TenantRelationship {
|
|
|
54
54
|
createdAt?: string | null;
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* The most ids one `__in` list may carry, and it is the SERVER's number.
|
|
59
|
+
*
|
|
60
|
+
* `apps/api/filters.py::MAX_ID_LIST` is 200 and `_uuid_list` answers a longer
|
|
61
|
+
* list with a 400 rather than truncating it — "bounded so the parameter cannot
|
|
62
|
+
* become an unpaginated read by another name". A client that sends more does not
|
|
63
|
+
* get a wider answer, it gets no answer, so the ids of a page are CHUNKED at
|
|
64
|
+
* this number rather than sent in one request and hoped for.
|
|
65
|
+
*/
|
|
66
|
+
export const MAX_ID_LIST = 200;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Parameters `/api/v1/relationships/` really applies (`RelationshipQuery._FILTERS`).
|
|
70
|
+
*
|
|
71
|
+
* Named here rather than inside either the adapter or the fixture, because both
|
|
72
|
+
* have to agree about it and a second copy is how they stop agreeing. It is a
|
|
73
|
+
* DESCRIPTION of a deployed server, not a contract this package enforces: a fork
|
|
74
|
+
* that has not shipped `RelationshipQuery` ignores every one of these, which is
|
|
75
|
+
* why `applied_filters` is read on the response rather than this list being
|
|
76
|
+
* trusted.
|
|
77
|
+
*/
|
|
78
|
+
export const RELATIONSHIP_FILTERS: ReadonlySet<string> = new Set([
|
|
79
|
+
'rel_type',
|
|
80
|
+
'source',
|
|
81
|
+
'target',
|
|
82
|
+
'entity',
|
|
83
|
+
'source__in',
|
|
84
|
+
'target__in',
|
|
85
|
+
'entity__in',
|
|
86
|
+
'source_type',
|
|
87
|
+
'target_type',
|
|
88
|
+
]);
|
|
89
|
+
|
|
90
|
+
/** Consumed by pagination and negotiation, never a filter (`filters.py::_RESERVED`). */
|
|
91
|
+
export const RESERVED_PARAMS: ReadonlySet<string> = new Set([
|
|
92
|
+
'page',
|
|
93
|
+
'page_size',
|
|
94
|
+
'format',
|
|
95
|
+
'type',
|
|
96
|
+
]);
|
|
97
|
+
|
|
57
98
|
/**
|
|
58
99
|
* Edges that mean "this party is ON this record", in the order they should be
|
|
59
100
|
* listed. DATA, not a closed set: a tenant that models its own edge passes its
|
package/src/corpus/fixtures.ts
CHANGED
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
toParty,
|
|
43
43
|
toSourceDocument,
|
|
44
44
|
} from './entities';
|
|
45
|
+
import { MAX_ID_LIST, RELATIONSHIP_FILTERS, RESERVED_PARAMS } from './entities';
|
|
45
46
|
import type { TenantEntity, TenantRelationship } from './entities';
|
|
46
47
|
import { splitSections, stem } from './text';
|
|
47
48
|
import type {
|
|
@@ -875,10 +876,22 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
|
|
|
875
876
|
* stands in for an OLDER fork — which is a useful thing to have a stand-in
|
|
876
877
|
* for, since that is the fork `listSources` refuses to print a count for
|
|
877
878
|
* (bd startsim-jevtu), but it is a lagging stub and not the contract;
|
|
878
|
-
* • `page_size`
|
|
879
|
-
*
|
|
880
|
-
*
|
|
881
|
-
*
|
|
879
|
+
* • `page_size` IS honoured, capped at 200. It was written here as ignored,
|
|
880
|
+
* citing a `PageNumberPagination` with no `page_size_query_param`, and that
|
|
881
|
+
* stopped being true with `TenantPageNumberPagination` (bd
|
|
882
|
+
* startsim-768w.38.12): "a client asking for one row to read a count got
|
|
883
|
+
* fifty, and a client asking for two hundred to walk a graph got fifty and
|
|
884
|
+
* no indication that its request had been overruled". A fixture that keeps
|
|
885
|
+
* ignoring it manufactures evidence for a cap the wire does not impose;
|
|
886
|
+
* • `/api/v1/relationships/` FILTERS — `rel_type`, `source`, `target`,
|
|
887
|
+
* `entity` and the `__in` forms of the last three, bounded at
|
|
888
|
+
* `MAX_ID_LIST` ids and reported in `applied_filters`. This too was written
|
|
889
|
+
* here as "every edge in the org, unfiltered, because the tenant has no
|
|
890
|
+
* `?source=` on it either", and bd startsim-te3r shipped `RelationshipQuery`
|
|
891
|
+
* for exactly the read that comment was excusing. An UNRECOGNISED parameter
|
|
892
|
+
* is still ignored and the page comes back as wide as it was — that part is
|
|
893
|
+
* real, it is what makes an unconfirmed filter dangerous, and it is what
|
|
894
|
+
* `applied_filters` exists to let a client detect.
|
|
882
895
|
*
|
|
883
896
|
* Every request is recorded, so "did the browser filter it" and "how many round
|
|
884
897
|
* trips did one screen cost" are both assertable.
|
|
@@ -886,7 +899,7 @@ export function createFixtureCorpusApi(options: FixtureCorpusOptions = {}): Corp
|
|
|
886
899
|
export function createFixtureCorpusHttp(options: {
|
|
887
900
|
entities?: TenantEntity[];
|
|
888
901
|
relationships?: TenantRelationship[];
|
|
889
|
-
/** Rows per page, matching the tenant's
|
|
902
|
+
/** Rows per page when the caller does not ask, matching the tenant's `PAGE_SIZE`. */
|
|
890
903
|
pageSize?: number;
|
|
891
904
|
/** Endpoints to fail, so the fail-soft paths are exercised. */
|
|
892
905
|
missing?: string[];
|
|
@@ -906,15 +919,90 @@ export function createFixtureCorpusHttp(options: {
|
|
|
906
919
|
|
|
907
920
|
function slice<T>(rows: T[], params: Record<string, unknown>) {
|
|
908
921
|
const page = Number(params.page ?? 1) || 1;
|
|
909
|
-
|
|
922
|
+
// `TenantPageNumberPagination.page_size_query_param` is `page_size` and its
|
|
923
|
+
// `max_page_size` is 200 — an OVERRULED request is served at the cap, not
|
|
924
|
+
// refused, which is the behaviour a client that walks a page has to survive.
|
|
925
|
+
const asked = Number(params.page_size);
|
|
926
|
+
const size = Number.isFinite(asked) && asked > 0 ? Math.min(asked, MAX_ID_LIST) : pageSize;
|
|
927
|
+
const start = (page - 1) * size;
|
|
910
928
|
return {
|
|
911
929
|
count: rows.length,
|
|
912
|
-
next: start +
|
|
930
|
+
next: start + size < rows.length ? `?page=${page + 1}` : null,
|
|
913
931
|
previous: page > 1 ? `?page=${page - 1}` : null,
|
|
914
|
-
results: rows.slice(start, start +
|
|
932
|
+
results: rows.slice(start, start + size),
|
|
915
933
|
};
|
|
916
934
|
}
|
|
917
935
|
|
|
936
|
+
/**
|
|
937
|
+
* `/api/v1/relationships/`, as `RelationshipQuery.apply` really answers it.
|
|
938
|
+
*
|
|
939
|
+
* THE SHAPE THAT MATTERS IS THE ONE IT REFUSES TO NARROW. A parameter this
|
|
940
|
+
* endpoint does not recognise is swept into `ignored_filters` and the page
|
|
941
|
+
* comes back exactly as wide as it was — so a fixture that returned `[]` for
|
|
942
|
+
* an unsupported filter would let a client pass here and read the whole edge
|
|
943
|
+
* table in production. Every filter below is one the server really applies,
|
|
944
|
+
* and everything else is really ignored.
|
|
945
|
+
*/
|
|
946
|
+
function edgeQuery(rows: TenantRelationship[], params: Record<string, unknown>) {
|
|
947
|
+
const applied: string[] = [];
|
|
948
|
+
const ignored: string[] = [];
|
|
949
|
+
const idList = (key: string): Set<string> => {
|
|
950
|
+
const parts = String(params[key]).split(',').map((part) => part.trim()).filter(Boolean);
|
|
951
|
+
// `_uuid_list` refuses a longer list with a 400 rather than truncating it,
|
|
952
|
+
// which is what keeps `__in` from becoming an unpaginated read by another
|
|
953
|
+
// name — and what a client that chunks its ids has to be tested against.
|
|
954
|
+
if (parts.length > MAX_ID_LIST) {
|
|
955
|
+
throw Object.assign(new Error(`${key}: at most ${MAX_ID_LIST} ids`), { status: 400 });
|
|
956
|
+
}
|
|
957
|
+
return new Set(parts);
|
|
958
|
+
};
|
|
959
|
+
const typeOf = (id: unknown): string =>
|
|
960
|
+
String(entities.find((entity) => String(entity.id) === String(id))?.entity_type ?? '');
|
|
961
|
+
let out = rows;
|
|
962
|
+
|
|
963
|
+
for (const [key, field] of [['source', 'source'], ['target', 'target']] as const) {
|
|
964
|
+
if (key in params) {
|
|
965
|
+
const wanted = String(params[key]);
|
|
966
|
+
out = out.filter((edge) => String(edge[field]) === wanted);
|
|
967
|
+
applied.push(key);
|
|
968
|
+
}
|
|
969
|
+
const listed = `${key}__in`;
|
|
970
|
+
if (listed in params) {
|
|
971
|
+
const wanted = idList(listed);
|
|
972
|
+
out = out.filter((edge) => wanted.has(String(edge[field])));
|
|
973
|
+
applied.push(listed);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
if ('entity' in params) {
|
|
977
|
+
const wanted = String(params.entity);
|
|
978
|
+
out = out.filter((edge) => String(edge.source) === wanted || String(edge.target) === wanted);
|
|
979
|
+
applied.push('entity');
|
|
980
|
+
}
|
|
981
|
+
if ('entity__in' in params) {
|
|
982
|
+
const wanted = idList('entity__in');
|
|
983
|
+
out = out.filter((edge) => wanted.has(String(edge.source)) || wanted.has(String(edge.target)));
|
|
984
|
+
applied.push('entity__in');
|
|
985
|
+
}
|
|
986
|
+
if ('rel_type' in params) {
|
|
987
|
+
const wanted = new Set(
|
|
988
|
+
String(params.rel_type).split(',').map((part) => part.trim()).filter(Boolean),
|
|
989
|
+
);
|
|
990
|
+
out = out.filter((edge) => wanted.has(String(edge.rel_type ?? edge.relType ?? '')));
|
|
991
|
+
applied.push('rel_type');
|
|
992
|
+
}
|
|
993
|
+
for (const [key, end] of [['source_type', 'source'], ['target_type', 'target']] as const) {
|
|
994
|
+
if (key in params) {
|
|
995
|
+
const wanted = String(params[key]);
|
|
996
|
+
out = out.filter((edge) => typeOf(edge[end]) === wanted);
|
|
997
|
+
applied.push(key);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
for (const key of Object.keys(params)) {
|
|
1001
|
+
if (!RESERVED_PARAMS.has(key) && !RELATIONSHIP_FILTERS.has(key)) ignored.push(key);
|
|
1002
|
+
}
|
|
1003
|
+
return { rows: out, applied, ignored };
|
|
1004
|
+
}
|
|
1005
|
+
|
|
918
1006
|
const http = {
|
|
919
1007
|
async get<T>(path: string, init?: { params?: Record<string, unknown> }): Promise<T> {
|
|
920
1008
|
const params = init?.params ?? {};
|
|
@@ -925,7 +1013,12 @@ export function createFixtureCorpusHttp(options: {
|
|
|
925
1013
|
throw Object.assign(new Error(`${path} responded 404`), { status: 404 });
|
|
926
1014
|
}
|
|
927
1015
|
if (path.includes('relationships')) {
|
|
928
|
-
|
|
1016
|
+
const query = edgeQuery(relationships, params);
|
|
1017
|
+
return {
|
|
1018
|
+
...slice(query.rows, params),
|
|
1019
|
+
applied_filters: query.applied.slice().sort(),
|
|
1020
|
+
ignored_filters: query.ignored.slice().sort(),
|
|
1021
|
+
} as T;
|
|
929
1022
|
}
|
|
930
1023
|
if (path.includes('entities')) {
|
|
931
1024
|
const id = /entities\/(.+)$/.exec(path)?.[1];
|
package/src/corpus/runs.ts
CHANGED
|
@@ -303,7 +303,16 @@ export async function startExtractionRun(
|
|
|
303
303
|
const at = now();
|
|
304
304
|
const driver = request.driverId ?? options.driverId ?? defaultDriverId();
|
|
305
305
|
|
|
306
|
-
|
|
306
|
+
// NOT `.catch(() => null)`. A store that could not answer is not a store that
|
|
307
|
+
// said "nothing has ever run", and this is the one place where confusing them
|
|
308
|
+
// is expensive: the branch below reads `null` as "start a new sweep", so a
|
|
309
|
+
// transient failure — or a read that could not see far enough (bd
|
|
310
|
+
// startsim-o0o6o.3) — put a SECOND driver on a live cursor, and two model
|
|
311
|
+
// calls over one document produce two items identity resolution cannot see are
|
|
312
|
+
// the same. `readRun` reports absence as `null` and everything else as a
|
|
313
|
+
// throw, precisely so this caller can tell them apart; swallowing the throw
|
|
314
|
+
// here threw that distinction away one line after it was made.
|
|
315
|
+
const existing = await deps.runs.readRun();
|
|
307
316
|
|
|
308
317
|
if (existing && !COMPLETE.has(existing.status) && !request.restart) {
|
|
309
318
|
if (isLeaseLive(existing, at) && existing.leaseOwner !== driver) {
|
package/src/corpus/types.ts
CHANGED
|
@@ -222,6 +222,34 @@ export interface SourceFreshness {
|
|
|
222
222
|
status: 'ok' | 'stale' | 'error' | 'never';
|
|
223
223
|
/** Why it is not `ok`, in the words the server used. */
|
|
224
224
|
detail?: string;
|
|
225
|
+
/**
|
|
226
|
+
* WHY THE LAST SYNC FAILED, in the vendor's own words, and a DIFFERENT FACT
|
|
227
|
+
* FROM THE CREDENTIAL'S VERDICT (bd startsim-ei8gm.2).
|
|
228
|
+
*
|
|
229
|
+
* The Tandem Granola card read "Failing — 20 syncs in a row failed, synced
|
|
230
|
+
* 16d ago" for sixteen days and nothing said why. It was our own 256 KiB
|
|
231
|
+
* egress cap — the per-note detail response peaked at 351 KB — which is a
|
|
232
|
+
* minutes-long diagnosis for anyone who could see the text and sixteen days
|
|
233
|
+
* for anyone who could not.
|
|
234
|
+
*
|
|
235
|
+
* `check_error` COULD NOT HAVE ANSWERED IT, which is why this is its own
|
|
236
|
+
* field rather than a wider read of that one. `check()` and `poll()` are
|
|
237
|
+
* different requests: Granola's check asks for `page_size=1` and passes,
|
|
238
|
+
* while its poll asks for thirty notes with transcripts and aborts. The
|
|
239
|
+
* credential verified cleanly throughout, so a strip built on `check_error`
|
|
240
|
+
* alone can count twenty failures and name none of them — and it sends the
|
|
241
|
+
* reader to re-enter a key that was never the problem.
|
|
242
|
+
*
|
|
243
|
+
* Central sends it only while the connection's NEWEST run is the failed one
|
|
244
|
+
* (`SubtreeConnectionsView._last_failure_reasons`), so it is never last
|
|
245
|
+
* Tuesday's error quoted over a fix that has already landed. It is SCRUBBED
|
|
246
|
+
* server-side against that connection's own stored credential plus the
|
|
247
|
+
* platform app_secret — a vendor can quote our key back inside its own error
|
|
248
|
+
* body — and a reason that could not be scrubbed arrives as a sentence saying
|
|
249
|
+
* so rather than as the raw text. Nothing here re-checks it; there is exactly
|
|
250
|
+
* one redaction path and it is not in the browser.
|
|
251
|
+
*/
|
|
252
|
+
lastError?: string;
|
|
225
253
|
}
|
|
226
254
|
|
|
227
255
|
/** DRF PageNumberPagination, which is what the tenant data API speaks. */
|