@startsimpli/integrations 0.2.13 → 0.2.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/__tests__/connections.test.tsx +165 -0
- package/src/__tests__/corpus-ask.test.ts +19 -1
- package/src/__tests__/corpus-copy.test.tsx +4 -1
- package/src/__tests__/corpus-field-gate.test.ts +246 -0
- package/src/__tests__/corpus-pages.test.tsx +58 -2
- package/src/__tests__/corpus-tracker.test.tsx +43 -0
- package/src/__tests__/corpus-transcript.test.ts +476 -0
- package/src/components/CredentialSchemaForm.tsx +55 -8
- package/src/components/pages/AskPage.tsx +11 -0
- package/src/components/pages/ConnectionsPage.tsx +142 -5
- package/src/components/pages/LibraryPage.tsx +56 -3
- package/src/components/pages/TrackerPage.tsx +17 -0
- package/src/corpus/ask.ts +33 -8
- package/src/corpus/entities.ts +56 -2
- package/src/corpus/extract.ts +340 -15
- package/src/corpus/fixtures.ts +9 -1
- package/src/corpus/llm.ts +80 -1
- package/src/corpus/retrieval.ts +28 -4
- package/src/corpus/text.ts +76 -0
- package/src/corpus/types.ts +108 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@startsimpli/integrations",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.16",
|
|
4
4
|
"description": "Provider-agnostic types, API clients, hooks, and UI primitives for email + calendar integrations across StartSimpli apps.",
|
|
5
5
|
"main": "./src/index.ts",
|
|
6
6
|
"types": "./src/index.ts",
|
|
@@ -48,7 +48,7 @@
|
|
|
48
48
|
"react-dom": "^19.0.0",
|
|
49
49
|
"typescript": "^6.0.0",
|
|
50
50
|
"vitest": "^3.0.0",
|
|
51
|
-
"@startsimpli/llm": "0.4.
|
|
51
|
+
"@startsimpli/llm": "0.4.16"
|
|
52
52
|
},
|
|
53
53
|
"scripts": {
|
|
54
54
|
"type-check": "tsc --noEmit",
|
|
@@ -627,3 +627,168 @@ describe('no per-vendor components', () => {
|
|
|
627
627
|
expect(offenders).toEqual([]);
|
|
628
628
|
});
|
|
629
629
|
});
|
|
630
|
+
|
|
631
|
+
// --------------------------------------------------------------------------
|
|
632
|
+
// settings on a CONNECTED card (bd startsim-768w.38.31)
|
|
633
|
+
// --------------------------------------------------------------------------
|
|
634
|
+
//
|
|
635
|
+
// A connector's `config_schema` has always shipped on every catalog row and has
|
|
636
|
+
// always been rendered — but only on the CONNECT form, once, before the account
|
|
637
|
+
// exists. After that there was no way to change a setting from the product at
|
|
638
|
+
// all. `ReplaceKeyForm` even says so, pointing at "the connection's own
|
|
639
|
+
// settings" as though that surface existed.
|
|
640
|
+
//
|
|
641
|
+
// It has to exist for `ingest_transcripts`, because the whole privacy posture is
|
|
642
|
+
// that transcripts are off until an admin deliberately turns them on — and a
|
|
643
|
+
// switch nobody can reach is not a decision anybody gets to make.
|
|
644
|
+
|
|
645
|
+
/** The Granola shape: a boolean, a number and a free-text list. */
|
|
646
|
+
const TRANSCRIPT_CONFIG = [
|
|
647
|
+
{
|
|
648
|
+
key: 'ingest_transcripts',
|
|
649
|
+
label: 'Store what was said, not just the summary',
|
|
650
|
+
type: 'boolean',
|
|
651
|
+
required: false,
|
|
652
|
+
default: false,
|
|
653
|
+
help: 'On: the full word-for-word transcript is stored too.',
|
|
654
|
+
},
|
|
655
|
+
{
|
|
656
|
+
key: 'transcript_retention_days',
|
|
657
|
+
label: 'Delete stored transcripts after (days)',
|
|
658
|
+
type: 'number',
|
|
659
|
+
required: false,
|
|
660
|
+
default: 90,
|
|
661
|
+
},
|
|
662
|
+
];
|
|
663
|
+
|
|
664
|
+
const CONFIGURABLE_CATALOG = {
|
|
665
|
+
results: [
|
|
666
|
+
{
|
|
667
|
+
...CATALOG.results[0],
|
|
668
|
+
config_schema: { version: 1, fields: TRANSCRIPT_CONFIG },
|
|
669
|
+
},
|
|
670
|
+
],
|
|
671
|
+
};
|
|
672
|
+
|
|
673
|
+
function configurableApi(overrides: Partial<ConnectionsApi> = {}): ConnectionsApi {
|
|
674
|
+
return stubApi({
|
|
675
|
+
connectorCatalog: vi.fn().mockResolvedValue(CONFIGURABLE_CATALOG),
|
|
676
|
+
connections: vi.fn().mockResolvedValue({
|
|
677
|
+
...CONNECTED,
|
|
678
|
+
results: [{ ...CONNECTED.results[0], config: { transcript_retention_days: 90 } }],
|
|
679
|
+
}),
|
|
680
|
+
...overrides,
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
async function openSettings(api: ConnectionsApi) {
|
|
685
|
+
render(<ConnectionsPage api={api} foundrySlug="acme" />);
|
|
686
|
+
const card = await screen.findByText('Ops n8n');
|
|
687
|
+
const section = card.closest('div[data-connection]') ?? card.parentElement!;
|
|
688
|
+
await userEvent.click(await within(section as HTMLElement).findByRole('button', { name: /Settings/i }));
|
|
689
|
+
return section as HTMLElement;
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
describe('CredentialSchemaForm — a boolean is a checkbox', () => {
|
|
693
|
+
it('renders a declared boolean as a checkbox, not a text box', () => {
|
|
694
|
+
// `bool("false")` is `True`. A privacy switch rendered as somewhere to type
|
|
695
|
+
// the word "false" is a switch that is on when the admin believes it is off.
|
|
696
|
+
renderForm(TRANSCRIPT_CONFIG);
|
|
697
|
+
const input = screen.getByLabelText(/Store what was said/i);
|
|
698
|
+
expect(input).toHaveAttribute('type', 'checkbox');
|
|
699
|
+
});
|
|
700
|
+
|
|
701
|
+
it('reports a boolean edit as a real boolean, not a string', () => {
|
|
702
|
+
const { onChange } = renderForm(TRANSCRIPT_CONFIG);
|
|
703
|
+
screen.getByLabelText(/Store what was said/i).click();
|
|
704
|
+
expect(onChange).toHaveBeenCalledWith('ingest_transcripts', true);
|
|
705
|
+
});
|
|
706
|
+
|
|
707
|
+
it('shows the declared default so an untouched field is not a guess', () => {
|
|
708
|
+
renderForm(TRANSCRIPT_CONFIG);
|
|
709
|
+
expect(screen.getByLabelText(/Store what was said/i)).not.toBeChecked();
|
|
710
|
+
expect(screen.getByLabelText(/Delete stored transcripts/i)).toHaveValue(90);
|
|
711
|
+
});
|
|
712
|
+
|
|
713
|
+
it('renders the help text, which is where the consequence is stated', () => {
|
|
714
|
+
renderForm(TRANSCRIPT_CONFIG);
|
|
715
|
+
expect(screen.getByText(/word-for-word transcript is stored/i)).toBeInTheDocument();
|
|
716
|
+
});
|
|
717
|
+
});
|
|
718
|
+
|
|
719
|
+
describe('ConnectionCard — a connected account has settings', () => {
|
|
720
|
+
it('renders the connector’s config schema for an existing connection', async () => {
|
|
721
|
+
const section = await openSettings(configurableApi());
|
|
722
|
+
expect(within(section).getByLabelText(/Store what was said/i)).toBeInTheDocument();
|
|
723
|
+
// The STORED value, not the declared default — a form that shows the default
|
|
724
|
+
// over a stored setting tells the admin their last change did not take.
|
|
725
|
+
expect(within(section).getByLabelText(/Delete stored transcripts/i)).toHaveValue(90);
|
|
726
|
+
});
|
|
727
|
+
|
|
728
|
+
it('does not offer settings for a connector that declares none', async () => {
|
|
729
|
+
render(<ConnectionsPage api={stubApi()} foundrySlug="acme" />);
|
|
730
|
+
const card = await screen.findByText('Ops n8n');
|
|
731
|
+
const section = (card.closest('div[data-connection]') ?? card.parentElement!) as HTMLElement;
|
|
732
|
+
expect(within(section).queryByRole('button', { name: /Settings/i })).toBeNull();
|
|
733
|
+
});
|
|
734
|
+
|
|
735
|
+
it('saves through updateConnection, sending only the fields it changed', async () => {
|
|
736
|
+
const updateConnection = vi.fn().mockResolvedValue(CONNECTED.results[0]);
|
|
737
|
+
const section = await openSettings(configurableApi({ updateConnection }));
|
|
738
|
+
|
|
739
|
+
await userEvent.click(within(section).getByLabelText(/Store what was said/i));
|
|
740
|
+
await userEvent.click(within(section).getByRole('button', { name: /^Save settings$/i }));
|
|
741
|
+
|
|
742
|
+
await waitFor(() => expect(updateConnection).toHaveBeenCalled());
|
|
743
|
+
const [, , payload] = updateConnection.mock.calls[0];
|
|
744
|
+
expect(payload.config).toEqual({ ingest_transcripts: true });
|
|
745
|
+
// The API merges, so an untouched field must NOT be echoed back — resending
|
|
746
|
+
// it turns "I changed one thing" into "I reassert everything", and races an
|
|
747
|
+
// edit made in another tab.
|
|
748
|
+
expect(payload.config).not.toHaveProperty('transcript_retention_days');
|
|
749
|
+
});
|
|
750
|
+
|
|
751
|
+
it('sends false when a checkbox is turned OFF, rather than omitting it', async () => {
|
|
752
|
+
// The connect form drops falsy values (`if (v) config[k] = v`), which for a
|
|
753
|
+
// checkbox means "off" is indistinguishable from "untouched" — and with a
|
|
754
|
+
// merging API that makes the off position unreachable.
|
|
755
|
+
const updateConnection = vi.fn().mockResolvedValue(CONNECTED.results[0]);
|
|
756
|
+
const section = await openSettings(
|
|
757
|
+
configurableApi({
|
|
758
|
+
updateConnection,
|
|
759
|
+
connections: vi.fn().mockResolvedValue({
|
|
760
|
+
...CONNECTED,
|
|
761
|
+
results: [{ ...CONNECTED.results[0], config: { ingest_transcripts: true } }],
|
|
762
|
+
}),
|
|
763
|
+
}),
|
|
764
|
+
);
|
|
765
|
+
|
|
766
|
+
await userEvent.click(within(section).getByLabelText(/Store what was said/i));
|
|
767
|
+
await userEvent.click(within(section).getByRole('button', { name: /^Save settings$/i }));
|
|
768
|
+
|
|
769
|
+
await waitFor(() => expect(updateConnection).toHaveBeenCalled());
|
|
770
|
+
expect(updateConnection.mock.calls[0][2].config).toEqual({ ingest_transcripts: false });
|
|
771
|
+
});
|
|
772
|
+
|
|
773
|
+
it('warns, before saving, what turning transcripts on will do', async () => {
|
|
774
|
+
const section = await openSettings(configurableApi());
|
|
775
|
+
await userEvent.click(within(section).getByLabelText(/Store what was said/i));
|
|
776
|
+
expect(
|
|
777
|
+
within(section).getByText(/word-for-word transcript is stored/i),
|
|
778
|
+
).toBeInTheDocument();
|
|
779
|
+
});
|
|
780
|
+
|
|
781
|
+
it('surfaces the API’s refusal rather than looking saved', async () => {
|
|
782
|
+
// The connector refuses `transcript_retention_days: 0` with nobody's name
|
|
783
|
+
// against it. That 400 has to land on the form, next to the field.
|
|
784
|
+
const updateConnection = vi.fn().mockRejectedValue(
|
|
785
|
+
Object.assign(new Error('transcript_retention_days=0 means keeping verbatim meeting speech forever'), {
|
|
786
|
+
status: 400,
|
|
787
|
+
}),
|
|
788
|
+
);
|
|
789
|
+
const section = await openSettings(configurableApi({ updateConnection }));
|
|
790
|
+
await userEvent.click(within(section).getByLabelText(/Store what was said/i));
|
|
791
|
+
await userEvent.click(within(section).getByRole('button', { name: /^Save settings$/i }));
|
|
792
|
+
expect(await within(section).findByText(/keeping verbatim meeting speech forever/i)).toBeInTheDocument();
|
|
793
|
+
});
|
|
794
|
+
});
|
|
@@ -89,7 +89,25 @@ describe('batching — N documents, ONE call', () => {
|
|
|
89
89
|
};
|
|
90
90
|
const text = documentText(long, 500);
|
|
91
91
|
expect(text.length).toBeLessThan(700);
|
|
92
|
-
|
|
92
|
+
// The marker now names the size of the cut, because "…[truncated]" told a
|
|
93
|
+
// reader that something was missing without telling them whether it was a
|
|
94
|
+
// sentence or forty minutes.
|
|
95
|
+
expect(text).toMatch(/\[truncated \d+ characters\]/);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it('spends the budget on the summary before the transcript', () => {
|
|
99
|
+
// A prefix cut over `body` + sections is what this used to be, and once
|
|
100
|
+
// `ingest_transcripts` lands `body` IS the transcript — so the entire 6,000
|
|
101
|
+
// character per-document budget would have gone on the opening minutes of
|
|
102
|
+
// the meeting and the summary would never have reached the model at all.
|
|
103
|
+
const long = {
|
|
104
|
+
...FIXTURE_DOCUMENTS[0],
|
|
105
|
+
summary: FIXTURE_DOCUMENTS[0].summary,
|
|
106
|
+
body: `${'x'.repeat(50_000)} the last thing anybody said`,
|
|
107
|
+
};
|
|
108
|
+
const text = documentText(long, 800);
|
|
109
|
+
expect(text).toContain('Agreed to ship the connector RED suite first');
|
|
110
|
+
expect(text).toContain('the last thing anybody said');
|
|
93
111
|
});
|
|
94
112
|
});
|
|
95
113
|
|
|
@@ -106,7 +106,10 @@ describe('Markdown is rendered, not printed', () => {
|
|
|
106
106
|
describe('LibraryPage renders a document’s own Markdown', () => {
|
|
107
107
|
it('shows the summary as prose, not as source', async () => {
|
|
108
108
|
const user = userEvent.setup();
|
|
109
|
-
|
|
109
|
+
// The SUMMARY is the field rendered as prose under the heading "Summary".
|
|
110
|
+
// It used to be `body`, which was only ever the summary because nothing had
|
|
111
|
+
// delivered a full text yet.
|
|
112
|
+
const document: SourceDocument = { ...FIXTURE_DOCUMENTS[0], summary: MARKDOWN_BODY };
|
|
110
113
|
const api = createFixtureCorpusApi({ scope: SCOPE, documents: [document], moneyEvents: [] });
|
|
111
114
|
render(<LibraryPage api={api} scope={SCOPE} />);
|
|
112
115
|
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE FIELD READ GATE, from the client's side (bd startsim-768w.38.33).
|
|
3
|
+
*
|
|
4
|
+
* `corpus-scope.test.ts` next door asks "can the pipeline be made to read
|
|
5
|
+
* another WORKSPACE's records". This asks the narrower and, on the tandem
|
|
6
|
+
* tenant, more likely question: can it be made to read a part of its OWN
|
|
7
|
+
* workspace's record that the server withheld from this reader.
|
|
8
|
+
*
|
|
9
|
+
* SAME ADVERSARIAL POSTURE, DELIBERATELY. Every case runs against a tenant that
|
|
10
|
+
* is LEAKING — it names `body` in `redacted_fields` and then sends `body`
|
|
11
|
+
* anyway, which is what a half-rolled deploy, a response cached from an admin's
|
|
12
|
+
* session, or a serializer edited without its gate looks like from out here. If
|
|
13
|
+
* the pipeline is only safe because the server is, these fail. That is the
|
|
14
|
+
* point; a test against a correctly-behaving server proves the server, not the
|
|
15
|
+
* client.
|
|
16
|
+
*
|
|
17
|
+
* THE BAIT is `doc_checkin`: a document whose SUMMARY is anodyne and whose
|
|
18
|
+
* withheld `body` contains the verbatim sentence that answers the question best.
|
|
19
|
+
* A mapper that reads `data.body` without checking `redacted_fields` puts that
|
|
20
|
+
* sentence into the prompt, the model quotes it, and the leak arrives in the
|
|
21
|
+
* most convincing possible format — cited, fluent, and attributed to the user's
|
|
22
|
+
* own corpus.
|
|
23
|
+
*/
|
|
24
|
+
import { describe, expect, it } from 'vitest';
|
|
25
|
+
|
|
26
|
+
import { askCorpus } from '../corpus/ask';
|
|
27
|
+
import { createCorpusApi } from '../corpus/client';
|
|
28
|
+
import { toSourceDocument, withheldFieldsOf } from '../corpus/entities';
|
|
29
|
+
import type { TenantEntity } from '../corpus/entities';
|
|
30
|
+
import { createFixtureCorpusHttp, createFixtureLlm, FIXTURE_ORG } from '../corpus/fixtures';
|
|
31
|
+
import type { CorpusHttp, CorpusLlm, CorpusScope } from '../corpus/types';
|
|
32
|
+
|
|
33
|
+
const SCOPE: CorpusScope = { orgId: FIXTURE_ORG };
|
|
34
|
+
|
|
35
|
+
/** The sentence nobody consented to re-host in a third system. */
|
|
36
|
+
const VERBATIM = 'Dana: I relapsed in March and I have been in therapy weekly since.';
|
|
37
|
+
const ANODYNE = 'Personal check-in. Agreed to revisit the lease in Q3.';
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* A tenant row as the LEAKING server sends it: the gate ran (it named what it
|
|
41
|
+
* withheld) and the payload came anyway.
|
|
42
|
+
*/
|
|
43
|
+
function leakingEntity(overrides: Partial<TenantEntity> = {}): TenantEntity {
|
|
44
|
+
return {
|
|
45
|
+
id: 'doc_checkin',
|
|
46
|
+
entity_type: 'source_document',
|
|
47
|
+
external_id: 'granola:not_checkin',
|
|
48
|
+
name: 'Personal check-in — therapy, substance use, and relationship progress',
|
|
49
|
+
org_id: FIXTURE_ORG,
|
|
50
|
+
created_at: '2026-08-01T10:00:00Z',
|
|
51
|
+
data: {
|
|
52
|
+
source: 'granola',
|
|
53
|
+
subtype: 'meeting',
|
|
54
|
+
title: 'Personal check-in',
|
|
55
|
+
summary: ANODYNE,
|
|
56
|
+
body: VERBATIM,
|
|
57
|
+
occurred_at: '2026-08-01T09:00:00Z',
|
|
58
|
+
},
|
|
59
|
+
redacted_fields: ['body'],
|
|
60
|
+
...overrides,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** The same row as a CORRECT server sends it: stripped, and saying so. */
|
|
65
|
+
function gatedEntity(): TenantEntity {
|
|
66
|
+
const row = leakingEntity();
|
|
67
|
+
const { body: _dropped, ...rest } = row.data as Record<string, unknown>;
|
|
68
|
+
return { ...row, data: rest };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function httpFor(entities: TenantEntity[]): CorpusHttp {
|
|
72
|
+
return createFixtureCorpusHttp({ entities, relationships: [] }).http as CorpusHttp;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function apiFor(entities: TenantEntity[]) {
|
|
76
|
+
return createCorpusApi({ http: httpFor(entities), scope: SCOPE, supportedFilters: [] });
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
describe('the mapper honours the server’s own declaration', () => {
|
|
80
|
+
it('drops a withheld key even when the server sent it anyway', () => {
|
|
81
|
+
const document = toSourceDocument(leakingEntity(), SCOPE);
|
|
82
|
+
expect(document.body).not.toContain('relapsed');
|
|
83
|
+
expect(JSON.stringify(document)).not.toContain('relapsed');
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('falls back to the summary rather than rendering an empty record', () => {
|
|
87
|
+
// A blank document would be the other failure: safe, and useless. The
|
|
88
|
+
// meeting is still a meeting, and its summary was never restricted.
|
|
89
|
+
const document = toSourceDocument(leakingEntity(), SCOPE);
|
|
90
|
+
expect(document.body).toBe(ANODYNE);
|
|
91
|
+
expect(document.summary).toBe(ANODYNE);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it('says WHICH field was withheld, so absence is not mistaken for emptiness', () => {
|
|
95
|
+
// The distinction the whole gate turns on: "this meeting has no transcript"
|
|
96
|
+
// and "you were not shown this meeting's transcript" are different facts,
|
|
97
|
+
// and only one of them is an answer.
|
|
98
|
+
expect(toSourceDocument(leakingEntity(), SCOPE).withheldFields).toEqual(['body']);
|
|
99
|
+
expect(withheldFieldsOf(leakingEntity())).toEqual(['body']);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it('accepts the camelCase spelling the api client produces', () => {
|
|
103
|
+
const row = leakingEntity();
|
|
104
|
+
const camel: TenantEntity = { ...row, redacted_fields: undefined, redactedFields: ['body'] };
|
|
105
|
+
expect(toSourceDocument(camel, SCOPE).body).not.toContain('relapsed');
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it('leaves a record with nothing withheld exactly as it was', () => {
|
|
109
|
+
// The migration property, from this side: a corpus with no policy anywhere
|
|
110
|
+
// must map identically to how it mapped before this existed.
|
|
111
|
+
const open = leakingEntity({ redacted_fields: undefined });
|
|
112
|
+
const document = toSourceDocument(open, SCOPE);
|
|
113
|
+
expect(document.body).toBe(VERBATIM);
|
|
114
|
+
expect(document.withheldFields).toBeUndefined();
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it('withholds a section as readily as a top-level field', () => {
|
|
118
|
+
const row = leakingEntity({
|
|
119
|
+
data: {
|
|
120
|
+
source: 'granola',
|
|
121
|
+
summary: ANODYNE,
|
|
122
|
+
transcript: VERBATIM,
|
|
123
|
+
},
|
|
124
|
+
redacted_fields: ['transcript'],
|
|
125
|
+
});
|
|
126
|
+
const document = toSourceDocument(row, SCOPE);
|
|
127
|
+
expect(JSON.stringify(document.sections)).not.toContain('relapsed');
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
describe('the corpus client never surfaces a withheld field', () => {
|
|
132
|
+
it('keeps it out of a listed document', async () => {
|
|
133
|
+
const page = await apiFor([leakingEntity()]).listDocuments({ page: 1, pageSize: 50 });
|
|
134
|
+
expect(page.results).toHaveLength(1);
|
|
135
|
+
expect(JSON.stringify(page.results)).not.toContain('relapsed');
|
|
136
|
+
expect(page.results[0]?.withheldFields).toEqual(['body']);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it('keeps it out of a fetched document', async () => {
|
|
140
|
+
const document = await apiFor([leakingEntity()]).getDocument('doc_checkin');
|
|
141
|
+
expect(JSON.stringify(document)).not.toContain('relapsed');
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
describe('askCorpus — the withheld text never reaches the model', () => {
|
|
146
|
+
/** A model that records exactly what it was shown. */
|
|
147
|
+
function spyLlm(): { llm: CorpusLlm; seen: string[] } {
|
|
148
|
+
const seen: string[] = [];
|
|
149
|
+
const inner = createFixtureLlm();
|
|
150
|
+
return {
|
|
151
|
+
seen,
|
|
152
|
+
llm: {
|
|
153
|
+
describe: inner.describe,
|
|
154
|
+
async answer(input) {
|
|
155
|
+
seen.push(JSON.stringify(input.documents));
|
|
156
|
+
return inner.answer(input);
|
|
157
|
+
},
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* A question answerable from the SUMMARY, so the document really is retrieved
|
|
164
|
+
* and really does reach the prompt. Asking something only the withheld text
|
|
165
|
+
* could answer would prove nothing: an empty prompt keeps every secret.
|
|
166
|
+
*/
|
|
167
|
+
const question = 'What did we agree about the lease?';
|
|
168
|
+
|
|
169
|
+
it('puts the document in the prompt WITHOUT the withheld text', async () => {
|
|
170
|
+
const { llm, seen } = spyLlm();
|
|
171
|
+
await askCorpus({ question, mode: 'ask', scope: SCOPE }, { corpus: apiFor([leakingEntity()]), llm });
|
|
172
|
+
|
|
173
|
+
expect(seen).toHaveLength(1); // ONE call, not one per document
|
|
174
|
+
expect(seen[0]).toContain('lease'); // the record DID reach the model...
|
|
175
|
+
expect(seen[0]).not.toContain('relapsed'); // ...without the part it may not read
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
it('never quotes it back in the answer or a citation', async () => {
|
|
179
|
+
const result = await askCorpus(
|
|
180
|
+
{ question, mode: 'ask', scope: SCOPE },
|
|
181
|
+
{ corpus: apiFor([leakingEntity()]), llm: createFixtureLlm() },
|
|
182
|
+
);
|
|
183
|
+
|
|
184
|
+
expect(result.status).toBe('answered');
|
|
185
|
+
expect(result.answer).not.toContain('relapsed');
|
|
186
|
+
expect(JSON.stringify(result.citations)).not.toContain('relapsed');
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it('reports that something was withheld rather than absorbing it', async () => {
|
|
190
|
+
// Same contract as `droppedOutOfScope`: an answer built from a partial
|
|
191
|
+
// record has to say so. Otherwise "your corpus has nothing about that" and
|
|
192
|
+
// "you were not shown the part that does" are the same sentence.
|
|
193
|
+
const result = await askCorpus(
|
|
194
|
+
{ question, mode: 'ask', scope: SCOPE },
|
|
195
|
+
{ corpus: apiFor([leakingEntity()]), llm: createFixtureLlm() },
|
|
196
|
+
);
|
|
197
|
+
expect(result.withheldFields).toEqual(['body']);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
it('tells the MODEL the record is partly withheld, so it cannot answer a confident no', async () => {
|
|
201
|
+
const { llm, seen } = spyLlm();
|
|
202
|
+
await askCorpus({ question, mode: 'ask', scope: SCOPE }, { corpus: apiFor([leakingEntity()]), llm });
|
|
203
|
+
expect(seen[0]).toContain('withheldFields');
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
it('does not retrieve on a term that exists ONLY in the withheld text', async () => {
|
|
207
|
+
// The client-side half of the search oracle the tenant API closes. If the
|
|
208
|
+
// withheld words were still rankable here, a reader could confirm what a
|
|
209
|
+
// transcript contains by watching which records come back — an answer about
|
|
210
|
+
// somebody's private speech, delivered without ever printing it.
|
|
211
|
+
const result = await askCorpus(
|
|
212
|
+
{ question: 'relapsed', mode: 'ask', scope: SCOPE },
|
|
213
|
+
{ corpus: apiFor([leakingEntity()]), llm: createFixtureLlm() },
|
|
214
|
+
);
|
|
215
|
+
expect(result.status).toBe('no_evidence');
|
|
216
|
+
expect(result.citations).toHaveLength(0);
|
|
217
|
+
expect(result.answer).not.toContain('relapsed');
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
it('answers identically whether the server leaked or behaved', async () => {
|
|
221
|
+
// The falsifiable version of "the client does not depend on the server
|
|
222
|
+
// getting it right": same question, one API stripping the field and one
|
|
223
|
+
// not, and the two answers must not differ.
|
|
224
|
+
const ask = (entity: TenantEntity) =>
|
|
225
|
+
askCorpus(
|
|
226
|
+
{ question, mode: 'ask', scope: SCOPE },
|
|
227
|
+
{ corpus: apiFor([entity]), llm: createFixtureLlm() },
|
|
228
|
+
);
|
|
229
|
+
const [leaked, gated] = await Promise.all([ask(leakingEntity()), ask(gatedEntity())]);
|
|
230
|
+
expect(leaked.answer).toBe(gated.answer);
|
|
231
|
+
expect(leaked.withheldFields).toEqual(gated.withheldFields);
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
it('still answers from the parts the reader IS allowed to see', async () => {
|
|
235
|
+
// Over-blocking would be the other way to fail: the summary was never
|
|
236
|
+
// restricted, and a corpus that goes silent because one field is gated has
|
|
237
|
+
// taken the product away in order to protect it.
|
|
238
|
+
const result = await askCorpus(
|
|
239
|
+
{ question, mode: 'ask', scope: SCOPE },
|
|
240
|
+
{ corpus: apiFor([leakingEntity()]), llm: createFixtureLlm() },
|
|
241
|
+
);
|
|
242
|
+
expect(result.status).toBe('answered');
|
|
243
|
+
expect(result.citations.length).toBeGreaterThan(0);
|
|
244
|
+
expect(result.answer).toContain('lease');
|
|
245
|
+
});
|
|
246
|
+
});
|
|
@@ -107,7 +107,7 @@ describe('LibraryPage — the only connected source is Dropbox', () => {
|
|
|
107
107
|
});
|
|
108
108
|
|
|
109
109
|
describe('LibraryPage — the same screen, Granola only', () => {
|
|
110
|
-
it('renders
|
|
110
|
+
it('renders the verbatim text separately from the summary, collapsed', async () => {
|
|
111
111
|
const user = userEvent.setup();
|
|
112
112
|
const { api } = singleSourceCorpus('granola', {
|
|
113
113
|
source: 'granola', label: 'Granola', documentCount: 4,
|
|
@@ -117,7 +117,16 @@ describe('LibraryPage — the same screen, Granola only', () => {
|
|
|
117
117
|
|
|
118
118
|
await user.click(await screen.findByText('Acme <> StartSimpli weekly sync'));
|
|
119
119
|
const detail = await screen.findByRole('region', { name: 'Record detail' });
|
|
120
|
-
|
|
120
|
+
// "Full text" and not "Transcript", because the canonical mapping puts a
|
|
121
|
+
// meeting transcript in the SAME field a PDF's extracted text goes in — the
|
|
122
|
+
// corpus is source-agnostic and this screen must stay that way. A source
|
|
123
|
+
// that wants its own word for it contributes a named `section`, which the
|
|
124
|
+
// Dropbox row above proves still works.
|
|
125
|
+
expect(within(detail).getByText('Full text')).toBeInTheDocument();
|
|
126
|
+
// The summary is still the thing rendered as prose, and the verbatim text
|
|
127
|
+
// is behind a disclosure rather than in front of whoever opened the record.
|
|
128
|
+
expect(within(detail).getByText('Summary')).toBeInTheDocument();
|
|
129
|
+
expect(within(detail).getByText(/Agreed to ship the connector RED suite/)).toBeInTheDocument();
|
|
121
130
|
});
|
|
122
131
|
});
|
|
123
132
|
|
|
@@ -302,6 +311,53 @@ function askApiOver(api: CorpusApi, extra: Partial<CorpusAskApi> = {}): CorpusAs
|
|
|
302
311
|
};
|
|
303
312
|
}
|
|
304
313
|
|
|
314
|
+
describe('a withheld field is SHOWN as withheld, not as empty', () => {
|
|
315
|
+
// bd startsim-768w.38.33. The tenant strips a restricted attribute and names
|
|
316
|
+
// it back; if the screen renders the gap silently, the reader concludes the
|
|
317
|
+
// meeting had no transcript — which is a stronger, and wrong, claim than the
|
|
318
|
+
// gate ever made.
|
|
319
|
+
|
|
320
|
+
it('LibraryPage names what was held back on the open record', async () => {
|
|
321
|
+
const withheld: SourceDocument = {
|
|
322
|
+
...ownDocuments()[0],
|
|
323
|
+
id: 'doc_withheld',
|
|
324
|
+
title: 'Personal check-in',
|
|
325
|
+
withheldFields: ['body'],
|
|
326
|
+
};
|
|
327
|
+
const api = createFixtureCorpusApi({
|
|
328
|
+
scope: SCOPE, documents: [withheld], moneyEvents: [],
|
|
329
|
+
});
|
|
330
|
+
render(<LibraryPage api={api} scope={SCOPE} initialDocumentId="doc_withheld" />);
|
|
331
|
+
|
|
332
|
+
expect(await screen.findByText(/not shared with you/i)).toBeInTheDocument();
|
|
333
|
+
expect(screen.getByText(/body/)).toBeInTheDocument();
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
it('AskPage says the answer may be incomplete', async () => {
|
|
337
|
+
const user = userEvent.setup();
|
|
338
|
+
const api: CorpusAskApi = {
|
|
339
|
+
async ask() {
|
|
340
|
+
return {
|
|
341
|
+
status: 'answered',
|
|
342
|
+
answer: 'They agreed to revisit the lease in Q3.',
|
|
343
|
+
citations: [],
|
|
344
|
+
considered: [],
|
|
345
|
+
unresolvedCitations: [],
|
|
346
|
+
droppedOutOfScope: 0,
|
|
347
|
+
withheldFields: ['body'],
|
|
348
|
+
provider: { provider: 'fixture', model: 'fixture-echo', live: false },
|
|
349
|
+
};
|
|
350
|
+
},
|
|
351
|
+
};
|
|
352
|
+
render(<AskPage api={api} scope={SCOPE} />);
|
|
353
|
+
await user.type(screen.getByRole('textbox'), 'lease');
|
|
354
|
+
await user.click(screen.getByRole('button', { name: 'Ask' }));
|
|
355
|
+
|
|
356
|
+
const answer = await screen.findByRole('region', { name: 'Answer' });
|
|
357
|
+
expect(within(answer).getByText(/only partly shared with you/i)).toBeInTheDocument();
|
|
358
|
+
});
|
|
359
|
+
});
|
|
360
|
+
|
|
305
361
|
describe('AskPage — an answer is only an answer when it is traceable', () => {
|
|
306
362
|
it('renders the answer with a citation per record, clickable through to it', async () => {
|
|
307
363
|
const user = userEvent.setup();
|
|
@@ -283,6 +283,9 @@ describe('TrackerPage — running the extraction', () => {
|
|
|
283
283
|
nextOffset: null,
|
|
284
284
|
fromRules: 24,
|
|
285
285
|
fromModel: 11,
|
|
286
|
+
ungrounded: 2,
|
|
287
|
+
unconfirmed: 3,
|
|
288
|
+
verified: true,
|
|
286
289
|
created: 30,
|
|
287
290
|
merged: 5,
|
|
288
291
|
resolved: 2,
|
|
@@ -333,3 +336,43 @@ describe('TrackerPage — running the extraction', () => {
|
|
|
333
336
|
expect(screen.queryByRole('button', { name: 'Pull from meetings' })).not.toBeInTheDocument();
|
|
334
337
|
});
|
|
335
338
|
});
|
|
339
|
+
|
|
340
|
+
// ---------------------------------------------------------------------------
|
|
341
|
+
// the run says what the verification threw away (bd startsim-768w.38.31)
|
|
342
|
+
// ---------------------------------------------------------------------------
|
|
343
|
+
|
|
344
|
+
describe('TrackerPage — the run reports its own precision work', () => {
|
|
345
|
+
const BASE = {
|
|
346
|
+
documentsProcessed: 16, totalDocuments: 16, nextOffset: null,
|
|
347
|
+
fromRules: 67, fromModel: 35, ungrounded: 2, unconfirmed: 59, verified: true,
|
|
348
|
+
created: 102, merged: 0, resolved: 0, needsReview: [], droppedOutOfScope: 0,
|
|
349
|
+
provider: { provider: 'anthropic', model: 'claude-sonnet-5', live: true },
|
|
350
|
+
};
|
|
351
|
+
|
|
352
|
+
async function runWith(result: Record<string, unknown>) {
|
|
353
|
+
const api = makeApi({ extract: vi.fn(async () => result) as never });
|
|
354
|
+
render(<TrackerPage api={api} scope={SCOPE} />);
|
|
355
|
+
await userEvent.click(await screen.findByRole('button', { name: /Pull from meetings/i }));
|
|
356
|
+
return screen;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
it('says how many proposed items were dropped, and why', async () => {
|
|
360
|
+
// Measured over the sixteen real meetings, two thirds of what the model
|
|
361
|
+
// proposes does not survive the check. Hiding that would make the model look
|
|
362
|
+
// better than it is and the list less arguable than it should be.
|
|
363
|
+
await runWith(BASE);
|
|
364
|
+
expect(await screen.findByText(/61 more were proposed and dropped/)).toBeInTheDocument();
|
|
365
|
+
expect(screen.getByText(/2 quoted something the record does not say/)).toBeInTheDocument();
|
|
366
|
+
expect(screen.getByText(/59 could not be confirmed/)).toBeInTheDocument();
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
it('says the check did not run rather than implying it did', async () => {
|
|
370
|
+
await runWith({ ...BASE, verified: false, unconfirmed: 0, ungrounded: 0 });
|
|
371
|
+
expect(await screen.findByText(/were NOT checked against the meetings/)).toBeInTheDocument();
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
it('stays quiet when the model contributed nothing at all', async () => {
|
|
375
|
+
await runWith({ ...BASE, fromModel: 0, ungrounded: 0, unconfirmed: 0 });
|
|
376
|
+
expect(screen.queryByText(/proposed and dropped/)).toBeNull();
|
|
377
|
+
});
|
|
378
|
+
});
|