@genesislcap/ai-assistant 15.15.2 → 15.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-assistant.api.json +271 -2
- package/dist/ai-assistant.d.ts +90 -1
- package/dist/chat-driver.cjs +72 -0
- package/dist/chat-driver.cjs.map +2 -2
- package/dist/chat-driver.mjs +72 -0
- package/dist/chat-driver.mjs.map +2 -2
- package/dist/custom-elements.json +221 -97
- package/dist/dts/components/ai-driver/ai-driver.d.ts +11 -0
- package/dist/dts/components/ai-driver/ai-driver.d.ts.map +1 -1
- package/dist/dts/components/chat-driver/chat-driver.d.ts +31 -1
- package/dist/dts/components/chat-driver/chat-driver.d.ts.map +1 -1
- package/dist/dts/components/chat-interaction-wrapper/chat-interaction-wrapper.d.ts +10 -0
- package/dist/dts/components/chat-interaction-wrapper/chat-interaction-wrapper.d.ts.map +1 -1
- package/dist/dts/components/orchestrating-driver/orchestrating-driver.cost.test.d.ts +2 -0
- package/dist/dts/components/orchestrating-driver/orchestrating-driver.cost.test.d.ts.map +1 -0
- package/dist/dts/components/orchestrating-driver/orchestrating-driver.d.ts +3 -0
- package/dist/dts/components/orchestrating-driver/orchestrating-driver.d.ts.map +1 -1
- package/dist/dts/index.d.ts +1 -1
- package/dist/dts/index.d.ts.map +1 -1
- package/dist/dts/main/file-attachments.test.d.ts +2 -0
- package/dist/dts/main/file-attachments.test.d.ts.map +1 -0
- package/dist/dts/main/interaction-cost.test.d.ts +2 -0
- package/dist/dts/main/interaction-cost.test.d.ts.map +1 -0
- package/dist/dts/main/main.d.ts +27 -0
- package/dist/dts/main/main.d.ts.map +1 -1
- package/dist/dts/main/main.template.d.ts.map +1 -1
- package/dist/dts/react.d.ts +2 -1
- package/dist/dts/state/debug-event-log.d.ts +1 -1
- package/dist/dts/state/debug-event-log.d.ts.map +1 -1
- package/dist/dts/types/ai-chat-widget.d.ts +8 -0
- package/dist/dts/types/ai-chat-widget.d.ts.map +1 -1
- package/dist/esm/components/chat-driver/chat-driver.js +75 -1
- package/dist/esm/components/chat-driver/chat-driver.test.js +90 -0
- package/dist/esm/components/chat-interaction-wrapper/chat-interaction-wrapper.js +22 -0
- package/dist/esm/components/orchestrating-driver/orchestrating-driver.cost.test.js +76 -0
- package/dist/esm/components/orchestrating-driver/orchestrating-driver.js +14 -0
- package/dist/esm/main/file-attachments.test.js +164 -0
- package/dist/esm/main/interaction-cost.test.js +107 -0
- package/dist/esm/main/main.js +193 -21
- package/dist/esm/main/main.template.js +1 -0
- package/dist/esm/state/debug-event-log.js +6 -0
- package/dist/react.cjs +6 -1
- package/dist/react.mjs +6 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +17 -17
- package/src/components/ai-driver/ai-driver.ts +11 -0
- package/src/components/chat-driver/chat-driver.test.ts +109 -0
- package/src/components/chat-driver/chat-driver.ts +78 -1
- package/src/components/chat-interaction-wrapper/chat-interaction-wrapper.ts +29 -0
- package/src/components/orchestrating-driver/orchestrating-driver.cost.test.ts +107 -0
- package/src/components/orchestrating-driver/orchestrating-driver.ts +17 -0
- package/src/index.ts +4 -1
- package/src/main/file-attachments.test.ts +215 -0
- package/src/main/interaction-cost.test.ts +140 -0
- package/src/main/main.template.ts +1 -0
- package/src/main/main.ts +204 -22
- package/src/state/debug-event-log.ts +8 -0
- package/src/types/ai-chat-widget.ts +8 -0
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import { assert, createLogicSuite } from '@genesislcap/foundation-testing';
|
|
2
|
+
import { FoundationAiAssistant } from './main';
|
|
3
|
+
|
|
4
|
+
// Hold a reference so the custom-element registration isn't tree-shaken.
|
|
5
|
+
FoundationAiAssistant;
|
|
6
|
+
|
|
7
|
+
// GENC-1508 — image files become IMAGE attachments, not text.
|
|
8
|
+
//
|
|
9
|
+
// `processFiles` read every accepted file with `readAsText`, so an accepted .png
|
|
10
|
+
// arrived as mojibake the model "read" as prose — an accepted image was worse than a
|
|
11
|
+
// rejected one. The layout-apply flow attaches the picked mockup's PNG beside its
|
|
12
|
+
// spec, which is what forced the branch: images now read as base64 and ship as
|
|
13
|
+
// `{kind:'image'}`, the shape the transports render as real image blocks.
|
|
14
|
+
//
|
|
15
|
+
// Unconnected elements (createElement upgrades but does not connect), same as
|
|
16
|
+
// blocked-state.test.ts — processFiles touches no session state.
|
|
17
|
+
|
|
18
|
+
const Suite = createLogicSuite('FoundationAiAssistant file attachments');
|
|
19
|
+
|
|
20
|
+
type ProcessFiles = {
|
|
21
|
+
processFiles(
|
|
22
|
+
files: File[],
|
|
23
|
+
existingBytes?: number,
|
|
24
|
+
): Promise<{ attachments: unknown[]; errors: string[] }>;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
function element(acceptedFiles: string): ProcessFiles {
|
|
28
|
+
const el = document.createElement('foundation-ai-assistant') as FoundationAiAssistant;
|
|
29
|
+
el.chatConfig = { ui: { acceptedFiles } };
|
|
30
|
+
return el as unknown as ProcessFiles;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** A tiny valid PNG header is unnecessary — processFiles never parses pixels. */
|
|
34
|
+
const PNG_BYTES = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]);
|
|
35
|
+
|
|
36
|
+
Suite('an accepted image file becomes a kind:image attachment with raw base64', async () => {
|
|
37
|
+
const el = element('.md,image/*');
|
|
38
|
+
const { attachments, errors } = await el.processFiles([
|
|
39
|
+
new File([PNG_BYTES], 'layout-mockup.png', { type: 'image/png' }),
|
|
40
|
+
]);
|
|
41
|
+
assert.equal(errors, []);
|
|
42
|
+
const [img] = attachments as Array<{
|
|
43
|
+
kind?: string;
|
|
44
|
+
name: string;
|
|
45
|
+
mimeType: string;
|
|
46
|
+
data?: string;
|
|
47
|
+
}>;
|
|
48
|
+
assert.is(img.kind, 'image');
|
|
49
|
+
assert.is(img.name, 'layout-mockup.png');
|
|
50
|
+
assert.is(img.mimeType, 'image/png');
|
|
51
|
+
// RAW base64 — a `data:` prefix left on would be sent to the vendor inside the payload.
|
|
52
|
+
assert.ok(img.data && !img.data.startsWith('data:'), 'data must be bare base64');
|
|
53
|
+
assert.is(atob(img.data!).length, PNG_BYTES.length);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
Suite('a text file on the same submit stays a text attachment', async () => {
|
|
57
|
+
const el = element('.md,image/*');
|
|
58
|
+
const { attachments } = await el.processFiles([
|
|
59
|
+
new File(['# spec'], 'layout-spec.md', { type: 'text/markdown' }),
|
|
60
|
+
new File([PNG_BYTES], 'layout-mockup.png', { type: 'image/png' }),
|
|
61
|
+
]);
|
|
62
|
+
const [text, img] = attachments as Array<{ kind?: string; content?: string; data?: string }>;
|
|
63
|
+
assert.is(text.kind, undefined);
|
|
64
|
+
assert.is(text.content, '# spec');
|
|
65
|
+
assert.is(img.kind, 'image');
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
Suite('an image type the host did not accept is still refused', async () => {
|
|
69
|
+
const el = element('.md,.txt');
|
|
70
|
+
const { attachments, errors } = await el.processFiles([
|
|
71
|
+
new File([PNG_BYTES], 'layout-mockup.png', { type: 'image/png' }),
|
|
72
|
+
]);
|
|
73
|
+
assert.equal(attachments, []);
|
|
74
|
+
assert.is(errors.length, 1);
|
|
75
|
+
assert.match(errors[0], /not an accepted file type/);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
// A ~70KB image: the base64 loop's chunk size is 32768, so this crosses the boundary twice.
|
|
79
|
+
// The 8-byte fixture above executes the loop body exactly once, which would not notice an
|
|
80
|
+
// off-by-one in the subarray bounds.
|
|
81
|
+
Suite('a multi-chunk image round-trips byte-for-byte across the base64 loop', async () => {
|
|
82
|
+
const big = new Uint8Array(70_000);
|
|
83
|
+
// A non-trivial pattern — an all-zero buffer survives most encoding mistakes.
|
|
84
|
+
for (let i = 0; i < big.length; i += 1) big[i] = (i * 31 + (i >> 8)) & 0xff;
|
|
85
|
+
const el = element('image/*');
|
|
86
|
+
const { attachments, errors } = await el.processFiles([
|
|
87
|
+
new File([big], 'wide.png', { type: 'image/png' }),
|
|
88
|
+
]);
|
|
89
|
+
assert.equal(errors, []);
|
|
90
|
+
const [img] = attachments as Array<{ data: string }>;
|
|
91
|
+
const back = atob(img.data);
|
|
92
|
+
assert.is(back.length, big.length);
|
|
93
|
+
let mismatch = -1;
|
|
94
|
+
for (let i = 0; i < big.length; i += 1) {
|
|
95
|
+
if (back.charCodeAt(i) !== big[i]) {
|
|
96
|
+
mismatch = i;
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
assert.is(mismatch, -1, 'every byte must survive the chunked encode');
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
Suite(
|
|
104
|
+
'an image over the size cap is refused by NAME and size, not sent to the vendor',
|
|
105
|
+
async () => {
|
|
106
|
+
// Refused here rather than at the API: a vendor 400 lands after the user has waited out the
|
|
107
|
+
// turn, and the encoded blob is persisted meanwhile — one oversized image can blow the storage
|
|
108
|
+
// quota and silently stop the session autosaving.
|
|
109
|
+
const el = element('image/*');
|
|
110
|
+
const { attachments, errors } = await el.processFiles([
|
|
111
|
+
new File([new Uint8Array(4_000_000)], 'huge.png', { type: 'image/png' }),
|
|
112
|
+
]);
|
|
113
|
+
assert.equal(attachments, []);
|
|
114
|
+
assert.is(errors.length, 1);
|
|
115
|
+
assert.match(errors[0], /huge\.png/);
|
|
116
|
+
assert.match(errors[0], /limited to/);
|
|
117
|
+
},
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
Suite(
|
|
121
|
+
'an image type the vendors do not accept falls back to TEXT, never a failed turn',
|
|
122
|
+
async () => {
|
|
123
|
+
// `image/*` covers HEIC, BMP, TIFF, SVG — none of which the APIs take, and sending one fails
|
|
124
|
+
// the whole turn. SVG especially: it IS text, so the text branch gives the model something it
|
|
125
|
+
// can actually read.
|
|
126
|
+
const el = element('image/*');
|
|
127
|
+
const { attachments, errors } = await el.processFiles([
|
|
128
|
+
new File(['<svg xmlns="http://www.w3.org/2000/svg"><rect /></svg>'], 'icon.svg', {
|
|
129
|
+
type: 'image/svg+xml',
|
|
130
|
+
}),
|
|
131
|
+
new File([new Uint8Array([0, 1, 2, 3])], 'photo.heic', { type: 'image/heic' }),
|
|
132
|
+
]);
|
|
133
|
+
assert.equal(errors, []);
|
|
134
|
+
const [svg, heic] = attachments as Array<{ kind?: string; content?: string; mimeType: string }>;
|
|
135
|
+
assert.is(svg.kind, undefined, 'SVG stays text — a model can read it');
|
|
136
|
+
assert.match(svg.content!, /<rect/);
|
|
137
|
+
assert.is(heic.kind, undefined, 'an unsupported binary degrades rather than failing the turn');
|
|
138
|
+
},
|
|
139
|
+
);
|
|
140
|
+
|
|
141
|
+
Suite('every supported spelling takes the image branch, including image/jpg', async () => {
|
|
142
|
+
const el = element('image/*');
|
|
143
|
+
const { attachments } = await el.processFiles(
|
|
144
|
+
['image/png', 'image/jpeg', 'image/jpg', 'image/gif', 'image/webp'].map(
|
|
145
|
+
(type, i) => new File([PNG_BYTES], `f${i}`, { type }),
|
|
146
|
+
),
|
|
147
|
+
);
|
|
148
|
+
assert.equal(
|
|
149
|
+
(attachments as Array<{ kind?: string }>).map((a) => a.kind),
|
|
150
|
+
['image', 'image', 'image', 'image', 'image'],
|
|
151
|
+
);
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
// The TOTAL, not just each file. `loadSelectedFiles` appends across successive picks and the
|
|
155
|
+
// per-file check is blind to what is already attached, so ten files each just under the per-image
|
|
156
|
+
// ceiling used to be ten passes and one ~34MB request — assembled entirely out of files that each
|
|
157
|
+
// passed. Reviewer catch on #2474.
|
|
158
|
+
|
|
159
|
+
Suite('the file that crosses the total is refused; the ones that fit are kept', async () => {
|
|
160
|
+
const el = element('image/*');
|
|
161
|
+
const mb = (n: number) => new Uint8Array(n * 1_000_000);
|
|
162
|
+
const { attachments, errors } = await el.processFiles([
|
|
163
|
+
new File([mb(3)], 'a.png', { type: 'image/png' }),
|
|
164
|
+
new File([mb(3)], 'b.png', { type: 'image/png' }),
|
|
165
|
+
new File([mb(3)], 'c.png', { type: 'image/png' }), // 9MB total — past the 8MB ceiling
|
|
166
|
+
]);
|
|
167
|
+
assert.is(attachments.length, 2, 'the two that fit are accepted');
|
|
168
|
+
assert.is(errors.length, 1);
|
|
169
|
+
assert.match(errors[0], /c\.png/);
|
|
170
|
+
assert.match(errors[0], /attachment limit/);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
Suite('the budget counts what is ALREADY attached, not just this pick', async () => {
|
|
174
|
+
// The picker's second pick has to see the first one's bytes, or "under the limit" is measured
|
|
175
|
+
// against zero every time and the limit means nothing.
|
|
176
|
+
const el = element('image/*');
|
|
177
|
+
const fresh = await el.processFiles([
|
|
178
|
+
new File([new Uint8Array(3_000_000)], 'a.png', { type: 'image/png' }),
|
|
179
|
+
]);
|
|
180
|
+
assert.is(fresh.attachments.length, 1, 'fits on its own');
|
|
181
|
+
|
|
182
|
+
const withHistory = await el.processFiles(
|
|
183
|
+
[new File([new Uint8Array(3_000_000)], 'b.png', { type: 'image/png' })],
|
|
184
|
+
6_000_000,
|
|
185
|
+
);
|
|
186
|
+
assert.is(
|
|
187
|
+
withHistory.attachments.length,
|
|
188
|
+
0,
|
|
189
|
+
'the same file is refused once 6MB is already committed',
|
|
190
|
+
);
|
|
191
|
+
assert.match(withHistory.errors[0], /6\.0MB already attached/);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
Suite('a refusal skips only the offending file — a later one that still fits is kept', async () => {
|
|
195
|
+
// Budgeted synchronously off `file.size`, in the user's own pick order, before anything is read
|
|
196
|
+
// (deciding inside the parallel reads would hand the last slot to whichever finished first).
|
|
197
|
+
// A file that does not fit is refused and the budget carries on, so a smaller one behind it is
|
|
198
|
+
// not punished for its neighbour.
|
|
199
|
+
const el = element('image/*');
|
|
200
|
+
const mb = (n: number) => new Uint8Array(n * 1_000_000);
|
|
201
|
+
const { attachments, errors } = await el.processFiles([
|
|
202
|
+
new File([mb(3)], 'a.png', { type: 'image/png' }), // running 3
|
|
203
|
+
new File([mb(3)], 'b.png', { type: 'image/png' }), // running 6
|
|
204
|
+
new File([mb(3)], 'big.png', { type: 'image/png' }), // 9 > 8 — refused
|
|
205
|
+
new File([mb(1)], 'small.png', { type: 'image/png' }), // 7 — still fits
|
|
206
|
+
]);
|
|
207
|
+
assert.equal(
|
|
208
|
+
(attachments as Array<{ name: string }>).map((a) => a.name),
|
|
209
|
+
['a.png', 'b.png', 'small.png'],
|
|
210
|
+
);
|
|
211
|
+
assert.is(errors.length, 1);
|
|
212
|
+
assert.match(errors[0], /big\.png/);
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
Suite.run();
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { assert, createLogicSuite } from '@genesislcap/foundation-testing';
|
|
2
|
+
import { getSessionStore } from '../state/session-store';
|
|
3
|
+
import { FoundationAiAssistant } from './main';
|
|
4
|
+
|
|
5
|
+
// Hold a reference so the custom-element registration isn't tree-shaken.
|
|
6
|
+
FoundationAiAssistant;
|
|
7
|
+
|
|
8
|
+
// GENC-1508 — the host hop of the post-resolve cost channel.
|
|
9
|
+
//
|
|
10
|
+
// The driver method and the orchestrator delegation both have tests; this pins the hop
|
|
11
|
+
// between them, which is the one most likely to break silently: a rename on either side of
|
|
12
|
+
// `handleInteractionCost` is a no-op that every other test still passes, and the "no spinner"
|
|
13
|
+
// guarantee is a design promise stated only in a comment.
|
|
14
|
+
//
|
|
15
|
+
// Unconnected element (createElement upgrades but does not connect), same as
|
|
16
|
+
// blocked-state.test.ts — this handler touches no session state.
|
|
17
|
+
|
|
18
|
+
const Suite = createLogicSuite('FoundationAiAssistant interaction cost');
|
|
19
|
+
|
|
20
|
+
type CostHost = {
|
|
21
|
+
handleInteractionCost(e: Event): void;
|
|
22
|
+
driver: unknown;
|
|
23
|
+
startLoadingTimer(): void;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
function host(): { el: CostHost; calls: Array<[string, number]>; spinners: number } {
|
|
27
|
+
const el = document.createElement('foundation-ai-assistant') as unknown as CostHost;
|
|
28
|
+
const calls: Array<[string, number]> = [];
|
|
29
|
+
const state = { spinners: 0 };
|
|
30
|
+
el.driver = {
|
|
31
|
+
recordExternalCost: (id: string, cost: number) => {
|
|
32
|
+
calls.push([id, cost]);
|
|
33
|
+
return true;
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
el.startLoadingTimer = () => {
|
|
37
|
+
state.spinners += 1;
|
|
38
|
+
};
|
|
39
|
+
return {
|
|
40
|
+
el,
|
|
41
|
+
calls,
|
|
42
|
+
get spinners() {
|
|
43
|
+
return state.spinners;
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const costEvent = (detail: unknown) => new CustomEvent('interaction-cost', { detail });
|
|
49
|
+
|
|
50
|
+
Suite('a cost report reaches the driver', () => {
|
|
51
|
+
const h = host();
|
|
52
|
+
h.el.handleInteractionCost(costEvent({ interactionId: 'abc', costUsd: 0.42 }));
|
|
53
|
+
assert.equal(h.calls, [['abc', 0.42]]);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
Suite('reporting cost NEVER starts the loading spinner', () => {
|
|
57
|
+
// The whole point of the channel: this is bookkeeping about a turn that already finished, so
|
|
58
|
+
// it must be invisible. A spinner would tell the user the assistant is working when nothing
|
|
59
|
+
// is pending — the one thing `handleInteractionCompleted` does that this must not copy.
|
|
60
|
+
const h = host();
|
|
61
|
+
h.el.handleInteractionCost(costEvent({ interactionId: 'abc', costUsd: 0.42 }));
|
|
62
|
+
assert.is(h.spinners, 0);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
Suite('an event with no interactionId is ignored', () => {
|
|
66
|
+
const h = host();
|
|
67
|
+
h.el.handleInteractionCost(costEvent({ costUsd: 0.42 }));
|
|
68
|
+
h.el.handleInteractionCost(costEvent(undefined));
|
|
69
|
+
assert.equal(h.calls, []);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
Suite('a driver that does not implement the optional method is not called into', () => {
|
|
73
|
+
// `recordExternalCost` is optional on `AiDriver`, so a driver predating this — or one that
|
|
74
|
+
// simply does not track spend — must be ignored rather than throw.
|
|
75
|
+
const h = host();
|
|
76
|
+
h.el.driver = {};
|
|
77
|
+
assert.not.throws(() =>
|
|
78
|
+
h.el.handleInteractionCost(costEvent({ interactionId: 'a', costUsd: 1 })),
|
|
79
|
+
);
|
|
80
|
+
h.el.driver = undefined;
|
|
81
|
+
assert.not.throws(() =>
|
|
82
|
+
h.el.handleInteractionCost(costEvent({ interactionId: 'a', costUsd: 1 })),
|
|
83
|
+
);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
Suite('a driver without the method leaves a TRACE, not a silent no-op', () => {
|
|
87
|
+
// Reviewer catch on #2474: every other hop traces its drops. A driver predating
|
|
88
|
+
// `recordExternalCost` made this one a silent no-op — money gone, nothing recorded — which is
|
|
89
|
+
// the invisible gap the whole channel exists to close, reintroduced at the last hop.
|
|
90
|
+
const h = host();
|
|
91
|
+
const meta: Array<[string, Record<string, unknown>]> = [];
|
|
92
|
+
(h.el as unknown as { logMeta(t: string, d: Record<string, unknown>): void }).logMeta = (
|
|
93
|
+
t,
|
|
94
|
+
d,
|
|
95
|
+
) => {
|
|
96
|
+
meta.push([t, d]);
|
|
97
|
+
};
|
|
98
|
+
h.el.driver = {};
|
|
99
|
+
h.el.handleInteractionCost(costEvent({ interactionId: 'abc', costUsd: 0.42 }));
|
|
100
|
+
assert.is(meta.length, 1);
|
|
101
|
+
assert.is(meta[0][0], 'interaction.external-cost-dropped');
|
|
102
|
+
assert.is(meta[0][1].costUsd, 0.42);
|
|
103
|
+
assert.is(meta[0][1].interactionId, 'abc');
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
// The OUTBOUND half: the driver announced, and the host has to turn that into a moved number.
|
|
107
|
+
// Tested because it is the same shape of hop as the one that already went wrong — the wrapper
|
|
108
|
+
// forward that was missing — and because it deliberately reads the DRIVER's history rather than
|
|
109
|
+
// `this.messages`, which is the part a reader would most likely "simplify" back.
|
|
110
|
+
|
|
111
|
+
let storeSeq = 0;
|
|
112
|
+
|
|
113
|
+
/** An element wired to its own real session store, so `sessionUsage` round-trips. */
|
|
114
|
+
function costHost(): FoundationAiAssistant {
|
|
115
|
+
const el = document.createElement('foundation-ai-assistant') as FoundationAiAssistant;
|
|
116
|
+
storeSeq += 1;
|
|
117
|
+
(el as unknown as { _sessionRef: unknown })._sessionRef = getSessionStore(
|
|
118
|
+
`cost-test-${(storeSeq += 1)}`,
|
|
119
|
+
false,
|
|
120
|
+
);
|
|
121
|
+
return el;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
Suite('the recorded spend reaches the session total from the DRIVER history', () => {
|
|
125
|
+
const el = costHost();
|
|
126
|
+
const sync = (el as unknown as { syncUsageTotals(m: unknown[]): void }).syncUsageTotals.bind(el);
|
|
127
|
+
|
|
128
|
+
sync([{ role: 'assistant', content: 'a', cost: 0.1, model: 'claude-sonnet-5' }]);
|
|
129
|
+
assert.is(el.sessionCostUsd, 0.1);
|
|
130
|
+
|
|
131
|
+
// The post-resolve delta lands on the interaction message as `externalCostUsd`; the total has
|
|
132
|
+
// to fold it in, or every hop before this one was wasted.
|
|
133
|
+
sync([
|
|
134
|
+
{ role: 'assistant', content: 'a', cost: 0.1, model: 'claude-sonnet-5' },
|
|
135
|
+
{ role: 'assistant', content: 'b', externalCostUsd: 0.25 },
|
|
136
|
+
]);
|
|
137
|
+
assert.is(el.sessionCostUsd, 0.35);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
Suite.run();
|
|
@@ -624,6 +624,7 @@ ${(tc) => (tc.foldPath?.length ? `${tc.foldPath.join(' › ')} › ` : '')}<stro
|
|
|
624
624
|
:interactionId=${(m) => m.interaction!.interactionId}
|
|
625
625
|
:resolved=${(m) => m.interaction!.resolved}
|
|
626
626
|
@interaction-completed=${(m, c) => c.parent.handleInteractionCompleted(c.event)}
|
|
627
|
+
@interaction-cost=${(m, c) => c.parent.handleInteractionCost(c.event)}
|
|
627
628
|
></ai-chat-interaction-wrapper>
|
|
628
629
|
`,
|
|
629
630
|
)}
|
package/src/main/main.ts
CHANGED
|
@@ -220,6 +220,67 @@ const SESSION_MENU_CLOSE_MS = 200;
|
|
|
220
220
|
*/
|
|
221
221
|
const LEDGER_FLUSH_DEBOUNCE_MS = 2000;
|
|
222
222
|
|
|
223
|
+
/**
|
|
224
|
+
* The image types BOTH vendors accept, and therefore the only ones sent as image blocks.
|
|
225
|
+
*
|
|
226
|
+
* A host's `acceptedFiles` may well say `image/*`, which covers HEIC, BMP, TIFF, AVIF and SVG —
|
|
227
|
+
* none of which the APIs take. Sending one fails the whole turn the user is waiting on, so
|
|
228
|
+
* anything outside this set falls through to the text branch instead: harmless for a binary, and
|
|
229
|
+
* genuinely useful for SVG, which is text a model can read.
|
|
230
|
+
*
|
|
231
|
+
* `image/jpg` is absent on purpose — `normalizeImageMime` rewrites it to `image/jpeg` at the
|
|
232
|
+
* transport, but a file arriving with that spelling is still a real JPEG, so it is accepted here.
|
|
233
|
+
*/
|
|
234
|
+
const VENDOR_IMAGE_MIME_TYPES = new Set([
|
|
235
|
+
'image/jpeg',
|
|
236
|
+
'image/jpg',
|
|
237
|
+
'image/png',
|
|
238
|
+
'image/gif',
|
|
239
|
+
'image/webp',
|
|
240
|
+
]);
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Ceiling on ONE image attachment, under Anthropic's ~5MB-per-image limit with room for the ~33%
|
|
244
|
+
* base64 overhead. Refused here, with the size named, rather than at the vendor: a 400 arrives
|
|
245
|
+
* only after the user has waited out the turn.
|
|
246
|
+
*
|
|
247
|
+
* A VENDOR guard, and only that. An earlier version of this comment also claimed it protected the
|
|
248
|
+
* session's storage quota — it does not, and the arithmetic says so: one file at this ceiling is
|
|
249
|
+
* ~4.7MB of base64, which alone can exceed a 5MB `localStorage` budget before the transcript is
|
|
250
|
+
* counted. Attachments riding into persisted storage is a real and separate problem; it is not
|
|
251
|
+
* this constant's to solve, and saying otherwise hid it.
|
|
252
|
+
*/
|
|
253
|
+
const MAX_IMAGE_ATTACHMENT_BYTES = 3_500_000;
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Ceiling on ALL attachments carried by one message, across every type and every successive pick.
|
|
257
|
+
*
|
|
258
|
+
* The per-file check above is blind to the total: the picker APPENDS (`loadSelectedFiles`), so ten
|
|
259
|
+
* files each just under the per-image limit are ten passes and one ~34MB request. Sized against
|
|
260
|
+
* the vendors' request ceilings — comfortably inside the smaller of them with room for the
|
|
261
|
+
* transcript itself — so a turn that would be rejected wholesale is refused here, where the user
|
|
262
|
+
* can still act on it, naming the file that crossed the line.
|
|
263
|
+
*
|
|
264
|
+
* Like its sibling, this does NOT make attachments safe to persist — 8MB of attachments is still
|
|
265
|
+
* ~10.7MB of base64. What a persisted session should keep of its images is GENC-1531; note the
|
|
266
|
+
* transports also REPLAY history attachments on every later call, so it is a recurring cost as
|
|
267
|
+
* well as a storage one.
|
|
268
|
+
*/
|
|
269
|
+
const MAX_ATTACHMENTS_TOTAL_BYTES = 8_000_000;
|
|
270
|
+
|
|
271
|
+
/** Base64 carries 3 bytes in every 4 characters, so decoded size is 3/4 of the encoded length. */
|
|
272
|
+
const BASE64_DECODED_RATIO = 0.75;
|
|
273
|
+
|
|
274
|
+
/** A rough byte size for an already-accepted attachment, for budgeting against the total cap. */
|
|
275
|
+
function attachmentBytes(a: ChatAttachment): number {
|
|
276
|
+
return a.kind === 'image'
|
|
277
|
+
? Math.ceil((a.data?.length ?? 0) * BASE64_DECODED_RATIO)
|
|
278
|
+
: (a.content?.length ?? 0);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Bytes per megabyte, for the size the refusal message quotes back to the user. */
|
|
282
|
+
const BYTES_PER_MB = 1_000_000;
|
|
283
|
+
|
|
223
284
|
/**
|
|
224
285
|
* Reporter id for this element's OWN session persister in `brokenSources` (GENC-1511).
|
|
225
286
|
*
|
|
@@ -2711,6 +2772,22 @@ export class FoundationAiAssistant extends GenesisElement {
|
|
|
2711
2772
|
};
|
|
2712
2773
|
driver.addEventListener('history-updated', onHistoryUpdated);
|
|
2713
2774
|
|
|
2775
|
+
// Post-resolve external cost. Deliberately does NOT reassign `messages`: that would rebuild
|
|
2776
|
+
// the trailing interaction row (a `recycle: false` repeat) and blank the widget that just
|
|
2777
|
+
// reported — see `recordExternalCost`. The totals come off the DRIVER's history because the
|
|
2778
|
+
// host's copy holds the pre-update message objects, and the persister already reads the
|
|
2779
|
+
// driver first, so the save picks the field up with no help from here.
|
|
2780
|
+
const onExternalCostRecorded = () => {
|
|
2781
|
+
if (driver.getRawHistory) this.syncUsageTotals(driver.getRawHistory());
|
|
2782
|
+
this.persister()?.scheduleSave();
|
|
2783
|
+
// Diagnostics too, for the same reason `onHistoryUpdated` schedules both: the forward-append
|
|
2784
|
+
// is what carries `interaction.external-cost` (and its `-dropped` sibling) into the persisted
|
|
2785
|
+
// stream. Without it the trace this change adds sits in memory waiting for activity that,
|
|
2786
|
+
// in this flow, never comes — the interaction is the last thing in the session.
|
|
2787
|
+
this.persister()?.scheduleDiagnostics();
|
|
2788
|
+
};
|
|
2789
|
+
driver.addEventListener('external-cost-recorded', onExternalCostRecorded);
|
|
2790
|
+
|
|
2714
2791
|
// Re-seed the rendered list from the driver's current history. `wireDriver`
|
|
2715
2792
|
// runs on (re)connect, popin, and when adopting a sibling's driver
|
|
2716
2793
|
// (GENC-1388); in those cases `messages` would otherwise stay stale until the
|
|
@@ -2791,6 +2868,7 @@ export class FoundationAiAssistant extends GenesisElement {
|
|
|
2791
2868
|
|
|
2792
2869
|
const cleanups: (() => void)[] = [
|
|
2793
2870
|
() => driver.removeEventListener('history-updated', onHistoryUpdated),
|
|
2871
|
+
() => driver.removeEventListener('external-cost-recorded', onExternalCostRecorded),
|
|
2794
2872
|
() => driver.removeEventListener('sub-agent-history-updated', onSubAgentHistoryUpdated),
|
|
2795
2873
|
() => driver.removeEventListener('sub-agent-start', onSubAgentStart),
|
|
2796
2874
|
() => driver.removeEventListener('sub-agent-stop', onSubAgentStop),
|
|
@@ -3281,6 +3359,39 @@ export class FoundationAiAssistant extends GenesisElement {
|
|
|
3281
3359
|
this.showingSplash = !!this.chatConfig.ui?.showSplash && this.messages.length === 0;
|
|
3282
3360
|
}
|
|
3283
3361
|
|
|
3362
|
+
/**
|
|
3363
|
+
* Recompute the session's cost and token buckets from a transcript.
|
|
3364
|
+
*
|
|
3365
|
+
* Takes the list rather than reading `this.messages`, because it has two callers with
|
|
3366
|
+
* different sources: the `messages` setter passes the rendered list, and post-resolve external
|
|
3367
|
+
* cost passes the DRIVER's history — spend recorded there must reach the total WITHOUT
|
|
3368
|
+
* reassigning `messages`, which would re-render the live interaction row. See
|
|
3369
|
+
* `onExternalCostRecorded`.
|
|
3370
|
+
*
|
|
3371
|
+
* Recomputing (rather than incrementing on append) keeps the totals correct under any mutation
|
|
3372
|
+
* of the message list — clear-chat, re-render, restore. `sumUsage` recurses into
|
|
3373
|
+
* `toolCall.subAgentTrace`, so work a sub-agent did (possibly on another provider at other
|
|
3374
|
+
* rates) counts, and it reads the usage a compaction banked onto its summary, so shrinking
|
|
3375
|
+
* history no longer shrinks the totals.
|
|
3376
|
+
*/
|
|
3377
|
+
private syncUsageTotals(messages: readonly ChatMessage[]): void {
|
|
3378
|
+
const running = sumUsage(messages);
|
|
3379
|
+
const current = this.sessionUsage;
|
|
3380
|
+
if (
|
|
3381
|
+
running.costUsd !== current.costUsd ||
|
|
3382
|
+
running.uncachedInputTokens !== current.uncachedInputTokens ||
|
|
3383
|
+
running.cacheReadTokens !== current.cacheReadTokens ||
|
|
3384
|
+
running.cacheWriteTokens !== current.cacheWriteTokens ||
|
|
3385
|
+
running.outputTokens !== current.outputTokens
|
|
3386
|
+
) {
|
|
3387
|
+
this.sessionUsage = running;
|
|
3388
|
+
// Keep this project's history row current as spend accrues, which is what lets
|
|
3389
|
+
// the Usage tab total be the sum of the rows alone. Idempotent (an upsert keyed
|
|
3390
|
+
// by the session key), so calling it per change is safe.
|
|
3391
|
+
this.finalizeCostSession();
|
|
3392
|
+
}
|
|
3393
|
+
}
|
|
3394
|
+
|
|
3284
3395
|
/**
|
|
3285
3396
|
* Runs side effects that were previously in `messagesChanged()`.
|
|
3286
3397
|
* Called from the `messages` setter after dispatching to the store.
|
|
@@ -3317,21 +3428,7 @@ export class FoundationAiAssistant extends GenesisElement {
|
|
|
3317
3428
|
// recurses into `toolCall.subAgentTrace`, so work a sub-agent did (possibly on
|
|
3318
3429
|
// another provider at other rates) counts, and it reads the usage a compaction
|
|
3319
3430
|
// banked onto its summary, so shrinking history no longer shrinks the totals.
|
|
3320
|
-
|
|
3321
|
-
const current = this.sessionUsage;
|
|
3322
|
-
if (
|
|
3323
|
-
running.costUsd !== current.costUsd ||
|
|
3324
|
-
running.uncachedInputTokens !== current.uncachedInputTokens ||
|
|
3325
|
-
running.cacheReadTokens !== current.cacheReadTokens ||
|
|
3326
|
-
running.cacheWriteTokens !== current.cacheWriteTokens ||
|
|
3327
|
-
running.outputTokens !== current.outputTokens
|
|
3328
|
-
) {
|
|
3329
|
-
this.sessionUsage = running;
|
|
3330
|
-
// Keep this project's history row current as spend accrues, which is what lets
|
|
3331
|
-
// the Usage tab total be the sum of the rows alone. Idempotent (an upsert keyed
|
|
3332
|
-
// by the session key), so calling it per change is safe.
|
|
3333
|
-
this.finalizeCostSession();
|
|
3334
|
-
}
|
|
3431
|
+
this.syncUsageTotals(this.messages);
|
|
3335
3432
|
// Record a context.updated meta event when the token count changes (≈once
|
|
3336
3433
|
// per LLM call, as a new usage-bearing message arrives), plus a one-shot
|
|
3337
3434
|
// threshold-crossed event the first time usage passes 80%.
|
|
@@ -4661,25 +4758,82 @@ export class FoundationAiAssistant extends GenesisElement {
|
|
|
4661
4758
|
);
|
|
4662
4759
|
}
|
|
4663
4760
|
|
|
4761
|
+
// Blob's own promise APIs, not FileReader: UTF-8 only (a `charset` parameter on the blob's
|
|
4762
|
+
// type is ignored, where FileReader would honour it), and they exist in the node test runtime
|
|
4763
|
+
// too — FileReader does not, so neither reader was testable before.
|
|
4664
4764
|
private readFileAsText(file: File): Promise<string> {
|
|
4665
|
-
return
|
|
4666
|
-
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
|
|
4765
|
+
return file.text();
|
|
4766
|
+
}
|
|
4767
|
+
|
|
4768
|
+
/** The base64 payload of a file, without any `data:` prefix — the raw form
|
|
4769
|
+
* `ChatImageAttachment.data` specifies (transports add their own framing). */
|
|
4770
|
+
private async readFileAsBase64(file: File): Promise<string> {
|
|
4771
|
+
const bytes = new Uint8Array(await file.arrayBuffer());
|
|
4772
|
+
let binary = '';
|
|
4773
|
+
const CHUNK = 0x8000;
|
|
4774
|
+
for (let i = 0; i < bytes.length; i += CHUNK) {
|
|
4775
|
+
binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
|
|
4776
|
+
}
|
|
4777
|
+
return btoa(binary);
|
|
4671
4778
|
}
|
|
4672
4779
|
|
|
4673
4780
|
private async processFiles(
|
|
4674
4781
|
files: File[],
|
|
4782
|
+
existingBytes = 0,
|
|
4675
4783
|
): Promise<{ attachments: ChatAttachment[]; errors: string[] }> {
|
|
4676
4784
|
type FileResult = { ok: true; attachment: ChatAttachment } | { ok: false; message: string };
|
|
4785
|
+
// Budget the batch BEFORE reading any of it. `file.size` needs no I/O, so the verdict is
|
|
4786
|
+
// deterministic and in the user's own pick order; deciding inside the parallel map below would
|
|
4787
|
+
// hand the last slot to whichever read happened to finish first. `existingBytes` is what the
|
|
4788
|
+
// caller has already committed to — the picker appends to a running list, so its files have to
|
|
4789
|
+
// count against what is already attached rather than starting from zero each time.
|
|
4790
|
+
let running = existingBytes;
|
|
4791
|
+
const overBudget = new Map<File, string>();
|
|
4792
|
+
for (const file of files) {
|
|
4793
|
+
if (running + file.size > MAX_ATTACHMENTS_TOTAL_BYTES) {
|
|
4794
|
+
overBudget.set(
|
|
4795
|
+
file,
|
|
4796
|
+
`"${file.name}" would take this message past the ` +
|
|
4797
|
+
`${MAX_ATTACHMENTS_TOTAL_BYTES / BYTES_PER_MB}MB attachment limit ` +
|
|
4798
|
+
`(${(running / BYTES_PER_MB).toFixed(1)}MB already attached).`,
|
|
4799
|
+
);
|
|
4800
|
+
continue;
|
|
4801
|
+
}
|
|
4802
|
+
running += file.size;
|
|
4803
|
+
}
|
|
4677
4804
|
const results = await Promise.all(
|
|
4678
4805
|
files.map(async (file): Promise<FileResult> => {
|
|
4679
4806
|
if (!this.isAcceptedFile(file)) {
|
|
4680
4807
|
return { ok: false, message: `"${file.name}" is not an accepted file type.` };
|
|
4681
4808
|
}
|
|
4809
|
+
const tooMuch = overBudget.get(file);
|
|
4810
|
+
if (tooMuch) return { ok: false, message: tooMuch };
|
|
4682
4811
|
try {
|
|
4812
|
+
// An image file becomes a real image attachment — the transports render those as
|
|
4813
|
+
// image blocks the model SEES. Reading it as text (the only path that existed)
|
|
4814
|
+
// produced mojibake the model "read" as prose: an accepted .png was worse than a
|
|
4815
|
+
// rejected one. Text stays the default for everything else.
|
|
4816
|
+
//
|
|
4817
|
+
// Only the types the vendors actually accept take this branch. `image/*` covers far
|
|
4818
|
+
// more than that, and sending one an API rejects fails the whole turn the user is
|
|
4819
|
+
// waiting on — where the text branch merely degrades. It also keeps SVG working:
|
|
4820
|
+
// SVG *is* text, and reading it as such gives the model something it can genuinely
|
|
4821
|
+
// read, which a base64 image block of it would not.
|
|
4822
|
+
if (VENDOR_IMAGE_MIME_TYPES.has(file.type.toLowerCase())) {
|
|
4823
|
+
if (file.size > MAX_IMAGE_ATTACHMENT_BYTES) {
|
|
4824
|
+
return {
|
|
4825
|
+
ok: false,
|
|
4826
|
+
message:
|
|
4827
|
+
`"${file.name}" is ${Math.ceil(file.size / BYTES_PER_MB)}MB; images are limited to ` +
|
|
4828
|
+
`${MAX_IMAGE_ATTACHMENT_BYTES / BYTES_PER_MB}MB.`,
|
|
4829
|
+
};
|
|
4830
|
+
}
|
|
4831
|
+
const data = await this.readFileAsBase64(file);
|
|
4832
|
+
return {
|
|
4833
|
+
ok: true,
|
|
4834
|
+
attachment: { kind: 'image', name: file.name, mimeType: file.type, data },
|
|
4835
|
+
};
|
|
4836
|
+
}
|
|
4683
4837
|
const content = await this.readFileAsText(file);
|
|
4684
4838
|
return {
|
|
4685
4839
|
ok: true,
|
|
@@ -4710,7 +4864,10 @@ export class FoundationAiAssistant extends GenesisElement {
|
|
|
4710
4864
|
const input = e.target as HTMLInputElement;
|
|
4711
4865
|
const files = Array.from(input.files ?? []);
|
|
4712
4866
|
input.value = '';
|
|
4713
|
-
const { attachments, errors } = await this.processFiles(
|
|
4867
|
+
const { attachments, errors } = await this.processFiles(
|
|
4868
|
+
files,
|
|
4869
|
+
this.attachments.reduce((n, a) => n + attachmentBytes(a), 0),
|
|
4870
|
+
);
|
|
4714
4871
|
if (attachments.length) {
|
|
4715
4872
|
this.attachments = [...this.attachments, ...attachments];
|
|
4716
4873
|
this.logMeta('attachment.added', {
|
|
@@ -4976,6 +5133,31 @@ export class FoundationAiAssistant extends GenesisElement {
|
|
|
4976
5133
|
}
|
|
4977
5134
|
}
|
|
4978
5135
|
|
|
5136
|
+
/**
|
|
5137
|
+
* A widget reporting spend it incurred AFTER resolving.
|
|
5138
|
+
*
|
|
5139
|
+
* Note what this deliberately does NOT do: no `startLoadingTimer()`. This is bookkeeping about a
|
|
5140
|
+
* turn that already finished, so it must be invisible — showing a spinner would tell the user the
|
|
5141
|
+
* assistant is working when nothing is pending.
|
|
5142
|
+
*/
|
|
5143
|
+
handleInteractionCost(e: Event) {
|
|
5144
|
+
const detail = (e as CustomEvent).detail;
|
|
5145
|
+
if (!detail?.interactionId) return;
|
|
5146
|
+
// Optional on `AiDriver`, so call it optionally — a driver that predates this, or one that
|
|
5147
|
+
// simply doesn't track spend, should ignore the report rather than throw on it. TRACED,
|
|
5148
|
+
// because the spend is real and already billed: an untraced no-op here is the same invisible
|
|
5149
|
+
// gap this whole channel exists to close, reintroduced at the last hop. Reviewer catch.
|
|
5150
|
+
if (!this.driver?.recordExternalCost) {
|
|
5151
|
+
this.logMeta('interaction.external-cost-dropped', {
|
|
5152
|
+
interactionId: detail.interactionId,
|
|
5153
|
+
costUsd: detail.costUsd,
|
|
5154
|
+
reason: 'driver does not implement recordExternalCost',
|
|
5155
|
+
});
|
|
5156
|
+
return;
|
|
5157
|
+
}
|
|
5158
|
+
this.driver.recordExternalCost(detail.interactionId, detail.costUsd);
|
|
5159
|
+
}
|
|
5160
|
+
|
|
4979
5161
|
/**
|
|
4980
5162
|
* The live {@link InteractionContext} for a pending interaction (GENC-1390).
|
|
4981
5163
|
* `AiChatInteractionWrapper` pierces this across the shadow boundary to build the
|