@theoremjs/agents 0.3.5-next.949.gac4d2f6 → 0.3.5-next.950.g9bb8f33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,10 +13,17 @@ declare function requireMediaLimits(profile: Profile): MediaLimits;
13
13
  declare function sanitizeCsvText(text: string): string;
14
14
  /** Whether a file of this type is text the detectors read once decoded. */
15
15
  declare function isTextFile(mimeType: string): boolean;
16
- /** A text file's base64 bytes as the model reads them: decoded, and a CSV's formula cells escaped. */
16
+ /** Whether a text file's bytes are base64, so it can be decoded and read. */
17
+ declare function isTextFileData(mimeType: string, data: string): boolean;
18
+ /** A text file's text; throws when its bytes are not base64, which every source of a file refuses first. */
17
19
  declare function textFileText(mimeType: string, data: string): string;
18
- /** Text as a text file's base64 bytes. */
19
- declare function textFileData(text: string): string;
20
+ /** `text` as the file's base64 bytes, in the encoding the file was read in; the given bytes when the text is unchanged. */
21
+ declare function rewriteTextFile(mimeType: string, data: string, text: string): string;
22
+ /** A text file as the model reads it: its text as UTF-8, under a charset that says so, with a CSV's formula cells escaped. */
23
+ declare function modelTextFile(mimeType: string, data: string): {
24
+ mimeType: string;
25
+ data: string;
26
+ };
20
27
  /** `sizeBytes` is absent for provider references, which carry no bytes. */
21
28
  export interface AttachmentFacts {
22
29
  name?: string;
@@ -49,4 +56,4 @@ declare function sanitizeTurnBlobs(profile: Profile, attachments: Array<TurnBlob
49
56
  attachments?: Array<TurnBlob | TurnMediaRef>;
50
57
  voice?: TurnBlob[];
51
58
  };
52
- export { assertTurnAttachments, attachmentIssueCopy, attachmentIssues, attachmentIssueText, attachmentsRefused, base64Bytes, isTextFile, isTurnMediaRef, maxBytesForMime, requireMediaLimits, resolveMediaLimits, sanitizeCsvText, sanitizeTurnBlobs, textFileData, textFileText, };
59
+ export { assertTurnAttachments, attachmentIssueCopy, attachmentIssues, attachmentIssueText, attachmentsRefused, base64Bytes, isTextFile, isTextFileData, isTurnMediaRef, maxBytesForMime, modelTextFile, requireMediaLimits, resolveMediaLimits, rewriteTextFile, sanitizeCsvText, sanitizeTurnBlobs, textFileText, };
@@ -62,13 +62,118 @@ function b64DecodedLen(data) {
62
62
  }
63
63
  return Math.floor((data.length * B64_TRIPLET) / B64_WORD) - pad;
64
64
  }
65
- function decodeText(bytes) {
65
+ const BOMS = [
66
+ ['utf-8', [0xef, 0xbb, 0xbf]],
67
+ ['utf-16le', [0xff, 0xfe]],
68
+ ['utf-16be', [0xfe, 0xff]],
69
+ ];
70
+ const CHUNK = 0x8000;
71
+ const CHARSET = /(;\s*charset\s*=\s*)("[^"]*"|[^;\s]+)/i;
72
+ /** why: what a browser reads text with no charset in when it is not UTF-8. */
73
+ const FALLBACK_CHARSET = 'windows-1252';
74
+ const tables = new Map();
75
+ /** The `charset` a MIME type names, lowercased. */
76
+ function charsetOf(mimeType) {
77
+ return CHARSET.exec(mimeType)?.[2]?.replaceAll('"', '').toLowerCase();
78
+ }
79
+ /** The encoding a charset label names, as the decoder knows it; `undefined` for one it does not know. */
80
+ function encodingName(label) {
66
81
  try {
67
- return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
82
+ return new TextDecoder(label).encoding;
68
83
  }
69
84
  catch {
70
- return new TextDecoder('latin1').decode(bytes);
85
+ return undefined;
86
+ }
87
+ }
88
+ /** A single-byte charset's table, each byte to one character and back; `null` for a charset that is not one. */
89
+ function byteTable(name) {
90
+ const known = tables.get(name);
91
+ if (known !== undefined)
92
+ return known;
93
+ const decoder = new TextDecoder(name);
94
+ const chars = Array.from({ length: 256 }, (_, byte) => decoder.decode(Uint8Array.of(byte)));
95
+ const bytes = new Map(chars.map((char, byte) => [char, byte]));
96
+ const table = chars.every((char) => char.length === 1) && bytes.size === 256 ? { chars, bytes } : null;
97
+ tables.set(name, table);
98
+ return table;
99
+ }
100
+ function validUtf8(bytes) {
101
+ try {
102
+ return new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes);
103
+ }
104
+ catch {
105
+ return undefined;
106
+ }
107
+ }
108
+ /** Text as its bytes spell it: a byte order mark first, then the charset the type names, then UTF-8 when it is valid. */
109
+ function decodeText(mimeType, bytes) {
110
+ for (const [kind, mark] of BOMS) {
111
+ if (mark.every((byte, index) => bytes[index] === byte)) {
112
+ const body = bytes.subarray(mark.length);
113
+ return { text: decodeAs({ kind }, body), encoding: { kind }, bom: true };
114
+ }
115
+ }
116
+ const label = charsetOf(mimeType);
117
+ const named = label === undefined ? undefined : encodingName(label);
118
+ if (named === 'utf-16le' || named === 'utf-16be')
119
+ return decodedAs({ kind: named }, bytes);
120
+ const table = named === undefined || named === 'utf-8' ? null : byteTable(named);
121
+ if (table)
122
+ return decodedAs({ kind: 'table', table }, bytes);
123
+ // why: a charset with no table, such as Shift_JIS, is read byte for byte, so its ASCII is read and every byte comes back as it was.
124
+ if (named !== undefined && named !== 'utf-8')
125
+ return decodedAs({ kind: 'bytes', name: named }, bytes);
126
+ const utf8 = validUtf8(bytes);
127
+ if (utf8 !== undefined)
128
+ return { text: utf8, encoding: { kind: 'utf-8' }, bom: false };
129
+ const fallback = byteTable(FALLBACK_CHARSET);
130
+ if (!fallback)
131
+ throw new TheoremError('internal', `${FALLBACK_CHARSET} has no byte table`); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
132
+ return decodedAs({ kind: 'table', table: fallback }, bytes);
133
+ }
134
+ function decodedAs(encoding, bytes) {
135
+ return { text: decodeAs(encoding, bytes), encoding, bom: false };
136
+ }
137
+ function decodeAs(encoding, bytes) {
138
+ if (encoding.kind === 'utf-8' || encoding.kind === 'utf-16le' || encoding.kind === 'utf-16be') {
139
+ return new TextDecoder(encoding.kind, { ignoreBOM: true }).decode(bytes);
140
+ }
141
+ const chars = encoding.kind === 'table' ? encoding.table.chars : undefined;
142
+ let text = '';
143
+ for (let at = 0; at < bytes.length; at += CHUNK) {
144
+ const chunk = bytes.subarray(at, at + CHUNK);
145
+ text += chars
146
+ ? Array.from(chunk, (byte) => chars[byte]).join('')
147
+ : String.fromCharCode(...chunk);
148
+ }
149
+ return text;
150
+ }
151
+ function encodeAs(encoding, text) {
152
+ if (encoding.kind === 'utf-8')
153
+ return new TextEncoder().encode(text);
154
+ if (encoding.kind === 'utf-16le' || encoding.kind === 'utf-16be') {
155
+ const bytes = new Uint8Array(text.length * 2);
156
+ const view = new DataView(bytes.buffer);
157
+ for (let index = 0; index < text.length; index++) {
158
+ view.setUint16(index * 2, text.charCodeAt(index), encoding.kind === 'utf-16le');
159
+ }
160
+ return bytes;
71
161
  }
162
+ const table = encoding.kind === 'table' ? encoding.table.bytes : undefined;
163
+ const bytes = [];
164
+ for (const char of text) {
165
+ const byte = table ? table.get(char) : latin1Byte(char);
166
+ // why: a character the charset has no byte for can only come from the guard's own placeholder, which is written as UTF-8.
167
+ if (byte === undefined)
168
+ bytes.push(...new TextEncoder().encode(char));
169
+ else
170
+ bytes.push(byte);
171
+ }
172
+ return Uint8Array.from(bytes);
173
+ }
174
+ function latin1Byte(char) {
175
+ const code = char.charCodeAt(0);
176
+ return char.length === 1 && code <= 0xff ? code : undefined;
72
177
  }
73
178
  /** CSV formula injection: formula-like cells get an apostrophe before they reach a model. */
74
179
  function sanitizeCsvText(text) {
@@ -81,13 +186,67 @@ function sanitizeCsvText(text) {
81
186
  function isTextFile(mimeType) {
82
187
  return TEXT_MIMES.has(mimeEssence(mimeType));
83
188
  }
84
- /** A text file's base64 bytes as the model reads them: decoded, and a CSV's formula cells escaped. */
189
+ /** A text file's base64 bytes, decoded; `null` when they are not base64. */
190
+ function decodeTextFile(mimeType, data) {
191
+ let bytes;
192
+ try {
193
+ bytes = base64ToBytes(data);
194
+ }
195
+ catch {
196
+ return null;
197
+ }
198
+ return decodeText(mimeType, bytes);
199
+ }
200
+ /** Whether a text file's bytes are base64, so it can be decoded and read. */
201
+ function isTextFileData(mimeType, data) {
202
+ return decodeTextFile(mimeType, data) !== null;
203
+ }
204
+ /** A text file's text; throws when its bytes are not base64, which every source of a file refuses first. */
85
205
  function textFileText(mimeType, data) {
86
- const text = decodeText(base64ToBytes(data));
87
- return mimeEssence(mimeType) === 'text/csv' ? sanitizeCsvText(text) : text;
206
+ const decoded = decodeTextFile(mimeType, data);
207
+ if (decoded === null) {
208
+ throw new TheoremError('internal', 'A text file was read before its bytes were checked'); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
209
+ }
210
+ return decoded.text;
88
211
  }
89
- /** Text as a text file's base64 bytes. */
90
- function textFileData(text) {
212
+ /** `text` as the file's base64 bytes, in the encoding the file was read in; the given bytes when the text is unchanged. */
213
+ function rewriteTextFile(mimeType, data, text) {
214
+ const decoded = decodeTextFile(mimeType, data);
215
+ if (decoded === null) {
216
+ throw new TheoremError('internal', 'A text file was written before its bytes were checked'); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
217
+ }
218
+ if (decoded.text === text)
219
+ return data;
220
+ const mark = decoded.bom
221
+ ? (BOMS.find(([kind]) => kind === decoded.encoding.kind)?.[1] ?? [])
222
+ : [];
223
+ const body = encodeAs(decoded.encoding, text);
224
+ const bytes = new Uint8Array(mark.length + body.length);
225
+ bytes.set(mark);
226
+ bytes.set(body, mark.length);
227
+ return bytesToBase64(bytes);
228
+ }
229
+ /** A text file as the model reads it: its text as UTF-8, under a charset that says so, with a CSV's formula cells escaped. */
230
+ function modelTextFile(mimeType, data) {
231
+ const decoded = decodeTextFile(mimeType, data);
232
+ if (decoded === null) {
233
+ throw new TheoremError('internal', 'A text file was read before its bytes were checked'); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
234
+ }
235
+ // why: a charset with no table is read byte for byte by the guard, and by its own decoder for the model, which reads UTF-8.
236
+ const read = decoded.encoding.kind === 'bytes'
237
+ ? new TextDecoder(decoded.encoding.name).decode(base64ToBytes(data))
238
+ : decoded.text;
239
+ const text = mimeEssence(mimeType) === 'text/csv' ? sanitizeCsvText(read) : read;
240
+ if (decoded.encoding.kind === 'utf-8' && !decoded.bom && text === decoded.text) {
241
+ return { mimeType, data };
242
+ }
243
+ return { mimeType: utf8MimeType(mimeType), data: utf8Data(text) };
244
+ }
245
+ /** The MIME type with a charset it names said as UTF-8. */
246
+ function utf8MimeType(mimeType) {
247
+ return charsetOf(mimeType) === undefined ? mimeType : mimeType.replace(CHARSET, '$1utf-8');
248
+ }
249
+ function utf8Data(text) {
91
250
  return bytesToBase64(new TextEncoder().encode(text));
92
251
  }
93
252
  function named(issue, name) {
@@ -207,7 +366,14 @@ function sanitizeAttachment(blob, reader) {
207
366
  if (!isTextFile(mimeType)) {
208
367
  return blob;
209
368
  }
210
- return { ...blob, data: textFileData(reader.read(textFileText(mimeType, data))) };
369
+ const model = modelTextFile(mimeType, data);
370
+ const text = textFileText(model.mimeType, model.data);
371
+ const read = reader.read(text);
372
+ return {
373
+ ...blob,
374
+ mimeType: model.mimeType,
375
+ data: rewriteTextFile(model.mimeType, model.data, read),
376
+ };
211
377
  }
212
378
  function hasTurnBlobs(attachments, voice) {
213
379
  return (attachments?.length ?? 0) > 0 || (voice?.length ?? 0) > 0;
@@ -222,4 +388,4 @@ function sanitizeTurnBlobs(profile, attachments, voice, at) {
222
388
  voice: voice?.length ? voice.map((blob) => sanitizeAttachment(blob, at.voice)) : voice,
223
389
  };
224
390
  }
225
- export { assertTurnAttachments, attachmentIssueCopy, attachmentIssues, attachmentIssueText, attachmentsRefused, base64Bytes, isTextFile, isTurnMediaRef, maxBytesForMime, requireMediaLimits, resolveMediaLimits, sanitizeCsvText, sanitizeTurnBlobs, textFileData, textFileText, };
391
+ export { assertTurnAttachments, attachmentIssueCopy, attachmentIssues, attachmentIssueText, attachmentsRefused, base64Bytes, isTextFile, isTextFileData, isTurnMediaRef, maxBytesForMime, modelTextFile, requireMediaLimits, resolveMediaLimits, rewriteTextFile, sanitizeCsvText, sanitizeTurnBlobs, textFileText, };
@@ -9,7 +9,7 @@ import { checkTaintGate, guardToolFailureText, guardToolResult, guardToolValue,
9
9
  import { recordToolCheck, startToolTrace, } from '../engine/tool-trace.js';
10
10
  import { sumTurnTokens } from '../engine/usage.js';
11
11
  import { jsonObjectSchema } from '../json.js';
12
- import { isTextFile, textFileData, textFileText } from '../registry/attachments.js';
12
+ import { isTextFile, rewriteTextFile, textFileText } from '../registry/attachments.js';
13
13
  import { isAwaitingUserInput, toolOutputPartRecord, } from '../turn-events.js';
14
14
  import { isRecord } from '../util/record.js';
15
15
  import { fillActivityLabel } from './activity-label.js';
@@ -1294,6 +1294,9 @@ function* guardResult(result, guard) {
1294
1294
  */
1295
1295
  function* readTextParts(result, guard, boundary) {
1296
1296
  const media = result.media;
1297
+ // invariant: a hidden tool's media is for the person only; the model reads none of it.
1298
+ if (result.hidden)
1299
+ return result;
1297
1300
  const texts = media?.map(partText) ?? [];
1298
1301
  if (!media || texts.every((text) => text === null))
1299
1302
  return result;
@@ -1313,7 +1316,11 @@ function* readTextParts(result, guard, boundary) {
1313
1316
  const text = kept.data[index];
1314
1317
  if (typeof text !== 'string')
1315
1318
  return part;
1316
- return part.type === 'text' ? { ...part, text } : { ...part, data: textFileData(text) };
1319
+ if (part.type === 'text')
1320
+ return { ...part, text };
1321
+ return 'data' in part
1322
+ ? { ...part, data: rewriteTextFile(part.mimeType, part.data, text) }
1323
+ : part;
1317
1324
  }));
1318
1325
  }
1319
1326
  /** The text the detectors read in a part: a text part's words, or a text file's decoded bytes. */
@@ -1330,18 +1337,17 @@ function* showParts(parts, guard) {
1330
1337
  const boundary = guard.kind && toolBoundary('tool_shown', guard.kind);
1331
1338
  const shown = [];
1332
1339
  for (const part of parts) {
1340
+ const file = 'data' in part ? part : undefined;
1333
1341
  // why: a text file's bytes are read decoded; other bytes are not text the detectors read. Every other field of the part is read as it is shown.
1334
- const { data, ...fields } = 'data' in part ? part : { ...part, data: undefined };
1335
- const text = data === undefined ? null : partText(part);
1336
- const read = text === null ? fields : { ...fields, data: text };
1337
- const checked = guardToolValue(read, guard.provenance, guard.policy, boundary);
1342
+ const text = file ? partText(file) : null;
1343
+ const checked = guardToolValue(file ? readable(file, text) : part, guard.provenance, guard.policy, boundary);
1338
1344
  if (checked.event)
1339
1345
  yield { type: 'guardrail', guardrail: checked.event };
1340
1346
  if (!('value' in checked))
1341
1347
  continue;
1342
- const kept = toolOutputPartRecord.safeParse(data === undefined || !isRecord(checked.value)
1343
- ? checked.value
1344
- : { ...checked.value, data: shownData(checked.value.data, text, data) });
1348
+ const kept = toolOutputPartRecord.safeParse(file && isRecord(checked.value)
1349
+ ? { ...checked.value, data: shownData(checked.value.data, text, file) }
1350
+ : checked.value);
1345
1351
  if (!kept.success) {
1346
1352
  throw new TheoremError('internal', 'A guard changed the shape of a part it read'); // labels-exempt: developer contract / internal diagnostic — not end-user or model copy (P2)
1347
1353
  }
@@ -1370,12 +1376,19 @@ function shownFiles(read, output) {
1370
1376
  ...read,
1371
1377
  parts: read.parts.map((part, index) => {
1372
1378
  const file = given[index];
1373
- if (!isFilePart(file) || partText(file) === null || !isRecord(part))
1379
+ if (!isFilePart(file) || !isRecord(part) || typeof part.data !== 'string')
1374
1380
  return part;
1375
- return typeof part.data === 'string' ? { ...part, data: textFileData(part.data) } : part;
1381
+ const text = partText(file);
1382
+ return text === null
1383
+ ? part
1384
+ : { ...part, data: rewriteTextFile(file.mimeType, file.data, part.data) };
1376
1385
  }),
1377
1386
  };
1378
1387
  }
1388
+ /** A part with bytes as the detectors read it: a text file's bytes as its text, other bytes left out. */
1389
+ function readable({ data: _data, ...fields }, text) {
1390
+ return text === null ? fields : { ...fields, data: text };
1391
+ }
1379
1392
  /** Whether `value` is a part that carries bytes. */
1380
1393
  function isFilePart(value) {
1381
1394
  return (isRecord(value) &&
@@ -1384,8 +1397,10 @@ function isFilePart(value) {
1384
1397
  typeof value.mimeType === 'string');
1385
1398
  }
1386
1399
  /** A shown part's bytes after its read: a text file's read text encoded again, any other part's bytes as they were. */
1387
- function shownData(read, text, data) {
1388
- return text === null ? data : typeof read === 'string' ? textFileData(read) : read;
1400
+ function shownData(read, text, given) {
1401
+ if (text === null)
1402
+ return given.data;
1403
+ return typeof read === 'string' ? rewriteTextFile(given.mimeType, given.data, read) : read;
1389
1404
  }
1390
1405
  /** What the person is shown of `value`, read at `tool_shown_<kind>`; a blocked one is the labels' words. */
1391
1406
  function* showValue(value, guard) {
@@ -1,9 +1,8 @@
1
1
  import { z } from 'zod';
2
2
  import { TheoremError } from '../../guardrails/error.js';
3
- import { isTextFile } from '../registry/attachments.js';
3
+ import { isTextFile, isTextFileData } from '../registry/attachments.js';
4
4
  import { MEDIA_INPUT_KIND_VALUES } from '../schema.js';
5
5
  import { toolOutputPartRecord } from '../turn-events.js';
6
- import { base64ToBytes } from '../util/base64.js';
7
6
  import { isRecord } from '../util/record.js';
8
7
  const mimeType = z.string().regex(/\S/);
9
8
  const metadata = z.record(z.string(), z.unknown()).optional();
@@ -16,20 +15,10 @@ const mediaPart = z
16
15
  sent: z.literal(true).optional(),
17
16
  metadata,
18
17
  })
19
- .refine((part) => !isTextFile(part.mimeType) || isBase64(part.data), {
18
+ .refine((part) => !isTextFile(part.mimeType) || isTextFileData(part.mimeType, part.data), {
20
19
  path: ['data'],
21
20
  message: 'A text file must be base64, so it can be decoded and read', // labels-exempt: developer contract error
22
21
  });
23
- /** Whether `data` decodes as base64. */
24
- function isBase64(data) {
25
- try {
26
- base64ToBytes(data);
27
- return true;
28
- }
29
- catch {
30
- return false;
31
- }
32
- }
33
22
  /** A reference part: the person is shown it through the app's own resolver. */
34
23
  export const toolOutputRefPartSchema = z.strictObject({
35
24
  type: z.enum(MEDIA_INPUT_KIND_VALUES),
@@ -4,6 +4,7 @@ import { fetchGuarded } from '../../guardrails/network.js';
4
4
  import { retryTransient } from '../../providers/shared/retry.js';
5
5
  import { refreshOAuthToken, tokenAudienceCovers } from '../auth/oauth.js';
6
6
  import { mapStrings } from '../engine/tree.js';
7
+ import { isTextFile, modelTextFile } from '../registry/attachments.js';
7
8
  import { mediaKindForMime } from '../registry/catalog.js';
8
9
  import { dispatchToolBody, guardToolTarget, messageOf, networkBlocked, networkBlockedEvent, requestChecks, startToolExecution, toolEvent, toolNetworkPolicy, } from './events.js';
9
10
  import { answeredIn, takeLimitPlace } from './http-limit.js';
@@ -279,7 +280,8 @@ function readParts(media) {
279
280
  return [part];
280
281
  if (!('data' in part))
281
282
  return [];
282
- return [{ type: part.type, mimeType: part.mimeType, data: part.data }];
283
+ const file = isTextFile(part.mimeType) ? modelTextFile(part.mimeType, part.data) : part;
284
+ return [{ type: part.type, mimeType: file.mimeType, data: file.data }];
283
285
  });
284
286
  }
285
287
  function failureOutcome(failure, callNotStarted) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@theoremjs/agents",
3
- "version": "0.3.5-next.949.gac4d2f6",
3
+ "version": "0.3.5-next.950.g9bb8f33",
4
4
  "description": "A flat TypeScript agent kernel for typed profiles, deterministic turn execution, registered tools, provider adapters, guardrails, and host-injected traces.",
5
5
  "keywords": [
6
6
  "agent",