@tiangong-ai/cli 0.0.20 → 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -2
- package/README.md +158 -13
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +11 -1
- package/dist/cli.js.map +1 -1
- package/dist/research/orchestration.js +166 -11
- package/dist/research/orchestration.js.map +1 -1
- package/dist/research/workspace/broker.js +407 -31
- package/dist/research/workspace/broker.js.map +1 -1
- package/dist/research/workspace/capabilities.js +65 -0
- package/dist/research/workspace/capabilities.js.map +1 -1
- package/dist/research/workspace/constants.d.ts +1 -1
- package/dist/research/workspace/constants.js +4 -0
- package/dist/research/workspace/constants.js.map +1 -1
- package/dist/research/workspace/evidence.d.ts +32 -0
- package/dist/research/workspace/evidence.js +235 -0
- package/dist/research/workspace/evidence.js.map +1 -0
- package/dist/research/workspace/executor.d.ts +11 -1
- package/dist/research/workspace/executor.js +678 -105
- package/dist/research/workspace/executor.js.map +1 -1
- package/dist/research/workspace/input-plan.d.ts +5 -0
- package/dist/research/workspace/input-plan.js +319 -0
- package/dist/research/workspace/input-plan.js.map +1 -0
- package/dist/research/workspace/journal.js +2 -1
- package/dist/research/workspace/journal.js.map +1 -1
- package/dist/research/workspace/preflight.d.ts +108 -0
- package/dist/research/workspace/preflight.js +261 -0
- package/dist/research/workspace/preflight.js.map +1 -0
- package/dist/research/workspace/projects.d.ts +5 -2
- package/dist/research/workspace/projects.js +290 -6
- package/dist/research/workspace/projects.js.map +1 -1
- package/dist/research/workspace/runtime.d.ts +5 -1
- package/dist/research/workspace/runtime.js +1314 -196
- package/dist/research/workspace/runtime.js.map +1 -1
- package/dist/research/workspace/sanitization.d.ts +5 -0
- package/dist/research/workspace/sanitization.js +72 -0
- package/dist/research/workspace/sanitization.js.map +1 -0
- package/dist/research/workspace/schemas.d.ts +17 -0
- package/dist/research/workspace/schemas.js +342 -0
- package/dist/research/workspace/schemas.js.map +1 -0
- package/dist/research/workspace/storage.js +4 -0
- package/dist/research/workspace/storage.js.map +1 -1
- package/dist/research/workspace/types.d.ts +160 -0
- package/dist/research/workspace/workspace.d.ts +14 -3
- package/dist/research/workspace/workspace.js +374 -12
- package/dist/research/workspace/workspace.js.map +1 -1
- package/package.json +2 -1
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { createServer } from "node:http";
|
|
3
3
|
import { readFile, writeFile } from "node:fs/promises";
|
|
4
|
-
import {
|
|
4
|
+
import { dirname } from "node:path";
|
|
5
5
|
import { CliError } from "../../errors.js";
|
|
6
6
|
import { loadCapabilityDeclarations, verifyCapabilities } from "./capabilities.js";
|
|
7
|
+
import { loadBrokerEvidenceCache, persistBrokerEvidence, storeBrokerEvidenceCache, } from "./evidence.js";
|
|
7
8
|
import { appendJournalEvent } from "./journal.js";
|
|
8
|
-
import {
|
|
9
|
+
import { sanitizeResearchRecord, sanitizeResearchText } from "./sanitization.js";
|
|
10
|
+
import { canonicalJson, ensureDirectory, isObject, pathExists, resolveContained, sha256Text, workspacePaths, } from "./storage.js";
|
|
11
|
+
import { loadWorkspaceConfig } from "./workspace.js";
|
|
9
12
|
const MAX_REQUEST_BYTES = 1024 * 1024;
|
|
10
|
-
const
|
|
13
|
+
const MAX_ERROR_RESPONSE_BYTES = 16 * 1024;
|
|
14
|
+
const BROKER_CONTEXT_BYTES_PER_TOKEN = 3;
|
|
11
15
|
export async function startCapabilityBroker(root, projectId, capsuleProject) {
|
|
12
16
|
const declarations = await loadCapabilityDeclarations(root);
|
|
13
17
|
const networkCapabilities = declarations.capabilities.filter((capability) => capability.permissions.includes("brokered-network"));
|
|
@@ -22,6 +26,7 @@ export async function startCapabilityBroker(root, projectId, capsuleProject) {
|
|
|
22
26
|
});
|
|
23
27
|
}
|
|
24
28
|
const credentialMap = await loadCredentialMap(root, declarations.capabilities);
|
|
29
|
+
const config = await loadWorkspaceConfig(root);
|
|
25
30
|
const routeToken = randomUUID().replaceAll("-", "");
|
|
26
31
|
const route = `/mcp/${routeToken}`;
|
|
27
32
|
const server = createServer((request, response) => {
|
|
@@ -34,6 +39,9 @@ export async function startCapabilityBroker(root, projectId, capsuleProject) {
|
|
|
34
39
|
capsuleProject,
|
|
35
40
|
capabilities: networkCapabilities,
|
|
36
41
|
credentialMap,
|
|
42
|
+
workspaceResponseBytes: config.budget.maxBrokerResponseBytes,
|
|
43
|
+
workspaceContextTokens: config.budget.maxBrokerContextTokens,
|
|
44
|
+
workspaceMaxItems: config.budget.maxBrokerItems,
|
|
37
45
|
});
|
|
38
46
|
});
|
|
39
47
|
await new Promise((resolvePromise, reject) => {
|
|
@@ -80,7 +88,7 @@ async function handleMcpRequest(input) {
|
|
|
80
88
|
tools: [
|
|
81
89
|
{
|
|
82
90
|
name: "fetch_candidate_source",
|
|
83
|
-
description: "Fetch one HTTPS candidate source through a locked capability and
|
|
91
|
+
description: "Fetch one bounded HTTPS candidate source through a locked capability, persist the raw response, and return content-addressed provenance plus a bounded context view.",
|
|
84
92
|
inputSchema: {
|
|
85
93
|
type: "object",
|
|
86
94
|
additionalProperties: false,
|
|
@@ -89,6 +97,10 @@ async function handleMcpRequest(input) {
|
|
|
89
97
|
capability_id: { type: "string" },
|
|
90
98
|
credential_id: { type: "string" },
|
|
91
99
|
url: { type: "string" },
|
|
100
|
+
json_pointer: { type: "string" },
|
|
101
|
+
item_offset: { type: "integer", minimum: 0 },
|
|
102
|
+
max_items: { type: "integer", minimum: 1 },
|
|
103
|
+
cache_mode: { enum: ["prefer", "bypass"] },
|
|
92
104
|
},
|
|
93
105
|
},
|
|
94
106
|
},
|
|
@@ -111,6 +123,9 @@ async function handleMcpRequest(input) {
|
|
|
111
123
|
capsuleProject: input.capsuleProject,
|
|
112
124
|
capabilities: input.capabilities,
|
|
113
125
|
credentialMap: input.credentialMap,
|
|
126
|
+
workspaceResponseBytes: input.workspaceResponseBytes,
|
|
127
|
+
workspaceContextTokens: input.workspaceContextTokens,
|
|
128
|
+
workspaceMaxItems: input.workspaceMaxItems,
|
|
114
129
|
arguments: params.arguments,
|
|
115
130
|
});
|
|
116
131
|
sendRpcResult(input.response, body, {
|
|
@@ -118,7 +133,12 @@ async function handleMcpRequest(input) {
|
|
|
118
133
|
});
|
|
119
134
|
}
|
|
120
135
|
catch (error) {
|
|
121
|
-
|
|
136
|
+
const detail = error instanceof CliError
|
|
137
|
+
? JSON.stringify({ code: error.code, message: error.message, details: error.details })
|
|
138
|
+
: error instanceof Error
|
|
139
|
+
? error.message
|
|
140
|
+
: String(error);
|
|
141
|
+
sendToolError(input.response, body, sanitizeResearchText(detail, [...input.credentialMap.values()]));
|
|
122
142
|
}
|
|
123
143
|
return;
|
|
124
144
|
}
|
|
@@ -126,7 +146,9 @@ async function handleMcpRequest(input) {
|
|
|
126
146
|
}
|
|
127
147
|
catch (error) {
|
|
128
148
|
sendJson(input.response, 500, {
|
|
129
|
-
error: error instanceof Error ? error.message : String(error),
|
|
149
|
+
error: sanitizeResearchText(error instanceof Error ? error.message : String(error), [
|
|
150
|
+
...input.credentialMap.values(),
|
|
151
|
+
]),
|
|
130
152
|
});
|
|
131
153
|
}
|
|
132
154
|
}
|
|
@@ -134,37 +156,96 @@ async function fetchCandidateSource(input) {
|
|
|
134
156
|
const capabilityId = input.arguments.capability_id;
|
|
135
157
|
const credentialId = input.arguments.credential_id;
|
|
136
158
|
const rawUrl = input.arguments.url;
|
|
159
|
+
const jsonPointer = input.arguments.json_pointer;
|
|
160
|
+
const requestedItemOffset = input.arguments.item_offset ?? 0;
|
|
161
|
+
const requestedMaxItems = input.arguments.max_items;
|
|
162
|
+
const cacheMode = input.arguments.cache_mode ?? "prefer";
|
|
137
163
|
if (typeof capabilityId !== "string" || typeof rawUrl !== "string") {
|
|
138
164
|
throw new Error("capability_id and url are required strings");
|
|
139
165
|
}
|
|
140
166
|
if (credentialId !== undefined && typeof credentialId !== "string") {
|
|
141
167
|
throw new Error("credential_id must be a string when provided");
|
|
142
168
|
}
|
|
169
|
+
if (jsonPointer !== undefined &&
|
|
170
|
+
(typeof jsonPointer !== "string" || !validJsonPointer(jsonPointer))) {
|
|
171
|
+
throw new Error("json_pointer must be an RFC 6901 JSON Pointer when provided");
|
|
172
|
+
}
|
|
173
|
+
if (typeof requestedItemOffset !== "number" ||
|
|
174
|
+
!Number.isInteger(requestedItemOffset) ||
|
|
175
|
+
requestedItemOffset < 0) {
|
|
176
|
+
throw new Error("item_offset must be a non-negative integer when provided");
|
|
177
|
+
}
|
|
178
|
+
if (requestedMaxItems !== undefined &&
|
|
179
|
+
(typeof requestedMaxItems !== "number" ||
|
|
180
|
+
!Number.isInteger(requestedMaxItems) ||
|
|
181
|
+
requestedMaxItems < 1)) {
|
|
182
|
+
throw new Error("max_items must be a positive integer when provided");
|
|
183
|
+
}
|
|
184
|
+
if (cacheMode !== "prefer" && cacheMode !== "bypass") {
|
|
185
|
+
throw new Error('cache_mode must be "prefer" or "bypass"');
|
|
186
|
+
}
|
|
143
187
|
const capability = input.capabilities.find((candidate) => candidate.id === capabilityId);
|
|
144
188
|
if (!capability)
|
|
145
189
|
throw new Error(`capability is not admitted for brokered network: ${capabilityId}`);
|
|
146
190
|
const target = validateHttpsUrl(rawUrl);
|
|
191
|
+
if (!capability.http)
|
|
192
|
+
throw new Error(`capability has no broker HTTP policy: ${capabilityId}`);
|
|
147
193
|
const credential = credentialId
|
|
148
194
|
? capability.credentials.find((candidate) => candidate.id === credentialId)
|
|
149
195
|
: undefined;
|
|
150
196
|
if (credentialId && !credential) {
|
|
151
197
|
throw new Error(`credential is not declared by capability ${capabilityId}: ${credentialId}`);
|
|
152
198
|
}
|
|
199
|
+
if (credential && cacheMode === "prefer") {
|
|
200
|
+
throw new Error("credentialed broker requests require cache_mode=bypass");
|
|
201
|
+
}
|
|
153
202
|
assertAllowedHost(target, capability.allowedHosts, "capability");
|
|
154
203
|
if (credential)
|
|
155
204
|
assertAllowedHost(target, credential.allowedHosts, "credential");
|
|
156
205
|
const attemptId = randomUUID();
|
|
206
|
+
const maxItems = Math.min(requestedMaxItems ?? capability.http.maxItems, capability.http.maxItems, input.workspaceMaxItems);
|
|
207
|
+
const cacheKeySha256 = sha256Text(canonicalJson({
|
|
208
|
+
capabilityId,
|
|
209
|
+
targetSha256: sha256Text(target.toString()),
|
|
210
|
+
accept: capability.http.accept,
|
|
211
|
+
}));
|
|
157
212
|
await appendJournalEvent(workspacePaths(input.root).journal, "capability.fetch.attempted", input.projectId, {
|
|
158
213
|
attemptId,
|
|
159
214
|
projectId: input.projectId,
|
|
160
215
|
capabilityId,
|
|
161
216
|
credentialId: credentialId ?? null,
|
|
162
217
|
targetSha256: sha256Text(target.toString()),
|
|
218
|
+
cacheMode,
|
|
219
|
+
cacheKeySha256,
|
|
163
220
|
});
|
|
164
221
|
try {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
222
|
+
if (cacheMode === "prefer") {
|
|
223
|
+
const cached = await loadBrokerEvidenceCache(input.root, cacheKeySha256);
|
|
224
|
+
if (cached) {
|
|
225
|
+
const raw = await readFile(resolveContained(workspacePaths(input.root).control, cached.locator));
|
|
226
|
+
const context = buildContextView(raw, cached.contentType, jsonPointer, requestedItemOffset, maxItems, input.workspaceContextTokens * BROKER_CONTEXT_BYTES_PER_TOKEN);
|
|
227
|
+
const receipt = await persistBrokerEvidence(input.root, {
|
|
228
|
+
attemptId,
|
|
229
|
+
projectId: input.projectId,
|
|
230
|
+
capabilityId,
|
|
231
|
+
credentialId: null,
|
|
232
|
+
status: cached.status,
|
|
233
|
+
contentType: cached.contentType,
|
|
234
|
+
sourceSha256: cached.sourceSha256,
|
|
235
|
+
contextItems: context.items,
|
|
236
|
+
contextOffset: context.offset,
|
|
237
|
+
contextTotalItems: context.totalItems,
|
|
238
|
+
contextNextOffset: context.nextOffset,
|
|
239
|
+
contextTruncated: context.truncated,
|
|
240
|
+
retrievedAt: cached.retrievedAt,
|
|
241
|
+
cacheHit: true,
|
|
242
|
+
}, raw, context.bytes);
|
|
243
|
+
await stageContextObject(input.capsuleProject, receipt.contextLocator, context.bytes);
|
|
244
|
+
await appendCompletedReceipt(input.root, input.projectId, receipt, cacheKeySha256);
|
|
245
|
+
return { ...receipt };
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
const headers = new Headers({ Accept: capability.http.accept });
|
|
168
249
|
if (credential) {
|
|
169
250
|
const value = input.credentialMap.get(credential.id);
|
|
170
251
|
if (!value)
|
|
@@ -172,50 +253,151 @@ async function fetchCandidateSource(input) {
|
|
|
172
253
|
headers.set(credential.headerName, `${credential.prefix}${value}`);
|
|
173
254
|
}
|
|
174
255
|
const { response, finalUrl } = await fetchWithRedirectPolicy(target, headers, capability.allowedHosts, credential?.allowedHosts);
|
|
256
|
+
const responseLimit = Math.min(capability.http.maxResponseBytes, input.workspaceResponseBytes);
|
|
175
257
|
const announcedLength = Number(response.headers.get("content-length") ?? "0");
|
|
176
|
-
if (announcedLength >
|
|
258
|
+
if (response.ok && announcedLength > responseLimit)
|
|
177
259
|
throw new Error("response exceeds the broker size limit");
|
|
178
|
-
const bytes = await readBoundedResponseBody(response);
|
|
260
|
+
const bytes = await readBoundedResponseBody(response, response.ok ? responseLimit : Math.min(responseLimit, MAX_ERROR_RESPONSE_BYTES), !response.ok);
|
|
179
261
|
for (const secret of input.credentialMap.values()) {
|
|
180
262
|
if (bytes.includes(Buffer.from(secret, "utf8"))) {
|
|
181
263
|
throw new Error("response failed credential disclosure screening");
|
|
182
264
|
}
|
|
183
265
|
}
|
|
184
|
-
if (!response.ok)
|
|
185
|
-
throw new Error(`HTTPS source returned status ${response.status}`);
|
|
186
266
|
const contentType = response.headers.get("content-type")?.split(";", 1)[0]?.trim() ?? "application/octet-stream";
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
: "
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
267
|
+
if (!response.ok) {
|
|
268
|
+
const retryAfterSeconds = parseRetryAfter(response.headers.get("retry-after"));
|
|
269
|
+
const excerpt = safeResponseExcerpt(bytes, contentType, [...input.credentialMap.values()]);
|
|
270
|
+
throw new CliError(`HTTPS source returned status ${response.status}.`, {
|
|
271
|
+
code: "RESEARCH_BROKER_HTTP_ERROR",
|
|
272
|
+
exitCode: 3,
|
|
273
|
+
details: {
|
|
274
|
+
status: response.status,
|
|
275
|
+
retryAfterSeconds,
|
|
276
|
+
responseExcerpt: excerpt,
|
|
277
|
+
requestId: safeResponseId(response.headers),
|
|
278
|
+
},
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
if (!contentTypeAllowed(contentType, capability.http.allowedContentTypes)) {
|
|
282
|
+
throw new CliError(`HTTPS source returned unsupported content type ${contentType}.`, {
|
|
283
|
+
code: "RESEARCH_BROKER_CONTENT_TYPE_INVALID",
|
|
284
|
+
exitCode: 3,
|
|
285
|
+
details: { contentType, allowedContentTypes: capability.http.allowedContentTypes },
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
assertNoSensitiveResponseMaterial(bytes, contentType);
|
|
289
|
+
const context = buildContextView(bytes, contentType, jsonPointer, requestedItemOffset, maxItems, input.workspaceContextTokens * BROKER_CONTEXT_BYTES_PER_TOKEN);
|
|
290
|
+
const receipt = await persistBrokerEvidence(input.root, {
|
|
196
291
|
attemptId,
|
|
292
|
+
projectId: input.projectId,
|
|
197
293
|
capabilityId,
|
|
198
294
|
credentialId: credentialId ?? null,
|
|
199
295
|
status: response.status,
|
|
200
296
|
contentType,
|
|
201
|
-
bytes: bytes.length,
|
|
202
|
-
sha256: sha256Bytes(bytes),
|
|
203
297
|
sourceSha256: sha256Text(finalUrl.toString()),
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
298
|
+
contextItems: context.items,
|
|
299
|
+
contextOffset: context.offset,
|
|
300
|
+
contextTotalItems: context.totalItems,
|
|
301
|
+
contextNextOffset: context.nextOffset,
|
|
302
|
+
contextTruncated: context.truncated,
|
|
303
|
+
retrievedAt: new Date().toISOString(),
|
|
304
|
+
cacheHit: false,
|
|
305
|
+
}, bytes, context.bytes);
|
|
306
|
+
await stageContextObject(input.capsuleProject, receipt.contextLocator, context.bytes);
|
|
307
|
+
if (!credential)
|
|
308
|
+
await storeBrokerEvidenceCache(input.root, cacheKeySha256, receipt);
|
|
309
|
+
await appendCompletedReceipt(input.root, input.projectId, receipt, cacheKeySha256);
|
|
310
|
+
return { ...receipt };
|
|
208
311
|
}
|
|
209
312
|
catch (error) {
|
|
313
|
+
const safeDetails = error instanceof CliError && isObject(error.details)
|
|
314
|
+
? sanitizeResearchRecord(error.details, [...input.credentialMap.values()])
|
|
315
|
+
: {};
|
|
210
316
|
await appendJournalEvent(workspacePaths(input.root).journal, "capability.fetch.failed", input.projectId, {
|
|
211
317
|
attemptId,
|
|
212
318
|
capabilityId,
|
|
213
319
|
credentialId: credentialId ?? null,
|
|
214
|
-
error: bounded(error instanceof Error ? error.message : String(error),
|
|
320
|
+
error: bounded(sanitizeResearchText(error instanceof Error ? error.message : String(error), [
|
|
321
|
+
...input.credentialMap.values(),
|
|
322
|
+
]), 500),
|
|
323
|
+
failureKind: brokerFailureKind(error),
|
|
324
|
+
...safeDetails,
|
|
215
325
|
});
|
|
216
326
|
throw error;
|
|
217
327
|
}
|
|
218
328
|
}
|
|
329
|
+
function assertNoSensitiveResponseMaterial(bytes, contentType) {
|
|
330
|
+
if (!contentType.includes("json") && !contentType.startsWith("text/"))
|
|
331
|
+
return;
|
|
332
|
+
const text = bytes.toString("utf8");
|
|
333
|
+
if (/\b(Bearer|Basic)\s+[A-Za-z0-9._~+\/-]+=*/i.test(text) ||
|
|
334
|
+
/\b(authorization|cookie|set-cookie|x-api-key|api-key)\s*:/i.test(text) ||
|
|
335
|
+
/\b(access_token|api[_-]?key|apikey|password|secret|session|token)\s*=/i.test(text)) {
|
|
336
|
+
throw new Error("response contains credential-like material and was not persisted");
|
|
337
|
+
}
|
|
338
|
+
if (!contentType.includes("json"))
|
|
339
|
+
return;
|
|
340
|
+
try {
|
|
341
|
+
const value = JSON.parse(text);
|
|
342
|
+
if (containsSensitiveJsonField(value)) {
|
|
343
|
+
throw new Error("response contains credential-like material and was not persisted");
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
catch (error) {
|
|
347
|
+
if (error instanceof SyntaxError)
|
|
348
|
+
return;
|
|
349
|
+
throw error;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
function containsSensitiveJsonField(value) {
|
|
353
|
+
if (Array.isArray(value))
|
|
354
|
+
return value.some(containsSensitiveJsonField);
|
|
355
|
+
if (!isObject(value))
|
|
356
|
+
return false;
|
|
357
|
+
const sensitive = /^(access_token|api[_-]?key|apikey|authorization|cookie|password|secret|session|token)$/i;
|
|
358
|
+
return Object.entries(value).some(([key, item]) => (sensitive.test(key) && item !== null && item !== "") || containsSensitiveJsonField(item));
|
|
359
|
+
}
|
|
360
|
+
async function stageContextObject(capsuleProject, locator, bytes) {
|
|
361
|
+
const destination = resolveContained(capsuleProject, locator);
|
|
362
|
+
await ensureDirectory(dirname(destination));
|
|
363
|
+
try {
|
|
364
|
+
await writeFile(destination, bytes, { mode: 0o600, flag: "wx" });
|
|
365
|
+
}
|
|
366
|
+
catch (error) {
|
|
367
|
+
if (error.code !== "EEXIST")
|
|
368
|
+
throw error;
|
|
369
|
+
if (!Buffer.from(await readFile(destination)).equals(Buffer.from(bytes))) {
|
|
370
|
+
throw new Error("staged broker context object failed its integrity check");
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
async function appendCompletedReceipt(root, projectId, receipt, cacheKeySha256) {
|
|
375
|
+
await appendJournalEvent(workspacePaths(root).journal, "capability.fetch.completed", projectId, {
|
|
376
|
+
attemptId: receipt.attemptId,
|
|
377
|
+
projectId: receipt.projectId,
|
|
378
|
+
capabilityId: receipt.capabilityId,
|
|
379
|
+
credentialId: receipt.credentialId,
|
|
380
|
+
status: receipt.status,
|
|
381
|
+
contentType: receipt.contentType,
|
|
382
|
+
bytes: receipt.bytes,
|
|
383
|
+
sha256: receipt.sha256,
|
|
384
|
+
sourceSha256: receipt.sourceSha256,
|
|
385
|
+
locator: receipt.locator,
|
|
386
|
+
contextLocator: receipt.contextLocator,
|
|
387
|
+
contextSha256: receipt.contextSha256,
|
|
388
|
+
contextBytes: receipt.contextBytes,
|
|
389
|
+
contextEstimatedTokens: receipt.contextEstimatedTokens,
|
|
390
|
+
contextItems: receipt.contextItems,
|
|
391
|
+
contextOffset: receipt.contextOffset ?? 0,
|
|
392
|
+
contextTotalItems: receipt.contextTotalItems ?? null,
|
|
393
|
+
contextNextOffset: receipt.contextNextOffset ?? null,
|
|
394
|
+
contextTruncated: receipt.contextTruncated,
|
|
395
|
+
retrievedAt: receipt.retrievedAt,
|
|
396
|
+
servedAt: receipt.servedAt,
|
|
397
|
+
cacheHit: receipt.cacheHit,
|
|
398
|
+
cacheKeySha256,
|
|
399
|
+
});
|
|
400
|
+
}
|
|
219
401
|
async function loadCredentialMap(root, capabilities) {
|
|
220
402
|
const path = workspacePaths(root).env;
|
|
221
403
|
if (!(await pathExists(path)))
|
|
@@ -280,7 +462,7 @@ function assertAllowedHost(url, allowedHosts, scope) {
|
|
|
280
462
|
throw new Error(`target host is outside ${scope} scope: ${url.host}`);
|
|
281
463
|
}
|
|
282
464
|
}
|
|
283
|
-
async function readBoundedResponseBody(response) {
|
|
465
|
+
async function readBoundedResponseBody(response, maxBytes, truncate = false) {
|
|
284
466
|
if (!response.body)
|
|
285
467
|
return Buffer.alloc(0);
|
|
286
468
|
const reader = response.body.getReader();
|
|
@@ -293,8 +475,14 @@ async function readBoundedResponseBody(response) {
|
|
|
293
475
|
break;
|
|
294
476
|
const chunk = Buffer.from(result.value);
|
|
295
477
|
bytes += chunk.length;
|
|
296
|
-
if (bytes >
|
|
478
|
+
if (bytes > maxBytes) {
|
|
297
479
|
await reader.cancel();
|
|
480
|
+
if (truncate) {
|
|
481
|
+
const remaining = maxBytes - chunks.reduce((sum, item) => sum + item.length, 0);
|
|
482
|
+
if (remaining > 0)
|
|
483
|
+
chunks.push(chunk.subarray(0, remaining));
|
|
484
|
+
break;
|
|
485
|
+
}
|
|
298
486
|
throw new Error("response exceeds the broker size limit");
|
|
299
487
|
}
|
|
300
488
|
chunks.push(chunk);
|
|
@@ -305,6 +493,194 @@ async function readBoundedResponseBody(response) {
|
|
|
305
493
|
}
|
|
306
494
|
return Buffer.concat(chunks);
|
|
307
495
|
}
|
|
496
|
+
function buildContextView(bytes, contentType, jsonPointer, itemOffset, maxItems, maxContextBytes) {
|
|
497
|
+
if (!Number.isInteger(maxContextBytes) || maxContextBytes < 1) {
|
|
498
|
+
throw new Error("broker context byte limit is invalid");
|
|
499
|
+
}
|
|
500
|
+
if (!contentType.includes("json")) {
|
|
501
|
+
if (itemOffset !== 0)
|
|
502
|
+
throw new Error("item_offset is supported only for JSON collections");
|
|
503
|
+
const boundedBytes = contentType.startsWith("text/")
|
|
504
|
+
? truncateUtf8(bytes, maxContextBytes)
|
|
505
|
+
: bytes.subarray(0, maxContextBytes);
|
|
506
|
+
return {
|
|
507
|
+
bytes: boundedBytes,
|
|
508
|
+
items: null,
|
|
509
|
+
offset: 0,
|
|
510
|
+
totalItems: null,
|
|
511
|
+
nextOffset: null,
|
|
512
|
+
truncated: boundedBytes.byteLength < bytes.byteLength,
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
let parsed;
|
|
516
|
+
try {
|
|
517
|
+
parsed = JSON.parse(bytes.toString("utf8"));
|
|
518
|
+
}
|
|
519
|
+
catch {
|
|
520
|
+
throw new Error("JSON response body is not valid JSON");
|
|
521
|
+
}
|
|
522
|
+
const selected = jsonPointer === undefined ? parsed : resolveJsonPointer(parsed, jsonPointer);
|
|
523
|
+
if (Array.isArray(selected)) {
|
|
524
|
+
const limited = [];
|
|
525
|
+
for (const item of selected.slice(itemOffset, itemOffset + maxItems)) {
|
|
526
|
+
const candidate = [...limited, item];
|
|
527
|
+
if (jsonBytes(candidate).byteLength > maxContextBytes)
|
|
528
|
+
break;
|
|
529
|
+
limited.push(item);
|
|
530
|
+
}
|
|
531
|
+
const nextOffset = itemOffset + limited.length < selected.length ? itemOffset + limited.length : null;
|
|
532
|
+
return {
|
|
533
|
+
bytes: jsonBytes(limited),
|
|
534
|
+
items: limited.length,
|
|
535
|
+
offset: itemOffset,
|
|
536
|
+
totalItems: selected.length,
|
|
537
|
+
nextOffset,
|
|
538
|
+
truncated: itemOffset > 0 || nextOffset !== null,
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
if (isObject(selected)) {
|
|
542
|
+
const entries = Object.entries(selected);
|
|
543
|
+
const limited = [];
|
|
544
|
+
for (const entry of entries.slice(itemOffset, itemOffset + maxItems)) {
|
|
545
|
+
const candidate = [...limited, entry];
|
|
546
|
+
if (jsonBytes(Object.fromEntries(candidate)).byteLength > maxContextBytes)
|
|
547
|
+
break;
|
|
548
|
+
limited.push(entry);
|
|
549
|
+
}
|
|
550
|
+
const nextOffset = itemOffset + limited.length < entries.length ? itemOffset + limited.length : null;
|
|
551
|
+
return {
|
|
552
|
+
bytes: jsonBytes(Object.fromEntries(limited)),
|
|
553
|
+
items: limited.length,
|
|
554
|
+
offset: itemOffset,
|
|
555
|
+
totalItems: entries.length,
|
|
556
|
+
nextOffset,
|
|
557
|
+
truncated: itemOffset > 0 || nextOffset !== null,
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
if (itemOffset !== 0)
|
|
561
|
+
throw new Error("item_offset is supported only for JSON collections");
|
|
562
|
+
const encoded = jsonBytes(selected);
|
|
563
|
+
if (encoded.byteLength <= maxContextBytes) {
|
|
564
|
+
return {
|
|
565
|
+
bytes: encoded,
|
|
566
|
+
items: 1,
|
|
567
|
+
offset: 0,
|
|
568
|
+
totalItems: 1,
|
|
569
|
+
nextOffset: null,
|
|
570
|
+
truncated: false,
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
if (typeof selected !== "string") {
|
|
574
|
+
throw new Error("selected JSON scalar exceeds the broker context token limit");
|
|
575
|
+
}
|
|
576
|
+
const characters = [...selected];
|
|
577
|
+
let lower = 0;
|
|
578
|
+
let upper = characters.length;
|
|
579
|
+
while (lower < upper) {
|
|
580
|
+
const middle = Math.ceil((lower + upper) / 2);
|
|
581
|
+
if (jsonBytes(characters.slice(0, middle).join("")).byteLength <= maxContextBytes) {
|
|
582
|
+
lower = middle;
|
|
583
|
+
}
|
|
584
|
+
else {
|
|
585
|
+
upper = middle - 1;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
return {
|
|
589
|
+
bytes: jsonBytes(characters.slice(0, lower).join("")),
|
|
590
|
+
items: 1,
|
|
591
|
+
offset: 0,
|
|
592
|
+
totalItems: 1,
|
|
593
|
+
nextOffset: null,
|
|
594
|
+
truncated: true,
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
function jsonBytes(value) {
|
|
598
|
+
return Buffer.from(`${JSON.stringify(value)}\n`, "utf8");
|
|
599
|
+
}
|
|
600
|
+
function truncateUtf8(bytes, maxBytes) {
|
|
601
|
+
if (bytes.byteLength <= maxBytes)
|
|
602
|
+
return bytes;
|
|
603
|
+
let end = maxBytes;
|
|
604
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
605
|
+
while (end > 0) {
|
|
606
|
+
const candidate = bytes.subarray(0, end);
|
|
607
|
+
try {
|
|
608
|
+
decoder.decode(candidate);
|
|
609
|
+
return Buffer.from(candidate);
|
|
610
|
+
}
|
|
611
|
+
catch {
|
|
612
|
+
end -= 1;
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
return Buffer.alloc(0);
|
|
616
|
+
}
|
|
617
|
+
function resolveJsonPointer(value, pointer) {
|
|
618
|
+
if (pointer === "")
|
|
619
|
+
return value;
|
|
620
|
+
let selected = value;
|
|
621
|
+
for (const rawPart of pointer.slice(1).split("/")) {
|
|
622
|
+
const part = rawPart.replaceAll("~1", "/").replaceAll("~0", "~");
|
|
623
|
+
if (Array.isArray(selected) &&
|
|
624
|
+
/^(0|[1-9][0-9]*)$/.test(part) &&
|
|
625
|
+
Number(part) < selected.length) {
|
|
626
|
+
selected = selected[Number(part)];
|
|
627
|
+
}
|
|
628
|
+
else if (isObject(selected) && Object.hasOwn(selected, part)) {
|
|
629
|
+
selected = selected[part];
|
|
630
|
+
}
|
|
631
|
+
else {
|
|
632
|
+
throw new Error("json_pointer does not resolve within the response");
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
return selected;
|
|
636
|
+
}
|
|
637
|
+
function validJsonPointer(value) {
|
|
638
|
+
return value === "" || (value.startsWith("/") && !/~(?:[^01]|$)/.test(value));
|
|
639
|
+
}
|
|
640
|
+
function contentTypeAllowed(contentType, allowed) {
|
|
641
|
+
const normalized = contentType.toLowerCase();
|
|
642
|
+
return allowed.some((pattern) => {
|
|
643
|
+
const candidate = pattern.toLowerCase();
|
|
644
|
+
if (candidate === "*/*" || candidate === normalized)
|
|
645
|
+
return true;
|
|
646
|
+
const escaped = candidate.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replaceAll("*", ".*");
|
|
647
|
+
return new RegExp(`^${escaped}$`).test(normalized);
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
function safeResponseExcerpt(bytes, contentType, secrets) {
|
|
651
|
+
if (!contentType.includes("json") && !contentType.startsWith("text/")) {
|
|
652
|
+
return `[${bytes.length} non-text byte(s)]`;
|
|
653
|
+
}
|
|
654
|
+
return bounded(sanitizeResearchText(bytes.toString("utf8"), secrets).replace(/\s+/g, " "), 1000);
|
|
655
|
+
}
|
|
656
|
+
function safeResponseId(headers) {
|
|
657
|
+
for (const name of ["x-request-id", "request-id", "x-amzn-requestid", "cf-ray"]) {
|
|
658
|
+
const value = headers.get(name)?.trim();
|
|
659
|
+
if (value && /^[A-Za-z0-9._:-]{1,200}$/.test(value))
|
|
660
|
+
return value;
|
|
661
|
+
}
|
|
662
|
+
return null;
|
|
663
|
+
}
|
|
664
|
+
function parseRetryAfter(value) {
|
|
665
|
+
if (!value)
|
|
666
|
+
return null;
|
|
667
|
+
const seconds = Number(value);
|
|
668
|
+
if (Number.isFinite(seconds) && seconds >= 0)
|
|
669
|
+
return Math.ceil(seconds);
|
|
670
|
+
const date = Date.parse(value);
|
|
671
|
+
return Number.isFinite(date) ? Math.max(0, Math.ceil((date - Date.now()) / 1000)) : null;
|
|
672
|
+
}
|
|
673
|
+
function brokerFailureKind(error) {
|
|
674
|
+
if (error instanceof CliError && error.code === "RESEARCH_BROKER_HTTP_ERROR") {
|
|
675
|
+
const status = isObject(error.details) ? error.details.status : undefined;
|
|
676
|
+
if (status === 429)
|
|
677
|
+
return "rate-limit";
|
|
678
|
+
if (typeof status === "number" && status >= 500)
|
|
679
|
+
return "server";
|
|
680
|
+
return "deterministic";
|
|
681
|
+
}
|
|
682
|
+
return error instanceof TypeError ? "transient" : "deterministic";
|
|
683
|
+
}
|
|
308
684
|
function validateHttpsUrl(value) {
|
|
309
685
|
const url = new URL(value);
|
|
310
686
|
if (url.protocol !== "https:" || url.username || url.password || !url.hostname) {
|