@agwab/pi-workflow 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +2 -0
  2. package/dist/compiler.d.ts +4 -6
  3. package/dist/compiler.js +70 -39
  4. package/dist/dynamic-decision.d.ts +0 -1
  5. package/dist/dynamic-decision.js +0 -7
  6. package/dist/dynamic-generated-task-runtime.d.ts +2 -0
  7. package/dist/dynamic-generated-task-runtime.js +21 -8
  8. package/dist/dynamic-profiles.d.ts +0 -1
  9. package/dist/dynamic-profiles.js +0 -3
  10. package/dist/engine-run-graph.d.ts +1 -0
  11. package/dist/engine-run-graph.js +142 -2
  12. package/dist/engine.d.ts +10 -6
  13. package/dist/engine.js +146 -77
  14. package/dist/extension.d.ts +2 -1
  15. package/dist/extension.js +38 -15
  16. package/dist/index.d.ts +3 -3
  17. package/dist/index.js +2 -1
  18. package/dist/store.d.ts +3 -1
  19. package/dist/store.js +189 -49
  20. package/dist/subagent-backend.d.ts +4 -0
  21. package/dist/subagent-backend.js +281 -31
  22. package/dist/types.d.ts +9 -1
  23. package/dist/workflow-runtime.d.ts +2 -0
  24. package/dist/workflow-runtime.js +40 -1
  25. package/dist/workflow-view.js +3 -1
  26. package/dist/workflow-web-source-extension.js +167 -48
  27. package/dist/workflow-web-source.d.ts +2 -1
  28. package/dist/workflow-web-source.js +84 -19
  29. package/docs/usage.md +11 -0
  30. package/node_modules/@agwab/pi-subagent/README.md +3 -3
  31. package/node_modules/@agwab/pi-subagent/api.mjs +1 -0
  32. package/node_modules/@agwab/pi-subagent/docs/usage.md +63 -12
  33. package/node_modules/@agwab/pi-subagent/package.json +2 -2
  34. package/node_modules/@agwab/pi-subagent/src/api.ts +54 -1
  35. package/node_modules/@agwab/pi-subagent/src/artifacts/registry.ts +9 -4
  36. package/node_modules/@agwab/pi-subagent/src/artifacts/result.ts +8 -0
  37. package/node_modules/@agwab/pi-subagent/src/core/constants.ts +9 -0
  38. package/node_modules/@agwab/pi-subagent/src/core/validation.ts +21 -0
  39. package/node_modules/@agwab/pi-subagent/src/index.ts +995 -573
  40. package/node_modules/@agwab/pi-subagent/src/orchestrate/async.ts +279 -156
  41. package/node_modules/@agwab/pi-subagent/src/orchestrate/interrupt.ts +165 -89
  42. package/node_modules/@agwab/pi-subagent/src/orchestrate/reconcile.ts +111 -65
  43. package/node_modules/@agwab/pi-subagent/src/orchestrate/run-ref.ts +219 -0
  44. package/node_modules/@agwab/pi-subagent/src/orchestrate/run.ts +88 -8
  45. package/node_modules/@agwab/pi-subagent/src/orchestrate/status.ts +614 -298
  46. package/node_modules/@agwab/pi-subagent/src/panel.ts +1352 -560
  47. package/node_modules/@agwab/pi-subagent/src/runners/headless-model.ts +53 -5
  48. package/node_modules/@agwab/pi-subagent/src/runners/tmux.ts +13 -6
  49. package/package.json +2 -2
  50. package/src/compiler.ts +127 -66
  51. package/src/dynamic-decision.ts +0 -11
  52. package/src/dynamic-generated-task-runtime.ts +47 -12
  53. package/src/dynamic-profiles.ts +0 -4
  54. package/src/engine-run-graph.ts +185 -2
  55. package/src/engine.ts +192 -107
  56. package/src/extension.ts +50 -17
  57. package/src/index.ts +3 -1
  58. package/src/store.ts +253 -55
  59. package/src/subagent-backend.ts +369 -32
  60. package/src/types.ts +13 -1
  61. package/src/workflow-runtime.ts +53 -2
  62. package/src/workflow-view.ts +2 -1
  63. package/src/workflow-web-source-extension.ts +621 -228
  64. package/src/workflow-web-source.ts +118 -28
  65. package/workflows/deep-research/helpers/claim-evidence-gate.mjs +56 -16
  66. package/workflows/deep-research/helpers/final-audit-packet.mjs +1 -4
  67. package/workflows/deep-research/helpers/normalize-input-packet.mjs +1 -1
  68. package/workflows/deep-research/helpers/render-executive.mjs +8 -21
  69. package/workflows/deep-research/helpers/sanitize-verification-candidates.mjs +89 -15
  70. package/workflows/deep-research/schemas/deep-research-executive-render-control.schema.json +0 -1
  71. package/workflows/deep-research/schemas/deep-research-verify-claims-control.schema.json +4 -1
  72. package/workflows/impact-review/spec.json +3 -3
  73. package/workflows/spec-review/helpers/spec-review-pipeline.mjs +1 -8
  74. package/dist/dynamic-loader.d.ts +0 -25
  75. package/dist/dynamic-loader.js +0 -13
  76. package/src/dynamic-loader.ts +0 -49
  77. package/workflows/impact-review/schemas/docs-release-impact-control.schema.json +0 -42
  78. package/workflows/impact-review/schemas/security-performance-impact-control.schema.json +0 -42
  79. package/workflows/impact-review/schemas/state-data-impact-control.schema.json +0 -42
@@ -11,7 +11,6 @@ import {
11
11
  buildWorkflowWebSourceCard,
12
12
  createWorkflowWebSource,
13
13
  createWorkflowWebVisibleBudget,
14
- DEFAULT_WORKFLOW_WEB_SECURITY_POLICY,
15
14
  errorToolResult,
16
15
  extractSearchCandidates,
17
16
  extractTextFromToolResult,
@@ -43,7 +42,8 @@ export interface WorkflowWebProviderLaunchConfig {
43
42
  extensionPath?: string;
44
43
  }
45
44
 
46
- export interface WorkflowWebSourceLaunchConfig extends WorkflowWebSourceCacheConfig {
45
+ export interface WorkflowWebSourceLaunchConfig
46
+ extends WorkflowWebSourceCacheConfig {
47
47
  schema: typeof WORKFLOW_WEB_SOURCE_LAUNCH_CONFIG_SCHEMA;
48
48
  workflowName?: string;
49
49
  stageId?: string;
@@ -111,14 +111,21 @@ export function registerWorkflowWebSourceExtension(
111
111
  ): void {
112
112
  const policy = normalizeWorkflowWebSourcePolicy(config.webSourcePolicy);
113
113
  const security = normalizeWorkflowWebSecurityPolicy(config.securityPolicy);
114
- const budget = createWorkflowWebVisibleBudget(policy.perTaskVisibleCharBudget);
114
+ const budget = createWorkflowWebVisibleBudget(
115
+ policy.perTaskVisibleCharBudget,
116
+ );
115
117
  const providerTools: CapturedProviderTools = new Map();
116
118
  const sourceCache: Map<string, WorkflowWebSource> = new Map();
117
- const fetchInFlight: Map<string, Promise<ReturnType<typeof toolResultFromJson>>> = new Map();
119
+ const fetchInFlight: Map<
120
+ string,
121
+ Promise<ReturnType<typeof toolResultFromJson>>
122
+ > = new Map();
118
123
  const fetchFailures: Map<string, FetchFailure> = new Map();
119
124
 
120
125
  if (providerExtension) {
121
- providerExtension(providerCapturePi(pi, providerTools, Boolean(config.exposeLegacyTools)));
126
+ providerExtension(
127
+ providerCapturePi(pi, providerTools, Boolean(config.exposeLegacyTools)),
128
+ );
122
129
  }
123
130
 
124
131
  pi.registerTool({
@@ -126,9 +133,15 @@ export function registerWorkflowWebSourceExtension(
126
133
  description:
127
134
  "Search the web through the workflow web-source provider and return compact candidate cards only.",
128
135
  parameters: Type.Object({
129
- query: Type.Optional(Type.String({ description: "Single search query." })),
130
- queries: Type.Optional(Type.Array(Type.String(), { description: "Multiple search queries." })),
131
- numResults: Type.Optional(Type.Number({ description: "Results per query." })),
136
+ query: Type.Optional(
137
+ Type.String({ description: "Single search query." }),
138
+ ),
139
+ queries: Type.Optional(
140
+ Type.Array(Type.String(), { description: "Multiple search queries." }),
141
+ ),
142
+ numResults: Type.Optional(
143
+ Type.Number({ description: "Results per query." }),
144
+ ),
132
145
  }),
133
146
  execute: async (toolCallId, params, signal, onUpdate, ctx) => {
134
147
  const providerTool = providerTools.get("web_search");
@@ -150,14 +163,19 @@ export function registerWorkflowWebSourceExtension(
150
163
  onUpdate,
151
164
  ctx,
152
165
  );
153
- const candidates = extractSearchCandidates(result, policy).map((candidate) => {
154
- const consumed = consumeText(candidate.snippet, policy.searchSnippetChars);
155
- return {
156
- ...candidate,
157
- snippet: consumed.text,
158
- budget: consumed.budget,
159
- };
160
- });
166
+ const candidates = extractSearchCandidates(result, policy).map(
167
+ (candidate) => {
168
+ const consumed = consumeText(
169
+ candidate.snippet,
170
+ policy.searchSnippetChars,
171
+ );
172
+ return {
173
+ ...candidate,
174
+ snippet: consumed.text,
175
+ budget: consumed.budget,
176
+ };
177
+ },
178
+ );
161
179
  await recordWorkflowWebSourceEvent(config, "search", {
162
180
  candidateCount: candidates.length,
163
181
  visibleChars: budget.used,
@@ -177,14 +195,44 @@ export function registerWorkflowWebSourceExtension(
177
195
  description:
178
196
  "Fetch one or more URLs into the workflow web-source cache and return compact source cards with sourceRefs.",
179
197
  parameters: Type.Object({
180
- url: Type.Optional(Type.String({ description: "Single URL to fetch into the workflow web-source cache." })),
181
- urls: Type.Optional(Type.Array(Type.String(), { description: "Multiple URLs to fetch in one tool call. Prefer this over repeated fetch calls when caching several promising sources." })),
182
- sources: Type.Optional(Type.Array(Type.Object({
183
- url: Type.String({ description: "URL to fetch into the workflow web-source cache." }),
184
- title: Type.Optional(Type.String({ description: "Optional source title override." })),
185
- }), { description: "Multiple URL/title objects to fetch in one tool call." })),
186
- title: Type.Optional(Type.String({ description: "Optional source title override for single-url fetches." })),
187
- titles: Type.Optional(Type.Array(Type.String(), { description: "Optional title overrides paired by index with urls." })),
198
+ url: Type.Optional(
199
+ Type.String({
200
+ description:
201
+ "Single URL to fetch into the workflow web-source cache.",
202
+ }),
203
+ ),
204
+ urls: Type.Optional(
205
+ Type.Array(Type.String(), {
206
+ description:
207
+ "Multiple URLs to fetch in one tool call. Prefer this over repeated fetch calls when caching several promising sources.",
208
+ }),
209
+ ),
210
+ sources: Type.Optional(
211
+ Type.Array(
212
+ Type.Object({
213
+ url: Type.String({
214
+ description: "URL to fetch into the workflow web-source cache.",
215
+ }),
216
+ title: Type.Optional(
217
+ Type.String({ description: "Optional source title override." }),
218
+ ),
219
+ }),
220
+ {
221
+ description:
222
+ "Multiple URL/title objects to fetch in one tool call.",
223
+ },
224
+ ),
225
+ ),
226
+ title: Type.Optional(
227
+ Type.String({
228
+ description: "Optional source title override for single-url fetches.",
229
+ }),
230
+ ),
231
+ titles: Type.Optional(
232
+ Type.Array(Type.String(), {
233
+ description: "Optional title overrides paired by index with urls.",
234
+ }),
235
+ ),
188
236
  }),
189
237
  execute: async (toolCallId, params, signal, onUpdate, ctx) => {
190
238
  const batchRequested = fetchSourceBatchRequested(params);
@@ -212,10 +260,15 @@ export function registerWorkflowWebSourceExtension(
212
260
  results.push({
213
261
  index,
214
262
  url: sanitizeUrlForModel(request.url),
215
- status: typeof payload.status === "string" ? payload.status : "unknown",
263
+ status:
264
+ typeof payload.status === "string" ? payload.status : "unknown",
216
265
  ...(typeof payload.code === "string" ? { code: payload.code } : {}),
217
- ...(typeof payload.message === "string" ? { message: payload.message } : {}),
218
- ...(typeof card?.sourceRef === "string" ? { sourceRef: card.sourceRef } : {}),
266
+ ...(typeof payload.message === "string"
267
+ ? { message: payload.message }
268
+ : {}),
269
+ ...(typeof card?.sourceRef === "string"
270
+ ? { sourceRef: card.sourceRef }
271
+ : {}),
219
272
  ...(card ? { cardIndex: cards.length - 1 } : {}),
220
273
  });
221
274
  }
@@ -239,7 +292,13 @@ export function registerWorkflowWebSourceExtension(
239
292
  next: "Use returned sourceRefs with workflow_web_source_read; batch snippets with reads:[...] or queries:[...] when possible.",
240
293
  });
241
294
  }
242
- return await fetchWorkflowWebSourceOnce(toolCallId, params, signal, onUpdate, ctx);
295
+ return await fetchWorkflowWebSourceOnce(
296
+ toolCallId,
297
+ params,
298
+ signal,
299
+ onUpdate,
300
+ ctx,
301
+ );
243
302
  },
244
303
  });
245
304
 
@@ -250,85 +309,132 @@ export function registerWorkflowWebSourceExtension(
250
309
  onUpdate?: unknown,
251
310
  ctx?: unknown,
252
311
  ): Promise<ToolResult> {
253
- const url = urlFromParams(params);
254
- if (!url) {
255
- return errorToolResult(
256
- "invalid_params",
257
- "workflow_web_fetch_source requires a url string parameter.",
258
- );
259
- }
260
- const checked = validateWorkflowWebUrl(url, security);
261
- if (!checked.ok) {
262
- await recordWorkflowWebSourceEvent(config, "blocked_url", {
263
- url: sanitizeUrlForModel(url),
264
- reason: checked.reason,
265
- });
266
- return errorToolResult("blocked_url", "URL blocked by workflow web-source security policy.", {
312
+ const url = urlFromParams(params);
313
+ if (!url) {
314
+ return errorToolResult(
315
+ "invalid_params",
316
+ "workflow_web_fetch_source requires a url string parameter.",
317
+ );
318
+ }
319
+ const checked = validateWorkflowWebUrl(url, security);
320
+ if (!checked.ok) {
321
+ await recordWorkflowWebSourceEvent(config, "blocked_url", {
322
+ url: sanitizeUrlForModel(url),
323
+ reason: checked.reason,
324
+ });
325
+ return errorToolResult(
326
+ "blocked_url",
327
+ "URL blocked by workflow web-source security policy.",
328
+ {
267
329
  reason: checked.reason,
268
330
  url: sanitizeUrlForModel(url),
269
- });
270
- }
271
- const fetchUrl = canonicalWorkflowWebFetchUrl(checked.normalizedUrl);
272
- const existing = await findWorkflowWebSourceByUrl(config, fetchUrl);
273
- if (existing) {
274
- sourceCache.set(existing.sourceRef, existing);
275
- const card = buildWorkflowWebSourceCard({
276
- source: existing,
277
- policy,
278
- budget,
279
- duplicate: true,
280
- });
281
- await recordWorkflowWebSourceEvent(config, "fetch_duplicate", {
282
- sourceRef: existing.sourceRef,
283
- url: existing.redactedUrl,
284
- visibleChars: budget.used,
285
- });
286
- return toolResultFromJson({ status: "ok", tool: "workflow_web_fetch_source", card });
287
- }
288
- const fetchKey = sourceUrlCacheKey(fetchUrl);
289
- const cachedFailure = fetchFailures.get(fetchKey) ?? await readDurableFetchFailure(config, fetchKey);
290
- if (cachedFailure) {
291
- fetchFailures.set(fetchKey, cachedFailure);
292
- await recordWorkflowWebSourceEvent(config, "fetch_negative_cache_hit", {
293
- url: sanitizeUrlForModel(fetchUrl),
294
- code: cachedFailure.code,
295
- });
296
- return errorToolResult(cachedFailure.code, cachedFailure.message, cachedFailure.extra);
297
- }
298
- const inFlight = fetchInFlight.get(fetchKey);
299
- if (inFlight) {
300
- const result = await inFlight;
301
- const source = await findWorkflowWebSourceByUrl(config, fetchUrl);
302
- if (!source) return result;
303
- sourceCache.set(source.sourceRef, source);
304
- const card = buildWorkflowWebSourceCard({ source, policy, budget, duplicate: true });
305
- await recordWorkflowWebSourceEvent(config, "fetch_duplicate", {
306
- sourceRef: source.sourceRef,
307
- url: source.redactedUrl,
308
- visibleChars: budget.used,
309
- });
310
- return toolResultFromJson({ status: "ok", tool: "workflow_web_fetch_source", card });
311
- }
312
- const fetchPromise = withWorkflowWebFetchLock(config, fetchKey, signal, async () => {
313
- const lockedExisting = await findWorkflowWebSourceByUrl(config, fetchUrl);
331
+ },
332
+ );
333
+ }
334
+ const fetchUrl = canonicalWorkflowWebFetchUrl(checked.normalizedUrl);
335
+ const existing = await findWorkflowWebSourceByUrl(config, fetchUrl);
336
+ if (existing) {
337
+ sourceCache.set(existing.sourceRef, existing);
338
+ const card = buildWorkflowWebSourceCard({
339
+ source: existing,
340
+ policy,
341
+ budget,
342
+ duplicate: true,
343
+ });
344
+ await recordWorkflowWebSourceEvent(config, "fetch_duplicate", {
345
+ sourceRef: existing.sourceRef,
346
+ url: existing.redactedUrl,
347
+ visibleChars: budget.used,
348
+ });
349
+ return toolResultFromJson({
350
+ status: "ok",
351
+ tool: "workflow_web_fetch_source",
352
+ card,
353
+ });
354
+ }
355
+ const fetchKey = sourceUrlCacheKey(fetchUrl);
356
+ const cachedFailure =
357
+ fetchFailures.get(fetchKey) ??
358
+ (await readDurableFetchFailure(config, fetchKey));
359
+ if (cachedFailure) {
360
+ fetchFailures.set(fetchKey, cachedFailure);
361
+ await recordWorkflowWebSourceEvent(config, "fetch_negative_cache_hit", {
362
+ url: sanitizeUrlForModel(fetchUrl),
363
+ code: cachedFailure.code,
364
+ });
365
+ return errorToolResult(
366
+ cachedFailure.code,
367
+ cachedFailure.message,
368
+ cachedFailure.extra,
369
+ );
370
+ }
371
+ const inFlight = fetchInFlight.get(fetchKey);
372
+ if (inFlight) {
373
+ const result = await inFlight;
374
+ const source = await findWorkflowWebSourceByUrl(config, fetchUrl);
375
+ if (!source) return result;
376
+ sourceCache.set(source.sourceRef, source);
377
+ const card = buildWorkflowWebSourceCard({
378
+ source,
379
+ policy,
380
+ budget,
381
+ duplicate: true,
382
+ });
383
+ await recordWorkflowWebSourceEvent(config, "fetch_duplicate", {
384
+ sourceRef: source.sourceRef,
385
+ url: source.redactedUrl,
386
+ visibleChars: budget.used,
387
+ });
388
+ return toolResultFromJson({
389
+ status: "ok",
390
+ tool: "workflow_web_fetch_source",
391
+ card,
392
+ });
393
+ }
394
+ const fetchPromise = withWorkflowWebFetchLock(
395
+ config,
396
+ fetchKey,
397
+ signal,
398
+ async () => {
399
+ const lockedExisting = await findWorkflowWebSourceByUrl(
400
+ config,
401
+ fetchUrl,
402
+ );
314
403
  if (lockedExisting) {
315
404
  sourceCache.set(lockedExisting.sourceRef, lockedExisting);
316
- const card = buildWorkflowWebSourceCard({ source: lockedExisting, policy, budget, duplicate: true });
405
+ const card = buildWorkflowWebSourceCard({
406
+ source: lockedExisting,
407
+ policy,
408
+ budget,
409
+ duplicate: true,
410
+ });
317
411
  await recordWorkflowWebSourceEvent(config, "fetch_duplicate", {
318
412
  sourceRef: lockedExisting.sourceRef,
319
413
  url: lockedExisting.redactedUrl,
320
414
  visibleChars: budget.used,
321
415
  });
322
- return toolResultFromJson({ status: "ok", tool: "workflow_web_fetch_source", card });
416
+ return toolResultFromJson({
417
+ status: "ok",
418
+ tool: "workflow_web_fetch_source",
419
+ card,
420
+ });
323
421
  }
324
422
  const lockedFailure = await readDurableFetchFailure(config, fetchKey);
325
423
  if (lockedFailure) {
326
424
  fetchFailures.set(fetchKey, lockedFailure);
327
- await recordWorkflowWebSourceEvent(config, "fetch_negative_cache_hit", {
328
- url: sanitizeUrlForModel(fetchUrl),
329
- code: lockedFailure.code,
330
- });
331
- return errorToolResult(lockedFailure.code, lockedFailure.message, lockedFailure.extra);
425
+ await recordWorkflowWebSourceEvent(
426
+ config,
427
+ "fetch_negative_cache_hit",
428
+ {
429
+ url: sanitizeUrlForModel(fetchUrl),
430
+ code: lockedFailure.code,
431
+ },
432
+ );
433
+ return errorToolResult(
434
+ lockedFailure.code,
435
+ lockedFailure.message,
436
+ lockedFailure.extra,
437
+ );
332
438
  }
333
439
  let text: string;
334
440
  let title = titleFromParams(params);
@@ -345,13 +451,21 @@ export function registerWorkflowWebSourceExtension(
345
451
  url: sanitizeUrlForModel(safeFetch.url),
346
452
  reason: safeFetch.reason,
347
453
  });
348
- return await cachedFetchFailureResult(config, fetchFailures, fetchKey, {
349
- code: "blocked_url",
350
- message:
351
- "URL was blocked by workflow web-source security policy before content fetch.",
352
- extra: { reason: safeFetch.reason, url: sanitizeUrlForModel(safeFetch.url) },
353
- reason: safeFetch.reason,
354
- });
454
+ return await cachedFetchFailureResult(
455
+ config,
456
+ fetchFailures,
457
+ fetchKey,
458
+ {
459
+ code: "blocked_url",
460
+ message:
461
+ "URL was blocked by workflow web-source security policy before content fetch.",
462
+ extra: {
463
+ reason: safeFetch.reason,
464
+ url: sanitizeUrlForModel(safeFetch.url),
465
+ },
466
+ reason: safeFetch.reason,
467
+ },
468
+ );
355
469
  }
356
470
  text = safeFetch.text;
357
471
  title = title ?? safeFetch.title;
@@ -368,31 +482,44 @@ export function registerWorkflowWebSourceExtension(
368
482
  return errorToolResult(missing.code, missing.message);
369
483
  }
370
484
  if (!security.allowPrivateHosts) {
371
- await recordWorkflowWebSourceEvent(config, "blocked_provider_fetch", {
372
- url: sanitizeUrlForModel(fetchUrl),
373
- reason: "untrusted_provider_fetch",
374
- });
485
+ await recordWorkflowWebSourceEvent(
486
+ config,
487
+ "blocked_provider_fetch",
488
+ {
489
+ url: sanitizeUrlForModel(fetchUrl),
490
+ reason: "untrusted_provider_fetch",
491
+ },
492
+ );
375
493
  return errorToolResult(
376
494
  "untrusted_provider_fetch",
377
495
  "Custom provider fetch_content is disabled unless securityPolicy.allowPrivateHosts is true; use the default safe fetch provider or a trusted provider configuration.",
378
496
  { url: sanitizeUrlForModel(fetchUrl) },
379
497
  );
380
498
  }
381
- const providerHostCheck = await validateResolvedHost(fetchUrl, security);
499
+ const providerHostCheck = await validateResolvedHost(
500
+ fetchUrl,
501
+ security,
502
+ );
382
503
  if (!providerHostCheck.ok) {
383
504
  await recordWorkflowWebSourceEvent(config, "blocked_provider_url", {
384
505
  url: sanitizeUrlForModel(providerHostCheck.url),
385
506
  reason: providerHostCheck.reason,
386
507
  });
387
- return await cachedFetchFailureResult(config, fetchFailures, fetchKey, {
388
- code: "blocked_url",
389
- message: "URL was blocked by workflow web-source security policy before provider fetch.",
390
- extra: {
508
+ return await cachedFetchFailureResult(
509
+ config,
510
+ fetchFailures,
511
+ fetchKey,
512
+ {
513
+ code: "blocked_url",
514
+ message:
515
+ "URL was blocked by workflow web-source security policy before provider fetch.",
516
+ extra: {
517
+ reason: providerHostCheck.reason,
518
+ url: sanitizeUrlForModel(providerHostCheck.url),
519
+ },
391
520
  reason: providerHostCheck.reason,
392
- url: sanitizeUrlForModel(providerHostCheck.url),
393
521
  },
394
- reason: providerHostCheck.reason,
395
- });
522
+ );
396
523
  }
397
524
  const result = await providerTool.execute(
398
525
  toolCallId,
@@ -401,21 +528,30 @@ export function registerWorkflowWebSourceExtension(
401
528
  onUpdate,
402
529
  ctx,
403
530
  );
404
- const providerUrlCheck = await validateProviderResultUrls(result, security);
531
+ const providerUrlCheck = await validateProviderResultUrls(
532
+ result,
533
+ security,
534
+ );
405
535
  if (!providerUrlCheck.ok) {
406
536
  await recordWorkflowWebSourceEvent(config, "blocked_provider_url", {
407
537
  url: sanitizeUrlForModel(providerUrlCheck.url),
408
538
  reason: providerUrlCheck.reason,
409
539
  });
410
- return await cachedFetchFailureResult(config, fetchFailures, fetchKey, {
411
- code: "blocked_url",
412
- message: "Provider result URL was blocked by workflow web-source security policy.",
413
- extra: {
540
+ return await cachedFetchFailureResult(
541
+ config,
542
+ fetchFailures,
543
+ fetchKey,
544
+ {
545
+ code: "blocked_url",
546
+ message:
547
+ "Provider result URL was blocked by workflow web-source security policy.",
548
+ extra: {
549
+ reason: providerUrlCheck.reason,
550
+ url: sanitizeUrlForModel(providerUrlCheck.url),
551
+ },
414
552
  reason: providerUrlCheck.reason,
415
- url: sanitizeUrlForModel(providerUrlCheck.url),
416
553
  },
417
- reason: providerUrlCheck.reason,
418
- });
554
+ );
419
555
  }
420
556
  text = extractTextFromToolResult(result);
421
557
  title = title ?? extractTitleFromToolResult(result);
@@ -424,12 +560,17 @@ export function registerWorkflowWebSourceExtension(
424
560
  await recordWorkflowWebSourceEvent(config, "fetch_empty", {
425
561
  url: sanitizeUrlForModel(fetchUrl),
426
562
  });
427
- return await cachedFetchFailureResult(config, fetchFailures, fetchKey, {
428
- code: "empty_source",
429
- message: "Provider returned no extractable text for this URL.",
430
- extra: { url: sanitizeUrlForModel(fetchUrl) },
431
- reason: "empty_source",
432
- });
563
+ return await cachedFetchFailureResult(
564
+ config,
565
+ fetchFailures,
566
+ fetchKey,
567
+ {
568
+ code: "empty_source",
569
+ message: "Provider returned no extractable text for this URL.",
570
+ extra: { url: sanitizeUrlForModel(fetchUrl) },
571
+ reason: "empty_source",
572
+ },
573
+ );
433
574
  }
434
575
  const source = createWorkflowWebSource({
435
576
  config,
@@ -448,24 +589,37 @@ export function registerWorkflowWebSourceExtension(
448
589
  textChars: source.textChars,
449
590
  visibleChars: budget.used,
450
591
  });
451
- return toolResultFromJson({ status: "ok", tool: "workflow_web_fetch_source", card });
452
- }).catch(async (error: unknown) => {
453
- const message = error instanceof Error ? error.message : "workflow_web_fetch_failed";
454
- const code = message === "fetch_lock_timeout" ? "fetch_lock_timeout" : "workflow_web_fetch_failed";
455
- await recordWorkflowWebSourceEvent(config, "fetch_failed", {
456
- url: sanitizeUrlForModel(fetchUrl),
457
- code,
458
- });
459
- return errorToolResult(code, "Workflow web-source fetch failed before a source could be cached.", {
460
- url: sanitizeUrlForModel(fetchUrl),
592
+ return toolResultFromJson({
593
+ status: "ok",
594
+ tool: "workflow_web_fetch_source",
595
+ card,
461
596
  });
597
+ },
598
+ ).catch(async (error: unknown) => {
599
+ const message =
600
+ error instanceof Error ? error.message : "workflow_web_fetch_failed";
601
+ const code =
602
+ message === "fetch_lock_timeout"
603
+ ? "fetch_lock_timeout"
604
+ : "workflow_web_fetch_failed";
605
+ await recordWorkflowWebSourceEvent(config, "fetch_failed", {
606
+ url: sanitizeUrlForModel(fetchUrl),
607
+ code,
462
608
  });
463
- fetchInFlight.set(fetchKey, fetchPromise);
464
- try {
465
- return await fetchPromise;
466
- } finally {
467
- fetchInFlight.delete(fetchKey);
468
- }
609
+ return errorToolResult(
610
+ code,
611
+ "Workflow web-source fetch failed before a source could be cached.",
612
+ {
613
+ url: sanitizeUrlForModel(fetchUrl),
614
+ },
615
+ );
616
+ });
617
+ fetchInFlight.set(fetchKey, fetchPromise);
618
+ try {
619
+ return await fetchPromise;
620
+ } finally {
621
+ fetchInFlight.delete(fetchKey);
622
+ }
469
623
  }
470
624
 
471
625
  pi.registerTool({
@@ -473,26 +627,90 @@ export function registerWorkflowWebSourceExtension(
473
627
  description:
474
628
  "Read one or more narrow exact/fuzzy/term-matched snippets from a cached workflow web source by sourceRef.",
475
629
  parameters: Type.Object({
476
- sourceRef: Type.String({ description: "Opaque sourceRef returned by workflow_web_fetch_source." }),
477
- query: Type.Optional(Type.String({ description: "Exact or fuzzy text to locate in the cached source." })),
478
- queries: Type.Optional(Type.Array(Type.String(), { description: "Multiple exact/fuzzy texts to locate in one cached source. Prefer this over repeated calls when reading several snippets from the same sourceRef." })),
479
- exact: Type.Optional(Type.String({ description: "Exact text to locate in the cached source." })),
480
- exactTexts: Type.Optional(Type.Array(Type.String(), { description: "Multiple exact texts to locate in one cached source." })),
481
- claim: Type.Optional(Type.String({ description: "Claim to locate when the exact quote is not known. Use with terms for deterministic quote harvesting." })),
482
- terms: Type.Optional(Type.Array(Type.String(), { description: "Important terms that should co-occur in the returned source window." })),
483
- reads: Type.Optional(Type.Array(Type.Object({
484
- query: Type.Optional(Type.String({ description: "Exact or fuzzy text to locate." })),
485
- exact: Type.Optional(Type.String({ description: "Exact text to locate." })),
486
- exactText: Type.Optional(Type.String({ description: "Exact text to locate." })),
487
- text: Type.Optional(Type.String({ description: "Text to locate." })),
488
- claim: Type.Optional(Type.String({ description: "Claim to locate when exact quote is unknown." })),
489
- terms: Type.Optional(Type.Array(Type.String(), { description: "Important terms for deterministic quote harvesting." })),
490
- maxChars: Type.Optional(Type.Number({ description: "Maximum visible snippet characters for this read." })),
491
- }), { description: "Mixed batch reads for one sourceRef; each item can use query or claim+terms." })),
492
- maxChars: Type.Optional(Type.Number({ description: "Maximum visible snippet characters per query." })),
630
+ sourceRef: Type.String({
631
+ description: "Opaque sourceRef returned by workflow_web_fetch_source.",
632
+ }),
633
+ query: Type.Optional(
634
+ Type.String({
635
+ description: "Exact or fuzzy text to locate in the cached source.",
636
+ }),
637
+ ),
638
+ queries: Type.Optional(
639
+ Type.Array(Type.String(), {
640
+ description:
641
+ "Multiple exact/fuzzy texts to locate in one cached source. Prefer this over repeated calls when reading several snippets from the same sourceRef.",
642
+ }),
643
+ ),
644
+ exact: Type.Optional(
645
+ Type.String({
646
+ description: "Exact text to locate in the cached source.",
647
+ }),
648
+ ),
649
+ exactTexts: Type.Optional(
650
+ Type.Array(Type.String(), {
651
+ description: "Multiple exact texts to locate in one cached source.",
652
+ }),
653
+ ),
654
+ claim: Type.Optional(
655
+ Type.String({
656
+ description:
657
+ "Claim to locate when the exact quote is not known. Use with terms for deterministic quote harvesting.",
658
+ }),
659
+ ),
660
+ terms: Type.Optional(
661
+ Type.Array(Type.String(), {
662
+ description:
663
+ "Important terms that should co-occur in the returned source window.",
664
+ }),
665
+ ),
666
+ reads: Type.Optional(
667
+ Type.Array(
668
+ Type.Object({
669
+ query: Type.Optional(
670
+ Type.String({ description: "Exact or fuzzy text to locate." }),
671
+ ),
672
+ exact: Type.Optional(
673
+ Type.String({ description: "Exact text to locate." }),
674
+ ),
675
+ exactText: Type.Optional(
676
+ Type.String({ description: "Exact text to locate." }),
677
+ ),
678
+ text: Type.Optional(
679
+ Type.String({ description: "Text to locate." }),
680
+ ),
681
+ claim: Type.Optional(
682
+ Type.String({
683
+ description: "Claim to locate when exact quote is unknown.",
684
+ }),
685
+ ),
686
+ terms: Type.Optional(
687
+ Type.Array(Type.String(), {
688
+ description:
689
+ "Important terms for deterministic quote harvesting.",
690
+ }),
691
+ ),
692
+ maxChars: Type.Optional(
693
+ Type.Number({
694
+ description:
695
+ "Maximum visible snippet characters for this read.",
696
+ }),
697
+ ),
698
+ }),
699
+ {
700
+ description:
701
+ "Mixed batch reads for one sourceRef; each item can use query or claim+terms.",
702
+ },
703
+ ),
704
+ ),
705
+ maxChars: Type.Optional(
706
+ Type.Number({
707
+ description: "Maximum visible snippet characters per query.",
708
+ }),
709
+ ),
493
710
  }),
494
711
  execute: async (_toolCallId, params) => {
495
- const sourceRef = stringParam(params, "sourceRef") ?? stringParam(params, "source_ref");
712
+ const sourceRef =
713
+ stringParam(params, "sourceRef") ?? stringParam(params, "source_ref");
496
714
  const requests = sourceReadRequestsFromParams(params);
497
715
  if (!sourceRef || requests.length === 0) {
498
716
  return errorToolResult(
@@ -502,18 +720,28 @@ export function registerWorkflowWebSourceExtension(
502
720
  }
503
721
  const source = await readCachedWorkflowWebSource(sourceRef);
504
722
  if (!source) {
505
- await recordWorkflowWebSourceEvent(config, "source_read_missing", { sourceRef });
506
- return errorToolResult("source_not_found", "No cached workflow web source exists for sourceRef.", {
723
+ await recordWorkflowWebSourceEvent(config, "source_read_missing", {
507
724
  sourceRef,
508
725
  });
726
+ return errorToolResult(
727
+ "source_not_found",
728
+ "No cached workflow web source exists for sourceRef.",
729
+ {
730
+ sourceRef,
731
+ },
732
+ );
509
733
  }
510
- const maxChars = positiveIntParam(params, "maxChars") ?? policy.sourceReadMaxChars;
734
+ const maxChars =
735
+ positiveIntParam(params, "maxChars") ?? policy.sourceReadMaxChars;
511
736
  const perQueryMaxChars = Math.min(maxChars, policy.sourceReadMaxChars);
512
737
  const reads = readWorkflowWebSourceSnippets({
513
738
  source,
514
739
  requests: requests.map((request) => ({
515
740
  ...request,
516
- maxChars: Math.min(request.maxChars ?? perQueryMaxChars, policy.sourceReadMaxChars),
741
+ maxChars: Math.min(
742
+ request.maxChars ?? perQueryMaxChars,
743
+ policy.sourceReadMaxChars,
744
+ ),
517
745
  })),
518
746
  maxChars: perQueryMaxChars,
519
747
  budget,
@@ -532,14 +760,20 @@ export function registerWorkflowWebSourceExtension(
532
760
  missingTerms: read.missingTerms,
533
761
  coverageRatio: read.coverageRatio,
534
762
  candidateOnly: read.candidateOnly,
763
+ truncated: read.truncated,
535
764
  quote: status === "budget_exhausted" ? undefined : read.quote,
536
765
  startOffset: read.startOffset,
537
766
  endOffset: read.endOffset,
538
767
  visibleChars: read.visibleChars,
539
768
  };
540
769
  });
541
- const responseStatus = aggregateSourceReadStatus(results.map((result) => result.status));
542
- const visibleChars = results.reduce((total, result) => total + result.visibleChars, 0);
770
+ const responseStatus = aggregateSourceReadStatus(
771
+ results.map((result) => result.status),
772
+ );
773
+ const visibleChars = results.reduce(
774
+ (total, result) => total + result.visibleChars,
775
+ 0,
776
+ );
543
777
  await recordWorkflowWebSourceEvent(config, "source_read", {
544
778
  sourceRef,
545
779
  status: responseStatus,
@@ -561,32 +795,48 @@ export function registerWorkflowWebSourceExtension(
561
795
  missingTerms: result.missingTerms,
562
796
  coverageRatio: result.coverageRatio,
563
797
  candidateOnly: result.candidateOnly,
564
- quote: result.status === "budget_exhausted" ? undefined : result.quote,
798
+ truncated: result.truncated,
799
+ quote:
800
+ result.status === "budget_exhausted" ? undefined : result.quote,
565
801
  startOffset: result.startOffset,
566
802
  endOffset: result.endOffset,
567
- budget: budgetSnapshot(result.status === "budget_exhausted"),
803
+ budget: budgetSnapshot(
804
+ result.status === "budget_exhausted" ||
805
+ result.status === "truncated",
806
+ ),
568
807
  next:
569
808
  result.status === "budget_exhausted"
570
809
  ? "Visible web-source budget is exhausted for this task; cite the sourceRef as an evidence gap or use a smaller query in a fresh task."
571
- : undefined,
810
+ : result.status === "truncated"
811
+ ? "The matched web-source snippet was truncated by the visible budget or maxChars; use a smaller exact query or a fresh task if the full quote is required."
812
+ : undefined,
572
813
  });
573
814
  }
815
+ const hasBudgetExhaustedRead = results.some(
816
+ (result) => result.status === "budget_exhausted",
817
+ );
818
+ const hasTruncatedRead = results.some(
819
+ (result) => result.status === "truncated",
820
+ );
574
821
  return toolResultFromJson({
575
822
  status: responseStatus,
576
823
  tool: "workflow_web_source_read",
577
824
  sourceRef,
578
825
  url: source.redactedUrl,
579
826
  results,
580
- budget: budgetSnapshot(results.some((result) => result.status === "budget_exhausted")),
581
- next:
582
- responseStatus === "budget_exhausted"
583
- ? "Visible web-source budget is exhausted for this task; cite missing quotes as evidence gaps or use smaller query batches in a fresh task."
827
+ budget: budgetSnapshot(hasBudgetExhaustedRead || hasTruncatedRead),
828
+ next: hasBudgetExhaustedRead
829
+ ? "Visible web-source budget is exhausted for this task; cite missing quotes as evidence gaps or use smaller query batches in a fresh task."
830
+ : hasTruncatedRead
831
+ ? "One or more matched web-source snippets were truncated by the visible budget or maxChars; use smaller exact queries or a fresh task if full quotes are required."
584
832
  : undefined,
585
833
  });
586
834
  },
587
835
  });
588
836
 
589
- async function readCachedWorkflowWebSource(sourceRef: string): Promise<WorkflowWebSource | undefined> {
837
+ async function readCachedWorkflowWebSource(
838
+ sourceRef: string,
839
+ ): Promise<WorkflowWebSource | undefined> {
590
840
  const cached = sourceCache.get(sourceRef);
591
841
  if (cached) return cached;
592
842
  const source = await readWorkflowWebSource(config, sourceRef);
@@ -669,7 +919,12 @@ async function cachedFetchFailureResult(
669
919
  config: WorkflowWebSourceCacheConfig,
670
920
  cache: Map<string, FetchFailure>,
671
921
  key: string,
672
- failure: { code: string; message: string; extra: Record<string, unknown>; reason: string },
922
+ failure: {
923
+ code: string;
924
+ message: string;
925
+ extra: Record<string, unknown>;
926
+ reason: string;
927
+ },
673
928
  ): Promise<ReturnType<typeof toolResultFromJson>> {
674
929
  const cached = {
675
930
  code: failure.code,
@@ -751,7 +1006,9 @@ async function readDurableFetchFailure(
751
1006
  key: string,
752
1007
  ): Promise<FetchFailure | undefined> {
753
1008
  try {
754
- const parsed = JSON.parse(await readFile(fetchFailurePath(config, key), "utf8")) as unknown;
1009
+ const parsed = JSON.parse(
1010
+ await readFile(fetchFailurePath(config, key), "utf8"),
1011
+ ) as unknown;
755
1012
  return normalizeFetchFailure(parsed);
756
1013
  } catch {
757
1014
  return undefined;
@@ -773,23 +1030,36 @@ async function writeDurableFetchFailure(
773
1030
 
774
1031
  function normalizeFetchFailure(value: unknown): FetchFailure | undefined {
775
1032
  if (!isRecord(value)) return undefined;
776
- if (typeof value.code !== "string" || typeof value.message !== "string") return undefined;
1033
+ if (typeof value.code !== "string" || typeof value.message !== "string")
1034
+ return undefined;
777
1035
  const extra = isRecord(value.extra) ? value.extra : {};
778
1036
  return {
779
1037
  code: value.code,
780
1038
  message: value.message,
781
1039
  extra,
782
1040
  ...(typeof value.reason === "string" ? { reason: value.reason } : {}),
783
- ...(typeof value.createdAt === "string" ? { createdAt: value.createdAt } : {}),
1041
+ ...(typeof value.createdAt === "string"
1042
+ ? { createdAt: value.createdAt }
1043
+ : {}),
784
1044
  };
785
1045
  }
786
1046
 
787
- function fetchLockPath(config: WorkflowWebSourceCacheConfig, key: string): string {
1047
+ function fetchLockPath(
1048
+ config: WorkflowWebSourceCacheConfig,
1049
+ key: string,
1050
+ ): string {
788
1051
  return resolve(config.cacheDir, "fetch-locks", fetchCacheFileKey(key));
789
1052
  }
790
1053
 
791
- function fetchFailurePath(config: WorkflowWebSourceCacheConfig, key: string): string {
792
- return resolve(config.cacheDir, "fetch-negative-cache", `${fetchCacheFileKey(key)}.json`);
1054
+ function fetchFailurePath(
1055
+ config: WorkflowWebSourceCacheConfig,
1056
+ key: string,
1057
+ ): string {
1058
+ return resolve(
1059
+ config.cacheDir,
1060
+ "fetch-negative-cache",
1061
+ `${fetchCacheFileKey(key)}.json`,
1062
+ );
793
1063
  }
794
1064
 
795
1065
  function fetchCacheFileKey(key: string): string {
@@ -813,7 +1083,11 @@ function shouldCacheFetchFailure(reason: string): boolean {
813
1083
  }
814
1084
 
815
1085
  function shouldCacheFetchFailureInMemory(reason: string): boolean {
816
- return reason === "empty_source" || reason === "dns_resolution_failed" || reason.includes("ENOTFOUND");
1086
+ return (
1087
+ reason === "empty_source" ||
1088
+ reason === "dns_resolution_failed" ||
1089
+ reason.includes("ENOTFOUND")
1090
+ );
817
1091
  }
818
1092
 
819
1093
  const WORKFLOW_WEB_FETCH_TIMEOUT_MS = 30_000;
@@ -824,25 +1098,46 @@ async function safeFetchWorkflowWebText(
824
1098
  security: WorkflowWebSecurityPolicy,
825
1099
  signal?: AbortSignal,
826
1100
  ): Promise<
827
- | { ok: true; url: string; text: string; title?: string; extractionLossy?: boolean }
1101
+ | {
1102
+ ok: true;
1103
+ url: string;
1104
+ text: string;
1105
+ title?: string;
1106
+ extractionLossy?: boolean;
1107
+ }
828
1108
  | { ok: false; reason: string; url: string }
829
1109
  > {
830
1110
  let current = url;
831
1111
  for (let redirectCount = 0; redirectCount < 6; redirectCount += 1) {
832
1112
  const checked = validateWorkflowWebUrl(current, security);
833
1113
  if (!checked.ok) return { ok: false, reason: checked.reason, url: current };
834
- const response = await safeFetchOnce(checked.normalizedUrl, security, signal);
1114
+ const response = await safeFetchOnce(
1115
+ checked.normalizedUrl,
1116
+ security,
1117
+ signal,
1118
+ );
835
1119
  if (!response.ok) return response;
836
1120
  if (response.status >= 300 && response.status < 400) {
837
1121
  if (!response.location)
838
- return { ok: false, reason: "redirect_without_location", url: checked.normalizedUrl };
1122
+ return {
1123
+ ok: false,
1124
+ reason: "redirect_without_location",
1125
+ url: checked.normalizedUrl,
1126
+ };
839
1127
  current = new URL(response.location, checked.normalizedUrl).href;
840
1128
  continue;
841
1129
  }
842
1130
  if (response.status < 200 || response.status >= 300) {
843
- return { ok: false, reason: `http_${response.status}`, url: checked.normalizedUrl };
1131
+ return {
1132
+ ok: false,
1133
+ reason: `http_${response.status}`,
1134
+ url: checked.normalizedUrl,
1135
+ };
844
1136
  }
845
- const extracted = extractWorkflowWebResponseText(response.text, response.contentType);
1137
+ const extracted = extractWorkflowWebResponseText(
1138
+ response.text,
1139
+ response.contentType,
1140
+ );
846
1141
  return {
847
1142
  ok: true,
848
1143
  url: checked.normalizedUrl,
@@ -859,7 +1154,14 @@ function safeFetchOnce(
859
1154
  security: WorkflowWebSecurityPolicy,
860
1155
  signal?: AbortSignal,
861
1156
  ): Promise<
862
- | { ok: true; status: number; location?: string; text: string; contentType?: string; truncated?: boolean }
1157
+ | {
1158
+ ok: true;
1159
+ status: number;
1160
+ location?: string;
1161
+ text: string;
1162
+ contentType?: string;
1163
+ truncated?: boolean;
1164
+ }
863
1165
  | { ok: false; reason: string; url: string }
864
1166
  > {
865
1167
  const parsed = new URL(url);
@@ -868,7 +1170,14 @@ function safeFetchOnce(
868
1170
  let settled = false;
869
1171
  const settle = (
870
1172
  result:
871
- | { ok: true; status: number; location?: string; text: string; contentType?: string; truncated?: boolean }
1173
+ | {
1174
+ ok: true;
1175
+ status: number;
1176
+ location?: string;
1177
+ text: string;
1178
+ contentType?: string;
1179
+ truncated?: boolean;
1180
+ }
872
1181
  | { ok: false; reason: string; url: string },
873
1182
  ) => {
874
1183
  if (settled) return;
@@ -880,20 +1189,26 @@ function safeFetchOnce(
880
1189
  {
881
1190
  method: "GET",
882
1191
  headers: {
883
- accept: "text/plain,text/html,application/json,application/xml;q=0.9,*/*;q=0.1",
1192
+ accept:
1193
+ "text/plain,text/html,application/json,application/xml;q=0.9,*/*;q=0.1",
884
1194
  "user-agent": "pi-workflow-web-source/1",
885
1195
  },
886
1196
  lookup(hostname, options, callback) {
887
1197
  lookupPublicAddress(hostname, security)
888
1198
  .then((address) => {
889
1199
  if (isLookupAllOptions(options)) {
890
- callback(null, [{ address: address.address, family: address.family }]);
1200
+ callback(null, [
1201
+ { address: address.address, family: address.family },
1202
+ ]);
891
1203
  return;
892
1204
  }
893
1205
  callback(null, address.address, address.family);
894
1206
  })
895
1207
  .catch((error: unknown) => {
896
- const reason = error instanceof Error ? error.message : "dns_resolution_failed";
1208
+ const reason =
1209
+ error instanceof Error
1210
+ ? error.message
1211
+ : "dns_resolution_failed";
897
1212
  callback(new Error(reason), "", 4);
898
1213
  });
899
1214
  },
@@ -906,7 +1221,12 @@ function safeFetchOnce(
906
1221
  ? res.headers["content-type"][0]
907
1222
  : res.headers["content-type"];
908
1223
  const status = res.statusCode ?? 0;
909
- if (status >= 200 && status < 300 && contentType && !isWorkflowWebTextContentType(contentType)) {
1224
+ if (
1225
+ status >= 200 &&
1226
+ status < 300 &&
1227
+ contentType &&
1228
+ !isWorkflowWebTextContentType(contentType)
1229
+ ) {
910
1230
  res.resume();
911
1231
  settle({ ok: false, reason: "unsupported_content_type", url });
912
1232
  return;
@@ -914,7 +1234,10 @@ function safeFetchOnce(
914
1234
  res.on("data", (chunk: string) => {
915
1235
  if (settled) return;
916
1236
  if (text.length + chunk.length > WORKFLOW_WEB_FETCH_MAX_CHARS) {
917
- text += chunk.slice(0, Math.max(0, WORKFLOW_WEB_FETCH_MAX_CHARS - text.length));
1237
+ text += chunk.slice(
1238
+ 0,
1239
+ Math.max(0, WORKFLOW_WEB_FETCH_MAX_CHARS - text.length),
1240
+ );
918
1241
  truncated = true;
919
1242
  req.destroy(new Error("workflow_fetch_truncated"));
920
1243
  return;
@@ -977,7 +1300,9 @@ async function lookupPublicAddress(
977
1300
  : privateIpReason(address.address);
978
1301
  if (!reason) return address;
979
1302
  }
980
- throw new Error(addresses.length > 0 ? "private_host_blocked" : "dns_resolution_failed");
1303
+ throw new Error(
1304
+ addresses.length > 0 ? "private_host_blocked" : "dns_resolution_failed",
1305
+ );
981
1306
  }
982
1307
 
983
1308
  async function validateResolvedHost(
@@ -992,7 +1317,10 @@ async function validateResolvedHost(
992
1317
  return { ok: false, reason: "invalid_url", url };
993
1318
  }
994
1319
  try {
995
- const addresses = await lookup(parsed.hostname, { all: true, verbatim: true });
1320
+ const addresses = await lookup(parsed.hostname, {
1321
+ all: true,
1322
+ verbatim: true,
1323
+ });
996
1324
  for (const address of addresses) {
997
1325
  const reason = privateIpReason(address.address);
998
1326
  if (reason) return { ok: false, reason, url };
@@ -1015,27 +1343,38 @@ function privateIpReason(address: string): string | undefined {
1015
1343
  if (hexMapped) {
1016
1344
  const high = Number.parseInt(hexMapped[1]!, 16);
1017
1345
  const low = Number.parseInt(hexMapped[2]!, 16);
1018
- return privateIpReason(`${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`);
1346
+ return privateIpReason(
1347
+ `${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`,
1348
+ );
1019
1349
  }
1020
1350
  if (isIP(lower) === 4) {
1021
1351
  const parts = lower.split(".").map((part) => Number(part));
1022
- if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return "private_host_blocked";
1352
+ if (
1353
+ parts.length !== 4 ||
1354
+ parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)
1355
+ )
1356
+ return "private_host_blocked";
1023
1357
  const [a, b, c, d] = parts as [number, number, number, number];
1024
- if (a === 0 || a === 10 || a === 127 || a >= 224) return "private_host_blocked";
1358
+ if (a === 0 || a === 10 || a === 127 || a >= 224)
1359
+ return "private_host_blocked";
1025
1360
  if (a === 100 && b >= 64 && b <= 127) return "private_host_blocked";
1026
1361
  if (a === 169 && b === 254) return "private_host_blocked";
1027
1362
  if (a === 172 && b >= 16 && b <= 31) return "private_host_blocked";
1028
1363
  if (a === 192 && b === 168) return "private_host_blocked";
1029
- if (a === 192 && b === 0 && (c === 0 || c === 2)) return "private_host_blocked";
1364
+ if (a === 192 && b === 0 && (c === 0 || c === 2))
1365
+ return "private_host_blocked";
1030
1366
  if (a === 198 && (b === 18 || b === 19)) return "private_host_blocked";
1031
1367
  if (a === 198 && b === 51 && c === 100) return "private_host_blocked";
1032
1368
  if (a === 203 && b === 0 && c === 113) return "private_host_blocked";
1033
- if (a === 255 && b === 255 && c === 255 && d === 255) return "private_host_blocked";
1369
+ if (a === 255 && b === 255 && c === 255 && d === 255)
1370
+ return "private_host_blocked";
1034
1371
  }
1035
1372
  if (isIP(lower) === 6) {
1036
1373
  if (lower === "::" || lower === "::1") return "private_host_blocked";
1037
- if (lower.startsWith("fc") || lower.startsWith("fd")) return "private_host_blocked";
1038
- if (lower.startsWith("fe80") || lower.startsWith("ff")) return "private_host_blocked";
1374
+ if (lower.startsWith("fc") || lower.startsWith("fd"))
1375
+ return "private_host_blocked";
1376
+ if (lower.startsWith("fe80") || lower.startsWith("ff"))
1377
+ return "private_host_blocked";
1039
1378
  if (lower.startsWith("2001:db8")) return "private_host_blocked";
1040
1379
  }
1041
1380
  return undefined;
@@ -1048,7 +1387,10 @@ async function validateProviderResultUrls(
1048
1387
  for (const url of providerResultUrls(result)) {
1049
1388
  const checked = validateWorkflowWebUrl(url, security);
1050
1389
  if (!checked.ok) return { ok: false, reason: checked.reason, url };
1051
- const resolved = await validateResolvedHost(checked.normalizedUrl, security);
1390
+ const resolved = await validateResolvedHost(
1391
+ checked.normalizedUrl,
1392
+ security,
1393
+ );
1052
1394
  if (!resolved.ok) return resolved;
1053
1395
  }
1054
1396
  if (!security.allowPrivateHosts) {
@@ -1151,11 +1493,12 @@ function canonicalWorkflowWebFetchUrl(url: string): string {
1151
1493
  if (parsed.pathname.length > 1 && parsed.pathname.endsWith("/")) {
1152
1494
  parsed.pathname = parsed.pathname.slice(0, -1);
1153
1495
  }
1154
- const sortedParams = [...parsed.searchParams.entries()].sort(([left], [right]) =>
1155
- left.localeCompare(right),
1496
+ const sortedParams = [...parsed.searchParams.entries()].sort(
1497
+ ([left], [right]) => left.localeCompare(right),
1156
1498
  );
1157
1499
  parsed.search = "";
1158
- for (const [key, value] of sortedParams) parsed.searchParams.append(key, value);
1500
+ for (const [key, value] of sortedParams)
1501
+ parsed.searchParams.append(key, value);
1159
1502
  return parsed.href;
1160
1503
  }
1161
1504
 
@@ -1165,13 +1508,20 @@ function shouldKeepWorkflowWebFragment(hash: string): boolean {
1165
1508
  return raw.startsWith("/") || raw.startsWith("!") || raw.includes("?");
1166
1509
  }
1167
1510
 
1168
- function fetchSourceRequestsFromParams(params: unknown): WorkflowWebFetchSourceRequest[] {
1511
+ function fetchSourceRequestsFromParams(
1512
+ params: unknown,
1513
+ ): WorkflowWebFetchSourceRequest[] {
1169
1514
  if (!isRecord(params)) return [];
1170
1515
  const requests: WorkflowWebFetchSourceRequest[] = [];
1171
1516
  const titles = Array.isArray(params.titles) ? params.titles : [];
1172
1517
  if (Array.isArray(params.sources)) {
1173
1518
  for (const source of params.sources) {
1174
- if (!isRecord(source) || typeof source.url !== "string" || !source.url.trim()) continue;
1519
+ if (
1520
+ !isRecord(source) ||
1521
+ typeof source.url !== "string" ||
1522
+ !source.url.trim()
1523
+ )
1524
+ continue;
1175
1525
  requests.push({
1176
1526
  url: source.url.trim(),
1177
1527
  ...(typeof source.title === "string" && source.title.trim()
@@ -1186,7 +1536,9 @@ function fetchSourceRequestsFromParams(params: unknown): WorkflowWebFetchSourceR
1186
1536
  const title = titles[index];
1187
1537
  requests.push({
1188
1538
  url: url.trim(),
1189
- ...(typeof title === "string" && title.trim() ? { title: title.trim() } : {}),
1539
+ ...(typeof title === "string" && title.trim()
1540
+ ? { title: title.trim() }
1541
+ : {}),
1190
1542
  });
1191
1543
  }
1192
1544
  }
@@ -1201,7 +1553,9 @@ function fetchSourceRequestsFromParams(params: unknown): WorkflowWebFetchSourceR
1201
1553
  return dedupeFetchSourceRequests(requests).slice(0, 20);
1202
1554
  }
1203
1555
 
1204
- function dedupeFetchSourceRequests(requests: WorkflowWebFetchSourceRequest[]): WorkflowWebFetchSourceRequest[] {
1556
+ function dedupeFetchSourceRequests(
1557
+ requests: WorkflowWebFetchSourceRequest[],
1558
+ ): WorkflowWebFetchSourceRequest[] {
1205
1559
  const deduped: WorkflowWebFetchSourceRequest[] = [];
1206
1560
  const seen = new Set<string>();
1207
1561
  for (const request of requests) {
@@ -1214,7 +1568,9 @@ function dedupeFetchSourceRequests(requests: WorkflowWebFetchSourceRequest[]): W
1214
1568
  }
1215
1569
 
1216
1570
  function payloadFromToolResult(result: ToolResult): Record<string, unknown> {
1217
- const text = result.content?.find((item) => typeof item.text === "string")?.text;
1571
+ const text = result.content?.find(
1572
+ (item) => typeof item.text === "string",
1573
+ )?.text;
1218
1574
  if (typeof text !== "string") return {};
1219
1575
  try {
1220
1576
  const payload = JSON.parse(text);
@@ -1237,7 +1593,9 @@ function titleFromParams(params: unknown): string | undefined {
1237
1593
  return stringParam(params, "title");
1238
1594
  }
1239
1595
 
1240
- function sourceReadRequestsFromParams(params: unknown): WorkflowWebSourceReadRequest[] {
1596
+ function sourceReadRequestsFromParams(
1597
+ params: unknown,
1598
+ ): WorkflowWebSourceReadRequest[] {
1241
1599
  const requests: WorkflowWebSourceReadRequest[] = [];
1242
1600
  if (isRecord(params) && Array.isArray(params.reads)) {
1243
1601
  for (const item of params.reads) {
@@ -1245,9 +1603,12 @@ function sourceReadRequestsFromParams(params: unknown): WorkflowWebSourceReadReq
1245
1603
  if (request) requests.push(request);
1246
1604
  }
1247
1605
  }
1248
- for (const query of stringArrayParam(params, "queries")) requests.push({ query });
1249
- for (const query of stringArrayParam(params, "exactTexts")) requests.push({ query });
1250
- for (const query of stringArrayParam(params, "texts")) requests.push({ query });
1606
+ for (const query of stringArrayParam(params, "queries"))
1607
+ requests.push({ query });
1608
+ for (const query of stringArrayParam(params, "exactTexts"))
1609
+ requests.push({ query });
1610
+ for (const query of stringArrayParam(params, "texts"))
1611
+ requests.push({ query });
1251
1612
  const query =
1252
1613
  stringParam(params, "query") ??
1253
1614
  stringParam(params, "exactText") ??
@@ -1255,11 +1616,14 @@ function sourceReadRequestsFromParams(params: unknown): WorkflowWebSourceReadReq
1255
1616
  stringParam(params, "text");
1256
1617
  const claim = stringParam(params, "claim");
1257
1618
  const terms = stringArrayParam(params, "terms");
1258
- if (query || claim || terms.length > 0) requests.push({ query, claim, terms });
1619
+ if (query || claim || terms.length > 0)
1620
+ requests.push({ query, claim, terms });
1259
1621
  return dedupeSourceReadRequests(requests).slice(0, 20);
1260
1622
  }
1261
1623
 
1262
- function sourceReadRequestFromRecord(value: unknown): WorkflowWebSourceReadRequest | undefined {
1624
+ function sourceReadRequestFromRecord(
1625
+ value: unknown,
1626
+ ): WorkflowWebSourceReadRequest | undefined {
1263
1627
  if (!isRecord(value)) return undefined;
1264
1628
  const query =
1265
1629
  stringParam(value, "query") ??
@@ -1273,7 +1637,9 @@ function sourceReadRequestFromRecord(value: unknown): WorkflowWebSourceReadReque
1273
1637
  return { query, claim, terms, maxChars };
1274
1638
  }
1275
1639
 
1276
- function dedupeSourceReadRequests(requests: WorkflowWebSourceReadRequest[]): WorkflowWebSourceReadRequest[] {
1640
+ function dedupeSourceReadRequests(
1641
+ requests: WorkflowWebSourceReadRequest[],
1642
+ ): WorkflowWebSourceReadRequest[] {
1277
1643
  const deduped: WorkflowWebSourceReadRequest[] = [];
1278
1644
  const seen = new Set<string>();
1279
1645
  for (const request of requests) {
@@ -1292,18 +1658,27 @@ function dedupeSourceReadRequests(requests: WorkflowWebSourceReadRequest[]): Wor
1292
1658
 
1293
1659
  function sourceReadBatchRequested(params: unknown): boolean {
1294
1660
  return (
1295
- (isRecord(params) && Array.isArray(params.reads) && params.reads.length > 0) ||
1661
+ (isRecord(params) &&
1662
+ Array.isArray(params.reads) &&
1663
+ params.reads.length > 0) ||
1296
1664
  stringArrayParam(params, "queries").length > 0 ||
1297
1665
  stringArrayParam(params, "exactTexts").length > 0 ||
1298
1666
  stringArrayParam(params, "texts").length > 0
1299
1667
  );
1300
1668
  }
1301
1669
 
1302
- type SourceReadToolStatus = "ok" | "candidate" | "budget_exhausted" | "not_found";
1670
+ type SourceReadToolStatus =
1671
+ | "ok"
1672
+ | "candidate"
1673
+ | "truncated"
1674
+ | "budget_exhausted"
1675
+ | "not_found";
1303
1676
 
1304
1677
  function sourceReadResponseStatus(
1305
1678
  read: WorkflowWebSourceReadResult,
1306
1679
  ): SourceReadToolStatus {
1680
+ if (read.status === "truncated" && !read.quote) return "budget_exhausted";
1681
+ if (read.status === "truncated") return "truncated";
1307
1682
  if (read.status === "matched" && !read.quote) return "budget_exhausted";
1308
1683
  if (read.status === "matched" && read.candidateOnly) return "candidate";
1309
1684
  if (read.status === "matched") return "ok";
@@ -1312,11 +1687,19 @@ function sourceReadResponseStatus(
1312
1687
 
1313
1688
  function aggregateSourceReadStatus(
1314
1689
  statuses: SourceReadToolStatus[],
1315
- ): "ok" | "candidate" | "partial" | "budget_exhausted" | "not_found" {
1690
+ ):
1691
+ | "ok"
1692
+ | "candidate"
1693
+ | "partial"
1694
+ | "truncated"
1695
+ | "budget_exhausted"
1696
+ | "not_found" {
1316
1697
  if (statuses.every((status) => status === "ok")) return "ok";
1317
1698
  if (statuses.every((status) => status === "candidate")) return "candidate";
1699
+ if (statuses.every((status) => status === "truncated")) return "truncated";
1318
1700
  if (statuses.every((status) => status === "not_found")) return "not_found";
1319
- if (statuses.every((status) => status === "budget_exhausted")) return "budget_exhausted";
1701
+ if (statuses.every((status) => status === "budget_exhausted"))
1702
+ return "budget_exhausted";
1320
1703
  return "partial";
1321
1704
  }
1322
1705
 
@@ -1345,18 +1728,25 @@ function positiveIntParam(params: unknown, key: string): number | undefined {
1345
1728
  }
1346
1729
 
1347
1730
  function isWorkflowWebTextContentType(contentType: string): boolean {
1348
- return /^(text\/|application\/(json|xml|xhtml\+xml|ld\+json)|[^;]+\+json\b|[^;]+\+xml\b)/i.test(contentType.trim());
1731
+ return /^(text\/|application\/(json|xml|xhtml\+xml|ld\+json)|[^;]+\+json\b|[^;]+\+xml\b)/i.test(
1732
+ contentType.trim(),
1733
+ );
1349
1734
  }
1350
1735
 
1351
1736
  function extractWorkflowWebResponseText(
1352
1737
  text: string,
1353
1738
  contentType?: string,
1354
1739
  ): { text: string; title?: string; lossy?: boolean } {
1355
- const looksHtml = /html/i.test(contentType ?? "") || /<html[\s>]|<body[\s>]|<title[\s>]/i.test(text);
1740
+ const looksHtml =
1741
+ /html/i.test(contentType ?? "") ||
1742
+ /<html[\s>]|<body[\s>]|<title[\s>]/i.test(text);
1356
1743
  if (!looksHtml) {
1357
1744
  return { text, title: titleFromPlainText(text) };
1358
1745
  }
1359
- const title = decodeHtmlEntities(text.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim() ?? "").slice(0, 200) || undefined;
1746
+ const title =
1747
+ decodeHtmlEntities(
1748
+ text.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim() ?? "",
1749
+ ).slice(0, 200) || undefined;
1360
1750
  const body = text
1361
1751
  .replace(/<script\b[\s\S]*?<\/script>/gi, " ")
1362
1752
  .replace(/<style\b[\s\S]*?<\/style>/gi, " ")
@@ -1407,5 +1797,8 @@ function extensionImportSpecifier(importPath: string): string {
1407
1797
  }
1408
1798
 
1409
1799
  export function workflowWebSourceModuleImportPath(modulePath: string): string {
1410
- return resolve(dirname(modulePath), `workflow-web-source-extension${extname(modulePath)}`);
1800
+ return resolve(
1801
+ dirname(modulePath),
1802
+ `workflow-web-source-extension${extname(modulePath)}`,
1803
+ );
1411
1804
  }