@qvac/harness 0.3.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +262 -0
  2. package/dist/harness.bundle +15 -11
  3. package/dist-lib/index.d.ts +1 -1
  4. package/dist-lib/lib/capabilities/completion.d.ts +5 -1
  5. package/dist-lib/lib/capabilities/completion.js +20 -8
  6. package/dist-lib/lib/helpers/tool-call-markup.js +15 -7
  7. package/dist-lib/lib/no-redirect-fetch-impl.d.ts +3 -2
  8. package/dist-lib/lib/no-redirect-fetch-impl.js +4 -1
  9. package/dist-lib/lib/tools/exec/adapters/adapter.d.ts +5 -1
  10. package/dist-lib/lib/tools/exec/adapters/python/distributions.d.ts +1 -0
  11. package/dist-lib/lib/tools/exec/adapters/python/distributions.js +4 -0
  12. package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.d.ts +72 -0
  13. package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.js +222 -0
  14. package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.d.ts +5 -0
  15. package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.js +29 -0
  16. package/dist-lib/lib/tools/exec/adapters/python/loader.js +2 -1
  17. package/dist-lib/lib/tools/exec/adapters/python/module-distributions.d.ts +1 -0
  18. package/dist-lib/lib/tools/exec/adapters/python/module-distributions.js +52 -0
  19. package/dist-lib/lib/tools/exec/adapters/python/named-hosts.d.ts +2 -0
  20. package/dist-lib/lib/tools/exec/adapters/python/named-hosts.js +15 -0
  21. package/dist-lib/lib/tools/exec/adapters/python/net-shim.d.ts +2 -0
  22. package/dist-lib/lib/tools/exec/adapters/python/net-shim.js +148 -0
  23. package/dist-lib/lib/tools/exec/adapters/python/output-policy.js +13 -1
  24. package/dist-lib/lib/tools/exec/adapters/python/pyodide.d.ts +4 -1
  25. package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +42 -7
  26. package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts +9 -3
  27. package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js +137 -21
  28. package/dist-lib/lib/tools/exec/adapters/python/run-session.d.ts +5 -0
  29. package/dist-lib/lib/tools/exec/adapters/python/run-session.js +33 -8
  30. package/dist-lib/lib/tools/exec/adapters/python/wheels.d.ts +6 -0
  31. package/dist-lib/lib/tools/exec/adapters/python/wheels.js +17 -3
  32. package/dist-lib/lib/tools/exec/adapters/python/worker/client.d.ts +2 -1
  33. package/dist-lib/lib/tools/exec/adapters/python/worker/client.js +16 -3
  34. package/dist-lib/lib/tools/exec/adapters/python/worker/entry.js +15 -2
  35. package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.d.ts +6 -0
  36. package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.js +6 -2
  37. package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.d.ts +1 -0
  38. package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.js +8 -0
  39. package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.d.ts +1 -0
  40. package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.js +8 -0
  41. package/dist-lib/lib/tools/exec/adapters/shell/shell-adapter.js +7 -1
  42. package/dist-lib/lib/tools/exec/shell-guidance.d.ts +10 -0
  43. package/dist-lib/lib/tools/exec/shell-guidance.js +123 -0
  44. package/dist-lib/lib/tools/exec/video-tools.d.ts +5 -0
  45. package/dist-lib/lib/tools/exec/video-tools.js +40 -0
  46. package/dist-lib/lib/tools/exec.d.ts +2 -1
  47. package/dist-lib/lib/tools/exec.js +93 -33
  48. package/dist-lib/lib/tools/http-request/truncate-response.d.ts +1 -0
  49. package/dist-lib/lib/tools/http-request/truncate-response.js +1 -1
  50. package/dist-lib/lib/tools/http-request.js +4 -1
  51. package/dist-lib/lib/tools/operations/builders.d.ts +1 -1
  52. package/dist-lib/lib/tools/operations/builders.js +9 -0
  53. package/dist-lib/lib/tools/operations/compile.js +41 -18
  54. package/dist-lib/lib/tools/operations/decoders.d.ts +6 -0
  55. package/dist-lib/lib/tools/operations/decoders.js +82 -0
  56. package/dist-lib/lib/tools/operations/manifest.d.ts +16 -2
  57. package/dist-lib/lib/tools/operations/manifest.js +6 -1
  58. package/dist-lib/lib/tools/skill.d.ts +3 -0
  59. package/dist-lib/lib/tools/skill.js +38 -5
  60. package/dist-lib/lib/tools/skills/manifest-parse.js +43 -1
  61. package/dist-lib/lib/tools/skills/manifest.d.ts +13 -0
  62. package/dist-lib/schema/harness.js +20 -1
  63. package/dist-lib/spec/hrpc/messages.js +93 -5
  64. package/dist-lib/spec/hyperschema/index.js +93 -5
  65. package/dist-lib/spec/hyperschema/schema.json +66 -1
  66. package/dist-lib/spec/hyperschema/types.d.ts +18 -1
  67. package/package.json +40 -2
  68. package/wheels/manifest.json +15 -0
@@ -44,7 +44,7 @@ export { requiredSettings, settingEnvName, type SkillSettingEntry, type SkillSet
44
44
  export type { ToolCatalogEntry } from './lib/tools/catalog.ts';
45
45
  export { TOOLSETS, toolsetEnabled, type Toolset } from './lib/tools/toolsets.ts';
46
46
  export { markCredentialRejected, markCredentialValid, recordCredentialHealth, resetAllCredentialHealth, resetCredentialHealth } from './lib/tools/skills/credential-health.ts';
47
- export type { InstallStep, SkillOAuthConfig, SkillRequires, SkillSetup, SkillSetupField, SkillSetupRoute } from './lib/tools/skills/manifest.ts';
47
+ export type { InstallStep, SkillOAuthConfig, SkillOAuthVerify, SkillOAuthVerifyRule, SkillRequires, SkillSetup, SkillSetupField, SkillSetupRoute } from './lib/tools/skills/manifest.ts';
48
48
  export { readSetup as normalizeSkillSetup } from './lib/tools/skills/manifest-parse.ts';
49
49
  export type { CredentialRecord, CredentialStoreValue } from './lib/tools/skills/auth-resolver.ts';
50
50
  export { isAvailableState, SKILL_STATE, type SkillState } from './lib/tools/skills/skill-state.ts';
@@ -139,6 +139,10 @@ export interface ToolExecuteContext {
139
139
  toolOps?: ReadonlyMap<string, readonly CliValidator[]> | undefined;
140
140
  runBudget?: RunBudget | undefined;
141
141
  reportBackend?: ((use: RunBackendUse) => void) | undefined;
142
+ uncappedResponse?: boolean | undefined;
143
+ sandboxMode?: boolean | undefined;
144
+ toolNames?: readonly string[] | undefined;
145
+ approval?: ToolDecision | undefined;
142
146
  }
143
147
  export interface RunBackendUse {
144
148
  tool: string;
@@ -178,7 +182,7 @@ export interface ToolDefinition {
178
182
  toolset: Toolset | null;
179
183
  validate?: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => string | null;
180
184
  effect: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => ToolEffect;
181
- resource?: (args: Record<string, unknown>) => ResourceDescriptor | null;
185
+ resource?: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => ResourceDescriptor | null;
182
186
  compiled?: true;
183
187
  execute: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => Promise<unknown>;
184
188
  }
@@ -19,6 +19,7 @@ import { annotateLastUserTime, skillsPromptSection, systemPromptParts } from '..
19
19
  import { isMetadataMethod } from '../tools/mcp/utils.js';
20
20
  import { unquoteArg } from '../tools/model-args.js';
21
21
  import { SKILL_TOOL_NAME, WEB_SEARCH_TOOL_NAME } from './tool-names.js';
22
+ import { mainFileRequested, skillFileArg, skillFileStem } from '../tools/skill.js';
22
23
  import { promptUsageParts, usageBreakdown } from './usage-breakdown.js';
23
24
  import { collectToolGrants, enabledTools, isSelected, skillIsActive, skillsAreSelected } from './tool-selection.js';
24
25
  import { guardToolCall } from '../tools/outbound-guard.js';
@@ -272,7 +273,7 @@ async function executeCall(tool, args, ctx, { approve, signal, denied, sources =
272
273
  type: 'tool-activity',
273
274
  name: tool.name,
274
275
  args,
275
- resource: tool.resource?.(args) ?? null,
276
+ resource: tool.resource?.(args, ctx) ?? null,
276
277
  effect: tool.effect(args, ctx),
277
278
  decision: 'blocked'
278
279
  });
@@ -284,7 +285,7 @@ async function executeCall(tool, args, ctx, { approve, signal, denied, sources =
284
285
  return deniedCall();
285
286
  }
286
287
  const effect = classified ?? tool.effect(args, ctx);
287
- const resource = tool.resource?.(args) ?? null;
288
+ const resource = tool.resource?.(args, ctx) ?? null;
288
289
  const autoRun = effect === 'read' || tool.autoApprove === true || skipApprovals === true;
289
290
  hlog('tool decision', {
290
291
  tool: tool.name,
@@ -326,7 +327,7 @@ async function executeCall(tool, args, ctx, { approve, signal, denied, sources =
326
327
  let output;
327
328
  let outcome = { kind: 'success' };
328
329
  try {
329
- output = await tool.execute(args, { ...ctx, signal });
330
+ output = await tool.execute(args, { ...ctx, signal, approval: decision });
330
331
  hlog('tool done', { tool: tool.name });
331
332
  }
332
333
  catch (err) {
@@ -709,6 +710,8 @@ export async function* completion(sdk, { modelId, acquireRound, model, agent, ct
709
710
  ? { ...resolvedGenerationParams, ...generationParamsOverride }
710
711
  : resolvedGenerationParams;
711
712
  const loadedThisTurn = new Set();
713
+ const loadedFilesThisTurn = new Set();
714
+ const fileKey = (skill, file) => `${skill}\0${skillFileStem(file)}`;
712
715
  let skillIcon = null;
713
716
  const skillEmoji = (name) => skillCatalog.find((entry) => entry.name === name)?.emoji ?? null;
714
717
  const nudgedSkills = new Set();
@@ -1004,13 +1007,20 @@ export async function* completion(sdk, { modelId, acquireRound, model, agent, ct
1004
1007
  toolOps: grantedOps,
1005
1008
  fsRoots,
1006
1009
  fsWriteRoots: writableRoots,
1007
- reportBackend: (use) => stats.backends.push(use)
1010
+ reportBackend: (use) => stats.backends.push(use),
1011
+ sandboxMode: agent?.sandboxMode === true,
1012
+ toolNames: enabled.map((tool) => tool.name)
1008
1013
  };
1009
1014
  const skillCall = call.name === SKILL_TOOL_NAME;
1010
1015
  const effect = tool?.effect(args, callCtx) ?? (skillCall ? 'read' : undefined);
1011
1016
  const resource = tool?.resource?.(args) ?? null;
1012
1017
  const skillName = unquoteArg(args?.name ?? args?.skill_name ?? args?.skill);
1013
- const skillReloaded = tool?.name === SKILL_TOOL_NAME && !args?.file && loadedThisTurn.has(skillName);
1018
+ const askedFile = skillFileArg(args);
1019
+ const mainRequested = mainFileRequested(skillName, askedFile);
1020
+ const skillReloaded = tool?.name === SKILL_TOOL_NAME &&
1021
+ (mainRequested
1022
+ ? loadedThisTurn.has(skillName)
1023
+ : loadedFilesThisTurn.has(fileKey(skillName, unquoteArg(askedFile))));
1014
1024
  const silent = searchCapped || skillReloaded || priorFailure !== undefined;
1015
1025
  const display = silent
1016
1026
  ? undefined
@@ -1034,9 +1044,10 @@ export async function* completion(sdk, { modelId, acquireRound, model, agent, ct
1034
1044
  execution = terminalCall(priorFailure.result, priorFailure.outcome, { executed: false });
1035
1045
  }
1036
1046
  else if (skillReloaded) {
1047
+ const loadedWhat = mainRequested ? skillName : `${skillName} ${unquoteArg(askedFile)}`;
1037
1048
  execution = terminalCall({
1038
1049
  name: skillName,
1039
- content: `${skillName} is already loaded. Follow the instructions you were given and do not load it again.`
1050
+ content: `${loadedWhat} is already loaded. Follow the instructions you were given and do not load it again.`
1040
1051
  }, { kind: 'guidance', message: `${skillName} is already loaded.` }, { executed: false });
1041
1052
  }
1042
1053
  else if (skillBlocked) {
@@ -1105,9 +1116,10 @@ export async function* completion(sdk, { modelId, acquireRound, model, agent, ct
1105
1116
  files.add(output.file);
1106
1117
  loadedSkillFiles.set(skillName, files);
1107
1118
  }
1108
- else {
1119
+ if (mainRequested)
1109
1120
  loadedThisTurn.add(skillName);
1110
- }
1121
+ else
1122
+ loadedFilesThisTurn.add(fileKey(skillName, unquoteArg(askedFile)));
1111
1123
  skillIcon = skillEmoji(skillName) ?? skillIcon;
1112
1124
  }
1113
1125
  if (!activity &&
@@ -1,7 +1,7 @@
1
1
  const FUNCTION_OPEN = '<function=';
2
2
  const FUNCTION_CLOSE = '</function>';
3
- const RESPONSE_OPEN = '<tool_response>';
4
- const RESPONSE_CLOSE = '</tool_response>';
3
+ const TOOL_RESPONSE = { open: '<tool_response>', close: '</tool_response>' };
4
+ const TOOL_CALL = { open: '<tool_call>', close: '</tool_call>' };
5
5
  const PARAMETER_OPEN = '<parameter=';
6
6
  const PARAMETER_CLOSE = '</parameter>';
7
7
  const MAX_NAME_LENGTH = 64;
@@ -66,26 +66,34 @@ export function stripToolCallMarkup(text) {
66
66
  out += text.slice(cursor);
67
67
  const dangling = danglingIndex(out);
68
68
  const withoutCalls = dangling === -1 ? out : out.slice(0, dangling);
69
- return stripToolResponseBlocks(withoutCalls)
69
+ const withoutSpans = stripSpans(stripToolResponseBlocks(withoutCalls), TOOL_CALL);
70
+ return withoutSpans
70
71
  .replace(/<\/?(tool_call|tool_response)>/gi, '')
71
72
  .replace(/\n{3,}/g, '\n\n')
72
73
  .trim();
73
74
  }
74
- export function stripToolResponseBlocks(text) {
75
+ function stripSpans(text, span) {
76
+ const hay = text.replace(/[A-Z]/g, lowerAscii);
75
77
  let out = '';
76
78
  let cursor = 0;
77
79
  for (;;) {
78
- const start = text.indexOf(RESPONSE_OPEN, cursor);
80
+ const start = hay.indexOf(span.open, cursor);
79
81
  if (start === -1)
80
82
  break;
81
83
  out += text.slice(cursor, start);
82
- const end = text.indexOf(RESPONSE_CLOSE, start + RESPONSE_OPEN.length);
84
+ const end = hay.indexOf(span.close, start + span.open.length);
83
85
  if (end === -1)
84
86
  return out;
85
- cursor = end + RESPONSE_CLOSE.length;
87
+ cursor = end + span.close.length;
86
88
  }
87
89
  return out + text.slice(cursor);
88
90
  }
91
+ function lowerAscii(char) {
92
+ return char.toLowerCase();
93
+ }
94
+ export function stripToolResponseBlocks(text) {
95
+ return stripSpans(text, TOOL_RESPONSE);
96
+ }
89
97
  function danglingIndex(text) {
90
98
  let cursor = 0;
91
99
  for (;;) {
@@ -1,6 +1,6 @@
1
1
  export interface FetchRequestLike {
2
2
  on(event: 'error', listener: (err: Error) => void): void;
3
- write(chunk: string): void;
3
+ write(chunk: string | Uint8Array): void;
4
4
  end(): void;
5
5
  destroy(err?: Error): void;
6
6
  }
@@ -19,7 +19,7 @@ export type BareRequestFn<Response extends FetchResponseLike = FetchResponseLike
19
19
  export interface NoRedirectFetchInit {
20
20
  method?: string;
21
21
  headers?: Record<string, string>;
22
- body?: string;
22
+ body?: string | Uint8Array;
23
23
  signal?: {
24
24
  readonly aborted: boolean;
25
25
  addEventListener(type: 'abort', listener: () => void): void;
@@ -32,6 +32,7 @@ export interface NoRedirectFetchResponse {
32
32
  status: number;
33
33
  headers: {
34
34
  get(name: string): string | null;
35
+ entries(): [string, string][];
35
36
  };
36
37
  text(): string;
37
38
  bytes(): Uint8Array;
@@ -29,7 +29,10 @@ export function runNoRedirectFetch(request, url, init = {}) {
29
29
  }
30
30
  resolve({
31
31
  status: res.statusCode,
32
- headers: { get: (name) => responseHeaders[name.toLowerCase()] ?? null },
32
+ headers: {
33
+ get: (name) => responseHeaders[name.toLowerCase()] ?? null,
34
+ entries: () => Object.entries(responseHeaders)
35
+ },
33
36
  text: () => b4a.toString(body, 'utf8'),
34
37
  bytes: () => body
35
38
  });
@@ -1,5 +1,5 @@
1
1
  import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
2
- import type { AttachFn, FetchAttachmentFn, RunBudget } from '../../../capabilities/completion.ts';
2
+ import type { AttachFn, FetchAttachmentFn, RunBudget, ToolDecision } from '../../../capabilities/completion.ts';
3
3
  import type { ExecResult } from '../runner.ts';
4
4
  export type ExecutionOptions = {
5
5
  readonly cwd?: string;
@@ -15,6 +15,10 @@ export type ExecutionContext = {
15
15
  path: string;
16
16
  }[];
17
17
  readonly runBudget?: RunBudget;
18
+ readonly sandboxMode?: boolean;
19
+ readonly toolNames?: readonly string[];
20
+ readonly approval?: ToolDecision;
21
+ readonly confirmConnect?: (host: string) => Promise<boolean>;
18
22
  };
19
23
  export type ExecutionPlan = {
20
24
  run(options: ExecutionOptions): Promise<ExecResult> | ExecResult;
@@ -0,0 +1 @@
1
+ export declare function moduleDistribution(module: string): string | null;
@@ -0,0 +1,4 @@
1
+ import { MODULE_DISTRIBUTIONS } from './module-distributions.js';
2
+ export function moduleDistribution(module) {
3
+ return Object.hasOwn(MODULE_DISTRIBUTIONS, module) ? (MODULE_DISTRIBUTIONS[module] ?? null) : null;
4
+ }
@@ -0,0 +1,72 @@
1
+ import { type HostResolver } from '../../../url-guard.ts';
2
+ import type { NoRedirectFetchInit, NoRedirectFetchResponse } from '../../../../no-redirect-fetch-impl.ts';
3
+ export declare const MAX_RESPONSE_BYTES: number;
4
+ type NoRedirectFetch = (url: URL, init?: NoRedirectFetchInit) => Promise<NoRedirectFetchResponse>;
5
+ declare const GUARDED: unique symbol;
6
+ export type ConnectPolicy = {
7
+ readonly named: readonly string[];
8
+ readonly ask: (host: string) => Promise<boolean>;
9
+ };
10
+ export declare function setConnectPolicy(next: ConnectPolicy | null): void;
11
+ export declare function isGuardedFetch(fetch: unknown): boolean;
12
+ type SignalLike = {
13
+ readonly aborted: boolean;
14
+ addEventListener(type: 'abort', listener: () => void): void;
15
+ };
16
+ type FetchInit = {
17
+ method?: string;
18
+ headers?: unknown;
19
+ body?: unknown;
20
+ signal?: SignalLike | null;
21
+ redirect?: string;
22
+ };
23
+ export type GuardedFetchOptions = {
24
+ readonly fetch?: NoRedirectFetch;
25
+ readonly resolveHost?: HostResolver;
26
+ readonly maxBytes?: number;
27
+ readonly timeoutMs?: number;
28
+ };
29
+ declare class GuardedHeaders {
30
+ #private;
31
+ constructor(entries: [string, string][]);
32
+ get(name: string): string | null;
33
+ has(name: string): boolean;
34
+ entries(): IterableIterator<[string, string]>;
35
+ keys(): IterableIterator<string>;
36
+ values(): IterableIterator<string>;
37
+ forEach(fn: (value: string, key: string) => void): void;
38
+ [Symbol.iterator](): IterableIterator<[string, string]>;
39
+ }
40
+ export declare class GuardedResponse {
41
+ #private;
42
+ readonly type = "basic";
43
+ readonly statusText = "";
44
+ readonly status: number;
45
+ readonly headers: GuardedHeaders;
46
+ readonly url: string;
47
+ readonly redirected: boolean;
48
+ constructor(status: number, headers: [string, string][], bytes: Uint8Array, url: string, redirected: boolean);
49
+ get ok(): boolean;
50
+ get bodyUsed(): boolean;
51
+ get body(): {
52
+ getReader(): {
53
+ read(): Promise<{
54
+ done: boolean;
55
+ value?: Uint8Array;
56
+ }>;
57
+ cancel(): Promise<void>;
58
+ releaseLock(): void;
59
+ };
60
+ };
61
+ arrayBuffer(): Promise<ArrayBuffer>;
62
+ bytes(): Promise<Uint8Array>;
63
+ text(): Promise<string>;
64
+ json(): Promise<unknown>;
65
+ clone(): GuardedResponse;
66
+ }
67
+ export declare function createGuardedFetch(options?: GuardedFetchOptions): ((input: string | URL | {
68
+ url: string;
69
+ }, init?: FetchInit) => Promise<GuardedResponse>) & {
70
+ [GUARDED]: true;
71
+ };
72
+ export {};
@@ -0,0 +1,222 @@
1
+ import { AbortController } from 'bare-abort-controller';
2
+ import b4a from 'b4a';
3
+ import defaultFetch from '#no-redirect-fetch';
4
+ import { lookupHost } from '../../../../dns-lookup.js';
5
+ import { hostIsNamed } from './named-hosts.js';
6
+ import { checkOutboundHost, resolveOutboundUrl } from '../../../url-guard.js';
7
+ export const MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
8
+ const MAX_REDIRECTS = 10;
9
+ const TIMEOUT_MS = 120_000;
10
+ const CHUNK_BYTES = 1024 * 1024;
11
+ const GUARDED = Symbol.for('qvac.harness.guardedFetch');
12
+ let policy = null;
13
+ const granted = new Set();
14
+ export function setConnectPolicy(next) {
15
+ policy = next;
16
+ granted.clear();
17
+ }
18
+ async function mayConnect(host) {
19
+ if (!policy || granted.has(host) || hostIsNamed(host, policy.named))
20
+ return true;
21
+ const allowed = await policy.ask(host);
22
+ if (allowed)
23
+ granted.add(host);
24
+ return allowed;
25
+ }
26
+ export function isGuardedFetch(fetch) {
27
+ return typeof fetch === 'function' && fetch[GUARDED] === true;
28
+ }
29
+ function isRedirect(status) {
30
+ return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
31
+ }
32
+ function headerRecord(headers) {
33
+ const out = {};
34
+ if (!headers || typeof headers !== 'object')
35
+ return out;
36
+ const pairs = Array.isArray(headers)
37
+ ? headers
38
+ : typeof headers.entries === 'function'
39
+ ? headers.entries()
40
+ : Object.entries(headers);
41
+ for (const pair of pairs) {
42
+ if (!Array.isArray(pair) || pair.length < 2)
43
+ continue;
44
+ out[String(pair[0])] = String(pair[1]);
45
+ }
46
+ return out;
47
+ }
48
+ function bodyBytes(body) {
49
+ if (body === undefined || body === null)
50
+ return undefined;
51
+ if (typeof body === 'string')
52
+ return body;
53
+ if (body instanceof ArrayBuffer)
54
+ return new Uint8Array(body);
55
+ if (ArrayBuffer.isView(body))
56
+ return new Uint8Array(body.buffer, body.byteOffset, body.byteLength);
57
+ return String(body);
58
+ }
59
+ function plain(bytes) {
60
+ return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
61
+ }
62
+ class GuardedHeaders {
63
+ #map;
64
+ constructor(entries) {
65
+ this.#map = new Map(entries.map(([k, v]) => [k.toLowerCase(), v]));
66
+ }
67
+ get(name) {
68
+ return this.#map.get(name.toLowerCase()) ?? null;
69
+ }
70
+ has(name) {
71
+ return this.#map.has(name.toLowerCase());
72
+ }
73
+ entries() {
74
+ return this.#map.entries();
75
+ }
76
+ keys() {
77
+ return this.#map.keys();
78
+ }
79
+ values() {
80
+ return this.#map.values();
81
+ }
82
+ forEach(fn) {
83
+ this.#map.forEach((value, key) => fn(value, key));
84
+ }
85
+ [Symbol.iterator]() {
86
+ return this.#map.entries();
87
+ }
88
+ }
89
+ export class GuardedResponse {
90
+ type = 'basic';
91
+ statusText = '';
92
+ status;
93
+ headers;
94
+ url;
95
+ redirected;
96
+ #bytes;
97
+ #used = false;
98
+ constructor(status, headers, bytes, url, redirected) {
99
+ this.status = status;
100
+ this.headers = new GuardedHeaders(headers);
101
+ this.#bytes = bytes;
102
+ this.url = url;
103
+ this.redirected = redirected;
104
+ }
105
+ get ok() {
106
+ return this.status >= 200 && this.status < 300;
107
+ }
108
+ get bodyUsed() {
109
+ return this.#used;
110
+ }
111
+ get body() {
112
+ let offset = 0;
113
+ const bytes = this.#bytes;
114
+ return {
115
+ getReader: () => {
116
+ this.#used = true;
117
+ return {
118
+ read: async () => {
119
+ if (offset >= bytes.byteLength)
120
+ return { done: true };
121
+ const value = plain(bytes.subarray(offset, offset + CHUNK_BYTES));
122
+ offset += value.byteLength;
123
+ return { done: false, value };
124
+ },
125
+ cancel: async () => {
126
+ offset = bytes.byteLength;
127
+ },
128
+ releaseLock: () => { }
129
+ };
130
+ }
131
+ };
132
+ }
133
+ async arrayBuffer() {
134
+ this.#used = true;
135
+ const { buffer, byteOffset, byteLength } = this.#bytes;
136
+ return buffer.slice(byteOffset, byteOffset + byteLength);
137
+ }
138
+ async bytes() {
139
+ return new Uint8Array(await this.arrayBuffer());
140
+ }
141
+ async text() {
142
+ this.#used = true;
143
+ return b4a.toString(this.#bytes, 'utf8');
144
+ }
145
+ async json() {
146
+ return JSON.parse(await this.text());
147
+ }
148
+ clone() {
149
+ return new GuardedResponse(this.status, [...this.headers.entries()], this.#bytes, this.url, this.redirected);
150
+ }
151
+ }
152
+ export function createGuardedFetch(options = {}) {
153
+ const send = options.fetch ?? defaultFetch;
154
+ const resolveHost = options.resolveHost ?? lookupHost;
155
+ const maxBytes = options.maxBytes ?? MAX_RESPONSE_BYTES;
156
+ const timeoutMs = options.timeoutMs ?? TIMEOUT_MS;
157
+ const guardedFetch = async function (input, init = {}) {
158
+ let url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url);
159
+ let method = (init.method ?? 'GET').toUpperCase();
160
+ let body = bodyBytes(init.body);
161
+ const headers = headerRecord(init.headers);
162
+ const controller = new AbortController();
163
+ const timer = setTimeout(() => controller.abort(undefined), timeoutMs);
164
+ if (init.signal?.aborted)
165
+ controller.abort(undefined);
166
+ init.signal?.addEventListener('abort', () => controller.abort(undefined));
167
+ try {
168
+ for (let hops = 0;; hops++) {
169
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') {
170
+ throw new TypeError(`fetch refused: unsupported protocol ${url.protocol}`);
171
+ }
172
+ const literal = checkOutboundHost(url);
173
+ if (literal)
174
+ throw new TypeError(`fetch refused: ${literal}`);
175
+ if (!(await mayConnect(url.hostname))) {
176
+ throw new TypeError(`fetch refused: ${url.hostname} was not approved for this run`);
177
+ }
178
+ const check = await resolveOutboundUrl(url, resolveHost, controller.signal);
179
+ if (check.blocked)
180
+ throw new TypeError(`fetch refused: ${check.blocked}`);
181
+ const response = await send(url, {
182
+ method,
183
+ headers,
184
+ body,
185
+ signal: controller.signal,
186
+ validatedAddresses: check.addresses,
187
+ maxBytes
188
+ });
189
+ const location = response.headers.get('location');
190
+ if (!isRedirect(response.status) || !location || init.redirect === 'manual') {
191
+ return new GuardedResponse(response.status, response.headers.entries(), plain(response.bytes()), url.href, hops > 0);
192
+ }
193
+ if (init.redirect === 'error')
194
+ throw new TypeError('fetch refused: redirect not allowed');
195
+ if (hops >= MAX_REDIRECTS)
196
+ throw new TypeError('fetch refused: too many redirects');
197
+ const next = new URL(location, url);
198
+ if (next.protocol !== url.protocol || next.host !== url.host) {
199
+ for (const name of Object.keys(headers)) {
200
+ if (/^(authorization|cookie)$/i.test(name))
201
+ delete headers[name];
202
+ }
203
+ }
204
+ if (response.status === 303 ||
205
+ ((response.status === 301 || response.status === 302) && method === 'POST')) {
206
+ method = 'GET';
207
+ body = undefined;
208
+ }
209
+ url = next;
210
+ }
211
+ }
212
+ finally {
213
+ clearTimeout(timer);
214
+ }
215
+ };
216
+ const fetch = (input, init) => {
217
+ const response = guardedFetch(input, init);
218
+ response.catch(() => { });
219
+ return response;
220
+ };
221
+ return Object.assign(fetch, { [GUARDED]: true });
222
+ }
@@ -0,0 +1,5 @@
1
+ export type InterruptTimer = {
2
+ readonly interrupt: Uint8Array;
3
+ disarm(): boolean;
4
+ };
5
+ export declare function startInterruptTimer(ms: number): InterruptTimer;
@@ -0,0 +1,29 @@
1
+ import { spawnInlineThread } from '#pyodide-worker-spawn';
2
+ const ARMED = 0;
3
+ const DISARMED = 1;
4
+ const FIRED = 2;
5
+ const SIGINT = 2;
6
+ const RESEND_MS = 100;
7
+ const SOURCE = `
8
+ const state = new Int32Array(data.state)
9
+ const interrupt = new Uint8Array(data.interrupt)
10
+ Atomics.wait(state, 0, ${ARMED}, data.ms)
11
+ if (Atomics.compareExchange(state, 0, ${ARMED}, ${FIRED}) === ${ARMED}) {
12
+ do Atomics.store(interrupt, 0, ${SIGINT})
13
+ while (Atomics.wait(state, 0, ${FIRED}, ${RESEND_MS}) === 'timed-out')
14
+ }
15
+ `;
16
+ export function startInterruptTimer(ms) {
17
+ const state = new Int32Array(new SharedArrayBuffer(4));
18
+ const interrupt = new Uint8Array(new SharedArrayBuffer(1));
19
+ const reap = spawnInlineThread(SOURCE, { state: state.buffer, interrupt: interrupt.buffer, ms });
20
+ return {
21
+ interrupt,
22
+ disarm() {
23
+ const fired = Atomics.exchange(state, 0, DISARMED) === FIRED;
24
+ Atomics.notify(state, 0);
25
+ reap();
26
+ return fired;
27
+ }
28
+ };
29
+ }
@@ -9,6 +9,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
9
9
  import b4a from 'b4a';
10
10
  import { constants as fsConstants } from 'fs';
11
11
  import process from 'process';
12
+ import { createGuardedFetch } from './guarded-fetch.js';
12
13
  import { seedPackageCache } from './wheels.js';
13
14
  class B4aTextDecoder {
14
15
  encoding;
@@ -53,7 +54,7 @@ async function shapeBareGlobals() {
53
54
  const g = globalThis;
54
55
  if (!g.fetch) {
55
56
  const bareFetch = await import('bare-fetch');
56
- g.fetch = bareFetch.default;
57
+ g.fetch = createGuardedFetch();
57
58
  g.Request = bareFetch.Request;
58
59
  g.Response = bareFetch.Response;
59
60
  g.Headers = bareFetch.Headers;
@@ -0,0 +1 @@
1
+ export declare const MODULE_DISTRIBUTIONS: Readonly<Record<string, string>>;
@@ -0,0 +1,52 @@
1
+ export const MODULE_DISTRIBUTIONS = {
2
+ Bio: 'biopython',
3
+ BioSQL: 'biopython',
4
+ Crypto: 'pycryptodome',
5
+ PIL: 'Pillow',
6
+ PhiSpyModules: 'phispy',
7
+ PhiSpyRepeatFinder: 'phispy',
8
+ _argon2_cffi_bindings: 'argon2-cffi-bindings',
9
+ _distutils_hack: 'setuptools',
10
+ _pyrsistent_version: 'pyrsistent',
11
+ _pytest: 'pytest',
12
+ _yaml: 'pyyaml',
13
+ _zengl: 'zengl',
14
+ argon2: 'argon2-cffi',
15
+ attr: 'attrs',
16
+ audioop: 'audioop-lts',
17
+ bigtests: 'screed',
18
+ bs4: 'beautifulsoup4',
19
+ cv2: 'opencv-python',
20
+ cvxpy: 'cvxpy-base',
21
+ dateutil: 'python-dateutil',
22
+ dns: 'dnspython',
23
+ docx: 'python-docx',
24
+ erfa: 'pyerfa',
25
+ fitz: 'PyMuPDF',
26
+ flint: 'python-flint',
27
+ flirt: 'python-flirt',
28
+ fpdf: 'fpdf2',
29
+ google: 'protobuf',
30
+ gridfs: 'pymongo',
31
+ isympy: 'sympy',
32
+ lakers: 'lakers-python',
33
+ mpl_toolkits: 'matplotlib',
34
+ mypyc: 'mypy',
35
+ nacl: 'pynacl',
36
+ pptx: 'python-pptx',
37
+ pygame: 'pygame-ce',
38
+ pylab: 'matplotlib',
39
+ pysat: 'python-sat',
40
+ pywt: 'pywavelets',
41
+ rpds: 'rpds-py',
42
+ ruamel: 'ruamel.yaml',
43
+ shapefile: 'pyshp',
44
+ sisl_toolbox: 'sisl',
45
+ skimage: 'scikit-image',
46
+ sklearn: 'scikit-learn',
47
+ taglib: 'pytaglib',
48
+ tiktoken_ext: 'tiktoken',
49
+ tlz: 'toolz',
50
+ unix_timezones: 'pyodide-unix-timezones',
51
+ yaml: 'pyyaml'
52
+ };
@@ -0,0 +1,2 @@
1
+ export declare function namedHosts(source: string): string[];
2
+ export declare function hostIsNamed(host: string, named: readonly string[]): boolean;
@@ -0,0 +1,15 @@
1
+ const URL_LITERAL = /https?:\/\/[^\s'"`\\<>(){}[\]]+/gi;
2
+ export function namedHosts(source) {
3
+ const hosts = new Set();
4
+ for (const [match] of source.matchAll(URL_LITERAL)) {
5
+ try {
6
+ hosts.add(new URL(match).hostname.toLowerCase());
7
+ }
8
+ catch { }
9
+ }
10
+ return [...hosts];
11
+ }
12
+ export function hostIsNamed(host, named) {
13
+ const bare = (value) => value.toLowerCase().replace(/^www\./, '');
14
+ return named.some((name) => bare(name) === bare(host));
15
+ }