@qvac/harness 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +262 -0
- package/dist/harness.bundle +15 -11
- package/dist-lib/index.d.ts +1 -1
- package/dist-lib/lib/capabilities/completion.d.ts +5 -1
- package/dist-lib/lib/capabilities/completion.js +20 -8
- package/dist-lib/lib/helpers/tool-call-markup.js +15 -7
- package/dist-lib/lib/no-redirect-fetch-impl.d.ts +3 -2
- package/dist-lib/lib/no-redirect-fetch-impl.js +4 -1
- package/dist-lib/lib/tools/exec/adapters/adapter.d.ts +5 -1
- package/dist-lib/lib/tools/exec/adapters/python/distributions.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/distributions.js +4 -0
- package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.d.ts +72 -0
- package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.js +222 -0
- package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.d.ts +5 -0
- package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.js +29 -0
- package/dist-lib/lib/tools/exec/adapters/python/loader.js +2 -1
- package/dist-lib/lib/tools/exec/adapters/python/module-distributions.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/module-distributions.js +52 -0
- package/dist-lib/lib/tools/exec/adapters/python/named-hosts.d.ts +2 -0
- package/dist-lib/lib/tools/exec/adapters/python/named-hosts.js +15 -0
- package/dist-lib/lib/tools/exec/adapters/python/net-shim.d.ts +2 -0
- package/dist-lib/lib/tools/exec/adapters/python/net-shim.js +148 -0
- package/dist-lib/lib/tools/exec/adapters/python/output-policy.js +13 -1
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.d.ts +4 -1
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +42 -7
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts +9 -3
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js +137 -21
- package/dist-lib/lib/tools/exec/adapters/python/run-session.d.ts +5 -0
- package/dist-lib/lib/tools/exec/adapters/python/run-session.js +33 -8
- package/dist-lib/lib/tools/exec/adapters/python/wheels.d.ts +6 -0
- package/dist-lib/lib/tools/exec/adapters/python/wheels.js +17 -3
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.d.ts +2 -1
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.js +16 -3
- package/dist-lib/lib/tools/exec/adapters/python/worker/entry.js +15 -2
- package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.d.ts +6 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.js +6 -2
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-bare.js +8 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.js +8 -0
- package/dist-lib/lib/tools/exec/adapters/shell/shell-adapter.js +7 -1
- package/dist-lib/lib/tools/exec/shell-guidance.d.ts +10 -0
- package/dist-lib/lib/tools/exec/shell-guidance.js +123 -0
- package/dist-lib/lib/tools/exec/video-tools.d.ts +5 -0
- package/dist-lib/lib/tools/exec/video-tools.js +40 -0
- package/dist-lib/lib/tools/exec.d.ts +2 -1
- package/dist-lib/lib/tools/exec.js +93 -33
- package/dist-lib/lib/tools/http-request/truncate-response.d.ts +1 -0
- package/dist-lib/lib/tools/http-request/truncate-response.js +1 -1
- package/dist-lib/lib/tools/http-request.js +4 -1
- package/dist-lib/lib/tools/operations/builders.d.ts +1 -1
- package/dist-lib/lib/tools/operations/builders.js +9 -0
- package/dist-lib/lib/tools/operations/compile.js +41 -18
- package/dist-lib/lib/tools/operations/decoders.d.ts +6 -0
- package/dist-lib/lib/tools/operations/decoders.js +82 -0
- package/dist-lib/lib/tools/operations/manifest.d.ts +16 -2
- package/dist-lib/lib/tools/operations/manifest.js +6 -1
- package/dist-lib/lib/tools/skill.d.ts +3 -0
- package/dist-lib/lib/tools/skill.js +38 -5
- package/dist-lib/lib/tools/skills/manifest-parse.js +43 -1
- package/dist-lib/lib/tools/skills/manifest.d.ts +13 -0
- package/dist-lib/schema/harness.js +20 -1
- package/dist-lib/spec/hrpc/messages.js +93 -5
- package/dist-lib/spec/hyperschema/index.js +93 -5
- package/dist-lib/spec/hyperschema/schema.json +66 -1
- package/dist-lib/spec/hyperschema/types.d.ts +18 -1
- package/package.json +40 -2
- package/wheels/manifest.json +15 -0
package/dist-lib/index.d.ts
CHANGED
|
@@ -44,7 +44,7 @@ export { requiredSettings, settingEnvName, type SkillSettingEntry, type SkillSet
|
|
|
44
44
|
export type { ToolCatalogEntry } from './lib/tools/catalog.ts';
|
|
45
45
|
export { TOOLSETS, toolsetEnabled, type Toolset } from './lib/tools/toolsets.ts';
|
|
46
46
|
export { markCredentialRejected, markCredentialValid, recordCredentialHealth, resetAllCredentialHealth, resetCredentialHealth } from './lib/tools/skills/credential-health.ts';
|
|
47
|
-
export type { InstallStep, SkillOAuthConfig, SkillRequires, SkillSetup, SkillSetupField, SkillSetupRoute } from './lib/tools/skills/manifest.ts';
|
|
47
|
+
export type { InstallStep, SkillOAuthConfig, SkillOAuthVerify, SkillOAuthVerifyRule, SkillRequires, SkillSetup, SkillSetupField, SkillSetupRoute } from './lib/tools/skills/manifest.ts';
|
|
48
48
|
export { readSetup as normalizeSkillSetup } from './lib/tools/skills/manifest-parse.ts';
|
|
49
49
|
export type { CredentialRecord, CredentialStoreValue } from './lib/tools/skills/auth-resolver.ts';
|
|
50
50
|
export { isAvailableState, SKILL_STATE, type SkillState } from './lib/tools/skills/skill-state.ts';
|
|
@@ -139,6 +139,10 @@ export interface ToolExecuteContext {
|
|
|
139
139
|
toolOps?: ReadonlyMap<string, readonly CliValidator[]> | undefined;
|
|
140
140
|
runBudget?: RunBudget | undefined;
|
|
141
141
|
reportBackend?: ((use: RunBackendUse) => void) | undefined;
|
|
142
|
+
uncappedResponse?: boolean | undefined;
|
|
143
|
+
sandboxMode?: boolean | undefined;
|
|
144
|
+
toolNames?: readonly string[] | undefined;
|
|
145
|
+
approval?: ToolDecision | undefined;
|
|
142
146
|
}
|
|
143
147
|
export interface RunBackendUse {
|
|
144
148
|
tool: string;
|
|
@@ -178,7 +182,7 @@ export interface ToolDefinition {
|
|
|
178
182
|
toolset: Toolset | null;
|
|
179
183
|
validate?: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => string | null;
|
|
180
184
|
effect: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => ToolEffect;
|
|
181
|
-
resource?: (args: Record<string, unknown
|
|
185
|
+
resource?: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => ResourceDescriptor | null;
|
|
182
186
|
compiled?: true;
|
|
183
187
|
execute: (args: Record<string, unknown>, ctx?: ToolExecuteContext) => Promise<unknown>;
|
|
184
188
|
}
|
|
@@ -19,6 +19,7 @@ import { annotateLastUserTime, skillsPromptSection, systemPromptParts } from '..
|
|
|
19
19
|
import { isMetadataMethod } from '../tools/mcp/utils.js';
|
|
20
20
|
import { unquoteArg } from '../tools/model-args.js';
|
|
21
21
|
import { SKILL_TOOL_NAME, WEB_SEARCH_TOOL_NAME } from './tool-names.js';
|
|
22
|
+
import { mainFileRequested, skillFileArg, skillFileStem } from '../tools/skill.js';
|
|
22
23
|
import { promptUsageParts, usageBreakdown } from './usage-breakdown.js';
|
|
23
24
|
import { collectToolGrants, enabledTools, isSelected, skillIsActive, skillsAreSelected } from './tool-selection.js';
|
|
24
25
|
import { guardToolCall } from '../tools/outbound-guard.js';
|
|
@@ -272,7 +273,7 @@ async function executeCall(tool, args, ctx, { approve, signal, denied, sources =
|
|
|
272
273
|
type: 'tool-activity',
|
|
273
274
|
name: tool.name,
|
|
274
275
|
args,
|
|
275
|
-
resource: tool.resource?.(args) ?? null,
|
|
276
|
+
resource: tool.resource?.(args, ctx) ?? null,
|
|
276
277
|
effect: tool.effect(args, ctx),
|
|
277
278
|
decision: 'blocked'
|
|
278
279
|
});
|
|
@@ -284,7 +285,7 @@ async function executeCall(tool, args, ctx, { approve, signal, denied, sources =
|
|
|
284
285
|
return deniedCall();
|
|
285
286
|
}
|
|
286
287
|
const effect = classified ?? tool.effect(args, ctx);
|
|
287
|
-
const resource = tool.resource?.(args) ?? null;
|
|
288
|
+
const resource = tool.resource?.(args, ctx) ?? null;
|
|
288
289
|
const autoRun = effect === 'read' || tool.autoApprove === true || skipApprovals === true;
|
|
289
290
|
hlog('tool decision', {
|
|
290
291
|
tool: tool.name,
|
|
@@ -326,7 +327,7 @@ async function executeCall(tool, args, ctx, { approve, signal, denied, sources =
|
|
|
326
327
|
let output;
|
|
327
328
|
let outcome = { kind: 'success' };
|
|
328
329
|
try {
|
|
329
|
-
output = await tool.execute(args, { ...ctx, signal });
|
|
330
|
+
output = await tool.execute(args, { ...ctx, signal, approval: decision });
|
|
330
331
|
hlog('tool done', { tool: tool.name });
|
|
331
332
|
}
|
|
332
333
|
catch (err) {
|
|
@@ -709,6 +710,8 @@ export async function* completion(sdk, { modelId, acquireRound, model, agent, ct
|
|
|
709
710
|
? { ...resolvedGenerationParams, ...generationParamsOverride }
|
|
710
711
|
: resolvedGenerationParams;
|
|
711
712
|
const loadedThisTurn = new Set();
|
|
713
|
+
const loadedFilesThisTurn = new Set();
|
|
714
|
+
const fileKey = (skill, file) => `${skill}\0${skillFileStem(file)}`;
|
|
712
715
|
let skillIcon = null;
|
|
713
716
|
const skillEmoji = (name) => skillCatalog.find((entry) => entry.name === name)?.emoji ?? null;
|
|
714
717
|
const nudgedSkills = new Set();
|
|
@@ -1004,13 +1007,20 @@ export async function* completion(sdk, { modelId, acquireRound, model, agent, ct
|
|
|
1004
1007
|
toolOps: grantedOps,
|
|
1005
1008
|
fsRoots,
|
|
1006
1009
|
fsWriteRoots: writableRoots,
|
|
1007
|
-
reportBackend: (use) => stats.backends.push(use)
|
|
1010
|
+
reportBackend: (use) => stats.backends.push(use),
|
|
1011
|
+
sandboxMode: agent?.sandboxMode === true,
|
|
1012
|
+
toolNames: enabled.map((tool) => tool.name)
|
|
1008
1013
|
};
|
|
1009
1014
|
const skillCall = call.name === SKILL_TOOL_NAME;
|
|
1010
1015
|
const effect = tool?.effect(args, callCtx) ?? (skillCall ? 'read' : undefined);
|
|
1011
1016
|
const resource = tool?.resource?.(args) ?? null;
|
|
1012
1017
|
const skillName = unquoteArg(args?.name ?? args?.skill_name ?? args?.skill);
|
|
1013
|
-
const
|
|
1018
|
+
const askedFile = skillFileArg(args);
|
|
1019
|
+
const mainRequested = mainFileRequested(skillName, askedFile);
|
|
1020
|
+
const skillReloaded = tool?.name === SKILL_TOOL_NAME &&
|
|
1021
|
+
(mainRequested
|
|
1022
|
+
? loadedThisTurn.has(skillName)
|
|
1023
|
+
: loadedFilesThisTurn.has(fileKey(skillName, unquoteArg(askedFile))));
|
|
1014
1024
|
const silent = searchCapped || skillReloaded || priorFailure !== undefined;
|
|
1015
1025
|
const display = silent
|
|
1016
1026
|
? undefined
|
|
@@ -1034,9 +1044,10 @@ export async function* completion(sdk, { modelId, acquireRound, model, agent, ct
|
|
|
1034
1044
|
execution = terminalCall(priorFailure.result, priorFailure.outcome, { executed: false });
|
|
1035
1045
|
}
|
|
1036
1046
|
else if (skillReloaded) {
|
|
1047
|
+
const loadedWhat = mainRequested ? skillName : `${skillName} ${unquoteArg(askedFile)}`;
|
|
1037
1048
|
execution = terminalCall({
|
|
1038
1049
|
name: skillName,
|
|
1039
|
-
content: `${
|
|
1050
|
+
content: `${loadedWhat} is already loaded. Follow the instructions you were given and do not load it again.`
|
|
1040
1051
|
}, { kind: 'guidance', message: `${skillName} is already loaded.` }, { executed: false });
|
|
1041
1052
|
}
|
|
1042
1053
|
else if (skillBlocked) {
|
|
@@ -1105,9 +1116,10 @@ export async function* completion(sdk, { modelId, acquireRound, model, agent, ct
|
|
|
1105
1116
|
files.add(output.file);
|
|
1106
1117
|
loadedSkillFiles.set(skillName, files);
|
|
1107
1118
|
}
|
|
1108
|
-
|
|
1119
|
+
if (mainRequested)
|
|
1109
1120
|
loadedThisTurn.add(skillName);
|
|
1110
|
-
|
|
1121
|
+
else
|
|
1122
|
+
loadedFilesThisTurn.add(fileKey(skillName, unquoteArg(askedFile)));
|
|
1111
1123
|
skillIcon = skillEmoji(skillName) ?? skillIcon;
|
|
1112
1124
|
}
|
|
1113
1125
|
if (!activity &&
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const FUNCTION_OPEN = '<function=';
|
|
2
2
|
const FUNCTION_CLOSE = '</function>';
|
|
3
|
-
const
|
|
4
|
-
const
|
|
3
|
+
const TOOL_RESPONSE = { open: '<tool_response>', close: '</tool_response>' };
|
|
4
|
+
const TOOL_CALL = { open: '<tool_call>', close: '</tool_call>' };
|
|
5
5
|
const PARAMETER_OPEN = '<parameter=';
|
|
6
6
|
const PARAMETER_CLOSE = '</parameter>';
|
|
7
7
|
const MAX_NAME_LENGTH = 64;
|
|
@@ -66,26 +66,34 @@ export function stripToolCallMarkup(text) {
|
|
|
66
66
|
out += text.slice(cursor);
|
|
67
67
|
const dangling = danglingIndex(out);
|
|
68
68
|
const withoutCalls = dangling === -1 ? out : out.slice(0, dangling);
|
|
69
|
-
|
|
69
|
+
const withoutSpans = stripSpans(stripToolResponseBlocks(withoutCalls), TOOL_CALL);
|
|
70
|
+
return withoutSpans
|
|
70
71
|
.replace(/<\/?(tool_call|tool_response)>/gi, '')
|
|
71
72
|
.replace(/\n{3,}/g, '\n\n')
|
|
72
73
|
.trim();
|
|
73
74
|
}
|
|
74
|
-
|
|
75
|
+
function stripSpans(text, span) {
|
|
76
|
+
const hay = text.replace(/[A-Z]/g, lowerAscii);
|
|
75
77
|
let out = '';
|
|
76
78
|
let cursor = 0;
|
|
77
79
|
for (;;) {
|
|
78
|
-
const start =
|
|
80
|
+
const start = hay.indexOf(span.open, cursor);
|
|
79
81
|
if (start === -1)
|
|
80
82
|
break;
|
|
81
83
|
out += text.slice(cursor, start);
|
|
82
|
-
const end =
|
|
84
|
+
const end = hay.indexOf(span.close, start + span.open.length);
|
|
83
85
|
if (end === -1)
|
|
84
86
|
return out;
|
|
85
|
-
cursor = end +
|
|
87
|
+
cursor = end + span.close.length;
|
|
86
88
|
}
|
|
87
89
|
return out + text.slice(cursor);
|
|
88
90
|
}
|
|
91
|
+
function lowerAscii(char) {
|
|
92
|
+
return char.toLowerCase();
|
|
93
|
+
}
|
|
94
|
+
export function stripToolResponseBlocks(text) {
|
|
95
|
+
return stripSpans(text, TOOL_RESPONSE);
|
|
96
|
+
}
|
|
89
97
|
function danglingIndex(text) {
|
|
90
98
|
let cursor = 0;
|
|
91
99
|
for (;;) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export interface FetchRequestLike {
|
|
2
2
|
on(event: 'error', listener: (err: Error) => void): void;
|
|
3
|
-
write(chunk: string): void;
|
|
3
|
+
write(chunk: string | Uint8Array): void;
|
|
4
4
|
end(): void;
|
|
5
5
|
destroy(err?: Error): void;
|
|
6
6
|
}
|
|
@@ -19,7 +19,7 @@ export type BareRequestFn<Response extends FetchResponseLike = FetchResponseLike
|
|
|
19
19
|
export interface NoRedirectFetchInit {
|
|
20
20
|
method?: string;
|
|
21
21
|
headers?: Record<string, string>;
|
|
22
|
-
body?: string;
|
|
22
|
+
body?: string | Uint8Array;
|
|
23
23
|
signal?: {
|
|
24
24
|
readonly aborted: boolean;
|
|
25
25
|
addEventListener(type: 'abort', listener: () => void): void;
|
|
@@ -32,6 +32,7 @@ export interface NoRedirectFetchResponse {
|
|
|
32
32
|
status: number;
|
|
33
33
|
headers: {
|
|
34
34
|
get(name: string): string | null;
|
|
35
|
+
entries(): [string, string][];
|
|
35
36
|
};
|
|
36
37
|
text(): string;
|
|
37
38
|
bytes(): Uint8Array;
|
|
@@ -29,7 +29,10 @@ export function runNoRedirectFetch(request, url, init = {}) {
|
|
|
29
29
|
}
|
|
30
30
|
resolve({
|
|
31
31
|
status: res.statusCode,
|
|
32
|
-
headers: {
|
|
32
|
+
headers: {
|
|
33
|
+
get: (name) => responseHeaders[name.toLowerCase()] ?? null,
|
|
34
|
+
entries: () => Object.entries(responseHeaders)
|
|
35
|
+
},
|
|
33
36
|
text: () => b4a.toString(body, 'utf8'),
|
|
34
37
|
bytes: () => body
|
|
35
38
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
|
|
2
|
-
import type { AttachFn, FetchAttachmentFn, RunBudget } from '../../../capabilities/completion.ts';
|
|
2
|
+
import type { AttachFn, FetchAttachmentFn, RunBudget, ToolDecision } from '../../../capabilities/completion.ts';
|
|
3
3
|
import type { ExecResult } from '../runner.ts';
|
|
4
4
|
export type ExecutionOptions = {
|
|
5
5
|
readonly cwd?: string;
|
|
@@ -15,6 +15,10 @@ export type ExecutionContext = {
|
|
|
15
15
|
path: string;
|
|
16
16
|
}[];
|
|
17
17
|
readonly runBudget?: RunBudget;
|
|
18
|
+
readonly sandboxMode?: boolean;
|
|
19
|
+
readonly toolNames?: readonly string[];
|
|
20
|
+
readonly approval?: ToolDecision;
|
|
21
|
+
readonly confirmConnect?: (host: string) => Promise<boolean>;
|
|
18
22
|
};
|
|
19
23
|
export type ExecutionPlan = {
|
|
20
24
|
run(options: ExecutionOptions): Promise<ExecResult> | ExecResult;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function moduleDistribution(module: string): string | null;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { type HostResolver } from '../../../url-guard.ts';
|
|
2
|
+
import type { NoRedirectFetchInit, NoRedirectFetchResponse } from '../../../../no-redirect-fetch-impl.ts';
|
|
3
|
+
export declare const MAX_RESPONSE_BYTES: number;
|
|
4
|
+
type NoRedirectFetch = (url: URL, init?: NoRedirectFetchInit) => Promise<NoRedirectFetchResponse>;
|
|
5
|
+
declare const GUARDED: unique symbol;
|
|
6
|
+
export type ConnectPolicy = {
|
|
7
|
+
readonly named: readonly string[];
|
|
8
|
+
readonly ask: (host: string) => Promise<boolean>;
|
|
9
|
+
};
|
|
10
|
+
export declare function setConnectPolicy(next: ConnectPolicy | null): void;
|
|
11
|
+
export declare function isGuardedFetch(fetch: unknown): boolean;
|
|
12
|
+
type SignalLike = {
|
|
13
|
+
readonly aborted: boolean;
|
|
14
|
+
addEventListener(type: 'abort', listener: () => void): void;
|
|
15
|
+
};
|
|
16
|
+
type FetchInit = {
|
|
17
|
+
method?: string;
|
|
18
|
+
headers?: unknown;
|
|
19
|
+
body?: unknown;
|
|
20
|
+
signal?: SignalLike | null;
|
|
21
|
+
redirect?: string;
|
|
22
|
+
};
|
|
23
|
+
export type GuardedFetchOptions = {
|
|
24
|
+
readonly fetch?: NoRedirectFetch;
|
|
25
|
+
readonly resolveHost?: HostResolver;
|
|
26
|
+
readonly maxBytes?: number;
|
|
27
|
+
readonly timeoutMs?: number;
|
|
28
|
+
};
|
|
29
|
+
declare class GuardedHeaders {
|
|
30
|
+
#private;
|
|
31
|
+
constructor(entries: [string, string][]);
|
|
32
|
+
get(name: string): string | null;
|
|
33
|
+
has(name: string): boolean;
|
|
34
|
+
entries(): IterableIterator<[string, string]>;
|
|
35
|
+
keys(): IterableIterator<string>;
|
|
36
|
+
values(): IterableIterator<string>;
|
|
37
|
+
forEach(fn: (value: string, key: string) => void): void;
|
|
38
|
+
[Symbol.iterator](): IterableIterator<[string, string]>;
|
|
39
|
+
}
|
|
40
|
+
export declare class GuardedResponse {
|
|
41
|
+
#private;
|
|
42
|
+
readonly type = "basic";
|
|
43
|
+
readonly statusText = "";
|
|
44
|
+
readonly status: number;
|
|
45
|
+
readonly headers: GuardedHeaders;
|
|
46
|
+
readonly url: string;
|
|
47
|
+
readonly redirected: boolean;
|
|
48
|
+
constructor(status: number, headers: [string, string][], bytes: Uint8Array, url: string, redirected: boolean);
|
|
49
|
+
get ok(): boolean;
|
|
50
|
+
get bodyUsed(): boolean;
|
|
51
|
+
get body(): {
|
|
52
|
+
getReader(): {
|
|
53
|
+
read(): Promise<{
|
|
54
|
+
done: boolean;
|
|
55
|
+
value?: Uint8Array;
|
|
56
|
+
}>;
|
|
57
|
+
cancel(): Promise<void>;
|
|
58
|
+
releaseLock(): void;
|
|
59
|
+
};
|
|
60
|
+
};
|
|
61
|
+
arrayBuffer(): Promise<ArrayBuffer>;
|
|
62
|
+
bytes(): Promise<Uint8Array>;
|
|
63
|
+
text(): Promise<string>;
|
|
64
|
+
json(): Promise<unknown>;
|
|
65
|
+
clone(): GuardedResponse;
|
|
66
|
+
}
|
|
67
|
+
export declare function createGuardedFetch(options?: GuardedFetchOptions): ((input: string | URL | {
|
|
68
|
+
url: string;
|
|
69
|
+
}, init?: FetchInit) => Promise<GuardedResponse>) & {
|
|
70
|
+
[GUARDED]: true;
|
|
71
|
+
};
|
|
72
|
+
export {};
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { AbortController } from 'bare-abort-controller';
|
|
2
|
+
import b4a from 'b4a';
|
|
3
|
+
import defaultFetch from '#no-redirect-fetch';
|
|
4
|
+
import { lookupHost } from '../../../../dns-lookup.js';
|
|
5
|
+
import { hostIsNamed } from './named-hosts.js';
|
|
6
|
+
import { checkOutboundHost, resolveOutboundUrl } from '../../../url-guard.js';
|
|
7
|
+
export const MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
|
|
8
|
+
const MAX_REDIRECTS = 10;
|
|
9
|
+
const TIMEOUT_MS = 120_000;
|
|
10
|
+
const CHUNK_BYTES = 1024 * 1024;
|
|
11
|
+
const GUARDED = Symbol.for('qvac.harness.guardedFetch');
|
|
12
|
+
let policy = null;
|
|
13
|
+
const granted = new Set();
|
|
14
|
+
export function setConnectPolicy(next) {
|
|
15
|
+
policy = next;
|
|
16
|
+
granted.clear();
|
|
17
|
+
}
|
|
18
|
+
async function mayConnect(host) {
|
|
19
|
+
if (!policy || granted.has(host) || hostIsNamed(host, policy.named))
|
|
20
|
+
return true;
|
|
21
|
+
const allowed = await policy.ask(host);
|
|
22
|
+
if (allowed)
|
|
23
|
+
granted.add(host);
|
|
24
|
+
return allowed;
|
|
25
|
+
}
|
|
26
|
+
export function isGuardedFetch(fetch) {
|
|
27
|
+
return typeof fetch === 'function' && fetch[GUARDED] === true;
|
|
28
|
+
}
|
|
29
|
+
function isRedirect(status) {
|
|
30
|
+
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
|
31
|
+
}
|
|
32
|
+
function headerRecord(headers) {
|
|
33
|
+
const out = {};
|
|
34
|
+
if (!headers || typeof headers !== 'object')
|
|
35
|
+
return out;
|
|
36
|
+
const pairs = Array.isArray(headers)
|
|
37
|
+
? headers
|
|
38
|
+
: typeof headers.entries === 'function'
|
|
39
|
+
? headers.entries()
|
|
40
|
+
: Object.entries(headers);
|
|
41
|
+
for (const pair of pairs) {
|
|
42
|
+
if (!Array.isArray(pair) || pair.length < 2)
|
|
43
|
+
continue;
|
|
44
|
+
out[String(pair[0])] = String(pair[1]);
|
|
45
|
+
}
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
function bodyBytes(body) {
|
|
49
|
+
if (body === undefined || body === null)
|
|
50
|
+
return undefined;
|
|
51
|
+
if (typeof body === 'string')
|
|
52
|
+
return body;
|
|
53
|
+
if (body instanceof ArrayBuffer)
|
|
54
|
+
return new Uint8Array(body);
|
|
55
|
+
if (ArrayBuffer.isView(body))
|
|
56
|
+
return new Uint8Array(body.buffer, body.byteOffset, body.byteLength);
|
|
57
|
+
return String(body);
|
|
58
|
+
}
|
|
59
|
+
function plain(bytes) {
|
|
60
|
+
return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
61
|
+
}
|
|
62
|
+
class GuardedHeaders {
|
|
63
|
+
#map;
|
|
64
|
+
constructor(entries) {
|
|
65
|
+
this.#map = new Map(entries.map(([k, v]) => [k.toLowerCase(), v]));
|
|
66
|
+
}
|
|
67
|
+
get(name) {
|
|
68
|
+
return this.#map.get(name.toLowerCase()) ?? null;
|
|
69
|
+
}
|
|
70
|
+
has(name) {
|
|
71
|
+
return this.#map.has(name.toLowerCase());
|
|
72
|
+
}
|
|
73
|
+
entries() {
|
|
74
|
+
return this.#map.entries();
|
|
75
|
+
}
|
|
76
|
+
keys() {
|
|
77
|
+
return this.#map.keys();
|
|
78
|
+
}
|
|
79
|
+
values() {
|
|
80
|
+
return this.#map.values();
|
|
81
|
+
}
|
|
82
|
+
forEach(fn) {
|
|
83
|
+
this.#map.forEach((value, key) => fn(value, key));
|
|
84
|
+
}
|
|
85
|
+
[Symbol.iterator]() {
|
|
86
|
+
return this.#map.entries();
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
export class GuardedResponse {
|
|
90
|
+
type = 'basic';
|
|
91
|
+
statusText = '';
|
|
92
|
+
status;
|
|
93
|
+
headers;
|
|
94
|
+
url;
|
|
95
|
+
redirected;
|
|
96
|
+
#bytes;
|
|
97
|
+
#used = false;
|
|
98
|
+
constructor(status, headers, bytes, url, redirected) {
|
|
99
|
+
this.status = status;
|
|
100
|
+
this.headers = new GuardedHeaders(headers);
|
|
101
|
+
this.#bytes = bytes;
|
|
102
|
+
this.url = url;
|
|
103
|
+
this.redirected = redirected;
|
|
104
|
+
}
|
|
105
|
+
get ok() {
|
|
106
|
+
return this.status >= 200 && this.status < 300;
|
|
107
|
+
}
|
|
108
|
+
get bodyUsed() {
|
|
109
|
+
return this.#used;
|
|
110
|
+
}
|
|
111
|
+
get body() {
|
|
112
|
+
let offset = 0;
|
|
113
|
+
const bytes = this.#bytes;
|
|
114
|
+
return {
|
|
115
|
+
getReader: () => {
|
|
116
|
+
this.#used = true;
|
|
117
|
+
return {
|
|
118
|
+
read: async () => {
|
|
119
|
+
if (offset >= bytes.byteLength)
|
|
120
|
+
return { done: true };
|
|
121
|
+
const value = plain(bytes.subarray(offset, offset + CHUNK_BYTES));
|
|
122
|
+
offset += value.byteLength;
|
|
123
|
+
return { done: false, value };
|
|
124
|
+
},
|
|
125
|
+
cancel: async () => {
|
|
126
|
+
offset = bytes.byteLength;
|
|
127
|
+
},
|
|
128
|
+
releaseLock: () => { }
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
async arrayBuffer() {
|
|
134
|
+
this.#used = true;
|
|
135
|
+
const { buffer, byteOffset, byteLength } = this.#bytes;
|
|
136
|
+
return buffer.slice(byteOffset, byteOffset + byteLength);
|
|
137
|
+
}
|
|
138
|
+
async bytes() {
|
|
139
|
+
return new Uint8Array(await this.arrayBuffer());
|
|
140
|
+
}
|
|
141
|
+
async text() {
|
|
142
|
+
this.#used = true;
|
|
143
|
+
return b4a.toString(this.#bytes, 'utf8');
|
|
144
|
+
}
|
|
145
|
+
async json() {
|
|
146
|
+
return JSON.parse(await this.text());
|
|
147
|
+
}
|
|
148
|
+
clone() {
|
|
149
|
+
return new GuardedResponse(this.status, [...this.headers.entries()], this.#bytes, this.url, this.redirected);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
export function createGuardedFetch(options = {}) {
|
|
153
|
+
const send = options.fetch ?? defaultFetch;
|
|
154
|
+
const resolveHost = options.resolveHost ?? lookupHost;
|
|
155
|
+
const maxBytes = options.maxBytes ?? MAX_RESPONSE_BYTES;
|
|
156
|
+
const timeoutMs = options.timeoutMs ?? TIMEOUT_MS;
|
|
157
|
+
const guardedFetch = async function (input, init = {}) {
|
|
158
|
+
let url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url);
|
|
159
|
+
let method = (init.method ?? 'GET').toUpperCase();
|
|
160
|
+
let body = bodyBytes(init.body);
|
|
161
|
+
const headers = headerRecord(init.headers);
|
|
162
|
+
const controller = new AbortController();
|
|
163
|
+
const timer = setTimeout(() => controller.abort(undefined), timeoutMs);
|
|
164
|
+
if (init.signal?.aborted)
|
|
165
|
+
controller.abort(undefined);
|
|
166
|
+
init.signal?.addEventListener('abort', () => controller.abort(undefined));
|
|
167
|
+
try {
|
|
168
|
+
for (let hops = 0;; hops++) {
|
|
169
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
170
|
+
throw new TypeError(`fetch refused: unsupported protocol ${url.protocol}`);
|
|
171
|
+
}
|
|
172
|
+
const literal = checkOutboundHost(url);
|
|
173
|
+
if (literal)
|
|
174
|
+
throw new TypeError(`fetch refused: ${literal}`);
|
|
175
|
+
if (!(await mayConnect(url.hostname))) {
|
|
176
|
+
throw new TypeError(`fetch refused: ${url.hostname} was not approved for this run`);
|
|
177
|
+
}
|
|
178
|
+
const check = await resolveOutboundUrl(url, resolveHost, controller.signal);
|
|
179
|
+
if (check.blocked)
|
|
180
|
+
throw new TypeError(`fetch refused: ${check.blocked}`);
|
|
181
|
+
const response = await send(url, {
|
|
182
|
+
method,
|
|
183
|
+
headers,
|
|
184
|
+
body,
|
|
185
|
+
signal: controller.signal,
|
|
186
|
+
validatedAddresses: check.addresses,
|
|
187
|
+
maxBytes
|
|
188
|
+
});
|
|
189
|
+
const location = response.headers.get('location');
|
|
190
|
+
if (!isRedirect(response.status) || !location || init.redirect === 'manual') {
|
|
191
|
+
return new GuardedResponse(response.status, response.headers.entries(), plain(response.bytes()), url.href, hops > 0);
|
|
192
|
+
}
|
|
193
|
+
if (init.redirect === 'error')
|
|
194
|
+
throw new TypeError('fetch refused: redirect not allowed');
|
|
195
|
+
if (hops >= MAX_REDIRECTS)
|
|
196
|
+
throw new TypeError('fetch refused: too many redirects');
|
|
197
|
+
const next = new URL(location, url);
|
|
198
|
+
if (next.protocol !== url.protocol || next.host !== url.host) {
|
|
199
|
+
for (const name of Object.keys(headers)) {
|
|
200
|
+
if (/^(authorization|cookie)$/i.test(name))
|
|
201
|
+
delete headers[name];
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
if (response.status === 303 ||
|
|
205
|
+
((response.status === 301 || response.status === 302) && method === 'POST')) {
|
|
206
|
+
method = 'GET';
|
|
207
|
+
body = undefined;
|
|
208
|
+
}
|
|
209
|
+
url = next;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
finally {
|
|
213
|
+
clearTimeout(timer);
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
const fetch = (input, init) => {
|
|
217
|
+
const response = guardedFetch(input, init);
|
|
218
|
+
response.catch(() => { });
|
|
219
|
+
return response;
|
|
220
|
+
};
|
|
221
|
+
return Object.assign(fetch, { [GUARDED]: true });
|
|
222
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { spawnInlineThread } from '#pyodide-worker-spawn';
|
|
2
|
+
const ARMED = 0;
|
|
3
|
+
const DISARMED = 1;
|
|
4
|
+
const FIRED = 2;
|
|
5
|
+
const SIGINT = 2;
|
|
6
|
+
const RESEND_MS = 100;
|
|
7
|
+
const SOURCE = `
|
|
8
|
+
const state = new Int32Array(data.state)
|
|
9
|
+
const interrupt = new Uint8Array(data.interrupt)
|
|
10
|
+
Atomics.wait(state, 0, ${ARMED}, data.ms)
|
|
11
|
+
if (Atomics.compareExchange(state, 0, ${ARMED}, ${FIRED}) === ${ARMED}) {
|
|
12
|
+
do Atomics.store(interrupt, 0, ${SIGINT})
|
|
13
|
+
while (Atomics.wait(state, 0, ${FIRED}, ${RESEND_MS}) === 'timed-out')
|
|
14
|
+
}
|
|
15
|
+
`;
|
|
16
|
+
export function startInterruptTimer(ms) {
|
|
17
|
+
const state = new Int32Array(new SharedArrayBuffer(4));
|
|
18
|
+
const interrupt = new Uint8Array(new SharedArrayBuffer(1));
|
|
19
|
+
const reap = spawnInlineThread(SOURCE, { state: state.buffer, interrupt: interrupt.buffer, ms });
|
|
20
|
+
return {
|
|
21
|
+
interrupt,
|
|
22
|
+
disarm() {
|
|
23
|
+
const fired = Atomics.exchange(state, 0, DISARMED) === FIRED;
|
|
24
|
+
Atomics.notify(state, 0);
|
|
25
|
+
reap();
|
|
26
|
+
return fired;
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
}
|
|
@@ -9,6 +9,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
|
|
|
9
9
|
import b4a from 'b4a';
|
|
10
10
|
import { constants as fsConstants } from 'fs';
|
|
11
11
|
import process from 'process';
|
|
12
|
+
import { createGuardedFetch } from './guarded-fetch.js';
|
|
12
13
|
import { seedPackageCache } from './wheels.js';
|
|
13
14
|
class B4aTextDecoder {
|
|
14
15
|
encoding;
|
|
@@ -53,7 +54,7 @@ async function shapeBareGlobals() {
|
|
|
53
54
|
const g = globalThis;
|
|
54
55
|
if (!g.fetch) {
|
|
55
56
|
const bareFetch = await import('bare-fetch');
|
|
56
|
-
g.fetch =
|
|
57
|
+
g.fetch = createGuardedFetch();
|
|
57
58
|
g.Request = bareFetch.Request;
|
|
58
59
|
g.Response = bareFetch.Response;
|
|
59
60
|
g.Headers = bareFetch.Headers;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const MODULE_DISTRIBUTIONS: Readonly<Record<string, string>>;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
export const MODULE_DISTRIBUTIONS = {
|
|
2
|
+
Bio: 'biopython',
|
|
3
|
+
BioSQL: 'biopython',
|
|
4
|
+
Crypto: 'pycryptodome',
|
|
5
|
+
PIL: 'Pillow',
|
|
6
|
+
PhiSpyModules: 'phispy',
|
|
7
|
+
PhiSpyRepeatFinder: 'phispy',
|
|
8
|
+
_argon2_cffi_bindings: 'argon2-cffi-bindings',
|
|
9
|
+
_distutils_hack: 'setuptools',
|
|
10
|
+
_pyrsistent_version: 'pyrsistent',
|
|
11
|
+
_pytest: 'pytest',
|
|
12
|
+
_yaml: 'pyyaml',
|
|
13
|
+
_zengl: 'zengl',
|
|
14
|
+
argon2: 'argon2-cffi',
|
|
15
|
+
attr: 'attrs',
|
|
16
|
+
audioop: 'audioop-lts',
|
|
17
|
+
bigtests: 'screed',
|
|
18
|
+
bs4: 'beautifulsoup4',
|
|
19
|
+
cv2: 'opencv-python',
|
|
20
|
+
cvxpy: 'cvxpy-base',
|
|
21
|
+
dateutil: 'python-dateutil',
|
|
22
|
+
dns: 'dnspython',
|
|
23
|
+
docx: 'python-docx',
|
|
24
|
+
erfa: 'pyerfa',
|
|
25
|
+
fitz: 'PyMuPDF',
|
|
26
|
+
flint: 'python-flint',
|
|
27
|
+
flirt: 'python-flirt',
|
|
28
|
+
fpdf: 'fpdf2',
|
|
29
|
+
google: 'protobuf',
|
|
30
|
+
gridfs: 'pymongo',
|
|
31
|
+
isympy: 'sympy',
|
|
32
|
+
lakers: 'lakers-python',
|
|
33
|
+
mpl_toolkits: 'matplotlib',
|
|
34
|
+
mypyc: 'mypy',
|
|
35
|
+
nacl: 'pynacl',
|
|
36
|
+
pptx: 'python-pptx',
|
|
37
|
+
pygame: 'pygame-ce',
|
|
38
|
+
pylab: 'matplotlib',
|
|
39
|
+
pysat: 'python-sat',
|
|
40
|
+
pywt: 'pywavelets',
|
|
41
|
+
rpds: 'rpds-py',
|
|
42
|
+
ruamel: 'ruamel.yaml',
|
|
43
|
+
shapefile: 'pyshp',
|
|
44
|
+
sisl_toolbox: 'sisl',
|
|
45
|
+
skimage: 'scikit-image',
|
|
46
|
+
sklearn: 'scikit-learn',
|
|
47
|
+
taglib: 'pytaglib',
|
|
48
|
+
tiktoken_ext: 'tiktoken',
|
|
49
|
+
tlz: 'toolz',
|
|
50
|
+
unix_timezones: 'pyodide-unix-timezones',
|
|
51
|
+
yaml: 'pyyaml'
|
|
52
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
const URL_LITERAL = /https?:\/\/[^\s'"`\\<>(){}[\]]+/gi;
|
|
2
|
+
export function namedHosts(source) {
|
|
3
|
+
const hosts = new Set();
|
|
4
|
+
for (const [match] of source.matchAll(URL_LITERAL)) {
|
|
5
|
+
try {
|
|
6
|
+
hosts.add(new URL(match).hostname.toLowerCase());
|
|
7
|
+
}
|
|
8
|
+
catch { }
|
|
9
|
+
}
|
|
10
|
+
return [...hosts];
|
|
11
|
+
}
|
|
12
|
+
export function hostIsNamed(host, named) {
|
|
13
|
+
const bare = (value) => value.toLowerCase().replace(/^www\./, '');
|
|
14
|
+
return named.some((name) => bare(name) === bare(host));
|
|
15
|
+
}
|