@qvac/harness 0.2.12 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CHANGELOG.md +213 -0
  2. package/dist/harness.bundle +16 -13
  3. package/dist-lib/index.d.ts +12 -2
  4. package/dist-lib/index.js +66 -5
  5. package/dist-lib/lib/capabilities/completion.d.ts +5 -0
  6. package/dist-lib/lib/capabilities/completion.js +17 -5
  7. package/dist-lib/lib/capabilities/enhance.d.ts +10 -0
  8. package/dist-lib/lib/capabilities/enhance.js +29 -0
  9. package/dist-lib/lib/capabilities/invoke.d.ts +11 -2
  10. package/dist-lib/lib/capabilities/invoke.js +32 -8
  11. package/dist-lib/lib/connect.js +1 -0
  12. package/dist-lib/lib/contract-types.d.ts +1 -0
  13. package/dist-lib/lib/helpers/clean-answer.d.ts +1 -0
  14. package/dist-lib/lib/helpers/clean-answer.js +16 -0
  15. package/dist-lib/lib/helpers/load-stall.d.ts +6 -0
  16. package/dist-lib/lib/helpers/load-stall.js +24 -0
  17. package/dist-lib/lib/helpers/tool-call-markup.js +15 -7
  18. package/dist-lib/lib/models/model-pool.d.ts +6 -0
  19. package/dist-lib/lib/models/model-pool.js +12 -0
  20. package/dist-lib/lib/models/nmt-catalog.d.ts +34 -0
  21. package/dist-lib/lib/models/nmt-catalog.js +166 -0
  22. package/dist-lib/lib/no-redirect-fetch-impl.d.ts +3 -2
  23. package/dist-lib/lib/no-redirect-fetch-impl.js +4 -1
  24. package/dist-lib/lib/serve.js +1 -0
  25. package/dist-lib/lib/tools/exec/adapters/adapter.d.ts +2 -0
  26. package/dist-lib/lib/tools/exec/adapters/python/distributions.d.ts +1 -0
  27. package/dist-lib/lib/tools/exec/adapters/python/distributions.js +4 -0
  28. package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.d.ts +67 -0
  29. package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.js +196 -0
  30. package/dist-lib/lib/tools/exec/adapters/python/loader.js +2 -1
  31. package/dist-lib/lib/tools/exec/adapters/python/module-distributions.d.ts +1 -0
  32. package/dist-lib/lib/tools/exec/adapters/python/module-distributions.js +52 -0
  33. package/dist-lib/lib/tools/exec/adapters/python/net-shim.d.ts +2 -0
  34. package/dist-lib/lib/tools/exec/adapters/python/net-shim.js +148 -0
  35. package/dist-lib/lib/tools/exec/adapters/python/output-policy.js +13 -1
  36. package/dist-lib/lib/tools/exec/adapters/python/pyodide.d.ts +1 -0
  37. package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +1 -0
  38. package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts +9 -3
  39. package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js +130 -20
  40. package/dist-lib/lib/tools/exec/adapters/python/run-session.d.ts +3 -0
  41. package/dist-lib/lib/tools/exec/adapters/python/run-session.js +25 -8
  42. package/dist-lib/lib/tools/exec/adapters/python/wheels.d.ts +6 -0
  43. package/dist-lib/lib/tools/exec/adapters/python/wheels.js +17 -3
  44. package/dist-lib/lib/tools/exec/adapters/shell/shell-adapter.js +7 -1
  45. package/dist-lib/lib/tools/exec/shell-guidance.d.ts +10 -0
  46. package/dist-lib/lib/tools/exec/shell-guidance.js +123 -0
  47. package/dist-lib/lib/tools/exec/video-tools.d.ts +5 -0
  48. package/dist-lib/lib/tools/exec/video-tools.js +40 -0
  49. package/dist-lib/lib/tools/exec.d.ts +2 -1
  50. package/dist-lib/lib/tools/exec.js +59 -32
  51. package/dist-lib/lib/tools/generate-image.js +2 -25
  52. package/dist-lib/lib/tools/http-request/truncate-response.d.ts +1 -0
  53. package/dist-lib/lib/tools/http-request/truncate-response.js +1 -1
  54. package/dist-lib/lib/tools/http-request.js +4 -1
  55. package/dist-lib/lib/tools/operations/builders.d.ts +1 -1
  56. package/dist-lib/lib/tools/operations/builders.js +9 -0
  57. package/dist-lib/lib/tools/operations/compile.js +41 -18
  58. package/dist-lib/lib/tools/operations/decoders.d.ts +6 -0
  59. package/dist-lib/lib/tools/operations/decoders.js +82 -0
  60. package/dist-lib/lib/tools/operations/manifest.d.ts +16 -2
  61. package/dist-lib/lib/tools/operations/manifest.js +6 -1
  62. package/dist-lib/lib/tools/skill.d.ts +3 -0
  63. package/dist-lib/lib/tools/skill.js +38 -5
  64. package/dist-lib/lib/tools/skills/manifest-parse.js +43 -1
  65. package/dist-lib/lib/tools/skills/manifest.d.ts +13 -0
  66. package/dist-lib/lib/tools/translate.d.ts +78 -0
  67. package/dist-lib/lib/tools/translate.js +228 -0
  68. package/dist-lib/schema/harness.js +22 -2
  69. package/dist-lib/spec/hrpc/messages.js +101 -8
  70. package/dist-lib/spec/hyperschema/index.js +101 -8
  71. package/dist-lib/spec/hyperschema/schema.json +71 -1
  72. package/dist-lib/spec/hyperschema/types.d.ts +19 -1
  73. package/package.json +35 -2
  74. package/wheels/manifest.json +15 -0
@@ -0,0 +1,166 @@
1
+ export const ENGLISH = 'en';
2
+ const TO_ENGLISH = [
3
+ 'ar',
4
+ 'az',
5
+ 'be',
6
+ 'bg',
7
+ 'bn',
8
+ 'bs',
9
+ 'ca',
10
+ 'cs',
11
+ 'da',
12
+ 'de',
13
+ 'el',
14
+ 'es',
15
+ 'et',
16
+ 'fa',
17
+ 'fi',
18
+ 'fr',
19
+ 'gu',
20
+ 'he',
21
+ 'hi',
22
+ 'hr',
23
+ 'hu',
24
+ 'id',
25
+ 'is',
26
+ 'it',
27
+ 'ja',
28
+ 'kn',
29
+ 'ko',
30
+ 'lt',
31
+ 'lv',
32
+ 'ml',
33
+ 'ms',
34
+ 'mt',
35
+ 'nb',
36
+ 'nl',
37
+ 'nn',
38
+ 'no',
39
+ 'pl',
40
+ 'pt',
41
+ 'ro',
42
+ 'ru',
43
+ 'sk',
44
+ 'sl',
45
+ 'sq',
46
+ 'sr',
47
+ 'sv',
48
+ 'ta',
49
+ 'te',
50
+ 'th',
51
+ 'tr',
52
+ 'uk',
53
+ 'vi',
54
+ 'zh'
55
+ ];
56
+ const FROM_ENGLISH = TO_ENGLISH.filter((code) => !['be', 'mt', 'nn'].includes(code));
57
+ export const NMT_TO_ENGLISH = TO_ENGLISH;
58
+ export const NMT_FROM_ENGLISH = FROM_ENGLISH;
59
+ const SLM_LANGUAGES = [
60
+ { iso: 'af', flores: 'afr_Latn', name: 'Afrikaans' },
61
+ { iso: 'am', flores: 'amh_Ethi', name: 'Amharic' },
62
+ { iso: 'ha', flores: 'hau_Latn', name: 'Hausa' },
63
+ { iso: 'ig', flores: 'ibo_Latn', name: 'Igbo' },
64
+ { iso: 'rw', flores: 'kin_Latn', name: 'Kinyarwanda' },
65
+ { iso: 'mg', flores: 'plt_Latn', name: 'Malagasy' },
66
+ { iso: 'ny', flores: 'nya_Latn', name: 'Nyanja' },
67
+ { iso: 'om', flores: 'gaz_Latn', name: 'Oromo' },
68
+ { iso: 'sn', flores: 'sna_Latn', name: 'Shona' },
69
+ { iso: 'so', flores: 'som_Latn', name: 'Somali' },
70
+ { iso: 'st', flores: 'sot_Latn', name: 'Southern Sotho' },
71
+ { iso: 'sw', flores: 'swh_Latn', name: 'Swahili' },
72
+ { iso: 'tn', flores: 'tsn_Latn', name: 'Tswana' },
73
+ { iso: 'xh', flores: 'xho_Latn', name: 'Xhosa' },
74
+ { iso: 'yo', flores: 'yor_Latn', name: 'Yoruba' },
75
+ { iso: 'zu', flores: 'zul_Latn', name: 'Zulu' }
76
+ ];
77
+ export const SLM_MODEL = 'TranslatePsy-AfriSLM-0.8B-Q4_K_M-imat';
78
+ export function isTranslationModel(model) {
79
+ return /(^|[/\\])TranslatePsy-/.test(model);
80
+ }
81
+ const EURONANO = 'https://huggingface.co/qvac/TranslatePsy-EuroNano/resolve/main';
82
+ const EURONANO_SIZE = 'Base';
83
+ export const EURONANO_LANGUAGES = [
84
+ 'cs',
85
+ 'de',
86
+ 'es',
87
+ 'fi',
88
+ 'fr',
89
+ 'it',
90
+ 'nl',
91
+ 'pt',
92
+ 'sv'
93
+ ];
94
+ function euroNanoPack(direction) {
95
+ const dir = `${EURONANO}/${direction}/${EURONANO_SIZE}/intgemm`;
96
+ return { model: `${dir}/model.intgemm.alphas.bin`, vocab: `${dir}/vocab.spm` };
97
+ }
98
+ export const SLM_LANGUAGE_CODES = SLM_LANGUAGES.map((entry) => entry.iso);
99
+ export function languageLabel(code) {
100
+ const wanted = code.toLowerCase();
101
+ return SLM_LANGUAGES.find((entry) => entry.flores.toLowerCase() === wanted || entry.iso === wanted)?.name;
102
+ }
103
+ export const SLM_LANGUAGE_NAMES = SLM_LANGUAGES.map(({ iso, name }) => ({ code: iso, name }));
104
+ export function slmLanguage(code) {
105
+ const wanted = code.toLowerCase();
106
+ return SLM_LANGUAGES.find((entry) => entry.flores.toLowerCase() === wanted || entry.iso === wanted)?.flores;
107
+ }
108
+ const modelFilter = (pair) => `model.${pair}.intgemm`;
109
+ export function sameLanguage(from, to) {
110
+ return canonical(from) === canonical(to);
111
+ }
112
+ function canonical(code) {
113
+ return slmLanguage(code) ?? code.toLowerCase();
114
+ }
115
+ export function translationPlanFor(requestedFrom, requestedTo) {
116
+ const from = requestedFrom.toLowerCase();
117
+ const to = requestedTo.toLowerCase();
118
+ if (sameLanguage(from, to))
119
+ return { error: `${from} is already the target language` };
120
+ const slmFrom = slmLanguage(from);
121
+ const slmTo = slmLanguage(to);
122
+ if (slmFrom || slmTo) {
123
+ return { route: 'slm', from: slmFrom ?? from, to: slmTo ?? to, model: SLM_MODEL };
124
+ }
125
+ if (to === ENGLISH && EURONANO_LANGUAGES.includes(from)) {
126
+ const pack = euroNanoPack('xx-en');
127
+ return {
128
+ route: 'nmt',
129
+ from,
130
+ to,
131
+ model: pack.model,
132
+ srcVocab: pack.vocab,
133
+ dstVocab: pack.vocab
134
+ };
135
+ }
136
+ if (from === ENGLISH && EURONANO_LANGUAGES.includes(to)) {
137
+ const pack = euroNanoPack('en-xx');
138
+ return {
139
+ route: 'nmt',
140
+ from,
141
+ to,
142
+ model: pack.model,
143
+ srcVocab: pack.vocab,
144
+ dstVocab: pack.vocab,
145
+ tag: `##${to.toUpperCase()}`
146
+ };
147
+ }
148
+ if (to === ENGLISH) {
149
+ if (!TO_ENGLISH.includes(from))
150
+ return { error: `cannot translate from ${from}` };
151
+ return { route: 'nmt', from, to, model: modelFilter(`${from}${ENGLISH}`) };
152
+ }
153
+ if (!FROM_ENGLISH.includes(to))
154
+ return { error: `cannot translate to ${to}` };
155
+ if (from === ENGLISH)
156
+ return { route: 'nmt', from, to, model: modelFilter(`${ENGLISH}${to}`) };
157
+ if (!TO_ENGLISH.includes(from))
158
+ return { error: `cannot translate from ${from}` };
159
+ return {
160
+ route: 'nmt',
161
+ from,
162
+ to,
163
+ model: modelFilter(`${from}${ENGLISH}`),
164
+ pivot: modelFilter(`${ENGLISH}${to}`)
165
+ };
166
+ }
@@ -1,6 +1,6 @@
1
1
  export interface FetchRequestLike {
2
2
  on(event: 'error', listener: (err: Error) => void): void;
3
- write(chunk: string): void;
3
+ write(chunk: string | Uint8Array): void;
4
4
  end(): void;
5
5
  destroy(err?: Error): void;
6
6
  }
@@ -19,7 +19,7 @@ export type BareRequestFn<Response extends FetchResponseLike = FetchResponseLike
19
19
  export interface NoRedirectFetchInit {
20
20
  method?: string;
21
21
  headers?: Record<string, string>;
22
- body?: string;
22
+ body?: string | Uint8Array;
23
23
  signal?: {
24
24
  readonly aborted: boolean;
25
25
  addEventListener(type: 'abort', listener: () => void): void;
@@ -32,6 +32,7 @@ export interface NoRedirectFetchResponse {
32
32
  status: number;
33
33
  headers: {
34
34
  get(name: string): string | null;
35
+ entries(): [string, string][];
35
36
  };
36
37
  text(): string;
37
38
  bytes(): Uint8Array;
@@ -29,7 +29,10 @@ export function runNoRedirectFetch(request, url, init = {}) {
29
29
  }
30
30
  resolve({
31
31
  status: res.statusCode,
32
- headers: { get: (name) => responseHeaders[name.toLowerCase()] ?? null },
32
+ headers: {
33
+ get: (name) => responseHeaders[name.toLowerCase()] ?? null,
34
+ entries: () => Object.entries(responseHeaders)
35
+ },
33
36
  text: () => b4a.toString(body, 'utf8'),
34
37
  bytes: () => body
35
38
  });
@@ -296,6 +296,7 @@ function runFrame(harness, frame, context) {
296
296
  return harness.invoke({
297
297
  capability,
298
298
  args: frame.arguments ?? {},
299
+ enhance: frame.enhance === 'auto' ? 'auto' : 'never',
299
300
  agent: frame.agent ?? undefined,
300
301
  attach: context.attach,
301
302
  fetchAttachment: context.fetchAttachment,
@@ -15,6 +15,8 @@ export type ExecutionContext = {
15
15
  path: string;
16
16
  }[];
17
17
  readonly runBudget?: RunBudget;
18
+ readonly sandboxMode?: boolean;
19
+ readonly toolNames?: readonly string[];
18
20
  };
19
21
  export type ExecutionPlan = {
20
22
  run(options: ExecutionOptions): Promise<ExecResult> | ExecResult;
@@ -0,0 +1 @@
1
+ export declare function moduleDistribution(module: string): string | null;
@@ -0,0 +1,4 @@
1
+ import { MODULE_DISTRIBUTIONS } from './module-distributions.js';
2
+ export function moduleDistribution(module) {
3
+ return Object.hasOwn(MODULE_DISTRIBUTIONS, module) ? (MODULE_DISTRIBUTIONS[module] ?? null) : null;
4
+ }
@@ -0,0 +1,67 @@
1
+ import { type HostResolver } from '../../../url-guard.ts';
2
+ import type { NoRedirectFetchInit, NoRedirectFetchResponse } from '../../../../no-redirect-fetch-impl.ts';
3
+ export declare const MAX_RESPONSE_BYTES: number;
4
+ type NoRedirectFetch = (url: URL, init?: NoRedirectFetchInit) => Promise<NoRedirectFetchResponse>;
5
+ declare const GUARDED: unique symbol;
6
+ export declare function isGuardedFetch(fetch: unknown): boolean;
7
+ type SignalLike = {
8
+ readonly aborted: boolean;
9
+ addEventListener(type: 'abort', listener: () => void): void;
10
+ };
11
+ type FetchInit = {
12
+ method?: string;
13
+ headers?: unknown;
14
+ body?: unknown;
15
+ signal?: SignalLike | null;
16
+ redirect?: string;
17
+ };
18
+ export type GuardedFetchOptions = {
19
+ readonly fetch?: NoRedirectFetch;
20
+ readonly resolveHost?: HostResolver;
21
+ readonly maxBytes?: number;
22
+ readonly timeoutMs?: number;
23
+ };
24
+ declare class GuardedHeaders {
25
+ #private;
26
+ constructor(entries: [string, string][]);
27
+ get(name: string): string | null;
28
+ has(name: string): boolean;
29
+ entries(): IterableIterator<[string, string]>;
30
+ keys(): IterableIterator<string>;
31
+ values(): IterableIterator<string>;
32
+ forEach(fn: (value: string, key: string) => void): void;
33
+ [Symbol.iterator](): IterableIterator<[string, string]>;
34
+ }
35
+ export declare class GuardedResponse {
36
+ #private;
37
+ readonly type = "basic";
38
+ readonly statusText = "";
39
+ readonly status: number;
40
+ readonly headers: GuardedHeaders;
41
+ readonly url: string;
42
+ readonly redirected: boolean;
43
+ constructor(status: number, headers: [string, string][], bytes: Uint8Array, url: string, redirected: boolean);
44
+ get ok(): boolean;
45
+ get bodyUsed(): boolean;
46
+ get body(): {
47
+ getReader(): {
48
+ read(): Promise<{
49
+ done: boolean;
50
+ value?: Uint8Array;
51
+ }>;
52
+ cancel(): Promise<void>;
53
+ releaseLock(): void;
54
+ };
55
+ };
56
+ arrayBuffer(): Promise<ArrayBuffer>;
57
+ bytes(): Promise<Uint8Array>;
58
+ text(): Promise<string>;
59
+ json(): Promise<unknown>;
60
+ clone(): GuardedResponse;
61
+ }
62
+ export declare function createGuardedFetch(options?: GuardedFetchOptions): ((input: string | URL | {
63
+ url: string;
64
+ }, init?: FetchInit) => Promise<GuardedResponse>) & {
65
+ [GUARDED]: true;
66
+ };
67
+ export {};
@@ -0,0 +1,196 @@
1
+ import { AbortController } from 'bare-abort-controller';
2
+ import b4a from 'b4a';
3
+ import defaultFetch from '#no-redirect-fetch';
4
+ import { lookupHost } from '../../../../dns-lookup.js';
5
+ import { resolveOutboundUrl } from '../../../url-guard.js';
6
+ export const MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
7
+ const MAX_REDIRECTS = 10;
8
+ const TIMEOUT_MS = 120_000;
9
+ const CHUNK_BYTES = 1024 * 1024;
10
+ const GUARDED = Symbol.for('qvac.harness.guardedFetch');
11
+ export function isGuardedFetch(fetch) {
12
+ return typeof fetch === 'function' && fetch[GUARDED] === true;
13
+ }
14
+ function isRedirect(status) {
15
+ return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
16
+ }
17
+ function headerRecord(headers) {
18
+ const out = {};
19
+ if (!headers || typeof headers !== 'object')
20
+ return out;
21
+ const pairs = Array.isArray(headers)
22
+ ? headers
23
+ : typeof headers.entries === 'function'
24
+ ? headers.entries()
25
+ : Object.entries(headers);
26
+ for (const pair of pairs) {
27
+ if (!Array.isArray(pair) || pair.length < 2)
28
+ continue;
29
+ out[String(pair[0])] = String(pair[1]);
30
+ }
31
+ return out;
32
+ }
33
+ function bodyBytes(body) {
34
+ if (body === undefined || body === null)
35
+ return undefined;
36
+ if (typeof body === 'string')
37
+ return body;
38
+ if (body instanceof ArrayBuffer)
39
+ return new Uint8Array(body);
40
+ if (ArrayBuffer.isView(body))
41
+ return new Uint8Array(body.buffer, body.byteOffset, body.byteLength);
42
+ return String(body);
43
+ }
44
+ function plain(bytes) {
45
+ return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
46
+ }
47
+ class GuardedHeaders {
48
+ #map;
49
+ constructor(entries) {
50
+ this.#map = new Map(entries.map(([k, v]) => [k.toLowerCase(), v]));
51
+ }
52
+ get(name) {
53
+ return this.#map.get(name.toLowerCase()) ?? null;
54
+ }
55
+ has(name) {
56
+ return this.#map.has(name.toLowerCase());
57
+ }
58
+ entries() {
59
+ return this.#map.entries();
60
+ }
61
+ keys() {
62
+ return this.#map.keys();
63
+ }
64
+ values() {
65
+ return this.#map.values();
66
+ }
67
+ forEach(fn) {
68
+ this.#map.forEach((value, key) => fn(value, key));
69
+ }
70
+ [Symbol.iterator]() {
71
+ return this.#map.entries();
72
+ }
73
+ }
74
+ export class GuardedResponse {
75
+ type = 'basic';
76
+ statusText = '';
77
+ status;
78
+ headers;
79
+ url;
80
+ redirected;
81
+ #bytes;
82
+ #used = false;
83
+ constructor(status, headers, bytes, url, redirected) {
84
+ this.status = status;
85
+ this.headers = new GuardedHeaders(headers);
86
+ this.#bytes = bytes;
87
+ this.url = url;
88
+ this.redirected = redirected;
89
+ }
90
+ get ok() {
91
+ return this.status >= 200 && this.status < 300;
92
+ }
93
+ get bodyUsed() {
94
+ return this.#used;
95
+ }
96
+ get body() {
97
+ let offset = 0;
98
+ const bytes = this.#bytes;
99
+ return {
100
+ getReader: () => {
101
+ this.#used = true;
102
+ return {
103
+ read: async () => {
104
+ if (offset >= bytes.byteLength)
105
+ return { done: true };
106
+ const value = plain(bytes.subarray(offset, offset + CHUNK_BYTES));
107
+ offset += value.byteLength;
108
+ return { done: false, value };
109
+ },
110
+ cancel: async () => {
111
+ offset = bytes.byteLength;
112
+ },
113
+ releaseLock: () => { }
114
+ };
115
+ }
116
+ };
117
+ }
118
+ async arrayBuffer() {
119
+ this.#used = true;
120
+ const { buffer, byteOffset, byteLength } = this.#bytes;
121
+ return buffer.slice(byteOffset, byteOffset + byteLength);
122
+ }
123
+ async bytes() {
124
+ return new Uint8Array(await this.arrayBuffer());
125
+ }
126
+ async text() {
127
+ this.#used = true;
128
+ return b4a.toString(this.#bytes, 'utf8');
129
+ }
130
+ async json() {
131
+ return JSON.parse(await this.text());
132
+ }
133
+ clone() {
134
+ return new GuardedResponse(this.status, [...this.headers.entries()], this.#bytes, this.url, this.redirected);
135
+ }
136
+ }
137
+ export function createGuardedFetch(options = {}) {
138
+ const send = options.fetch ?? defaultFetch;
139
+ const resolveHost = options.resolveHost ?? lookupHost;
140
+ const maxBytes = options.maxBytes ?? MAX_RESPONSE_BYTES;
141
+ const timeoutMs = options.timeoutMs ?? TIMEOUT_MS;
142
+ const guardedFetch = async function (input, init = {}) {
143
+ let url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url);
144
+ let method = (init.method ?? 'GET').toUpperCase();
145
+ let body = bodyBytes(init.body);
146
+ const headers = headerRecord(init.headers);
147
+ const controller = new AbortController();
148
+ const timer = setTimeout(() => controller.abort(undefined), timeoutMs);
149
+ if (init.signal?.aborted)
150
+ controller.abort(undefined);
151
+ init.signal?.addEventListener('abort', () => controller.abort(undefined));
152
+ try {
153
+ for (let hops = 0;; hops++) {
154
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') {
155
+ throw new TypeError(`fetch refused: unsupported protocol ${url.protocol}`);
156
+ }
157
+ const check = await resolveOutboundUrl(url, resolveHost, controller.signal);
158
+ if (check.blocked)
159
+ throw new TypeError(`fetch refused: ${check.blocked}`);
160
+ const response = await send(url, {
161
+ method,
162
+ headers,
163
+ body,
164
+ signal: controller.signal,
165
+ validatedAddresses: check.addresses,
166
+ maxBytes
167
+ });
168
+ const location = response.headers.get('location');
169
+ if (!isRedirect(response.status) || !location || init.redirect === 'manual') {
170
+ return new GuardedResponse(response.status, response.headers.entries(), plain(response.bytes()), url.href, hops > 0);
171
+ }
172
+ if (init.redirect === 'error')
173
+ throw new TypeError('fetch refused: redirect not allowed');
174
+ if (hops >= MAX_REDIRECTS)
175
+ throw new TypeError('fetch refused: too many redirects');
176
+ const next = new URL(location, url);
177
+ if (next.protocol !== url.protocol || next.host !== url.host) {
178
+ for (const name of Object.keys(headers)) {
179
+ if (/^(authorization|cookie)$/i.test(name))
180
+ delete headers[name];
181
+ }
182
+ }
183
+ if (response.status === 303 ||
184
+ ((response.status === 301 || response.status === 302) && method === 'POST')) {
185
+ method = 'GET';
186
+ body = undefined;
187
+ }
188
+ url = next;
189
+ }
190
+ }
191
+ finally {
192
+ clearTimeout(timer);
193
+ }
194
+ };
195
+ return Object.assign(guardedFetch, { [GUARDED]: true });
196
+ }
@@ -9,6 +9,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
9
9
  import b4a from 'b4a';
10
10
  import { constants as fsConstants } from 'fs';
11
11
  import process from 'process';
12
+ import { createGuardedFetch } from './guarded-fetch.js';
12
13
  import { seedPackageCache } from './wheels.js';
13
14
  class B4aTextDecoder {
14
15
  encoding;
@@ -53,7 +54,7 @@ async function shapeBareGlobals() {
53
54
  const g = globalThis;
54
55
  if (!g.fetch) {
55
56
  const bareFetch = await import('bare-fetch');
56
- g.fetch = bareFetch.default;
57
+ g.fetch = createGuardedFetch();
57
58
  g.Request = bareFetch.Request;
58
59
  g.Response = bareFetch.Response;
59
60
  g.Headers = bareFetch.Headers;
@@ -0,0 +1 @@
1
+ export declare const MODULE_DISTRIBUTIONS: Readonly<Record<string, string>>;
@@ -0,0 +1,52 @@
1
+ export const MODULE_DISTRIBUTIONS = {
2
+ Bio: 'biopython',
3
+ BioSQL: 'biopython',
4
+ Crypto: 'pycryptodome',
5
+ PIL: 'Pillow',
6
+ PhiSpyModules: 'phispy',
7
+ PhiSpyRepeatFinder: 'phispy',
8
+ _argon2_cffi_bindings: 'argon2-cffi-bindings',
9
+ _distutils_hack: 'setuptools',
10
+ _pyrsistent_version: 'pyrsistent',
11
+ _pytest: 'pytest',
12
+ _yaml: 'pyyaml',
13
+ _zengl: 'zengl',
14
+ argon2: 'argon2-cffi',
15
+ attr: 'attrs',
16
+ audioop: 'audioop-lts',
17
+ bigtests: 'screed',
18
+ bs4: 'beautifulsoup4',
19
+ cv2: 'opencv-python',
20
+ cvxpy: 'cvxpy-base',
21
+ dateutil: 'python-dateutil',
22
+ dns: 'dnspython',
23
+ docx: 'python-docx',
24
+ erfa: 'pyerfa',
25
+ fitz: 'PyMuPDF',
26
+ flint: 'python-flint',
27
+ flirt: 'python-flirt',
28
+ fpdf: 'fpdf2',
29
+ google: 'protobuf',
30
+ gridfs: 'pymongo',
31
+ isympy: 'sympy',
32
+ lakers: 'lakers-python',
33
+ mpl_toolkits: 'matplotlib',
34
+ mypyc: 'mypy',
35
+ nacl: 'pynacl',
36
+ pptx: 'python-pptx',
37
+ pygame: 'pygame-ce',
38
+ pylab: 'matplotlib',
39
+ pysat: 'python-sat',
40
+ pywt: 'pywavelets',
41
+ rpds: 'rpds-py',
42
+ ruamel: 'ruamel.yaml',
43
+ shapefile: 'pyshp',
44
+ sisl_toolbox: 'sisl',
45
+ skimage: 'scikit-image',
46
+ sklearn: 'scikit-learn',
47
+ taglib: 'pytaglib',
48
+ tiktoken_ext: 'tiktoken',
49
+ tlz: 'toolz',
50
+ unix_timezones: 'pyodide-unix-timezones',
51
+ yaml: 'pyyaml'
52
+ };
@@ -0,0 +1,2 @@
1
+ export declare function pythonReachesNetwork(): boolean;
2
+ export declare const NET_SHIM = "\ndef _qvac_install_net():\n import errno, http.client, io\n import js\n from pyodide.ffi import can_run_sync, run_sync, to_js\n\n if getattr(http.client, '_qvac_net', False):\n return\n\n class _Response(http.client.HTTPResponse):\n # the body is read in full by the fetch, so there is no socket behind it\n def __init__(self, status, headers, body, url):\n self.status = self.code = status\n self.reason = http.client.responses.get(status, '')\n self.headers = self.msg = http.client.HTTPMessage()\n for name, value in headers:\n self.headers[name] = value\n self.fp = io.BytesIO(body)\n self.version = 11\n self.will_close = True\n self.chunked = False\n self.length = len(body)\n self.url = url\n self.debuglevel = 0\n\n def read(self, amt=None):\n return self.fp.read() if amt is None or amt < 0 else self.fp.read(amt)\n\n def read1(self, n=-1):\n return self.read(n)\n\n def readinto(self, b):\n data = self.fp.read(len(b))\n b[: len(data)] = data\n return len(data)\n\n def peek(self, n=0):\n pos = self.fp.tell()\n data = self.fp.read(max(n, 1))\n self.fp.seek(pos)\n return data\n\n def isclosed(self):\n return self.fp.closed\n\n def close(self):\n self.fp.close()\n\n def fileno(self):\n raise OSError('this response has no socket')\n\n class _Connection(http.client.HTTPConnection):\n scheme = 'http'\n\n def __init__(self, host, port=None, timeout=None, source_address=None, blocksize=8192, **kw):\n self.host, self.port, self.timeout = host, port, timeout\n self.sock = None\n self.debuglevel = 0\n self._tunnel_host = None\n self._request = None\n self._response = None\n\n def set_debuglevel(self, level):\n pass\n\n def set_tunnel(self, host, port=None, headers=None):\n self._tunnel_host = host\n\n def connect(self):\n pass\n\n def close(self):\n pass\n\n def putrequest(self, method, url, skip_host=False, skip_accept_encoding=False):\n self._request = [method, url, {}, b'']\n self._response = None\n\n def putheader(self, header, *values):\n self._request[2][header] = ', '.join(str(v) for v in values)\n\n # A socket connects while the request is sent, so that is where a failure lands and\n # where urllib turns it into URLError. A body sent after the headers waits for it.\n def endheaders(self, message_body=None, *, encode_chunked=False):\n if message_body is not None:\n self.send(message_body)\n if message_body is not None or self._request[0] in ('GET', 'HEAD', 'DELETE', 'OPTIONS'):\n self._response = self._fetch()\n\n def send(self, data):\n if hasattr(data, 'read'):\n data = data.read()\n if isinstance(data, str):\n data = data.encode()\n self._request[3] += bytes(data)\n\n def request(self, method, url, body=None, headers={}, *, encode_chunked=False):\n self.putrequest(method, url)\n for name, value in dict(headers).items():\n self.putheader(name, value)\n self.endheaders(body)\n\n def getresponse(self):\n response = self._response or self._fetch()\n self._response = None\n return response\n\n def _fetch(self):\n method, url, headers, body = self._request\n if not url.startswith(('http://', 'https://')):\n port = '' if self.port in (None, self.default_port) else ':%d' % self.port\n url = '%s://%s%s%s' % (self.scheme, self.host, port, url)\n if not can_run_sync():\n raise OSError(errno.EHOSTUNREACH, 'Host is unreachable')\n # the fetch hands back decoded bytes, so no encoding is asked for and none reported\n skip = ('accept-encoding', 'connection', 'host', 'content-length')\n headers = {k: v for k, v in headers.items() if k.lower() not in skip}\n headers['Accept-Encoding'] = 'identity'\n init = {'method': method, 'headers': headers, 'redirect': 'manual'}\n if body:\n init['body'] = to_js(body)\n try:\n response = run_sync(js.fetch(url, to_js(init, dict_converter=js.Object.fromEntries)))\n data = bytes(run_sync(response.arrayBuffer()).to_py())\n except Exception as error:\n # the guard's own refusal, or the transport's: either says more than an errno\n raise OSError(getattr(error, 'message', None) or str(error)) from None\n entries = [(str(k), str(v)) for k, v in response.headers.entries()]\n drop = ('content-encoding', 'transfer-encoding', 'content-length')\n kept = [(k, v) for k, v in entries if k.lower() not in drop]\n kept.append(('Content-Length', str(len(data))))\n return _Response(int(response.status), kept, data, url)\n\n class _HTTPSConnection(_Connection):\n scheme = 'https'\n default_port = 443\n\n http.client.HTTPConnection = _Connection\n http.client.HTTPSConnection = _HTTPSConnection\n http.client._qvac_net = True\n\n_qvac_install_net()\ndel _qvac_install_net\n";