@mettlelabs/mettle 0.0.0-beta.1-linux-arm64 → 0.0.0-beta.2-darwin-arm64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +503 -8
- package/bin/mettle +2 -2
- package/harness/audit-investigation.mjs +16 -16
- package/harness/chunk-3wt8a8fq.mjs +1716 -0
- package/harness/{chunk-13tgmdey.mjs → chunk-49ytr8d5.mjs} +1 -1
- package/harness/chunk-4vx34w5h.mjs +2357 -0
- package/harness/{chunk-xqyc4ca2.mjs → chunk-9dg1btk1.mjs} +1220 -292
- package/harness/chunk-c7dx5mdd.mjs +10807 -0
- package/harness/{chunk-msg872ea.mjs → chunk-h1td5kq9.mjs} +97 -9
- package/harness/{chunk-4jtnt9k5.mjs → chunk-jp69h6sj.mjs} +1 -1
- package/harness/chunk-rd80vvc5.mjs +22047 -0
- package/harness/{chunk-5cvb1jnb.mjs → chunk-w0qwc5z2.mjs} +2 -2
- package/harness/issue-api-token.mjs +2 -2
- package/harness/launcher.mjs +318 -18
- package/harness/sandbox.mjs +8 -7
- package/harness/server.mjs +22883 -1891
- package/package.json +2 -5
- package/harness/chunk-2gt4cy7x.mjs +0 -2290
- package/harness/chunk-32y07kkp.mjs +0 -3747
- package/harness/chunk-7bg2gnv2.mjs +0 -7540
|
@@ -1,154 +1,132 @@
|
|
|
1
1
|
import {
|
|
2
|
+
IdSchema,
|
|
3
|
+
SourceLocationSchema,
|
|
2
4
|
HarnessError,
|
|
3
5
|
canonicalJson,
|
|
4
6
|
sha256,
|
|
5
|
-
digest
|
|
6
|
-
|
|
7
|
-
|
|
7
|
+
digest
|
|
8
|
+
} from "./chunk-rd80vvc5.mjs";
|
|
9
|
+
import {
|
|
10
|
+
CHILD_TIMEOUT_MAX_MS,
|
|
11
|
+
CommitSchema,
|
|
12
|
+
RepositoryPathSchema,
|
|
13
|
+
SemgrepProfileSchema,
|
|
14
|
+
GVISOR_RUNTIME,
|
|
15
|
+
SandboxRuntimeSchema,
|
|
16
|
+
SandboxConfigSchema,
|
|
17
|
+
sandboxRuntime,
|
|
18
|
+
sandboxAllowsExecution,
|
|
19
|
+
TaskNameSchema,
|
|
20
|
+
PROBE_TOOL,
|
|
21
|
+
SHELL_TOOL,
|
|
22
|
+
HTTP_HEADER_LIMIT,
|
|
23
|
+
HTTP_RESPONSE_LIMIT,
|
|
24
|
+
WEBSITE_RESPONSE_LIMIT,
|
|
25
|
+
withinHttpScope,
|
|
26
|
+
PublicAddressSchema,
|
|
27
|
+
PLAYWRIGHT_VERSION
|
|
28
|
+
} from "./chunk-c7dx5mdd.mjs";
|
|
8
29
|
import {
|
|
9
30
|
exports_iso
|
|
10
31
|
} from "./chunk-b71gpkxg.mjs";
|
|
11
32
|
import {
|
|
33
|
+
toJSONSchema,
|
|
12
34
|
string,
|
|
35
|
+
uuid,
|
|
13
36
|
url,
|
|
14
37
|
number,
|
|
15
38
|
boolean,
|
|
39
|
+
_null,
|
|
16
40
|
unknown,
|
|
17
41
|
array,
|
|
18
42
|
object,
|
|
19
43
|
strictObject,
|
|
20
44
|
looseObject,
|
|
45
|
+
union,
|
|
21
46
|
discriminatedUnion,
|
|
22
47
|
record,
|
|
23
48
|
_enum,
|
|
24
49
|
literal
|
|
25
50
|
} from "./chunk-1sygmp11.mjs";
|
|
26
51
|
|
|
27
|
-
// apps/harness/src/storage/
|
|
28
|
-
var
|
|
29
|
-
var
|
|
30
|
-
var
|
|
31
|
-
var
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
commit: CommitSchema,
|
|
39
|
-
paths: array(RepositoryPathSchema).min(1).max(1e4)
|
|
52
|
+
// apps/harness/src/storage/joern.ts
|
|
53
|
+
var JOERN_TIMEOUT_MS = 180000;
|
|
54
|
+
var JOERN_TASK_TIMEOUT_MS = 210000;
|
|
55
|
+
var JOERN_QUERY_ID = "mettle.source-trace-dataflow-v1";
|
|
56
|
+
var JoernProfileSchema = strictObject({
|
|
57
|
+
version: literal("4.0.647")
|
|
58
|
+
});
|
|
59
|
+
var JoernTraceSchema = strictObject({
|
|
60
|
+
source: SourceLocationSchema,
|
|
61
|
+
sink: SourceLocationSchema,
|
|
62
|
+
locations: array(SourceLocationSchema).min(2).max(16)
|
|
40
63
|
});
|
|
41
|
-
function
|
|
42
|
-
|
|
64
|
+
function joernCandidate(finding, binding) {
|
|
65
|
+
if (finding.surface !== "code" || finding.claim !== "vulnerability" || finding.status !== "candidate" || finding.verification.result !== "pending" || finding.subject.kind !== "source" || finding.subject.repository !== binding.repository || finding.subject.revision !== binding.revision || !finding.sourceTrace || !finding.evidence.toolCalls.some((call) => ["semgrep_scan", "secret_scan", "dependency_scan"].includes(call.tool)))
|
|
66
|
+
return;
|
|
67
|
+
const steps = finding.sourceTrace.pathToImpact;
|
|
68
|
+
const inputs = steps.filter((step) => step.kind === "input");
|
|
69
|
+
const sinks = steps.filter((step) => step.kind === "sink");
|
|
70
|
+
const locations = steps.map((step) => step.location);
|
|
71
|
+
if (inputs.length !== 1 || sinks.length !== 1 || locations.some((location) => location.revision !== binding.revision || !binding.paths.includes(location.path) || !/\.(?:[cm]?js|jsx|[cm]?ts|tsx)$/.test(location.path)))
|
|
72
|
+
return;
|
|
73
|
+
return JoernTraceSchema.parse({
|
|
74
|
+
source: inputs[0].location,
|
|
75
|
+
sink: sinks[0].location,
|
|
76
|
+
locations
|
|
77
|
+
});
|
|
43
78
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
79
|
+
|
|
80
|
+
// apps/harness/src/storage/scanners.ts
|
|
81
|
+
var EcosystemSchema = _enum([
|
|
82
|
+
"npm",
|
|
83
|
+
"PyPI",
|
|
84
|
+
"Go",
|
|
85
|
+
"crates.io",
|
|
86
|
+
"Maven",
|
|
87
|
+
"Packagist",
|
|
88
|
+
"RubyGems"
|
|
89
|
+
]);
|
|
90
|
+
var MAX_DATABASE_BYTES = 268435456;
|
|
91
|
+
var SEMGREP_VERSION_TIMEOUT_MS = 60000;
|
|
92
|
+
var SEMGREP_SCAN_TIMEOUT_MS = 120000;
|
|
93
|
+
var SEMGREP_TASK_TIMEOUT_MS = SEMGREP_VERSION_TIMEOUT_MS + SEMGREP_SCAN_TIMEOUT_MS + 30000;
|
|
94
|
+
var SecretScanProfileSchema = strictObject({
|
|
95
|
+
version: literal("8.30.1")
|
|
48
96
|
});
|
|
49
|
-
var
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
97
|
+
var DependencyScanProfileSchema = strictObject({
|
|
98
|
+
version: literal("2.6.0"),
|
|
99
|
+
paths: array(RepositoryPathSchema).min(1).max(64).refine((paths) => new Set(paths).size === paths.length),
|
|
100
|
+
database: strictObject({
|
|
101
|
+
revision: string().min(1).max(128),
|
|
102
|
+
capturedAt: exports_iso.datetime(),
|
|
103
|
+
ecosystems: array(strictObject({
|
|
104
|
+
name: EcosystemSchema,
|
|
105
|
+
sha256: string().regex(/^[a-f0-9]{64}$/)
|
|
106
|
+
})).min(1).max(EcosystemSchema.options.length).refine((items) => new Set(items.map((item) => item.name)).size === items.length)
|
|
107
|
+
})
|
|
58
108
|
});
|
|
59
|
-
var
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
body: string().max(8192)
|
|
109
|
+
var SOURCE_SCANNER_LABEL = "org.mettle.source-scanners";
|
|
110
|
+
var ScannerImageProfilesSchema = strictObject({
|
|
111
|
+
secretScan: SecretScanProfileSchema,
|
|
112
|
+
semgrep: SemgrepProfileSchema,
|
|
113
|
+
joern: JoernProfileSchema.optional(),
|
|
114
|
+
dependencyDatabase: DependencyScanProfileSchema.shape.database.optional()
|
|
66
115
|
});
|
|
67
|
-
var
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
id: IdSchema,
|
|
81
|
-
identity: string().min(1).max(128),
|
|
82
|
-
resource: string().min(1).max(512),
|
|
83
|
-
action: string().min(1).max(128),
|
|
84
|
-
expected: _enum(["allow", "deny"])
|
|
85
|
-
})).min(1).max(64)
|
|
86
|
-
}).superRefine((policy, context) => {
|
|
87
|
-
const identities = new Set(policy.identities.map((identity) => identity.id));
|
|
88
|
-
const rules = new Set(policy.rules.map((rule) => rule.id));
|
|
89
|
-
if (identities.size !== policy.identities.length || rules.size !== policy.rules.length || policy.rules.some((rule) => !identities.has(rule.identity)))
|
|
90
|
-
context.addIssue({
|
|
91
|
-
code: "custom",
|
|
92
|
-
message: "Policy identities and rules must be unique and bound."
|
|
93
|
-
});
|
|
94
|
-
});
|
|
95
|
-
var GVISOR_RUNTIME = "mettle-runsc";
|
|
96
|
-
var SandboxRuntimeSchema = _enum([GVISOR_RUNTIME, "runc"]);
|
|
97
|
-
var SandboxConfigSchema = strictObject({
|
|
98
|
-
socket: string().regex(/^\/(?:[^\0\r\n]+)$/).max(100),
|
|
99
|
-
image: string().regex(/^[a-z0-9][a-z0-9./:_-]*@sha256:[a-f0-9]{64}$/),
|
|
100
|
-
runtime: SandboxRuntimeSchema.optional(),
|
|
101
|
-
engineId: string().min(1).max(128).optional(),
|
|
102
|
-
location: _enum(["local", "vm"]).optional()
|
|
103
|
-
}).refine((config) => config.location !== "vm" || sandboxRuntime(config) === GVISOR_RUNTIME && Boolean(config.engineId), "VM execution requires a pinned engine identity and mettle-runsc.");
|
|
104
|
-
function sandboxRuntime(config) {
|
|
105
|
-
return config.runtime ?? GVISOR_RUNTIME;
|
|
106
|
-
}
|
|
107
|
-
function sandboxAllowsExecution(config, tool) {
|
|
108
|
-
return sandboxRuntime(config) === GVISOR_RUNTIME || !["sandbox_shell", "bash_run", "test_run"].includes(tool);
|
|
109
|
-
}
|
|
110
|
-
var TaskNameSchema = _enum([
|
|
111
|
-
"web_capture_read",
|
|
112
|
-
"web_capture_compare",
|
|
113
|
-
"repo_read",
|
|
114
|
-
"repo_search",
|
|
115
|
-
"change_inspect",
|
|
116
|
-
"test_run",
|
|
117
|
-
"semgrep_scan",
|
|
118
|
-
"secret_scan",
|
|
119
|
-
"dependency_scan",
|
|
120
|
-
"bash_run",
|
|
121
|
-
"http_request",
|
|
122
|
-
"web_discover",
|
|
123
|
-
"web_request",
|
|
124
|
-
"web_browse",
|
|
125
|
-
"session_health",
|
|
126
|
-
"browser_workflow",
|
|
127
|
-
"nuclei_scan",
|
|
128
|
-
"sqli_confirm",
|
|
129
|
-
"zap_scan"
|
|
116
|
+
var SCANNER_FINDINGS_MAX = 40;
|
|
117
|
+
var SOURCE_SCANNER_INPUT_LIMIT = 33554432;
|
|
118
|
+
var SOURCE_SCANNER_TIMEOUT_MS = 120000;
|
|
119
|
+
var SOURCE_SCANNER_VERSION_TIMEOUT_MS = 1e4;
|
|
120
|
+
var SOURCE_SCANNER_TASK_TIMEOUT_MS = 150000;
|
|
121
|
+
var ScannerStatusSchema = _enum([
|
|
122
|
+
"completed",
|
|
123
|
+
"failed",
|
|
124
|
+
"timed_out",
|
|
125
|
+
"output_limit",
|
|
126
|
+
"unavailable",
|
|
127
|
+
"version_mismatch",
|
|
128
|
+
"database_mismatch"
|
|
130
129
|
]);
|
|
131
|
-
var SCANNER_CAPABILITIES = [
|
|
132
|
-
{ profile: "secretScan", capability: "secrets" },
|
|
133
|
-
{ profile: "dependencyScan", capability: "supply_chain" }
|
|
134
|
-
];
|
|
135
|
-
var SUBJECT_CAPABILITIES = {
|
|
136
|
-
runtime: ROUTE_CAPABILITIES.web,
|
|
137
|
-
source: [
|
|
138
|
-
...new Set([
|
|
139
|
-
...ROUTE_CAPABILITIES.repository,
|
|
140
|
-
...ROUTE_CAPABILITIES.pull_request,
|
|
141
|
-
...SCANNER_CAPABILITIES.map(({ capability }) => capability)
|
|
142
|
-
])
|
|
143
|
-
]
|
|
144
|
-
};
|
|
145
|
-
var PROBE_TOOL = "sandbox_probe";
|
|
146
|
-
var SHELL_TOOL = "sandbox_shell";
|
|
147
|
-
var SANDBOX_TOOLS = [
|
|
148
|
-
PROBE_TOOL,
|
|
149
|
-
SHELL_TOOL,
|
|
150
|
-
...TaskNameSchema.options
|
|
151
|
-
];
|
|
152
130
|
|
|
153
131
|
// apps/harness/src/sandbox/worker.ts
|
|
154
132
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
@@ -193,9 +171,20 @@ class ToolRegistry {
|
|
|
193
171
|
schema
|
|
194
172
|
}));
|
|
195
173
|
}
|
|
174
|
+
inventory() {
|
|
175
|
+
return [...this.tools.values()].map((tool) => ({
|
|
176
|
+
name: tool.name,
|
|
177
|
+
version: "1",
|
|
178
|
+
inputDigest: digest(toJSONSchema(tool.schema, { unrepresentable: "any" })),
|
|
179
|
+
outputDigest: digest(toJSONSchema(tool.outputSchema, { unrepresentable: "any" }))
|
|
180
|
+
})).sort((a, b) => a.name.localeCompare(b.name));
|
|
181
|
+
}
|
|
196
182
|
select(names) {
|
|
197
183
|
return new ToolRegistry([...this.tools.values()].filter((tool) => names.includes(tool.name)));
|
|
198
184
|
}
|
|
185
|
+
unavailable(names, reason) {
|
|
186
|
+
return new ToolRegistry([...this.tools.values()].map((tool) => names.includes(tool.name) ? { ...tool, unavailableReason: reason } : tool));
|
|
187
|
+
}
|
|
199
188
|
admit(name, input, canonical) {
|
|
200
189
|
const tool = this.tools.get(name);
|
|
201
190
|
if (!tool)
|
|
@@ -210,64 +199,22 @@ class ToolRegistry {
|
|
|
210
199
|
// apps/harness/src/sandbox/docker.ts
|
|
211
200
|
import { request } from "node:http";
|
|
212
201
|
|
|
213
|
-
// apps/harness/src/storage/http-scope.ts
|
|
214
|
-
var HTTP_HEADER_LIMIT = 16384;
|
|
215
|
-
var HTTP_RESPONSE_LIMIT = 65536;
|
|
216
|
-
var WEBSITE_RESPONSE_LIMIT = 1048576;
|
|
217
|
-
function withinHttpScope(scope, url) {
|
|
218
|
-
return url.origin === scope.origin && !url.username && !url.password && !url.hash && !/%2f|%5c|%00|%25/i.test(url.pathname) && (url.pathname === scope.pathname || url.pathname.startsWith(scope.pathname.endsWith("/") ? scope.pathname : `${scope.pathname}/`));
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
// apps/harness/src/storage/scanners.ts
|
|
222
|
-
var EcosystemSchema = _enum([
|
|
223
|
-
"npm",
|
|
224
|
-
"PyPI",
|
|
225
|
-
"Go",
|
|
226
|
-
"crates.io",
|
|
227
|
-
"Maven",
|
|
228
|
-
"Packagist",
|
|
229
|
-
"RubyGems"
|
|
230
|
-
]);
|
|
231
|
-
var MAX_DATABASE_BYTES = 268435456;
|
|
232
|
-
var SEMGREP_VERSION_TIMEOUT_MS = 60000;
|
|
233
|
-
var SEMGREP_SCAN_TIMEOUT_MS = 120000;
|
|
234
|
-
var SEMGREP_TASK_TIMEOUT_MS = SEMGREP_VERSION_TIMEOUT_MS + SEMGREP_SCAN_TIMEOUT_MS + 30000;
|
|
235
|
-
var SecretScanProfileSchema = strictObject({
|
|
236
|
-
version: literal("8.30.1")
|
|
237
|
-
});
|
|
238
|
-
var DependencyScanProfileSchema = strictObject({
|
|
239
|
-
version: literal("2.6.0"),
|
|
240
|
-
paths: array(RepositoryPathSchema).min(1).max(64).refine((paths) => new Set(paths).size === paths.length),
|
|
241
|
-
database: strictObject({
|
|
242
|
-
revision: string().min(1).max(128),
|
|
243
|
-
capturedAt: exports_iso.datetime(),
|
|
244
|
-
ecosystems: array(strictObject({
|
|
245
|
-
name: EcosystemSchema,
|
|
246
|
-
sha256: string().regex(/^[a-f0-9]{64}$/)
|
|
247
|
-
})).min(1).max(EcosystemSchema.options.length).refine((items) => new Set(items.map((item) => item.name)).size === items.length)
|
|
248
|
-
})
|
|
249
|
-
});
|
|
250
|
-
var SCANNER_FINDINGS_MAX = 40;
|
|
251
|
-
var SOURCE_SCANNER_TIMEOUT_MS = 120000;
|
|
252
|
-
var SOURCE_SCANNER_VERSION_TIMEOUT_MS = 1e4;
|
|
253
|
-
var SOURCE_SCANNER_TASK_TIMEOUT_MS = 150000;
|
|
254
|
-
var ScannerStatusSchema = _enum([
|
|
255
|
-
"completed",
|
|
256
|
-
"failed",
|
|
257
|
-
"timed_out",
|
|
258
|
-
"output_limit",
|
|
259
|
-
"unavailable",
|
|
260
|
-
"version_mismatch",
|
|
261
|
-
"database_mismatch"
|
|
262
|
-
]);
|
|
263
|
-
|
|
264
202
|
// apps/harness/src/sandbox/limits.ts
|
|
265
203
|
var TASK_TIMEOUT_MS = 30000;
|
|
266
204
|
var TASK_INPUT_LIMIT = 1048576;
|
|
205
|
+
var TASK_INPUT_LIMITS = {
|
|
206
|
+
semgrep_scan: SOURCE_SCANNER_INPUT_LIMIT,
|
|
207
|
+
secret_scan: SOURCE_SCANNER_INPUT_LIMIT,
|
|
208
|
+
dependency_scan: SOURCE_SCANNER_INPUT_LIMIT,
|
|
209
|
+
joern_confirm: SOURCE_SCANNER_INPUT_LIMIT
|
|
210
|
+
};
|
|
211
|
+
var TASK_INPUT_MAX = Math.max(TASK_INPUT_LIMIT, ...Object.values(TASK_INPUT_LIMITS));
|
|
267
212
|
var TASK_RESULT_LIMIT = 32768;
|
|
213
|
+
var PDF_BYTES_LIMIT = 524288;
|
|
214
|
+
var PDF_RESULT_LIMIT = 1048576;
|
|
268
215
|
var BROKER_RESPONSE_LIMIT = 4 * Math.ceil(WEBSITE_RESPONSE_LIMIT / 3) + 12 * HTTP_HEADER_LIMIT + 1024;
|
|
269
216
|
var CHILD_OUTPUT_KEEP = 4096;
|
|
270
|
-
var TASK_CONTAINER_LIFETIME_S = Math.ceil(SEMGREP_TASK_TIMEOUT_MS / 1000) + 15;
|
|
217
|
+
var TASK_CONTAINER_LIFETIME_S = Math.ceil(Math.max(SEMGREP_TASK_TIMEOUT_MS, JOERN_TASK_TIMEOUT_MS) / 1000) + 15;
|
|
271
218
|
var SANDBOX_UID = 65534;
|
|
272
219
|
var REPO_SEARCH_FILES = 100;
|
|
273
220
|
var REPO_SEARCH_MATCHES = 20;
|
|
@@ -306,7 +253,7 @@ class DockerExchange {
|
|
|
306
253
|
stream: true
|
|
307
254
|
});
|
|
308
255
|
this.framed = this.framed.subarray(length + 8);
|
|
309
|
-
if (Buffer.byteLength(this.lines) >
|
|
256
|
+
if (Buffer.byteLength(this.lines) > 524288)
|
|
310
257
|
throw new Error;
|
|
311
258
|
while (this.lines.includes(`
|
|
312
259
|
`)) {
|
|
@@ -354,8 +301,8 @@ class DockerEngine {
|
|
|
354
301
|
constructor(socket) {
|
|
355
302
|
this.socket = socket;
|
|
356
303
|
}
|
|
357
|
-
startExec(id, input, signal, timeoutMs, exchange) {
|
|
358
|
-
if (!/^[a-f0-9]{64}$/.test(id) || input.length >
|
|
304
|
+
startExec(id, input, signal, timeoutMs, exchange, outputLimit = 65536) {
|
|
305
|
+
if (!/^[a-f0-9]{64}$/.test(id) || input.length > TASK_INPUT_MAX || !Number.isSafeInteger(outputLimit) || outputLimit < 65536 || outputLimit > 1064960)
|
|
359
306
|
throw new HarnessError("sandbox_input_limit", "Invalid isolated exec input.");
|
|
360
307
|
return new Promise((resolve, reject) => {
|
|
361
308
|
let complete;
|
|
@@ -430,7 +377,7 @@ class DockerEngine {
|
|
|
430
377
|
return;
|
|
431
378
|
}
|
|
432
379
|
size += chunk.length;
|
|
433
|
-
if (size >
|
|
380
|
+
if (size > outputLimit)
|
|
434
381
|
fail("sandbox_output_limit");
|
|
435
382
|
else
|
|
436
383
|
chunks.push(chunk);
|
|
@@ -590,6 +537,13 @@ var RESOURCE_PROFILES = {
|
|
|
590
537
|
NanoCpus: 2000000000,
|
|
591
538
|
PidsLimit: 512,
|
|
592
539
|
Tmpfs: { "/tmp": "rw,noexec,nosuid,nodev,size=512m" }
|
|
540
|
+
},
|
|
541
|
+
analysis: {
|
|
542
|
+
Memory: 4294967296,
|
|
543
|
+
MemorySwap: 4294967296,
|
|
544
|
+
NanoCpus: 2000000000,
|
|
545
|
+
PidsLimit: 512,
|
|
546
|
+
Tmpfs: { "/tmp": "rw,noexec,nosuid,nodev,size=1024m" }
|
|
593
547
|
}
|
|
594
548
|
};
|
|
595
549
|
var ShellCommandSchema = strictObject({
|
|
@@ -632,10 +586,12 @@ function sandboxConfig(environment = process.env) {
|
|
|
632
586
|
const parsed = SandboxConfigSchema.safeParse({
|
|
633
587
|
socket: environment.HARNESS_SANDBOX_SOCKET ?? "/var/run/docker.sock",
|
|
634
588
|
image: environment.HARNESS_SANDBOX_IMAGE,
|
|
635
|
-
...environment.HARNESS_SANDBOX_RUNTIME !== undefined ? { runtime: environment.HARNESS_SANDBOX_RUNTIME } : {}
|
|
589
|
+
...environment.HARNESS_SANDBOX_RUNTIME !== undefined ? { runtime: environment.HARNESS_SANDBOX_RUNTIME } : {},
|
|
590
|
+
...environment.HARNESS_SANDBOX_ENGINE_ID !== undefined ? { engineId: environment.HARNESS_SANDBOX_ENGINE_ID } : {},
|
|
591
|
+
...environment.HARNESS_SANDBOX_LOCATION !== undefined ? { location: environment.HARNESS_SANDBOX_LOCATION } : {}
|
|
636
592
|
});
|
|
637
593
|
if (!parsed.success)
|
|
638
|
-
throw new HarnessError("sandbox_config", "Use an approved repository@sha256 image, a local Unix socket, and HARNESS_SANDBOX_RUNTIME=mettle-runsc or runc.");
|
|
594
|
+
throw new HarnessError("sandbox_config", "Use an approved repository@sha256 image or full local sha256 image ID, a local Unix socket, and HARNESS_SANDBOX_RUNTIME=mettle-runsc or runc.");
|
|
639
595
|
return Object.freeze(parsed.data);
|
|
640
596
|
}
|
|
641
597
|
function launchSpec(image, script, labels, profile, runtime) {
|
|
@@ -725,6 +681,7 @@ var ACTIVE_SCAN_TOOLS = [
|
|
|
725
681
|
];
|
|
726
682
|
var common = {
|
|
727
683
|
id: IdSchema,
|
|
684
|
+
identity: IdSchema.optional(),
|
|
728
685
|
url: url().max(2048),
|
|
729
686
|
maxRequests: number().int().min(1).max(128),
|
|
730
687
|
requestsPerSecond: number().int().min(1).max(5),
|
|
@@ -750,7 +707,15 @@ var ActiveScanProfileSchema = discriminatedUnion("tool", [
|
|
|
750
707
|
tool: literal("sqli_confirm"),
|
|
751
708
|
version: literal("1.10"),
|
|
752
709
|
parameter: string().regex(/^[a-zA-Z0-9_]{1,64}$/),
|
|
753
|
-
technique: _enum(["B", "E"])
|
|
710
|
+
technique: _enum(["B", "E"]),
|
|
711
|
+
request: strictObject({
|
|
712
|
+
method: _enum(["POST", "PUT", "PATCH"]),
|
|
713
|
+
contentType: _enum([
|
|
714
|
+
"application/json",
|
|
715
|
+
"application/x-www-form-urlencoded"
|
|
716
|
+
]),
|
|
717
|
+
body: string().min(1).max(4096)
|
|
718
|
+
}).optional()
|
|
754
719
|
}),
|
|
755
720
|
strictObject({
|
|
756
721
|
...common,
|
|
@@ -777,6 +742,18 @@ var ActiveScannersSchema = strictObject({
|
|
|
777
742
|
})).min(1).max(4),
|
|
778
743
|
profiles: array(ActiveScanProfileSchema).min(1).max(8)
|
|
779
744
|
});
|
|
745
|
+
function bodyParameters(body, contentType) {
|
|
746
|
+
if (contentType === "application/x-www-form-urlencoded") {
|
|
747
|
+
const entries = [...new URLSearchParams(body)];
|
|
748
|
+
return entries.length <= 32 && entries.every(([key]) => /^[a-zA-Z0-9_]{1,64}$/.test(key)) ? entries : undefined;
|
|
749
|
+
}
|
|
750
|
+
try {
|
|
751
|
+
const value = record(string().regex(/^[a-zA-Z0-9_]{1,64}$/), union([string(), number().finite(), boolean(), _null()])).parse(JSON.parse(body));
|
|
752
|
+
return Object.keys(value).length <= 32 && JSON.stringify(value) === body ? Object.entries(value) : undefined;
|
|
753
|
+
} catch {
|
|
754
|
+
return;
|
|
755
|
+
}
|
|
756
|
+
}
|
|
780
757
|
function scannerEndpoint(config, target, proposed) {
|
|
781
758
|
const url2 = new URL(proposed);
|
|
782
759
|
const scope = new URL(target);
|
|
@@ -798,8 +775,13 @@ function validateActiveScanners(config, target, buildIdentityRef) {
|
|
|
798
775
|
if (profile.tool === "nuclei_scan")
|
|
799
776
|
for (const path of profile.template.paths)
|
|
800
777
|
scannerEndpoint(config, target, new URL(path, profile.url).href);
|
|
801
|
-
if (profile.tool === "sqli_confirm"
|
|
802
|
-
|
|
778
|
+
if (profile.tool === "sqli_confirm") {
|
|
779
|
+
const query = new URL(profile.url).searchParams.getAll(profile.parameter);
|
|
780
|
+
const body = profile.request ? bodyParameters(profile.request.body, profile.request.contentType) : undefined;
|
|
781
|
+
const selected = body?.filter(([key]) => key === profile.parameter);
|
|
782
|
+
if (profile.request ? !body || selected?.length !== 1 || typeof selected[0]?.[1] !== "string" || query.length !== 0 : query.length !== 1)
|
|
783
|
+
throw new HarnessError("scanner_denied", "SQLMap requires one exact query or body parameter; JSON bodies must be canonical flat objects with a string test value.");
|
|
784
|
+
}
|
|
803
785
|
}
|
|
804
786
|
}
|
|
805
787
|
var SCANNER_REQUEST_STRIP_HEADERS = [
|
|
@@ -813,12 +795,20 @@ var SCANNER_REQUEST_STRIP_HEADERS = [
|
|
|
813
795
|
"keep-alive",
|
|
814
796
|
"accept-encoding"
|
|
815
797
|
];
|
|
816
|
-
function scannerRequestAllowed(profile, url2, method) {
|
|
798
|
+
function scannerRequestAllowed(profile, url2, method, request) {
|
|
817
799
|
const approved = new URL(profile.url);
|
|
818
800
|
if (url2.origin !== approved.origin)
|
|
819
801
|
return false;
|
|
820
802
|
if (profile.tool === "zap_scan")
|
|
821
803
|
return true;
|
|
804
|
+
if (profile.tool === "sqli_confirm" && profile.request) {
|
|
805
|
+
if (url2.href !== profile.url || method !== profile.request.method || request?.body === undefined || request.headers["content-type"] !== profile.request.contentType)
|
|
806
|
+
return false;
|
|
807
|
+
const actual = bodyParameters(request.body, profile.request.contentType);
|
|
808
|
+
const original = bodyParameters(profile.request.body, profile.request.contentType);
|
|
809
|
+
const fixed = (entries) => entries.filter(([key]) => key !== profile.parameter);
|
|
810
|
+
return Boolean(actual && original && actual.filter(([key]) => key === profile.parameter).length === 1 && digest(fixed(actual)) === digest(fixed(original)));
|
|
811
|
+
}
|
|
822
812
|
if (method !== "GET" && method !== "HEAD")
|
|
823
813
|
return false;
|
|
824
814
|
if (profile.tool === "nuclei_scan")
|
|
@@ -829,7 +819,38 @@ function scannerRequestAllowed(profile, url2, method) {
|
|
|
829
819
|
return JSON.stringify(fixed(url2)) === JSON.stringify(fixed(approved));
|
|
830
820
|
}
|
|
831
821
|
function sqlmapCandidate(finding, profile) {
|
|
832
|
-
return finding.status === "candidate" && finding.verification.result === "pending" && finding.cwe === "CWE-89" && finding.subject.kind === "runtime" && ("endpoint" in finding && finding.endpoint ? finding.endpoint.url === profile.url && finding.endpoint.method === "GET" && finding.endpoint.parameter === profile.parameter : finding.location === profile.url);
|
|
822
|
+
return finding.status === "candidate" && finding.verification.result === "pending" && finding.cwe === "CWE-89" && finding.subject.kind === "runtime" && ("endpoint" in finding && finding.endpoint ? finding.endpoint.url === profile.url && finding.endpoint.method === (profile.request?.method ?? "GET") && finding.endpoint.parameter === profile.parameter : !profile.request && finding.location === profile.url);
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
// apps/harness/src/storage/port-scan.ts
|
|
826
|
+
var PORT_CONNECT_TIMEOUT_MS = 2000;
|
|
827
|
+
var PORT_SCAN_TIMEOUT_MS = 120000;
|
|
828
|
+
var PORT_SCAN_TASK_TIMEOUT_MS = 150000;
|
|
829
|
+
var PortScanProfileSchema = strictObject({
|
|
830
|
+
version: literal("2.6.1"),
|
|
831
|
+
approvedBy: string().min(1).max(256),
|
|
832
|
+
expiresAt: exports_iso.datetime(),
|
|
833
|
+
host: string().min(1).max(253),
|
|
834
|
+
address: PublicAddressSchema,
|
|
835
|
+
ports: array(number().int().min(1).max(65535)).min(1).max(32).refine((ports) => new Set(ports).size === ports.length),
|
|
836
|
+
connectionsPerSecond: number().int().min(1).max(5)
|
|
837
|
+
});
|
|
838
|
+
var PortRequestSchema = strictObject({
|
|
839
|
+
address: PublicAddressSchema,
|
|
840
|
+
port: number().int().min(1).max(65535)
|
|
841
|
+
});
|
|
842
|
+
var PortOutcomeSchema = _enum([
|
|
843
|
+
"open",
|
|
844
|
+
"closed",
|
|
845
|
+
"timed_out",
|
|
846
|
+
"unreachable",
|
|
847
|
+
"cancelled"
|
|
848
|
+
]);
|
|
849
|
+
function portDestination(profile, target, input) {
|
|
850
|
+
const request = PortRequestSchema.parse(input);
|
|
851
|
+
if (new URL(target).hostname !== profile.host || request.address !== profile.address || !profile.ports.includes(request.port))
|
|
852
|
+
throw new HarnessError("scanner_denied", "TCP discovery requires the exact approved host, IPv4 address and port.");
|
|
853
|
+
return request;
|
|
833
854
|
}
|
|
834
855
|
|
|
835
856
|
// apps/harness/src/sandbox/active-program.ts
|
|
@@ -925,7 +946,7 @@ async function activeScan(tool, input, send) {
|
|
|
925
946
|
const version = await run("/usr/bin/python3", ["/opt/mettle/sqlmap/sqlmap.py", "--version"], 10000);
|
|
926
947
|
if (version.code !== 0 || version.stdout.split("\n")[0].trim() !== profile.version + "#stable") throw new Error("version_mismatch");
|
|
927
948
|
await fs.writeFile(root + "/targets.txt", profile.url + "\n");
|
|
928
|
-
const scanned = await run("/usr/bin/python3", ["/opt/mettle/sqlmap/sqlmap.py", "-m", root + "/targets.txt", "-p", profile.parameter, "--technique=" + profile.technique, "--batch", "--skip-waf", "--level=1", "--risk=1", "--threads=1", "--timeout=5", "--retries=0", "--delay=" + String(1 / profile.requestsPerSecond), "--proxy=" + proxyUrl, "--ignore-redirects", "--disable-coloring", "--flush-session", "--output-dir=" + root + "/sqlmap", "--results-file=" + root + "/sqlmap.csv", "-v", "0"]);
|
|
949
|
+
const scanned = await run("/usr/bin/python3", ["/opt/mettle/sqlmap/sqlmap.py", "-m", root + "/targets.txt", "-p", profile.parameter, ...(profile.request ? ["--method=" + profile.request.method, "--data=" + profile.request.body, "--headers=Content-Type: " + profile.request.contentType] : []), "--technique=" + profile.technique, "--batch", "--skip-waf", "--level=1", "--risk=1", "--threads=1", "--timeout=5", "--retries=0", "--delay=" + String(1 / profile.requestsPerSecond), "--proxy=" + proxyUrl, "--ignore-redirects", "--disable-coloring", "--flush-session", "--output-dir=" + root + "/sqlmap", "--results-file=" + root + "/sqlmap.csv", "-v", "0"]);
|
|
929
950
|
const errors = (scanned.stdout + scanned.stderr).split("\n").filter((line) => /\[(?:CRITICAL|ERROR)\]/.test(line));
|
|
930
951
|
if (scanned.code !== 0 || errors.some((line) => !/\[(?:CRITICAL|ERROR)\] all tested parameters do not appear to be injectable\./.test(line))) throw new Error("scan_failed");
|
|
931
952
|
const parsed = await run("/usr/bin/python3", ["-c", "import csv,json,sys; print(json.dumps(list(csv.DictReader(open(sys.argv[1],newline='')))))", root + "/sqlmap.csv"], 10000);
|
|
@@ -933,7 +954,8 @@ async function activeScan(tool, input, send) {
|
|
|
933
954
|
findings = JSON.parse(parsed.stdout).flatMap((item) => {
|
|
934
955
|
if (item['Target URL'] !== profile.url) throw new Error();
|
|
935
956
|
if (!item['Technique(s)'] || item['Note(s)']) return [];
|
|
936
|
-
|
|
957
|
+
const json = profile.request?.contentType === "application/json";
|
|
958
|
+
if (item.Place !== (profile.request ? json ? "(custom) POST" : "POST" : "GET") || item.Parameter !== (json ? "JSON " : "") + profile.parameter) throw new Error();
|
|
937
959
|
return [{ ruleId: "sqlmap-" + profile.technique, url: profile.url, title: "SQL injection observation", parameter: profile.parameter }];
|
|
938
960
|
});
|
|
939
961
|
} else if (tool === "zap_scan") {
|
|
@@ -970,8 +992,375 @@ async function activeScan(tool, input, send) {
|
|
|
970
992
|
}
|
|
971
993
|
`;
|
|
972
994
|
|
|
995
|
+
// apps/harness/src/sandbox/browser-page-program.ts
|
|
996
|
+
var daemon = String.raw`
|
|
997
|
+
const crypto = require("node:crypto"), net = require("node:net"), readline = require("node:readline");
|
|
998
|
+
const retained = {};
|
|
999
|
+
let busy = false;
|
|
1000
|
+
const server = net.createServer(async (socket) => {
|
|
1001
|
+
if (busy) { socket.destroy(); return; }
|
|
1002
|
+
busy = true;
|
|
1003
|
+
const lines = readline.createInterface({ input: socket, crlfDelay: Infinity });
|
|
1004
|
+
const iterator = lines[Symbol.asyncIterator]();
|
|
1005
|
+
let disconnected = false;
|
|
1006
|
+
socket.on("close", () => { disconnected = true; });
|
|
1007
|
+
socket.on("error", () => {});
|
|
1008
|
+
const next = async () => {
|
|
1009
|
+
const line = await iterator.next();
|
|
1010
|
+
if (line.done || Buffer.byteLength(line.value) > ${BROKER_RESPONSE_LIMIT}) throw new Error("browser_page_protocol");
|
|
1011
|
+
return JSON.parse(line.value);
|
|
1012
|
+
};
|
|
1013
|
+
let queue = Promise.resolve();
|
|
1014
|
+
const send = (request) => {
|
|
1015
|
+
const result = queue.then(async () => {
|
|
1016
|
+
if (disconnected) throw new Error("browser_page_disconnected");
|
|
1017
|
+
socket.write(JSON.stringify({ request }) + "\n");
|
|
1018
|
+
const response = await next();
|
|
1019
|
+
if (!Object.hasOwn(response, "response")) throw new Error("browser_page_refused");
|
|
1020
|
+
return response.response;
|
|
1021
|
+
});
|
|
1022
|
+
queue = result;
|
|
1023
|
+
return result;
|
|
1024
|
+
};
|
|
1025
|
+
try {
|
|
1026
|
+
const { input } = await next();
|
|
1027
|
+
if ((input.page.operation === "open") === Boolean(retained.page)) throw new Error("browser_page_lost");
|
|
1028
|
+
const value = await websiteBrowse(input, send, retained);
|
|
1029
|
+
await queue;
|
|
1030
|
+
socket.end(JSON.stringify({ value }) + "\n");
|
|
1031
|
+
if (input.page.operation === "close") server.close();
|
|
1032
|
+
} catch {
|
|
1033
|
+
if (retained.browser) await retained.browser.close().catch(() => {});
|
|
1034
|
+
socket.end(JSON.stringify({ error: "browser_page_failed" }) + "\n");
|
|
1035
|
+
server.close();
|
|
1036
|
+
} finally { lines.close(); busy = false; }
|
|
1037
|
+
});
|
|
1038
|
+
server.listen("/tmp/mettle-browser-page.sock", () => { process.send({ ready: true }); process.disconnect(); });
|
|
1039
|
+
server.on("error", () => process.exit(1));
|
|
1040
|
+
`;
|
|
1041
|
+
var BROWSER_PAGE_PROGRAM = String.raw`
|
|
1042
|
+
async function browseRetainedPage(input, send) {
|
|
1043
|
+
if (input.page.operation === "open") {
|
|
1044
|
+
const definitions = [websiteScope, restoreBrowserPrivateState, captureBrowserPrivateState, websiteBrowse].map((fn) => fn.toString()).join("\n");
|
|
1045
|
+
const child = require("node:child_process").spawn(process.execPath, ["-e", definitions + "\n" + ${JSON.stringify(daemon)}], { detached: true, stdio: ["ignore", "ignore", "ignore", "ipc"] });
|
|
1046
|
+
await new Promise((resolve, reject) => {
|
|
1047
|
+
const timer = setTimeout(() => { child.kill(); reject(new Error("browser_page_start")); }, 10000);
|
|
1048
|
+
child.once("error", (error) => { clearTimeout(timer); reject(error); });
|
|
1049
|
+
child.once("exit", () => { clearTimeout(timer); reject(new Error("browser_page_start")); });
|
|
1050
|
+
child.once("message", (message) => { clearTimeout(timer); message.ready === true ? resolve() : reject(new Error("browser_page_start")); });
|
|
1051
|
+
});
|
|
1052
|
+
child.unref();
|
|
1053
|
+
}
|
|
1054
|
+
const socket = require("node:net").connect("/tmp/mettle-browser-page.sock");
|
|
1055
|
+
await new Promise((resolve, reject) => { socket.once("connect", resolve); socket.once("error", reject); });
|
|
1056
|
+
const lines = readline.createInterface({ input: socket, crlfDelay: Infinity });
|
|
1057
|
+
const encoded = JSON.stringify({ input });
|
|
1058
|
+
if (Buffer.byteLength(encoded) > ${TASK_INPUT_LIMIT}) throw new Error("browser_page_input");
|
|
1059
|
+
socket.write(encoded + "\n");
|
|
1060
|
+
try {
|
|
1061
|
+
for await (const line of lines) {
|
|
1062
|
+
if (Buffer.byteLength(line) > ${BROKER_RESPONSE_LIMIT}) throw new Error("browser_page_output");
|
|
1063
|
+
const message = JSON.parse(line);
|
|
1064
|
+
if (Object.keys(message).length !== 1) throw new Error("browser_page_protocol");
|
|
1065
|
+
if (Object.hasOwn(message, "request")) socket.write(JSON.stringify({ response: await send(message.request) }) + "\n");
|
|
1066
|
+
else if (Object.hasOwn(message, "value")) return message.value;
|
|
1067
|
+
else throw new Error("browser_page_failed");
|
|
1068
|
+
}
|
|
1069
|
+
throw new Error("browser_page_lost");
|
|
1070
|
+
} finally { lines.close(); socket.destroy(); }
|
|
1071
|
+
}
|
|
1072
|
+
`;
|
|
1073
|
+
|
|
1074
|
+
// apps/harness/src/sandbox/port-program.ts
|
|
1075
|
+
var PORT_PROGRAM = String.raw`
|
|
1076
|
+
async function portScan(input, send) {
|
|
1077
|
+
const fs = require("node:fs/promises");
|
|
1078
|
+
const net = require("node:net");
|
|
1079
|
+
const { spawn } = require("node:child_process");
|
|
1080
|
+
const root = await fs.mkdtemp("/tmp/mettle-ports-");
|
|
1081
|
+
const profile = input.profile;
|
|
1082
|
+
const sockets = new Set();
|
|
1083
|
+
const children = new Set();
|
|
1084
|
+
const jobs = new Set();
|
|
1085
|
+
let failure;
|
|
1086
|
+
const run = (args, timeout) => new Promise((resolve, reject) => {
|
|
1087
|
+
const child = spawn("/usr/local/bin/naabu", args, { cwd: root, env: { PATH: "/usr/local/bin:/usr/bin:/bin", HOME: root, TMPDIR: root }, shell: false, stdio: ["ignore", "pipe", "pipe"] });
|
|
1088
|
+
children.add(child);
|
|
1089
|
+
let stdout = "", stderr = "", size = 0, error;
|
|
1090
|
+
const timer = setTimeout(() => { error = new Error("timed_out"); child.kill("SIGKILL"); }, timeout);
|
|
1091
|
+
for (const [stream, field] of [[child.stdout, "stdout"], [child.stderr, "stderr"]]) stream.on("data", (chunk) => {
|
|
1092
|
+
size += chunk.length;
|
|
1093
|
+
if (size > 65536) { error = new Error("output_limit"); child.kill("SIGKILL"); return; }
|
|
1094
|
+
if (field === "stdout") stdout += chunk.toString(); else stderr += chunk.toString();
|
|
1095
|
+
});
|
|
1096
|
+
child.once("error", (cause) => { error = cause; });
|
|
1097
|
+
child.once("close", (code) => { clearTimeout(timer); children.delete(child); if (error) reject(error); else resolve({ code, stdout, stderr }); });
|
|
1098
|
+
});
|
|
1099
|
+
const proxy = net.createServer((socket) => {
|
|
1100
|
+
let bytes = Buffer.alloc(0), phase = "greeting";
|
|
1101
|
+
sockets.add(socket);
|
|
1102
|
+
socket.on("close", () => sockets.delete(socket));
|
|
1103
|
+
socket.on("error", (error) => { if (phase !== "done" || error.code !== "ECONNRESET") failure ??= error; });
|
|
1104
|
+
socket.setTimeout(10000, () => { failure ??= new Error("proxy_timeout"); socket.destroy(); });
|
|
1105
|
+
socket.on("data", (chunk) => {
|
|
1106
|
+
try {
|
|
1107
|
+
if (phase === "connecting" || phase === "done") throw new Error("payload_denied");
|
|
1108
|
+
bytes = Buffer.concat([bytes, chunk]);
|
|
1109
|
+
if (bytes.length > 512) throw new Error("proxy_input_limit");
|
|
1110
|
+
if (phase === "greeting") {
|
|
1111
|
+
if (bytes.length < 2) return;
|
|
1112
|
+
if (bytes[0] !== 5 || bytes[1] !== 1) throw new Error("auth_denied");
|
|
1113
|
+
if (bytes.length < 3) return;
|
|
1114
|
+
if (bytes[2] !== 0) throw new Error("auth_denied");
|
|
1115
|
+
bytes = bytes.subarray(3);
|
|
1116
|
+
phase = "request";
|
|
1117
|
+
socket.write(Buffer.from([5, 0]));
|
|
1118
|
+
}
|
|
1119
|
+
if (phase !== "request" || bytes.length < 4) return;
|
|
1120
|
+
if (bytes[0] !== 5 || bytes[1] !== 1 || bytes[2] !== 0 || bytes[3] !== 1) throw new Error("connect_only_ipv4");
|
|
1121
|
+
if (bytes.length < 10) return;
|
|
1122
|
+
if (bytes.length !== 10) throw new Error("payload_denied");
|
|
1123
|
+
const address = [...bytes.subarray(4, 8)].join(".");
|
|
1124
|
+
const port = bytes.readUInt16BE(8);
|
|
1125
|
+
if (address !== profile.address || !profile.ports.includes(port)) throw new Error("scope_denied");
|
|
1126
|
+
phase = "connecting";
|
|
1127
|
+
const job = send({ address, port }).then((response) => {
|
|
1128
|
+
phase = "done";
|
|
1129
|
+
const code = response.outcome === "open" ? 0 : response.outcome === "closed" ? 5 : 4;
|
|
1130
|
+
socket.end(Buffer.from([5, code, 0, 1, 0, 0, 0, 0, 0, 0]));
|
|
1131
|
+
}).catch((error) => { failure ??= error; socket.destroy(); });
|
|
1132
|
+
jobs.add(job);
|
|
1133
|
+
void job.finally(() => jobs.delete(job));
|
|
1134
|
+
} catch (error) { failure ??= error; socket.destroy(); }
|
|
1135
|
+
});
|
|
1136
|
+
});
|
|
1137
|
+
try {
|
|
1138
|
+
const version = await run(["-version"], 10000);
|
|
1139
|
+
if (version.code !== 0 || !(version.stdout + version.stderr).split(/\s+/).includes(profile.version)) throw new Error("version_mismatch");
|
|
1140
|
+
await fs.writeFile(root + "/config.yaml", "{}\n", { flag: "wx", mode: 0o600 });
|
|
1141
|
+
await new Promise((resolve, reject) => { proxy.once("error", reject); proxy.listen(0, "127.0.0.1", resolve); });
|
|
1142
|
+
const result = await run(["-host", profile.address, "-p", profile.ports.join(","), "-s", "c", "-iv", "4", "-Pn", "-proxy", "127.0.0.1:" + proxy.address().port, "-config", root + "/config.yaml", "-no-stdin", "-disable-update-check", "-c", "1", "-rate", String(profile.connectionsPerSecond), "-retries", "1", "-timeout", "5000", "-warm-up-time", "0", "-silent", "-json"], ${PORT_SCAN_TIMEOUT_MS});
|
|
1143
|
+
await Promise.all(jobs);
|
|
1144
|
+
if (failure) throw failure;
|
|
1145
|
+
if (result.code !== 0 || /\[(?:ERR|FTL)\]/.test(result.stderr)) throw new Error("scan_failed");
|
|
1146
|
+
const openPorts = result.stdout.split("\n").filter(Boolean).map((line) => {
|
|
1147
|
+
const entry = JSON.parse(line);
|
|
1148
|
+
if (entry.ip !== profile.address || entry.protocol !== "tcp" || !profile.ports.includes(entry.port)) throw new Error("scan_output");
|
|
1149
|
+
return entry.port;
|
|
1150
|
+
});
|
|
1151
|
+
return { status: "completed", version: profile.version, openPorts: [...new Set(openPorts)].sort((a, b) => a - b) };
|
|
1152
|
+
} finally {
|
|
1153
|
+
for (const child of children) child.kill("SIGKILL");
|
|
1154
|
+
for (const socket of sockets) socket.destroy();
|
|
1155
|
+
await new Promise((resolve) => proxy.close(resolve));
|
|
1156
|
+
await fs.rm(root, { recursive: true, force: true });
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
`;
|
|
1160
|
+
|
|
1161
|
+
// apps/harness/src/sandbox/browser-state-program.ts
|
|
1162
|
+
var BROWSER_STATE_PROGRAM = `
|
|
1163
|
+
async function restoreBrowserPrivateState(page, context, state, url) {
|
|
1164
|
+
const origin = new URL(url).origin;
|
|
1165
|
+
const saved = state.origins.find((entry) => entry.origin === origin);
|
|
1166
|
+
if (!saved || (!saved.sessionStorage?.length && !saved.opfs?.length)) return;
|
|
1167
|
+
const bootstrap = new URL(url);
|
|
1168
|
+
bootstrap.searchParams.set("__mettle_restore", crypto.randomBytes(16).toString("hex"));
|
|
1169
|
+
const route = async (request) => request.fulfill({ status: 200, contentType: "text/html", body: "<!doctype html><title>Restore private state</title>" });
|
|
1170
|
+
await page.route(bootstrap.href, route);
|
|
1171
|
+
try {
|
|
1172
|
+
await page.goto(bootstrap.href, { waitUntil: "domcontentloaded" });
|
|
1173
|
+
await page.evaluate(async (saved) => {
|
|
1174
|
+
sessionStorage.clear();
|
|
1175
|
+
for (const { name, value } of saved.sessionStorage || []) sessionStorage.setItem(name, value);
|
|
1176
|
+
if (!saved.opfs?.length) return;
|
|
1177
|
+
const root = await navigator.storage.getDirectory();
|
|
1178
|
+
for (const entry of saved.opfs) {
|
|
1179
|
+
const parts = entry.path.split("/"), name = parts.pop();
|
|
1180
|
+
let parent = root;
|
|
1181
|
+
for (const part of parts) parent = await parent.getDirectoryHandle(part, { create: true });
|
|
1182
|
+
if (entry.kind === "directory") await parent.getDirectoryHandle(name, { create: true });
|
|
1183
|
+
else {
|
|
1184
|
+
const handle = await parent.getFileHandle(name, { create: true });
|
|
1185
|
+
const writer = await handle.createWritable();
|
|
1186
|
+
await writer.write(Uint8Array.from(atob(entry.base64), (char) => char.charCodeAt(0)));
|
|
1187
|
+
await writer.close();
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
}, saved);
|
|
1191
|
+
} finally { await page.unroute(bootstrap.href, route); }
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
async function captureBrowserPrivateState(page, context, target) {
|
|
1195
|
+
if (new URL(page.url()).origin !== new URL(target).origin) throw new Error("browser_state_origin");
|
|
1196
|
+
const extras = await page.evaluate(async () => {
|
|
1197
|
+
const session = Object.entries(sessionStorage).map(([name, value]) => ({ name, value }));
|
|
1198
|
+
if (session.length > 64 || session.some((entry) => entry.name.length > 256 || entry.value.length > 16384)) throw new Error("browser_state_limit");
|
|
1199
|
+
const opfs = [];
|
|
1200
|
+
let bytes = 0;
|
|
1201
|
+
const visit = async (directory, prefix, depth) => {
|
|
1202
|
+
if (depth > 8) throw new Error("browser_state_limit");
|
|
1203
|
+
for await (const [name, handle] of directory.entries()) {
|
|
1204
|
+
if (opfs.length >= 64) throw new Error("browser_state_limit");
|
|
1205
|
+
const path = prefix + name;
|
|
1206
|
+
if (handle.kind === "directory") {
|
|
1207
|
+
opfs.push({ kind: "directory", path });
|
|
1208
|
+
await visit(handle, path + "/", depth + 1);
|
|
1209
|
+
} else {
|
|
1210
|
+
const file = await handle.getFile();
|
|
1211
|
+
bytes += file.size;
|
|
1212
|
+
if (file.size > 65536 || bytes > 131072) throw new Error("browser_state_limit");
|
|
1213
|
+
const buffer = new Uint8Array(await file.arrayBuffer());
|
|
1214
|
+
let binary = "";
|
|
1215
|
+
for (const byte of buffer) binary += String.fromCharCode(byte);
|
|
1216
|
+
opfs.push({ kind: "file", path, base64: btoa(binary) });
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
};
|
|
1220
|
+
if (navigator.storage?.getDirectory) await visit(await navigator.storage.getDirectory(), "", 1);
|
|
1221
|
+
return { origin: location.origin, sessionStorage: session, opfs };
|
|
1222
|
+
});
|
|
1223
|
+
const state = await context.storageState({ indexedDB: true });
|
|
1224
|
+
let entry = state.origins.find((entry) => entry.origin === extras.origin);
|
|
1225
|
+
if (!entry) { entry = { origin: extras.origin, localStorage: [] }; state.origins.push(entry); }
|
|
1226
|
+
Object.assign(entry, extras);
|
|
1227
|
+
return state;
|
|
1228
|
+
}
|
|
1229
|
+
`;
|
|
1230
|
+
|
|
1231
|
+
// apps/harness/src/sandbox/javascript-program.ts
|
|
1232
|
+
var JAVASCRIPT_PROGRAM = String.raw`
|
|
1233
|
+
const node = (value) => value && typeof value === "object" && ("type" in value) && typeof value.type === "string" ? value : undefined;
|
|
1234
|
+
function literal(value, depth = 0) {
|
|
1235
|
+
const entry = node(value);
|
|
1236
|
+
if (!entry || depth > 8)
|
|
1237
|
+
return;
|
|
1238
|
+
if (entry.type === "StringLiteral")
|
|
1239
|
+
return typeof entry.value === "string" ? entry.value : undefined;
|
|
1240
|
+
if (entry.type === "TemplateLiteral" && Array.isArray(entry.expressions) && !entry.expressions.length) {
|
|
1241
|
+
const quasis = entry.quasis;
|
|
1242
|
+
return quasis[0]?.value.cooked ?? undefined;
|
|
1243
|
+
}
|
|
1244
|
+
if (entry.type === "BinaryExpression" && entry.operator === "+") {
|
|
1245
|
+
const left = literal(entry.left, depth + 1), right = literal(entry.right, depth + 1);
|
|
1246
|
+
return left !== undefined && right !== undefined && left.length + right.length <= 2048 ? left + right : undefined;
|
|
1247
|
+
}
|
|
1248
|
+
return;
|
|
1249
|
+
}
|
|
1250
|
+
const property = (value) => {
|
|
1251
|
+
const entry = node(value);
|
|
1252
|
+
return entry?.type === "Identifier" ? String(entry.name) : literal(entry);
|
|
1253
|
+
};
|
|
1254
|
+
function javascriptDiscovery(source, base, contentType) {
|
|
1255
|
+
const scripts = /text\/html/i.test(contentType) ? [...source.matchAll(/<script\b[^>]*>([\s\S]*?)<\/script\s*>/gi)].slice(0, 32).map((entry) => entry[1]) : /(?:javascript|ecmascript)/i.test(contentType) ? [source] : [];
|
|
1256
|
+
if (!scripts.length)
|
|
1257
|
+
return { leads: [], omitted: 0 };
|
|
1258
|
+
const { parse } = require("/opt/mettle-parser/lib/index.js");
|
|
1259
|
+
const leads = [];
|
|
1260
|
+
let omitted = 0;
|
|
1261
|
+
for (const script of scripts) {
|
|
1262
|
+
if (Buffer.byteLength(script) > 262144) {
|
|
1263
|
+
omitted++;
|
|
1264
|
+
continue;
|
|
1265
|
+
}
|
|
1266
|
+
let ast;
|
|
1267
|
+
try {
|
|
1268
|
+
ast = parse(script, {
|
|
1269
|
+
sourceType: "unambiguous",
|
|
1270
|
+
errorRecovery: false,
|
|
1271
|
+
allowReturnOutsideFunction: true
|
|
1272
|
+
});
|
|
1273
|
+
} catch {
|
|
1274
|
+
omitted++;
|
|
1275
|
+
continue;
|
|
1276
|
+
}
|
|
1277
|
+
const pending = [ast];
|
|
1278
|
+
let visited = 0;
|
|
1279
|
+
while (pending.length && visited++ < 40000) {
|
|
1280
|
+
const current = node(pending.pop());
|
|
1281
|
+
if (!current)
|
|
1282
|
+
continue;
|
|
1283
|
+
for (const [key, value] of Object.entries(current)) {
|
|
1284
|
+
if (["loc", "start", "end", "comments", "tokens", "extra"].includes(key))
|
|
1285
|
+
continue;
|
|
1286
|
+
if (Array.isArray(value))
|
|
1287
|
+
pending.push(...value);
|
|
1288
|
+
else if (node(value))
|
|
1289
|
+
pending.push(value);
|
|
1290
|
+
}
|
|
1291
|
+
if (current.type !== "CallExpression")
|
|
1292
|
+
continue;
|
|
1293
|
+
const callee = node(current.callee);
|
|
1294
|
+
const name = callee?.type === "Identifier" ? String(callee.name) : callee?.type === "MemberExpression" ? property(callee.property) : undefined;
|
|
1295
|
+
const receiver = callee?.type === "MemberExpression" ? property(callee.object) : undefined;
|
|
1296
|
+
const args = Array.isArray(current.arguments) ? current.arguments : [];
|
|
1297
|
+
let method, address;
|
|
1298
|
+
if (name === "fetch" && (!receiver || ["window", "globalThis"].includes(receiver))) {
|
|
1299
|
+
address = literal(args[0]);
|
|
1300
|
+
method = "GET";
|
|
1301
|
+
if (args[1] !== undefined) {
|
|
1302
|
+
const options = node(args[1]);
|
|
1303
|
+
if (options?.type !== "ObjectExpression" || !Array.isArray(options.properties)) {
|
|
1304
|
+
omitted++;
|
|
1305
|
+
continue;
|
|
1306
|
+
}
|
|
1307
|
+
for (const value of options.properties) {
|
|
1308
|
+
const option = node(value);
|
|
1309
|
+
if (option?.type !== "ObjectProperty" || option.computed) {
|
|
1310
|
+
method = undefined;
|
|
1311
|
+
break;
|
|
1312
|
+
}
|
|
1313
|
+
if (property(option.key) === "method")
|
|
1314
|
+
method = literal(option.value)?.toUpperCase();
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
} else if (name === "open" && receiver) {
|
|
1318
|
+
method = literal(args[0])?.toUpperCase();
|
|
1319
|
+
address = literal(args[1]);
|
|
1320
|
+
} else if (receiver === "axios" && name && ["get", "head", "options", "post", "put", "patch", "delete"].includes(name)) {
|
|
1321
|
+
method = name.toUpperCase();
|
|
1322
|
+
address = literal(args[0]);
|
|
1323
|
+
} else
|
|
1324
|
+
continue;
|
|
1325
|
+
if (!address || address.length > 2048 || !method || !["GET", "HEAD", "OPTIONS", "POST", "PUT", "PATCH", "DELETE"].includes(method)) {
|
|
1326
|
+
omitted++;
|
|
1327
|
+
continue;
|
|
1328
|
+
}
|
|
1329
|
+
try {
|
|
1330
|
+
const url = new URL(address, base);
|
|
1331
|
+
url.hash = "";
|
|
1332
|
+
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.href.length > 2048 || leads.length >= 200) {
|
|
1333
|
+
omitted++;
|
|
1334
|
+
continue;
|
|
1335
|
+
}
|
|
1336
|
+
leads.push({
|
|
1337
|
+
url: url.href,
|
|
1338
|
+
method,
|
|
1339
|
+
kind: "javascript",
|
|
1340
|
+
inputs: [],
|
|
1341
|
+
contentType: null,
|
|
1342
|
+
operation: null,
|
|
1343
|
+
authentication: "unknown",
|
|
1344
|
+
stateChanging: !["GET", "HEAD", "OPTIONS"].includes(method),
|
|
1345
|
+
limitations: [
|
|
1346
|
+
"Static syntax-tree request lead; execution, body values, authentication and dynamic bindings remain unproven."
|
|
1347
|
+
]
|
|
1348
|
+
});
|
|
1349
|
+
} catch {
|
|
1350
|
+
omitted++;
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
if (pending.length)
|
|
1354
|
+
omitted++;
|
|
1355
|
+
}
|
|
1356
|
+
return { leads, omitted };
|
|
1357
|
+
}
|
|
1358
|
+
`;
|
|
1359
|
+
|
|
973
1360
|
// apps/harness/src/sandbox/website-program.ts
|
|
974
1361
|
var WEBSITE_PROGRAM = String.raw`
|
|
1362
|
+
${JAVASCRIPT_PROGRAM}
|
|
1363
|
+
${BROWSER_STATE_PROGRAM}
|
|
975
1364
|
function websiteScope(input, raw, method = "GET") {
|
|
976
1365
|
try {
|
|
977
1366
|
const url = new URL(raw);
|
|
@@ -983,59 +1372,93 @@ function websiteScope(input, raw, method = "GET") {
|
|
|
983
1372
|
});
|
|
984
1373
|
} catch { return false; }
|
|
985
1374
|
}
|
|
986
|
-
function
|
|
987
|
-
const
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
1375
|
+
function websiteAttribute(attributes, key) {
|
|
1376
|
+
const match = new RegExp('(?:^|\\s)' + key + '\\s*=\\s*(?:"([^"<>]*)"|\'([^\'<>]*)\'|([^\\s<>]+))', 'i').exec(attributes);
|
|
1377
|
+
return match ? (match[1] ?? match[2] ?? match[3]).replace(/&/g, "&") : undefined;
|
|
1378
|
+
}
|
|
1379
|
+
function websiteLeads(body, base, contentType) {
|
|
1380
|
+
const leads = [];
|
|
1381
|
+
let omitted = 0;
|
|
1382
|
+
const add = (raw, method, kind, fields = [], media = null, limitations = []) => {
|
|
993
1383
|
try {
|
|
994
|
-
const url = new URL(
|
|
995
|
-
if (!["https:", "http:"].includes(url.protocol) || url.href.length > 2048)
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
1384
|
+
const url = new URL(raw, base); url.hash = "";
|
|
1385
|
+
if (!["https:", "http:"].includes(url.protocol) || url.username || url.password || url.href.length > 2048 || leads.length >= 200) { omitted++; return; }
|
|
1386
|
+
leads.push({ url: url.href, method, kind, inputs: fields, contentType: media, operation: null, authentication: "unknown", stateChanging: !["GET", "HEAD", "OPTIONS"].includes(method), limitations });
|
|
1387
|
+
} catch { omitted++; }
|
|
1388
|
+
};
|
|
1389
|
+
if (/text\/html/i.test(contentType)) {
|
|
1390
|
+
for (const match of body.matchAll(/<(a|script)\b([^>]{0,8192})>/gi)) {
|
|
1391
|
+
const tag = match[1].toLowerCase();
|
|
1392
|
+
const value = websiteAttribute(match[2], tag === "script" ? "src" : "href");
|
|
1393
|
+
if (value !== undefined) add(value, "GET", tag === "script" ? "script" : "page");
|
|
1394
|
+
}
|
|
1395
|
+
for (const match of body.matchAll(/<form\b([^>]{0,8192})>([\s\S]{0,32768}?)(?:<\/form\s*>|$)/gi)) {
|
|
1396
|
+
const method = (websiteAttribute(match[1], "method") || "GET").toUpperCase();
|
|
1397
|
+
if (!["GET", "POST"].includes(method)) { omitted++; continue; }
|
|
1398
|
+
const fields = [], names = new Set();
|
|
1399
|
+
let missing = 0;
|
|
1400
|
+
for (const control of match[2].matchAll(/<(?:input|select|textarea|button)\b([^>]{0,8192})>/gi)) {
|
|
1401
|
+
const key = websiteAttribute(control[1], "name");
|
|
1402
|
+
if (!key || /(?:^|\s)disabled(?:\s|=|$)/i.test(control[1]) || names.has(key)) continue;
|
|
1403
|
+
names.add(key);
|
|
1404
|
+
if (fields.length >= 64 || key.length > 256) { missing++; continue; }
|
|
1405
|
+
fields.push({ location: method === "GET" ? "query" : "body", name: key, required: /(?:^|\s)required(?:\s|=|$)/i.test(control[1]), type: "string", identifier: false });
|
|
1406
|
+
}
|
|
1407
|
+
const media = method === "POST" ? (websiteAttribute(match[1], "enctype") || "application/x-www-form-urlencoded").toLowerCase() : null;
|
|
1408
|
+
add(websiteAttribute(match[1], "action") || base, method, "form", fields, media, ["Form controls are leads; values, login state and workflow prerequisites are not synthesized.", ...(missing ? [missing + " form input names omitted by bounds."] : [])]);
|
|
1409
|
+
omitted += missing;
|
|
1410
|
+
}
|
|
999
1411
|
}
|
|
1000
|
-
|
|
1412
|
+
const priority = { page: 0, javascript: 1, script: 2, form: 3 };
|
|
1413
|
+
return { leads: leads.sort((a, b) => priority[a.kind] - priority[b.kind]), omitted };
|
|
1001
1414
|
}
|
|
1415
|
+
function websiteRequestKey(request) { return request.method + " " + request.url; }
|
|
1416
|
+
function websiteLeadKey(lead) { return websiteRequestKey(lead) + " " + (lead.contentType || "") + " " + JSON.stringify((lead.inputs || []).map((input) => input.location + ":" + input.name).sort()); }
|
|
1002
1417
|
async function websiteObserve(tool, input, send) {
|
|
1003
|
-
const initial = { url: input.url || input.target, method: input.method || "GET", ...(input.body !== undefined ? { body: input.body } : {}), ...(input.headers ? { headers: input.headers } : {}) };
|
|
1418
|
+
const initial = { url: input.url || input.target, method: input.method || "GET", ...(input.body !== undefined ? { body: input.body } : {}), ...(input.bodyBase64 !== undefined ? { bodyBase64: input.bodyBase64 } : {}), ...(input.headers ? { headers: input.headers } : {}) };
|
|
1004
1419
|
const queue = [{ request: initial, depth: 0 }];
|
|
1005
1420
|
const seen = new Set(), pages = [], endpoints = new Map(), leads = new Map();
|
|
1006
1421
|
let omitted = 0, redirects = 0;
|
|
1007
1422
|
while (queue.length && pages.length < input.maxPages && seen.size < 100) {
|
|
1008
|
-
const next = queue.shift();
|
|
1009
|
-
if (seen.has(
|
|
1010
|
-
seen.add(
|
|
1423
|
+
const next = queue.shift(), key = websiteRequestKey(next.request);
|
|
1424
|
+
if (seen.has(key)) continue;
|
|
1425
|
+
seen.add(key);
|
|
1011
1426
|
const response = await send(next.request);
|
|
1012
|
-
const
|
|
1427
|
+
const textual = /^(text\/|application\/(.*json|.*xml|javascript|x-javascript))/i.test(response.contentType);
|
|
1428
|
+
const body = textual ? Buffer.from(response.body, "base64").toString("utf8") : "";
|
|
1013
1429
|
if ([301,302,303,307,308].includes(response.status)) {
|
|
1014
1430
|
const location = response.headers && response.headers.location;
|
|
1015
1431
|
if (typeof location === "string" && redirects++ < 5 && ["GET", "HEAD"].includes(next.request.method)) {
|
|
1016
1432
|
const redirect = new URL(location, next.request.url); redirect.hash = "";
|
|
1017
|
-
|
|
1018
|
-
|
|
1433
|
+
const lead = { url: redirect.href, method: next.request.method, kind: "redirect", inScope: websiteScope(input, redirect.href, next.request.method) };
|
|
1434
|
+
if (leads.size < 40) leads.set(websiteLeadKey(lead), lead); else omitted++;
|
|
1435
|
+
if (lead.inScope) queue.unshift({ request: { url: redirect.href, method: next.request.method }, depth: next.depth });
|
|
1436
|
+
else omitted++;
|
|
1019
1437
|
} else omitted++;
|
|
1020
1438
|
}
|
|
1021
1439
|
const url = new URL(next.request.url);
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1440
|
+
const parameters = [...new Set(url.searchParams.keys())];
|
|
1441
|
+
omitted += Math.max(0, parameters.length - 16);
|
|
1442
|
+
if (endpoints.size < 40 || endpoints.has(key)) endpoints.set(key, { id: crypto.createHash("sha256").update(key).digest("hex"), url: url.href, method: next.request.method, parameters: parameters.slice(0,16), observed: true }); else omitted++;
|
|
1443
|
+
pages.push({ url: url.href, status: response.status, contentType: response.contentType, excerpt: textual ? (response.excerpt ?? body).slice(0, tool === "web_request" ? 2048 : 512) : "[Binary response; body not retained as text]", truncated: Boolean(response.bodyTruncated) || textual && body.length > (tool === "web_request" ? 2048 : 512), ...(response.bodyTruncated ? { bodyTruncated: true } : {}), ...(response.evidenceRef ? { evidenceRef: response.evidenceRef } : {}) });
|
|
1444
|
+
if (response.bodyTruncated) omitted++;
|
|
1445
|
+
const discovered = websiteLeads(body, url.href, response.contentType);
|
|
1446
|
+
const scripts = javascriptDiscovery(body, url.href, response.contentType);
|
|
1447
|
+
discovered.leads.push(...scripts.leads);
|
|
1448
|
+
omitted += discovered.omitted + scripts.omitted;
|
|
1449
|
+
for (const link of discovered.leads) {
|
|
1450
|
+
const inScope = websiteScope(input, link.url, link.method), linkKey = websiteRequestKey(link), leadKey = websiteLeadKey(link);
|
|
1451
|
+
if (leads.size < 40 || leads.has(leadKey)) leads.set(leadKey, { ...link, inScope }); else omitted++;
|
|
1029
1452
|
if (!inScope) { omitted++; continue; }
|
|
1030
|
-
if (endpoints.size < 40 && !endpoints.has(
|
|
1031
|
-
if (tool !== "web_discover" || link.kind
|
|
1453
|
+
if (endpoints.size < 40 && !endpoints.has(linkKey)) endpoints.set(linkKey, { id: crypto.createHash("sha256").update(linkKey).digest("hex"), url: link.url, method: link.method, parameters: [], observed: false });
|
|
1454
|
+
if (tool !== "web_discover" || !["page", "script", "javascript"].includes(link.kind) || !["GET", "HEAD"].includes(link.method)) continue;
|
|
1032
1455
|
if (/\.(?:css|png|jpe?g|gif|svg|ico|avif|webp|woff2?|ttf|mp4|pdf)(?:\?|$)/i.test(link.url)) continue;
|
|
1033
|
-
if (next.depth < input.maxDepth && queue.length < 40 && !seen.has(
|
|
1034
|
-
else if (!seen.has(
|
|
1456
|
+
if (next.depth < input.maxDepth && queue.length < 40 && !seen.has(linkKey)) queue.push({ request: { url: link.url, method: link.method, ...(link.kind === "script" ? { resourceType: "script" } : {}) }, depth: next.depth + 1 });
|
|
1457
|
+
else if (!seen.has(linkKey)) omitted++;
|
|
1035
1458
|
}
|
|
1036
1459
|
}
|
|
1037
1460
|
const value = { pages, endpoints: [...endpoints.values()], leads: [...leads.values()], omitted: omitted + queue.length,
|
|
1038
|
-
coverage: "Bounded HTTP observations and
|
|
1461
|
+
coverage: "Bounded HTTP observations, page/form inputs and literal JavaScript/XHR hints. Hints remain leads until concrete responses are retained. External origins do not grant access; dynamic routes, omitted inputs and unexercised business workflows remain explicit gaps.", complete: false, verification: "not-independent" };
|
|
1039
1462
|
while (Buffer.byteLength(JSON.stringify(value)) > 16000) {
|
|
1040
1463
|
if (value.leads.length) value.leads.pop(); else if (value.endpoints.length) value.endpoints.pop(); else value.pages.pop();
|
|
1041
1464
|
value.omitted++;
|
|
@@ -1043,27 +1466,56 @@ async function websiteObserve(tool, input, send) {
|
|
|
1043
1466
|
return value;
|
|
1044
1467
|
}
|
|
1045
1468
|
|
|
1046
|
-
async function websiteBrowse(input, send) {
|
|
1469
|
+
async function websiteBrowse(input, send, retained) {
|
|
1047
1470
|
const version = require("/opt/mettle-browser/node_modules/playwright/package.json").version;
|
|
1048
1471
|
if (version !== input.version) throw new Error("browser_version");
|
|
1049
|
-
const { chromium } = require("/opt/mettle-browser/node_modules/playwright");
|
|
1050
|
-
const browser = await chromium.launch({ headless: true, chromiumSandbox: false, timeout: 10000, args: ["--disable-dev-shm-usage"] });
|
|
1472
|
+
const { chromium, errors } = require("/opt/mettle-browser/node_modules/playwright");
|
|
1473
|
+
const browser = retained?.browser || await chromium.launch({ headless: true, chromiumSandbox: false, timeout: 10000, args: ["--disable-dev-shm-usage"] });
|
|
1051
1474
|
const blocked = [];
|
|
1052
|
-
|
|
1475
|
+
const websocketObserver = retained || {};
|
|
1476
|
+
let context, page, routeHandler, consoleHandler, dialogHandler, pageHandler, keep = false;
|
|
1477
|
+
let failure, requests = 0, pending = Promise.resolve(), accepting = true, blockedOmitted = 0;
|
|
1478
|
+
websocketObserver.blockedWebSocket = (url) => {
|
|
1479
|
+
if (blocked.length < 20) blocked.push({ url: url.slice(0, 2048), reason: "websocket" }); else blockedOmitted++;
|
|
1480
|
+
};
|
|
1053
1481
|
try {
|
|
1054
|
-
const
|
|
1482
|
+
const restored = input.persistSession ? await send({ sessionState: "load" }) : undefined;
|
|
1483
|
+
if (retained?.context && (retained.startUrl !== input.url || retained.credentialRevision !== restored?.credentialRevision || retained.stateDigest !== restored?.stateDigest)) throw new Error("browser_page_binding");
|
|
1484
|
+
context = retained?.context || await browser.newContext({ serviceWorkers: "block", acceptDownloads: false, ...(restored ? { storageState: { cookies: restored.state.cookies, origins: restored.state.origins.map(({ sessionStorage, opfs, ...origin }) => origin) } } : {}) });
|
|
1485
|
+
const executionEvents = [];
|
|
1486
|
+
let executionEventsOmitted = 0;
|
|
1487
|
+
const executionEvent = (channel, kind, text, url) => {
|
|
1488
|
+
if (executionEvents.length >= 32) { executionEventsOmitted++; return; }
|
|
1489
|
+
executionEvents.push({ channel, kind: kind.slice(0,32), text: text.slice(0,256), url: url.slice(0,256), truncated: text.length > 256 || url.length > 256 });
|
|
1490
|
+
};
|
|
1491
|
+
consoleHandler = (message) => executionEvent("console", message.type(), message.text(), message.page()?.url() || "");
|
|
1492
|
+
context.on("console", consoleHandler);
|
|
1493
|
+
dialogHandler = (dialog) => {
|
|
1494
|
+
executionEvent("dialog", dialog.type(), dialog.message(), dialog.page()?.url() || "");
|
|
1495
|
+
void dialog.dismiss().catch((error) => { failure = error; });
|
|
1496
|
+
};
|
|
1497
|
+
context.on("dialog", dialogHandler);
|
|
1498
|
+
pageHandler = (event) => executionEvent("error", "pageerror", event.error().message, event.page()?.url() || "");
|
|
1499
|
+
context.on("weberror", pageHandler);
|
|
1055
1500
|
context.setDefaultTimeout(5000);
|
|
1056
|
-
|
|
1057
|
-
|
|
1501
|
+
if (!retained?.context) {
|
|
1502
|
+
await context.routeWebSocket("**/*", (socket) => {
|
|
1503
|
+
websocketObserver.blockedWebSocket?.(socket.url());
|
|
1504
|
+
socket.close();
|
|
1505
|
+
});
|
|
1506
|
+
if (retained) await context.route("**/*", (route) => route.abort());
|
|
1507
|
+
}
|
|
1508
|
+
routeHandler = async (route) => {
|
|
1058
1509
|
const request = route.request();
|
|
1059
1510
|
const url = request.url();
|
|
1060
1511
|
const hints = request.headers();
|
|
1061
1512
|
if (hints["next-router-prefetch"] === "1" || /(?:^|[\s,;])prefetch(?:$|[\s,;])/.test(hints["sec-purpose"] || hints.purpose || "")) {
|
|
1062
|
-
if (blocked.length < 20) blocked.push({ url: url.slice(0, 2048), reason: "prefetch" });
|
|
1513
|
+
if (blocked.length < 20) blocked.push({ url: url.slice(0, 2048), reason: "prefetch" }); else blockedOmitted++;
|
|
1063
1514
|
await route.abort(); return;
|
|
1064
1515
|
}
|
|
1065
1516
|
if (!accepting || !websiteScope(input, url, request.method()) || ["image", "font", "media"].includes(request.resourceType()) || requests >= 40) {
|
|
1066
1517
|
if (blocked.length < 20) blocked.push({ url: url.slice(0, 2048), reason: !accepting ? "workflow_complete" : !websiteScope(input, url, request.method()) ? "scope" : requests >= 40 ? "request_limit" : "asset" });
|
|
1518
|
+
else blockedOmitted++;
|
|
1067
1519
|
await route.abort(); return;
|
|
1068
1520
|
}
|
|
1069
1521
|
requests++;
|
|
@@ -1072,33 +1524,105 @@ async function websiteBrowse(input, send) {
|
|
|
1072
1524
|
if (failure) { await route.abort(); return; }
|
|
1073
1525
|
const headers = Object.fromEntries(Object.entries(await request.allHeaders()).filter(([name]) => input.requestHeaders.includes(name)));
|
|
1074
1526
|
const resourceType = request.resourceType();
|
|
1075
|
-
const
|
|
1527
|
+
const body = request.postDataBuffer();
|
|
1528
|
+
const response = await send({ url, method: request.method(), headers, ...(["script", "stylesheet"].includes(resourceType) ? { resourceType } : {}), ...(body !== null ? { bodyBase64: body.toString("base64") } : {}) });
|
|
1076
1529
|
const responseHeaders = Object.fromEntries(Object.entries(response.headers).filter(([, value]) => value !== undefined).map(([name, value]) => [name, Array.isArray(value) ? value.join(name === "set-cookie" ? "\n" : ", ") : value]));
|
|
1077
1530
|
await route.fulfill({ status: response.status, headers: responseHeaders, body: Buffer.from(response.body, "base64") });
|
|
1078
1531
|
} catch (error) { failure = error; await route.abort().catch(() => {}); }
|
|
1079
1532
|
});
|
|
1080
1533
|
pending = work;
|
|
1081
1534
|
await work;
|
|
1082
|
-
}
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1535
|
+
};
|
|
1536
|
+
await context.route("**/*", routeHandler);
|
|
1537
|
+
page = retained?.page || await context.newPage();
|
|
1538
|
+
if (restored && !retained?.context) await restoreBrowserPrivateState(page, context, restored.state, input.url);
|
|
1539
|
+
let workflow = { outcome: "completed" }, actions = 0, phase = "navigation";
|
|
1540
|
+
try {
|
|
1541
|
+
if (!retained?.page) await page.goto(input.url, { waitUntil: "domcontentloaded", timeout: 60000 });
|
|
1542
|
+
phase = "action";
|
|
1543
|
+
for (const action of input.actions) {
|
|
1544
|
+
if (action.type === "click") await page.locator(action.selector).click();
|
|
1545
|
+
else if (action.type === "fill") await page.locator(action.selector).fill(action.value);
|
|
1546
|
+
else if (action.type === "waitFor") await page.locator(action.selector).filter({ hasText: action.text }).waitFor({ state: "visible" });
|
|
1547
|
+
else if (action.type === "select") await page.locator(action.selector).selectOption({ label: action.label });
|
|
1548
|
+
else if (action.type === "check") await page.locator(action.selector).setChecked(action.checked);
|
|
1549
|
+
else if (action.type === "press") await page.locator(action.selector).press(action.key);
|
|
1550
|
+
else throw new Error("browser_action");
|
|
1551
|
+
actions++;
|
|
1552
|
+
}
|
|
1553
|
+
} catch (error) {
|
|
1554
|
+
if (!(error instanceof errors.TimeoutError)) throw error;
|
|
1555
|
+
workflow = { outcome: "timed_out", phase, ...(phase === "action" ? { actionIndex: actions } : {}) };
|
|
1090
1556
|
}
|
|
1091
1557
|
await pending;
|
|
1092
1558
|
if (failure) throw failure;
|
|
1093
|
-
const
|
|
1094
|
-
const
|
|
1095
|
-
|
|
1559
|
+
const observe = { selector: "body", textOffset: 0, controlsOffset: 0, ...input.observe };
|
|
1560
|
+
const snapshot = await page.locator(observe.selector).evaluateAll((regions, offsets) => {
|
|
1561
|
+
const root = regions[0], items = [];
|
|
1562
|
+
const controlSelector = 'input:not([type="hidden"]), button, select, textarea, a[href], [role="button"]';
|
|
1563
|
+
const elements = root ? [...(root.matches(controlSelector) ? [root] : []), ...root.querySelectorAll(controlSelector)] : [];
|
|
1564
|
+
let total = 0;
|
|
1565
|
+
for (const element of elements) {
|
|
1566
|
+
if (!element.getClientRects().length || getComputedStyle(element).visibility === "hidden") continue;
|
|
1567
|
+
const index = total++;
|
|
1568
|
+
if (index < offsets.controlsOffset || items.length >= 20) continue;
|
|
1569
|
+
let selector = element.id ? "#" + CSS.escape(element.id) : "";
|
|
1570
|
+
if (!selector || document.querySelectorAll(selector).length !== 1) {
|
|
1571
|
+
const parts = [];
|
|
1572
|
+
for (let node = element; node; node = node.parentElement) {
|
|
1573
|
+
let index = 1;
|
|
1574
|
+
for (let sibling = node.previousElementSibling; sibling; sibling = sibling.previousElementSibling) if (sibling.localName === node.localName) index++;
|
|
1575
|
+
parts.unshift(CSS.escape(node.localName) + ":nth-of-type(" + index + ")");
|
|
1576
|
+
}
|
|
1577
|
+
selector = parts.join(" > ");
|
|
1578
|
+
}
|
|
1579
|
+
if (selector.length > 256) continue;
|
|
1580
|
+
const label = element.getAttribute("aria-label") || Array.from(element.labels || []).map((label) => label.innerText).join(" ") || element.getAttribute("placeholder") || (element.localName === "input" ? "" : element.innerText) || "";
|
|
1581
|
+
items.push({ index, selector, tag: element.localName, type: (element.getAttribute("type") || "").slice(0, 32), label: label.trim().slice(0, 128), disabled: element.matches(":disabled") || element.getAttribute("aria-disabled") === "true", ...(element.localName === "a" && element.href.length <= 2048 ? { href: element.href } : {}) });
|
|
1582
|
+
}
|
|
1583
|
+
const text = Array.from(root?.innerText || "");
|
|
1584
|
+
return { controls: items, totalControls: total, matchedRegions: regions.length, text: text.slice(offsets.textOffset, offsets.textOffset + 4096).join(""), totalTextChars: text.length };
|
|
1585
|
+
}, observe);
|
|
1586
|
+
const url = page.url();
|
|
1587
|
+
const value = { url: url.slice(0, 2048), title: (await page.title()).slice(0, 256), text: snapshot.text, truncated: snapshot.totalTextChars > observe.textOffset + Array.from(snapshot.text).length || observe.textOffset > 0 || snapshot.matchedRegions !== 1 || url.length > 2048, actions, workflow, controls: snapshot.controls, controlsOmitted: snapshot.totalControls - snapshot.controls.length, blocked, blockedOmitted,
|
|
1588
|
+
observation: { ...observe, matchedRegions: snapshot.matchedRegions, totalTextChars: snapshot.totalTextChars, totalControls: snapshot.totalControls, nextTextOffset: null, nextControlsOffset: null },
|
|
1589
|
+
complete: false, coverage: "One isolated browser workflow and a snapshot of the first matching DOM region. Missing or additional regions and omitted text/controls are not inspected. Offsets refer to this DOM snapshot. Continue a retained page to observe it without replaying actions; a call without page reloads the URL. A timed-out action may have had effects; inspect recorded traffic before deciding whether to retry. Remaining actions were not attempted. " + (input.persistSession ? "The approved identity retains bounded cookies, localStorage, IndexedDB, sessionStorage and OPFS across fresh contexts." : "Storage lasts for this browser page only."), verification: "not-independent" };
|
|
1096
1590
|
accepting = false;
|
|
1097
1591
|
await pending;
|
|
1098
1592
|
if (failure) throw failure;
|
|
1099
|
-
await context.
|
|
1593
|
+
const saved = restored ? await send({ sessionState: "save", expectedRevision: restored.revision, state: await captureBrowserPrivateState(page, context, input.target) }) : undefined;
|
|
1594
|
+
if (retained) {
|
|
1595
|
+
Object.assign(retained, { browser, context, page, startUrl: input.url, credentialRevision: restored?.credentialRevision, stateDigest: saved?.stateDigest });
|
|
1596
|
+
value.page = { id: input.page.id, state: input.page.operation === "close" ? "closed" : "retained" };
|
|
1597
|
+
keep = input.page.operation !== "close";
|
|
1598
|
+
} else await context.close();
|
|
1599
|
+
value.blockedOmitted = blockedOmitted;
|
|
1600
|
+
value.executionEvents = executionEvents;
|
|
1601
|
+
value.executionEventsOmitted = executionEventsOmitted;
|
|
1602
|
+
while (Buffer.byteLength(JSON.stringify(value)) > 15936) {
|
|
1603
|
+
if (value.executionEvents.length) { value.executionEvents.pop(); value.executionEventsOmitted++; }
|
|
1604
|
+
else if (value.blocked.length) { value.blocked.pop(); value.blockedOmitted++; }
|
|
1605
|
+
else if (Array.from(value.text).length > 512) { value.text = Array.from(value.text).slice(0, Math.floor(Array.from(value.text).length / 2)).join(""); value.truncated = true; }
|
|
1606
|
+
else if (value.controls.length) { value.controls.pop(); value.controlsOmitted++; }
|
|
1607
|
+
else { value.text = Array.from(value.text).slice(0, Math.floor(Array.from(value.text).length / 2)).join(""); value.truncated = true; }
|
|
1608
|
+
}
|
|
1609
|
+
const textEnd = observe.textOffset + Array.from(value.text).length;
|
|
1610
|
+
value.observation.nextTextOffset = textEnd < snapshot.totalTextChars ? textEnd : null;
|
|
1611
|
+
const controlsEnd = value.controls.length ? value.controls.at(-1).index + 1 : snapshot.totalControls;
|
|
1612
|
+
value.observation.nextControlsOffset = controlsEnd < snapshot.totalControls ? controlsEnd : null;
|
|
1613
|
+
if (retained) value.coverage += " The live page retains DOM and JavaScript state until close, worker loss or expiry. HTTP is denied between commands; no idle-time execution assertion is made.";
|
|
1100
1614
|
return value;
|
|
1101
|
-
} finally {
|
|
1615
|
+
} finally {
|
|
1616
|
+
accepting = false;
|
|
1617
|
+
delete websocketObserver.blockedWebSocket;
|
|
1618
|
+
if (retained && context) {
|
|
1619
|
+
if (routeHandler) await context.unroute("**/*", routeHandler);
|
|
1620
|
+
if (consoleHandler) context.off("console", consoleHandler);
|
|
1621
|
+
if (dialogHandler) context.off("dialog", dialogHandler);
|
|
1622
|
+
if (pageHandler) context.off("weberror", pageHandler);
|
|
1623
|
+
}
|
|
1624
|
+
if (!keep) await browser.close();
|
|
1625
|
+
}
|
|
1102
1626
|
}
|
|
1103
1627
|
`;
|
|
1104
1628
|
|
|
@@ -1110,6 +1634,8 @@ const lines = readline.createInterface({ input: process.stdin, crlfDelay: Infini
|
|
|
1110
1634
|
const iterator = lines[Symbol.asyncIterator]();
|
|
1111
1635
|
${ACTIVE_PROGRAM}
|
|
1112
1636
|
${WEBSITE_PROGRAM}
|
|
1637
|
+
${PORT_PROGRAM}
|
|
1638
|
+
${BROWSER_PAGE_PROGRAM}
|
|
1113
1639
|
(async () => {
|
|
1114
1640
|
if (process.getuid() !== ${SANDBOX_UID}) throw new Error();
|
|
1115
1641
|
const first = await iterator.next();
|
|
@@ -1129,10 +1655,12 @@ ${WEBSITE_PROGRAM}
|
|
|
1129
1655
|
return result;
|
|
1130
1656
|
};
|
|
1131
1657
|
let value;
|
|
1132
|
-
if (tool === "
|
|
1658
|
+
if (tool === "port_scan") {
|
|
1659
|
+
value = await portScan(input, send);
|
|
1660
|
+
} else if (tool === "web_discover" || tool === "web_request") {
|
|
1133
1661
|
value = await websiteObserve(tool, input, send);
|
|
1134
1662
|
} else if (tool === "web_browse") {
|
|
1135
|
-
value = await websiteBrowse(input, send);
|
|
1663
|
+
value = input.page ? await browseRetainedPage(input, send) : await websiteBrowse(input, send);
|
|
1136
1664
|
} else if (tool === "http_request") {
|
|
1137
1665
|
const response = await send({ ...input.request, requestId: input.requestId });
|
|
1138
1666
|
const body = Buffer.from(response.body, "base64").toString("utf8");
|
|
@@ -1152,7 +1680,8 @@ ${WEBSITE_PROGRAM}
|
|
|
1152
1680
|
await context.route("**/*", async (route) => {
|
|
1153
1681
|
try {
|
|
1154
1682
|
const request = route.request();
|
|
1155
|
-
const
|
|
1683
|
+
const body = request.postDataBuffer();
|
|
1684
|
+
const response = await send({ url: request.url(), method: request.method(), ...(body !== null ? { bodyBase64: body.toString("base64") } : {}) });
|
|
1156
1685
|
await route.fulfill({ status: response.status, contentType: response.contentType, body: Buffer.from(response.body, "base64") });
|
|
1157
1686
|
} catch (error) { failure = error; await route.abort(); }
|
|
1158
1687
|
});
|
|
@@ -1182,6 +1711,327 @@ ${WEBSITE_PROGRAM}
|
|
|
1182
1711
|
})().catch(() => { lines.close(); process.stdin.destroy(); process.exitCode = 1; });
|
|
1183
1712
|
`;
|
|
1184
1713
|
|
|
1714
|
+
// apps/harness/src/sandbox/environment-program.ts
|
|
1715
|
+
var ENVIRONMENT_PROGRAM = String.raw`
|
|
1716
|
+
function environmentCheck(input) {
|
|
1717
|
+
const child = require("node:child_process");
|
|
1718
|
+
const checks = [];
|
|
1719
|
+
const missing = [];
|
|
1720
|
+
const run = (binary, args) => child.spawnSync(binary, args, { env: { PATH: "/usr/local/bin:/usr/bin:/bin", HOME: "/tmp", TMPDIR: "/tmp", LC_ALL: "C", PLAYWRIGHT_BROWSERS_PATH: "/opt/mettle-browser/browsers", PYTHONDONTWRITEBYTECODE: "1", JAVA_TOOL_OPTIONS: "-Djava.awt.headless=true -Duser.home=/tmp" }, stdio: ["ignore", "pipe", "pipe"], timeout: 20000, killSignal: "SIGKILL", maxBuffer: 8192, encoding: "utf8", shell: false });
|
|
1721
|
+
if (input.browserVersion) {
|
|
1722
|
+
const script = "(async () => { const p = require('/opt/mettle-browser/node_modules/playwright'); if (require('/opt/mettle-browser/node_modules/playwright/package.json').version !== process.argv[1]) process.exit(2); const browser = await p.chromium.launch({headless:true,chromiumSandbox:false,args:['--disable-dev-shm-usage']}); await browser.close(); process.stdout.write('ready'); })().catch(() => process.exit(1));";
|
|
1723
|
+
const result = run("/usr/local/bin/node", ["-e", script, input.browserVersion]);
|
|
1724
|
+
(result.status === 0 && result.stdout === "ready" ? checks : missing).push("playwright/" + input.browserVersion);
|
|
1725
|
+
}
|
|
1726
|
+
for (const profile of input.scanners) {
|
|
1727
|
+
let result, matched = false;
|
|
1728
|
+
if (profile.tool === "port_scan") {
|
|
1729
|
+
result = run("/usr/local/bin/naabu", ["-version"]);
|
|
1730
|
+
matched = (result.stdout + result.stderr).split(/\s+/).includes(profile.version);
|
|
1731
|
+
} else if (profile.tool === "nuclei_scan") {
|
|
1732
|
+
result = run("/usr/local/bin/nuclei", ["-version"]);
|
|
1733
|
+
matched = (result.stdout + result.stderr).split(/\s+/).includes("v" + profile.version);
|
|
1734
|
+
} else if (profile.tool === "sqli_confirm") {
|
|
1735
|
+
result = run("/usr/bin/python3", ["/opt/mettle/sqlmap/sqlmap.py", "--version"]);
|
|
1736
|
+
matched = String(result.stdout).split("\n")[0].trim() === profile.version + "#stable";
|
|
1737
|
+
} else if (profile.tool === "zap_scan") {
|
|
1738
|
+
result = run("/zap/zap.sh", ["-Xmx768m", "-silent", "-cmd", "-dir", "/tmp/zap-preflight", "-version"]);
|
|
1739
|
+
matched = String(result.stdout).split(/\s+/).includes(profile.version);
|
|
1740
|
+
} else throw new Error("Unsupported admitted prerequisite");
|
|
1741
|
+
(result.status === 0 && !result.error && matched ? checks : missing).push(profile.tool + "/" + profile.version);
|
|
1742
|
+
}
|
|
1743
|
+
return { status: missing.length ? "unavailable" : "ready", checks, missing };
|
|
1744
|
+
}
|
|
1745
|
+
`;
|
|
1746
|
+
|
|
1747
|
+
// apps/harness/src/sandbox/experiment-program.ts
|
|
1748
|
+
var EXPERIMENT_PROGRAM = String.raw`
|
|
1749
|
+
function experimentRun(args, input) {
|
|
1750
|
+
const path = require("node:path"), child = require("node:child_process");
|
|
1751
|
+
const { policy } = input;
|
|
1752
|
+
if (!policy.runtimes.includes(args.runtime) || !["node", "bash"].includes(args.runtime) || !Number.isInteger(policy.timeoutMs) || policy.timeoutMs < 100 || policy.timeoutMs > 10000 || !Number.isInteger(policy.maxScriptBytes) || policy.maxScriptBytes > 16384) throw new Error("experiment_policy");
|
|
1753
|
+
const hash = (value) => crypto.createHash("sha256").update(value).digest("hex");
|
|
1754
|
+
const run = (program) => {
|
|
1755
|
+
if (typeof program !== "string" || !program.length || program.includes("\0") || Buffer.byteLength(program) > policy.maxScriptBytes) throw new Error("experiment_program");
|
|
1756
|
+
const root = fs.mkdtempSync("/tmp/mettle-experiment-"), workspace = path.join(root, "workspace");
|
|
1757
|
+
try {
|
|
1758
|
+
materializeWorkspace(workspace, input.files);
|
|
1759
|
+
const script = path.join(root, args.runtime === "node" ? "experiment.cjs" : "experiment.bash");
|
|
1760
|
+
fs.writeFileSync(script, program, { flag: "wx", mode: 0o600 });
|
|
1761
|
+
const result = child.spawnSync(args.runtime === "node" ? "/usr/local/bin/node" : "/bin/bash", args.runtime === "node" ? ["--", script] : ["--noprofile", "--norc", "-p", "--", script], {
|
|
1762
|
+
cwd: workspace, env: { PATH: "/usr/local/bin:/usr/bin:/bin", HOME: root, TMPDIR: root, LC_ALL: "C" },
|
|
1763
|
+
stdio: ["ignore", "pipe", "pipe"], timeout: policy.timeoutMs, killSignal: "SIGKILL", maxBuffer: ${TASK_INPUT_LIMIT}, encoding: "utf8", shell: false
|
|
1764
|
+
});
|
|
1765
|
+
const stdout = result.stdout || "", stderr = result.stderr || "", exceeded = result.error?.code === "ENOBUFS";
|
|
1766
|
+
return { scriptSha256: hash(program), status: spawnStatus(result, "completed", exceeded), exitCode: result.status, stdout: stdout.slice(0, ${CHILD_OUTPUT_KEEP}), stderr: stderr.slice(0, ${CHILD_OUTPUT_KEEP}), truncated: exceeded || stdout.length > ${CHILD_OUTPUT_KEEP} || stderr.length > ${CHILD_OUTPUT_KEEP} };
|
|
1767
|
+
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
|
1768
|
+
};
|
|
1769
|
+
const control = run(args.negativeControl), probe = run(args.program);
|
|
1770
|
+
return { subjectDigest: hash(JSON.stringify(input.subject)), filesDigest: hash(JSON.stringify(input.files)), runtime: args.runtime, control, probe, assertion: "inconclusive", verification: "not-independent" };
|
|
1771
|
+
}
|
|
1772
|
+
`;
|
|
1773
|
+
|
|
1774
|
+
// apps/harness/src/sandbox/joern-program.ts
|
|
1775
|
+
var JOERN_QUERY = `
|
|
1776
|
+
import io.joern.dataflowengineoss.language.*
|
|
1777
|
+
import io.shiftleft.codepropertygraph.generated.nodes.CfgNode
|
|
1778
|
+
import io.shiftleft.semanticcpg.language.*
|
|
1779
|
+
import java.nio.file.{Files, Paths}
|
|
1780
|
+
|
|
1781
|
+
@main def exec(cpgFile: String, traceFile: String, outFile: String): Unit = {
|
|
1782
|
+
val request = ujson.read(Files.readString(Paths.get(traceFile)))
|
|
1783
|
+
val expected = request("files").arr.map(_.str).toSet
|
|
1784
|
+
val workspace = request("workspace").str + "/"
|
|
1785
|
+
def relative(name: String): String = name.stripPrefix(workspace).stripPrefix("./")
|
|
1786
|
+
val version = io.joern.joerncli.console.JoernConsole.version
|
|
1787
|
+
if (version != request("version").str) throw new IllegalArgumentException("version_mismatch")
|
|
1788
|
+
importCpg(cpgFile).getOrElse(throw new IllegalArgumentException("invalid_cpg"))
|
|
1789
|
+
def matches(node: CfgNode, location: ujson.Value): Boolean =
|
|
1790
|
+
relative(node.method.filename) == location("path").str && node.lineNumber.exists(line =>
|
|
1791
|
+
line >= location("startLine").num.toInt && line <= location("endLine").num.toInt)
|
|
1792
|
+
val sources = cpg.all.collect { case node: CfgNode if matches(node, request("source")) => node }.take(129).toList
|
|
1793
|
+
val sinks = cpg.call.filter(node => matches(node, request("sink"))).argument.filter(_.argumentIndex > 0).take(129).toList
|
|
1794
|
+
val parsed = cpg.file.name.map(relative).filter(expected.contains).toSet
|
|
1795
|
+
var omitted = sources.size > 128 || sinks.size > 128
|
|
1796
|
+
val flows = sinks.take(128).iterator.reachableByFlows(sources.take(128)).take(17).toList
|
|
1797
|
+
omitted ||= flows.size > 16
|
|
1798
|
+
val paths = flows.take(16).flatMap { flow =>
|
|
1799
|
+
val located = flow.elements.collect {
|
|
1800
|
+
case node: CfgNode if node.lineNumber.exists(_ > 0) && expected.contains(relative(node.method.filename)) =>
|
|
1801
|
+
(relative(node.method.filename), node.lineNumber.get)
|
|
1802
|
+
}.toList
|
|
1803
|
+
val points = located.distinct
|
|
1804
|
+
if (located.size != flow.elements.size || points.size > 32) omitted = true
|
|
1805
|
+
if (points.size < 2 || points.size > 32) {
|
|
1806
|
+
omitted = true
|
|
1807
|
+
None
|
|
1808
|
+
} else Some(ujson.Arr.from(points.map { case (path, line) => ujson.Obj("path" -> path, "line" -> line) }))
|
|
1809
|
+
}
|
|
1810
|
+
val result = ujson.Obj(
|
|
1811
|
+
"version" -> version,
|
|
1812
|
+
"sourceNodes" -> sources.size,
|
|
1813
|
+
"sinkNodes" -> sinks.size,
|
|
1814
|
+
"parsedFiles" -> ujson.Arr.from(parsed.toList.sorted),
|
|
1815
|
+
"paths" -> ujson.Arr.from(paths),
|
|
1816
|
+
"omitted" -> omitted,
|
|
1817
|
+
"complete" -> (sources.nonEmpty && sinks.nonEmpty && parsed == expected && !omitted)
|
|
1818
|
+
)
|
|
1819
|
+
Files.writeString(Paths.get(outFile), result.toString)
|
|
1820
|
+
}
|
|
1821
|
+
`;
|
|
1822
|
+
var JOERN_PROGRAM = `
|
|
1823
|
+
function scanJoern(input, workspace, root) {
|
|
1824
|
+
const path = require("node:path");
|
|
1825
|
+
const child = require("node:child_process");
|
|
1826
|
+
const query = ${JSON.stringify(JOERN_QUERY)};
|
|
1827
|
+
const queryDigest = crypto.createHash("sha256").update(query).digest("hex");
|
|
1828
|
+
const binding = { version: input.version, ruleId: ${JSON.stringify(JOERN_QUERY_ID)}, queryDigest };
|
|
1829
|
+
const incomplete = (status) => ({ ...binding, status, complete: false, sourceNodes: 0, sinkNodes: 0, parsedFiles: [], paths: [], omitted: false });
|
|
1830
|
+
const deadline = Date.now() + ${JOERN_TIMEOUT_MS};
|
|
1831
|
+
const run = (binary, args) => child.spawnSync(binary, args, {
|
|
1832
|
+
cwd: root, env: { PATH: "/usr/local/bin:/usr/bin:/bin", HOME: root, TMPDIR: root, LC_ALL: "C.UTF-8", JAVA_TOOL_OPTIONS: "-Xmx2304m -XX:ActiveProcessorCount=2 -DZstdNativePath=/opt/mettle/joern/native/libzstd-jni.so -Djava.io.tmpdir=" + root + " -Duser.home=" + root, JOERN_HOME: "/opt/mettle/joern" },
|
|
1833
|
+
stdio: ["ignore", "pipe", "pipe"], timeout: Math.max(1, deadline - Date.now()), killSignal: "SIGKILL", maxBuffer: 1048576, encoding: "utf8", shell: false,
|
|
1834
|
+
});
|
|
1835
|
+
const cpg = path.join(root, "cpg.bin");
|
|
1836
|
+
const trace = path.join(root, "trace.json");
|
|
1837
|
+
const script = path.join(root, "query.sc");
|
|
1838
|
+
const report = path.join(root, "report.json");
|
|
1839
|
+
fs.writeFileSync(trace, JSON.stringify({ ...input.trace, version: input.version, workspace, files: input.files.map((file) => file.path) }), { flag: "wx", mode: 0o600 });
|
|
1840
|
+
fs.writeFileSync(script, query, { flag: "wx", mode: 0o600 });
|
|
1841
|
+
const parsed = run("/opt/mettle/joern/frontends/jssrc2cpg/bin/jssrc2cpg", ["-J-Xmx2304m", "-J-XX:ActiveProcessorCount=2", "-Dlog4j.configurationFile=/opt/mettle/joern/conf/log4j2.xml", workspace, "--output", cpg, "--no-tsTypes"]);
|
|
1842
|
+
if (parsed.error || parsed.status !== 0) return incomplete(spawnStatus(parsed));
|
|
1843
|
+
const analyzed = run("/opt/mettle/joern/bin/repl-bridge", ["-J-Xmx2304m", "-J-XX:ActiveProcessorCount=2", "-Dlog4j.configurationFile=/opt/mettle/joern/conf/log4j2.xml", "--script", script, "--param", "cpgFile=" + cpg, "--param", "traceFile=" + trace, "--param", "outFile=" + report]);
|
|
1844
|
+
if (analyzed.error || analyzed.status !== 0) return incomplete(String(analyzed.stdout).includes("version_mismatch") ? "version_mismatch" : spawnStatus(analyzed));
|
|
1845
|
+
try {
|
|
1846
|
+
if (fs.statSync(report).size > 24576) return incomplete("output_limit");
|
|
1847
|
+
return { ...binding, ...JSON.parse(fs.readFileSync(report, "utf8")), status: "completed" };
|
|
1848
|
+
} catch { return incomplete("failed"); }
|
|
1849
|
+
}
|
|
1850
|
+
`;
|
|
1851
|
+
|
|
1852
|
+
// apps/harness/src/storage/package-probe.ts
|
|
1853
|
+
var PACKAGE_IMPORT_TIMEOUT_MS = 3000;
|
|
1854
|
+
var PACKAGE_CHANGE_LIMIT = 128;
|
|
1855
|
+
function packageJavaScript(path) {
|
|
1856
|
+
return /\.[cm]?js$/.test(path);
|
|
1857
|
+
}
|
|
1858
|
+
var PackageImportObservationSchema = strictObject({
|
|
1859
|
+
status: _enum([
|
|
1860
|
+
"loaded",
|
|
1861
|
+
"load_failed",
|
|
1862
|
+
"timed_out",
|
|
1863
|
+
"output_limit",
|
|
1864
|
+
"unavailable",
|
|
1865
|
+
"failed"
|
|
1866
|
+
]),
|
|
1867
|
+
changes: array(strictObject({
|
|
1868
|
+
target: string().min(1).max(64),
|
|
1869
|
+
key: string().max(256),
|
|
1870
|
+
kind: _enum(["added", "removed", "changed"]),
|
|
1871
|
+
enumerable: boolean().nullable()
|
|
1872
|
+
})).max(PACKAGE_CHANGE_LIMIT),
|
|
1873
|
+
error: string().max(1024).nullable(),
|
|
1874
|
+
truncated: boolean(),
|
|
1875
|
+
stdout: string().max(4096),
|
|
1876
|
+
stderr: string().max(4096)
|
|
1877
|
+
});
|
|
1878
|
+
var PackageImportResultSchema = PackageImportObservationSchema.extend({
|
|
1879
|
+
repository: string().min(1).max(256),
|
|
1880
|
+
revision: CommitSchema,
|
|
1881
|
+
path: RepositoryPathSchema,
|
|
1882
|
+
verification: literal("not-independent"),
|
|
1883
|
+
complete: boolean()
|
|
1884
|
+
}).refine((value) => value.complete === (value.status === "loaded" && !value.truncated), { message: "Only an untruncated loaded import has complete observations." });
|
|
1885
|
+
|
|
1886
|
+
// apps/harness/src/sandbox/package-program.ts
|
|
1887
|
+
var PACKAGE_IMPORT_PROGRAM = `
|
|
1888
|
+
const fs = require("node:fs");
|
|
1889
|
+
const { pathToFileURL } = require("node:url");
|
|
1890
|
+
const { isMainThread, Worker, MessageChannel, receiveMessageOnPort, parentPort, workerData } = require("node:worker_threads");
|
|
1891
|
+
if (isMainThread) {
|
|
1892
|
+
const { port1, port2 } = new MessageChannel();
|
|
1893
|
+
const worker = new Worker(__filename, { workerData: { path: process.argv[2], port: port2 }, transferList: [port2] });
|
|
1894
|
+
let error = "The import worker did not return a complete observation.";
|
|
1895
|
+
worker.on("error", cause => { error = String(cause).slice(0, 1024); });
|
|
1896
|
+
worker.on("exit", code => {
|
|
1897
|
+
const before = receiveMessageOnPort(port1)?.message;
|
|
1898
|
+
const after = receiveMessageOnPort(port1)?.message;
|
|
1899
|
+
port1.close();
|
|
1900
|
+
const changes = [];
|
|
1901
|
+
let truncated = false;
|
|
1902
|
+
if (code !== 0 || !before || !after) {
|
|
1903
|
+
fs.writeFileSync(3, JSON.stringify({ status: "failed", changes, error, truncated }));
|
|
1904
|
+
process.exit(0);
|
|
1905
|
+
}
|
|
1906
|
+
for (let i = 0; i < before.length; i++) {
|
|
1907
|
+
const [name, entries] = before[i];
|
|
1908
|
+
const previous = new Map(entries.map(entry => [entry[0], entry]));
|
|
1909
|
+
const current = new Map(after.targets[i][1].map(entry => [entry[0], entry]));
|
|
1910
|
+
for (const key of new Set([...current.keys(), ...previous.keys()])) {
|
|
1911
|
+
const old = previous.get(key), next = current.get(key);
|
|
1912
|
+
if (old && next && old[2].every((value, index) => Object.is(value, next[2][index]))) continue;
|
|
1913
|
+
if (changes.length === ${PACKAGE_CHANGE_LIMIT}) { truncated = true; continue; }
|
|
1914
|
+
changes.push({ target: name, key: (next || old)[1], kind: !old ? "added" : !next ? "removed" : "changed", enumerable: next?.[3] ?? null });
|
|
1915
|
+
}
|
|
1916
|
+
}
|
|
1917
|
+
fs.writeFileSync(3, JSON.stringify({ status: after.status, changes, error: after.error, truncated }));
|
|
1918
|
+
process.exit(0);
|
|
1919
|
+
});
|
|
1920
|
+
} else {
|
|
1921
|
+
const { path, port } = workerData;
|
|
1922
|
+
delete workerData.port;
|
|
1923
|
+
parentPort.close();
|
|
1924
|
+
const post = port.postMessage.bind(port);
|
|
1925
|
+
const ownKeys = Reflect.ownKeys;
|
|
1926
|
+
const descriptor = Object.getOwnPropertyDescriptor;
|
|
1927
|
+
const same = Object.is;
|
|
1928
|
+
const string = String;
|
|
1929
|
+
const slice = Function.call.bind(String.prototype.slice);
|
|
1930
|
+
const append = Function.call.bind(Array.prototype.push);
|
|
1931
|
+
const get = Function.call.bind(Map.prototype.get);
|
|
1932
|
+
const set = Function.call.bind(Map.prototype.set);
|
|
1933
|
+
const exit = process.exit.bind(process);
|
|
1934
|
+
const identities = new Map();
|
|
1935
|
+
let nextIdentity = 1;
|
|
1936
|
+
function identity(value) {
|
|
1937
|
+
if (same(value, -0)) return 0;
|
|
1938
|
+
let id = get(identities, value);
|
|
1939
|
+
if (id === undefined) { id = nextIdentity++; set(identities, value, id); }
|
|
1940
|
+
return id;
|
|
1941
|
+
}
|
|
1942
|
+
const targets = [
|
|
1943
|
+
["globalThis", globalThis], ["Object.prototype", Object.prototype],
|
|
1944
|
+
["Array.prototype", Array.prototype], ["Function.prototype", Function.prototype],
|
|
1945
|
+
["String.prototype", String.prototype], ["Number.prototype", Number.prototype],
|
|
1946
|
+
["Boolean.prototype", Boolean.prototype], ["Promise.prototype", Promise.prototype],
|
|
1947
|
+
["Map.prototype", Map.prototype], ["Set.prototype", Set.prototype],
|
|
1948
|
+
["RegExp.prototype", RegExp.prototype], ["Date.prototype", Date.prototype]
|
|
1949
|
+
];
|
|
1950
|
+
function snapshot() {
|
|
1951
|
+
const result = [];
|
|
1952
|
+
for (let i = 0; i < targets.length; i++) {
|
|
1953
|
+
const [name, target] = targets[i];
|
|
1954
|
+
const keys = ownKeys(target), entries = [];
|
|
1955
|
+
for (let j = 0; j < keys.length; j++) {
|
|
1956
|
+
const key = keys[j], value = descriptor(target, key);
|
|
1957
|
+
append(entries, [identity(key), slice(string(key), 0, 256), [identity(value.value), identity(value.get), identity(value.set), value.writable, value.enumerable, value.configurable], value.enumerable]);
|
|
1958
|
+
}
|
|
1959
|
+
append(result, [name, entries]);
|
|
1960
|
+
}
|
|
1961
|
+
return result;
|
|
1962
|
+
}
|
|
1963
|
+
for (const key of ownKeys(globalThis)) descriptor(globalThis, key);
|
|
1964
|
+
post(snapshot());
|
|
1965
|
+
(async () => {
|
|
1966
|
+
let status = "loaded", error = null;
|
|
1967
|
+
try { await import(pathToFileURL(path).href); }
|
|
1968
|
+
catch (cause) { status = "load_failed"; error = slice(string(cause), 0, 1024); }
|
|
1969
|
+
post({ status, error, targets: snapshot() });
|
|
1970
|
+
exit(0);
|
|
1971
|
+
})().catch(() => exit(1));
|
|
1972
|
+
}
|
|
1973
|
+
`;
|
|
1974
|
+
var PACKAGE_PROBE_PROGRAM = String.raw`
|
|
1975
|
+
function packageImportProbe(input, args, workspace, root) {
|
|
1976
|
+
const child = require("node:child_process");
|
|
1977
|
+
const path = require("node:path");
|
|
1978
|
+
if (args.path !== input.path || !input.files.some(file => file.path === args.path) || !/\.[cm]?js$/.test(args.path)) throw new Error();
|
|
1979
|
+
const script = path.join(root, "import-probe.cjs");
|
|
1980
|
+
fs.writeFileSync(script, ${JSON.stringify(PACKAGE_IMPORT_PROGRAM)}, { flag: "wx", mode: 0o600 });
|
|
1981
|
+
const result = child.spawnSync("/usr/local/bin/node", ["--max-old-space-size=128", script, path.join(workspace, args.path)], {
|
|
1982
|
+
cwd: workspace, env: { PATH: "/usr/local/bin:/usr/bin:/bin", HOME: root, TMPDIR: root },
|
|
1983
|
+
timeout: ${PACKAGE_IMPORT_TIMEOUT_MS}, killSignal: "SIGKILL", maxBuffer: ${TASK_INPUT_LIMIT},
|
|
1984
|
+
stdio: ["ignore", "pipe", "pipe", "pipe"], encoding: "utf8", shell: false
|
|
1985
|
+
});
|
|
1986
|
+
let value = { status: spawnStatus(result), changes: [], error: null, truncated: false };
|
|
1987
|
+
if (value.status === "completed") {
|
|
1988
|
+
try { value = JSON.parse(result.output[3]); }
|
|
1989
|
+
catch { value.status = "failed"; value.error = "The import probe did not return a complete observation."; }
|
|
1990
|
+
}
|
|
1991
|
+
const stdout = result.stdout || "", stderr = result.stderr || "";
|
|
1992
|
+
const observation = { ...value, stdout: stdout.slice(0, ${CHILD_OUTPUT_KEEP}), stderr: stderr.slice(0, ${CHILD_OUTPUT_KEEP}),
|
|
1993
|
+
truncated: value.truncated || stdout.length > ${CHILD_OUTPUT_KEEP} || stderr.length > ${CHILD_OUTPUT_KEEP} || result.error?.code === "ENOBUFS" };
|
|
1994
|
+
return Buffer.byteLength(JSON.stringify(observation)) > ${TASK_RESULT_LIMIT - 1024}
|
|
1995
|
+
? { status: "output_limit", changes: [], error: "The import observation exceeded the task evidence limit.", truncated: true, stdout: "", stderr: "" }
|
|
1996
|
+
: observation;
|
|
1997
|
+
}
|
|
1998
|
+
`;
|
|
1999
|
+
|
|
2000
|
+
// apps/harness/src/sandbox/report-program.ts
|
|
2001
|
+
var PDF_BROWSER_PROGRAM = `
|
|
2002
|
+
(async () => {
|
|
2003
|
+
const fs = require("node:fs");
|
|
2004
|
+
const crypto = require("node:crypto");
|
|
2005
|
+
const input = JSON.parse(fs.readFileSync(0, "utf8"));
|
|
2006
|
+
if (require("/opt/mettle-browser/node_modules/playwright/package.json").version !== input.version) throw new Error();
|
|
2007
|
+
const { chromium } = require("/opt/mettle-browser/node_modules/playwright");
|
|
2008
|
+
const browser = await chromium.launch({ headless: true, chromiumSandbox: false, timeout: 10000, args: ["--disable-dev-shm-usage"] });
|
|
2009
|
+
try {
|
|
2010
|
+
const context = await browser.newContext({ javaScriptEnabled: false, serviceWorkers: "block", acceptDownloads: false });
|
|
2011
|
+
let blockedRequests = 0;
|
|
2012
|
+
await context.route("**/*", async route => { blockedRequests++; await route.abort(); });
|
|
2013
|
+
await context.routeWebSocket("**/*", socket => socket.close());
|
|
2014
|
+
const page = await context.newPage();
|
|
2015
|
+
await page.setContent(input.html, { waitUntil: "load", timeout: 10000 });
|
|
2016
|
+
const body = await page.pdf({ format: "A4", printBackground: true, preferCSSPageSize: true, tagged: true, displayHeaderFooter: true, headerTemplate: "<span></span>", footerTemplate: '<div style="font-size:9px;width:100%;text-align:center">Mettle - <span class="pageNumber"></span> / <span class="totalPages"></span></div>', timeout: 10000 });
|
|
2017
|
+
if (body.length > ${PDF_BYTES_LIMIT} || body.subarray(0, 5).toString() !== "%PDF-") throw new Error();
|
|
2018
|
+
process.stdout.write(JSON.stringify({ content: body.toString("base64"), sha256: crypto.createHash("sha256").update(body).digest("hex"), htmlDigest: crypto.createHash("sha256").update(input.html).digest("hex"), blockedRequests }));
|
|
2019
|
+
await context.close();
|
|
2020
|
+
} finally { await browser.close(); }
|
|
2021
|
+
})().catch(() => { process.exitCode = 1; });
|
|
2022
|
+
`;
|
|
2023
|
+
var REPORT_PROGRAM = `
|
|
2024
|
+
function reportPdf(input) {
|
|
2025
|
+
if (typeof input.html !== "string" || Buffer.byteLength(input.html) > ${PDF_BYTES_LIMIT} || input.version !== ${JSON.stringify(PLAYWRIGHT_VERSION)}) throw new Error();
|
|
2026
|
+
const result = require("node:child_process").spawnSync("/usr/local/bin/node", ["-e", ${JSON.stringify(PDF_BROWSER_PROGRAM)}], {
|
|
2027
|
+
input: JSON.stringify(input), encoding: "utf8", shell: false, timeout: 25000, killSignal: "SIGKILL", maxBuffer: ${PDF_RESULT_LIMIT},
|
|
2028
|
+
env: { PATH: "/usr/local/bin:/usr/bin:/bin", HOME: "/tmp", TMPDIR: "/tmp", PLAYWRIGHT_BROWSERS_PATH: "/opt/mettle-browser/browsers" }, stdio: ["pipe", "pipe", "pipe"]
|
|
2029
|
+
});
|
|
2030
|
+
if (result.status !== 0 || result.error) throw new Error();
|
|
2031
|
+
return JSON.parse(result.stdout);
|
|
2032
|
+
}
|
|
2033
|
+
`;
|
|
2034
|
+
|
|
1185
2035
|
// apps/harness/src/sandbox/scanner-program.ts
|
|
1186
2036
|
var SPAWN_STATUS_PROGRAM = `
|
|
1187
2037
|
function spawnStatus(result, success = "completed", exceeded = false) {
|
|
@@ -1273,19 +2123,46 @@ function scanSource(tool, input, workspace, root) {
|
|
|
1273
2123
|
}
|
|
1274
2124
|
`;
|
|
1275
2125
|
|
|
2126
|
+
// apps/harness/src/sandbox/workspace-program.ts
|
|
2127
|
+
var WORKSPACE_PROGRAM = String.raw`
|
|
2128
|
+
function materializeWorkspace(workspace, files) {
|
|
2129
|
+
const path = require("node:path");
|
|
2130
|
+
fs.mkdirSync(workspace, { recursive: true });
|
|
2131
|
+
for (const file of files) {
|
|
2132
|
+
if (!file.path || file.path.startsWith("/") || file.path.includes("\\") || file.path.split("/").some((part) => !part || part === "." || part === "..")) throw new Error("workspace_path");
|
|
2133
|
+
const destination = path.join(workspace, file.path);
|
|
2134
|
+
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
2135
|
+
fs.writeFileSync(destination, file.content, { flag: "wx", mode: 0o600 });
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
`;
|
|
2139
|
+
|
|
1276
2140
|
// apps/harness/src/sandbox/task-program.ts
|
|
1277
2141
|
var TASK_PROGRAM = String.raw`
|
|
1278
2142
|
const fs = require("node:fs");
|
|
1279
2143
|
const crypto = require("node:crypto");
|
|
1280
2144
|
${SPAWN_STATUS_PROGRAM}
|
|
2145
|
+
${WORKSPACE_PROGRAM}
|
|
2146
|
+
${EXPERIMENT_PROGRAM}
|
|
1281
2147
|
${SOURCE_SCANNER_PROGRAM}
|
|
2148
|
+
${JOERN_PROGRAM}
|
|
2149
|
+
${ENVIRONMENT_PROGRAM}
|
|
2150
|
+
${REPORT_PROGRAM}
|
|
2151
|
+
${PACKAGE_PROBE_PROGRAM}
|
|
1282
2152
|
try {
|
|
1283
2153
|
const bytes = fs.readFileSync(0);
|
|
1284
|
-
if (bytes.length > ${
|
|
2154
|
+
if (bytes.length > ${TASK_INPUT_MAX} || process.getuid() !== ${SANDBOX_UID}) throw new Error();
|
|
1285
2155
|
const request = JSON.parse(bytes.toString("utf8"));
|
|
1286
2156
|
const { tool, args, input } = request;
|
|
2157
|
+
if (bytes.length > (${JSON.stringify(TASK_INPUT_LIMITS)}[tool] ?? ${TASK_INPUT_LIMIT})) throw new Error();
|
|
1287
2158
|
let value;
|
|
1288
|
-
if (tool === "
|
|
2159
|
+
if (tool === "environment_check") {
|
|
2160
|
+
value = environmentCheck(input);
|
|
2161
|
+
} else if (tool === "experiment_run") {
|
|
2162
|
+
value = experimentRun(args, input);
|
|
2163
|
+
} else if (tool === "report_pdf") {
|
|
2164
|
+
value = reportPdf(input);
|
|
2165
|
+
} else if (tool === "web_capture_read") {
|
|
1289
2166
|
if (args.id !== input.capture.id) throw new Error();
|
|
1290
2167
|
value = { ...input.capture, provenance: "operator-supplied-unverified" };
|
|
1291
2168
|
} else if (tool === "web_capture_compare") {
|
|
@@ -1299,19 +2176,17 @@ try {
|
|
|
1299
2176
|
};
|
|
1300
2177
|
} else if (tool === "change_inspect") {
|
|
1301
2178
|
value = input;
|
|
1302
|
-
} else if (["test_run", "semgrep_scan", "bash_run", "secret_scan", "dependency_scan"].includes(tool)) {
|
|
2179
|
+
} else if (["test_run", "package_import_probe", "semgrep_scan", "bash_run", "secret_scan", "dependency_scan", "joern_confirm"].includes(tool)) {
|
|
1303
2180
|
const path = require("node:path");
|
|
1304
2181
|
const child = require("node:child_process");
|
|
1305
2182
|
const root = fs.mkdtempSync("/tmp/mettle-task-");
|
|
1306
2183
|
const workspace = path.join(root, "workspace");
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
}
|
|
1314
|
-
if (tool === "secret_scan" || tool === "dependency_scan") {
|
|
2184
|
+
materializeWorkspace(workspace, input.files);
|
|
2185
|
+
if (tool === "package_import_probe") {
|
|
2186
|
+
value = packageImportProbe(input, args, workspace, root);
|
|
2187
|
+
} else if (tool === "joern_confirm") {
|
|
2188
|
+
value = scanJoern(input, workspace, root);
|
|
2189
|
+
} else if (tool === "secret_scan" || tool === "dependency_scan") {
|
|
1315
2190
|
value = scanSource(tool, input, workspace, root);
|
|
1316
2191
|
} else if (tool === "bash_run") {
|
|
1317
2192
|
const profile = input.profile;
|
|
@@ -1401,24 +2276,32 @@ try {
|
|
|
1401
2276
|
}
|
|
1402
2277
|
} else throw new Error();
|
|
1403
2278
|
const result = JSON.stringify({ inputDigest: crypto.createHash("sha256").update(bytes).digest("hex"), value });
|
|
1404
|
-
if (Buffer.byteLength(result) > ${TASK_RESULT_LIMIT}) throw new Error();
|
|
2279
|
+
if (Buffer.byteLength(result) > (tool === "report_pdf" ? ${PDF_RESULT_LIMIT} : ${TASK_RESULT_LIMIT})) throw new Error();
|
|
1405
2280
|
fs.writeFileSync(1, result);
|
|
1406
2281
|
} catch { process.exitCode = 1; }
|
|
1407
2282
|
`;
|
|
1408
2283
|
|
|
1409
2284
|
// apps/harness/src/sandbox/tasks.ts
|
|
1410
|
-
var
|
|
2285
|
+
var sandboxReceipt = strictObject({
|
|
1411
2286
|
runtime: SandboxRuntimeSchema,
|
|
1412
2287
|
image: string(),
|
|
1413
2288
|
policyDigest: string(),
|
|
1414
2289
|
inputDigest: string(),
|
|
1415
2290
|
cleanupVerified: literal(true)
|
|
1416
2291
|
});
|
|
2292
|
+
var SandboxReceiptSchema = union([
|
|
2293
|
+
sandboxReceipt,
|
|
2294
|
+
sandboxReceipt.extend({
|
|
2295
|
+
cleanupVerified: literal(false),
|
|
2296
|
+
retainedPageId: uuid()
|
|
2297
|
+
})
|
|
2298
|
+
]);
|
|
1417
2299
|
function isolatedTask(tool, args, input, session) {
|
|
1418
2300
|
TaskNameSchema.parse(tool);
|
|
1419
2301
|
const bytes = Buffer.from(canonicalJson({ tool, args, input }), "utf8");
|
|
1420
|
-
|
|
1421
|
-
|
|
2302
|
+
const limit = TASK_INPUT_LIMITS[tool] ?? TASK_INPUT_LIMIT;
|
|
2303
|
+
if (bytes.length > limit)
|
|
2304
|
+
throw new HarnessError("sandbox_input_limit", `The isolated ${tool} task exceeds its ${limit}-byte input limit.`);
|
|
1422
2305
|
return {
|
|
1423
2306
|
tool,
|
|
1424
2307
|
args,
|
|
@@ -1428,9 +2311,23 @@ function isolatedTask(tool, args, input, session) {
|
|
|
1428
2311
|
};
|
|
1429
2312
|
}
|
|
1430
2313
|
function taskProfile(tool) {
|
|
1431
|
-
|
|
2314
|
+
if (tool === "joern_confirm")
|
|
2315
|
+
return "analysis";
|
|
2316
|
+
return [
|
|
2317
|
+
"browser_workflow",
|
|
2318
|
+
"web_browse",
|
|
2319
|
+
"zap_scan",
|
|
2320
|
+
"environment_check",
|
|
2321
|
+
"report_pdf"
|
|
2322
|
+
].includes(tool) ? "browser" : "tools";
|
|
1432
2323
|
}
|
|
1433
2324
|
function taskTimeoutMs(tool) {
|
|
2325
|
+
if (tool === "joern_confirm")
|
|
2326
|
+
return JOERN_TASK_TIMEOUT_MS;
|
|
2327
|
+
if (tool === "port_scan")
|
|
2328
|
+
return PORT_SCAN_TASK_TIMEOUT_MS;
|
|
2329
|
+
if (tool === "environment_check")
|
|
2330
|
+
return 90000;
|
|
1434
2331
|
if (["web_discover", "web_browse"].includes(tool))
|
|
1435
2332
|
return ACTIVE_SCAN_TASK_TIMEOUT_MS;
|
|
1436
2333
|
if (ACTIVE_SCAN_TOOLS.some((name) => name === tool))
|
|
@@ -1439,11 +2336,14 @@ function taskTimeoutMs(tool) {
|
|
|
1439
2336
|
return SOURCE_SCANNER_TASK_TIMEOUT_MS;
|
|
1440
2337
|
return tool === "semgrep_scan" ? SEMGREP_TASK_TIMEOUT_MS : TASK_TIMEOUT_MS;
|
|
1441
2338
|
}
|
|
1442
|
-
function
|
|
2339
|
+
function taskResultLimit(tool) {
|
|
2340
|
+
return tool === "report_pdf" ? PDF_RESULT_LIMIT : TASK_RESULT_LIMIT;
|
|
2341
|
+
}
|
|
2342
|
+
function taskSpec(image, labels, profile, runtime, retained = false) {
|
|
1443
2343
|
return {
|
|
1444
2344
|
...launchSpec(image, "", labels, profile, runtime),
|
|
1445
2345
|
Entrypoint: ["/bin/sleep"],
|
|
1446
|
-
Cmd: [String(TASK_CONTAINER_LIFETIME_S)]
|
|
2346
|
+
Cmd: [String(retained ? 600 : TASK_CONTAINER_LIFETIME_S)]
|
|
1447
2347
|
};
|
|
1448
2348
|
}
|
|
1449
2349
|
function taskExecSpec(live, runtime) {
|
|
@@ -1470,7 +2370,8 @@ function taskRuntimeDigest(image, runtime) {
|
|
|
1470
2370
|
return digest({
|
|
1471
2371
|
specs: {
|
|
1472
2372
|
tools: taskSpec(image, {}, "tools", runtime),
|
|
1473
|
-
browser: taskSpec(image, {}, "browser", runtime)
|
|
2373
|
+
browser: taskSpec(image, {}, "browser", runtime),
|
|
2374
|
+
analysis: taskSpec(image, {}, "analysis", runtime)
|
|
1474
2375
|
},
|
|
1475
2376
|
exec: taskExecSpec(false, runtime),
|
|
1476
2377
|
liveExec: taskExecSpec(true, runtime),
|
|
@@ -1481,7 +2382,7 @@ function taskPolicyDigest(image, task, runtime) {
|
|
|
1481
2382
|
const profile = taskProfile(task.tool);
|
|
1482
2383
|
return digest({
|
|
1483
2384
|
profile,
|
|
1484
|
-
spec: taskSpec(image, {}, profile, runtime),
|
|
2385
|
+
spec: taskSpec(image, {}, profile, runtime, Boolean(task.session?.page)),
|
|
1485
2386
|
exec: taskExecSpec(isLiveTask(task.tool), runtime),
|
|
1486
2387
|
tool: task.tool,
|
|
1487
2388
|
inputDigest: task.inputDigest,
|
|
@@ -1490,6 +2391,7 @@ function taskPolicyDigest(image, task, runtime) {
|
|
|
1490
2391
|
}
|
|
1491
2392
|
function isLiveTask(tool) {
|
|
1492
2393
|
return [
|
|
2394
|
+
"port_scan",
|
|
1493
2395
|
"http_request",
|
|
1494
2396
|
"web_discover",
|
|
1495
2397
|
"web_request",
|
|
@@ -1510,21 +2412,25 @@ async function checkSandbox(config, engine) {
|
|
|
1510
2412
|
if (selected === "runc" && (!Array.isArray(info.SecurityOptions) || !info.SecurityOptions.some((option) => typeof option === "string" && option.split(",").includes("name=seccomp"))))
|
|
1511
2413
|
throw new HarnessError("sandbox_unavailable", "Local Docker requires kernel seccomp support and the explicit builtin profile.");
|
|
1512
2414
|
const image = dockerJson(await engine.call("GET", `/images/${encodeURIComponent(config.image)}/json`));
|
|
1513
|
-
if (!Array.isArray(image?.RepoDigests) || !image.RepoDigests.includes(config.image) || Object.keys(image.Config?.Volumes ?? {}).length !== 0)
|
|
2415
|
+
if ((config.image.startsWith("sha256:") ? image?.Id !== config.image : !Array.isArray(image?.RepoDigests) || !image.RepoDigests.includes(config.image)) || Object.keys(image.Config?.Volumes ?? {}).length !== 0)
|
|
1514
2416
|
throw new HarnessError("sandbox_image", "Preload the approved digest-pinned probe image without declared volumes.");
|
|
1515
2417
|
return info.ID;
|
|
1516
2418
|
}
|
|
1517
2419
|
async function sandboxFromEnvironment(chat, environment = process.env) {
|
|
1518
2420
|
if (!environment.HARNESS_SANDBOX_IMAGE) {
|
|
1519
|
-
if (environment.HARNESS_SANDBOX_RUNTIME !== undefined)
|
|
1520
|
-
throw new HarnessError("sandbox_config", "
|
|
2421
|
+
if (environment.HARNESS_SANDBOX_RUNTIME !== undefined || environment.HARNESS_SANDBOX_ENGINE_ID !== undefined || environment.HARNESS_SANDBOX_LOCATION !== undefined)
|
|
2422
|
+
throw new HarnessError("sandbox_config", "Explicit sandbox settings require HARNESS_SANDBOX_IMAGE.");
|
|
1521
2423
|
return;
|
|
1522
2424
|
}
|
|
1523
2425
|
if (!chat)
|
|
1524
2426
|
throw new HarnessError("sandbox_config", "HARNESS_SANDBOX_IMAGE requires HARNESS_CHAT=1.");
|
|
1525
2427
|
const config = sandboxConfig(environment);
|
|
1526
|
-
await checkSandbox(config, new DockerEngine(config.socket));
|
|
1527
|
-
return
|
|
2428
|
+
const engineId = await checkSandbox(config, new DockerEngine(config.socket));
|
|
2429
|
+
return Object.freeze({
|
|
2430
|
+
...config,
|
|
2431
|
+
engineId,
|
|
2432
|
+
location: config.location ?? "local"
|
|
2433
|
+
});
|
|
1528
2434
|
}
|
|
1529
2435
|
var CONTAINMENT_FINDINGS = new Set([
|
|
1530
2436
|
"sandbox_policy",
|
|
@@ -1542,7 +2448,7 @@ function cancelled(error) {
|
|
|
1542
2448
|
function jobSpec(record2, job, runtime) {
|
|
1543
2449
|
const labels = sandboxLabels(record2);
|
|
1544
2450
|
if (job?.kind === "task")
|
|
1545
|
-
return taskSpec(record2.image, labels, taskProfile(job.task.tool), runtime);
|
|
2451
|
+
return taskSpec(record2.image, labels, taskProfile(job.task.tool), runtime, Boolean(job.task.session?.page));
|
|
1546
2452
|
if (job?.kind === "shell")
|
|
1547
2453
|
return shellSpec(record2.image, job.command, labels, runtime);
|
|
1548
2454
|
return launchSpec(record2.image, probeScript(runtime), labels, "probe", runtime);
|
|
@@ -1730,7 +2636,11 @@ class SandboxWorker {
|
|
|
1730
2636
|
const workerId = await checkSandbox(this.config, this.engine);
|
|
1731
2637
|
if (signal.aborted)
|
|
1732
2638
|
throw new HarnessError("stop_requested", "Sandbox execution was stopped before allocation.");
|
|
1733
|
-
const
|
|
2639
|
+
const page = task?.tool === "web_browse" ? task.session?.page : undefined;
|
|
2640
|
+
if (page && (!this.journal.acquirePage || !this.journal.releasePage))
|
|
2641
|
+
throw new HarnessError("browser_page_denied", "Retained pages require durable lifecycle support.");
|
|
2642
|
+
const reused = page && page.operation !== "open";
|
|
2643
|
+
const record2 = reused ? await this.journal.acquirePage(callId, task.args, page) : await this.journal.reserve(callId, workerId, this.config.image, task ? taskPolicyDigest(this.config.image, task, this.runtime) : job?.kind === "shell" ? shellPolicyDigest(this.config.image, job.command, this.runtime) : probePolicyDigest(this.config.image, this.runtime), task ? { tool: task.tool, args: task.args } : job?.kind === "shell" ? { tool: SHELL_TOOL, args: { command: job.command } } : undefined, page);
|
|
1734
2644
|
const deadline = AbortSignal.timeout(task ? taskTimeoutMs(task.tool) : PROBE_TIMEOUT_MS);
|
|
1735
2645
|
const bounded = AbortSignal.any([signal, deadline]);
|
|
1736
2646
|
let primary;
|
|
@@ -1738,23 +2648,27 @@ class SandboxWorker {
|
|
|
1738
2648
|
let exitCode = null;
|
|
1739
2649
|
let output = Buffer.alloc(0);
|
|
1740
2650
|
try {
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
2651
|
+
if (!reused)
|
|
2652
|
+
await this.journal.provision(record2, async () => {
|
|
2653
|
+
const created = dockerCreated(await this.engine.call("POST", `/containers/create?name=${record2.name}`, jobSpec(record2, job, this.runtime), bounded, ENGINE_LAUNCH_TIMEOUT_MS));
|
|
2654
|
+
if (!/^[a-f0-9]{64}$/.test(created?.Id ?? ""))
|
|
2655
|
+
throw new HarnessError("sandbox_protocol", "Container creation did not return a valid identity.");
|
|
2656
|
+
return created.Id;
|
|
2657
|
+
});
|
|
1747
2658
|
const container = await this.inspect(record2);
|
|
1748
2659
|
if (!container)
|
|
1749
2660
|
throw new HarnessError("sandbox_missing", "The created sandbox disappeared before execution.");
|
|
2661
|
+
if (record2.workerId !== workerId || record2.image !== this.config.image)
|
|
2662
|
+
throw new HarnessError("sandbox_identity", "The retained page belongs to a different engine or image.");
|
|
1750
2663
|
this.assertPolicy(container, record2, job);
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
2664
|
+
if (!reused)
|
|
2665
|
+
await this.journal.start(record2, async () => {
|
|
2666
|
+
if (bounded.aborted)
|
|
2667
|
+
throw new HarnessError("stop_requested", "Sandbox dispatch was cancelled.");
|
|
2668
|
+
dockerStatus(await this.engine.call("POST", `/containers/${record2.containerId}/start`, undefined, bounded, ENGINE_LAUNCH_TIMEOUT_MS), 204);
|
|
2669
|
+
});
|
|
1756
2670
|
if (task)
|
|
1757
|
-
value = await this.runIsolated(record2, task, bounded);
|
|
2671
|
+
value = await this.runIsolated(record2, task, bounded, callId);
|
|
1758
2672
|
else {
|
|
1759
2673
|
const exit = await this.engine.call("POST", `/containers/${record2.containerId}/wait?condition=not-running`, undefined, bounded, PROBE_TIMEOUT_MS).then((response) => dockerJson(response), (error) => {
|
|
1760
2674
|
if (shell && deadline.aborted && !signal.aborted && cancelled(error))
|
|
@@ -1779,8 +2693,16 @@ class SandboxWorker {
|
|
|
1779
2693
|
} catch (error) {
|
|
1780
2694
|
primary = containmentFinding(error) ? error : signal.aborted ? new HarnessError("stop_requested", "Sandbox execution was cancelled; no result is invented.") : deadline.aborted && cancelled(error) ? new HarnessError("sandbox_timeout", "The sandbox exceeded its wall deadline.") : error;
|
|
1781
2695
|
}
|
|
2696
|
+
const retained = Boolean(page && page.operation !== "close" && primary === undefined);
|
|
1782
2697
|
try {
|
|
1783
|
-
if (
|
|
2698
|
+
if (retained) {
|
|
2699
|
+
try {
|
|
2700
|
+
await this.journal.releasePage(record2, callId);
|
|
2701
|
+
} catch (error) {
|
|
2702
|
+
await this.cleanup(record2, workerId);
|
|
2703
|
+
throw error;
|
|
2704
|
+
}
|
|
2705
|
+
} else if (primary instanceof HarnessError && primary.code === "sandbox_rejected" && !record2.containerId)
|
|
1784
2706
|
await this.journal.removed(record2);
|
|
1785
2707
|
else
|
|
1786
2708
|
await this.cleanup(record2, workerId);
|
|
@@ -1798,7 +2720,8 @@ class SandboxWorker {
|
|
|
1798
2720
|
image: record2.image,
|
|
1799
2721
|
policyDigest: record2.policyDigest,
|
|
1800
2722
|
inputDigest: task.inputDigest,
|
|
1801
|
-
cleanupVerified:
|
|
2723
|
+
cleanupVerified: !retained,
|
|
2724
|
+
...retained ? { retainedPageId: page.id } : {}
|
|
1802
2725
|
}
|
|
1803
2726
|
};
|
|
1804
2727
|
if (job?.kind === "shell") {
|
|
@@ -1821,7 +2744,7 @@ class SandboxWorker {
|
|
|
1821
2744
|
cleanupVerified: true
|
|
1822
2745
|
};
|
|
1823
2746
|
}
|
|
1824
|
-
async runIsolated(record2, task, signal) {
|
|
2747
|
+
async runIsolated(record2, task, signal, callId) {
|
|
1825
2748
|
const live = isLiveTask(task.tool);
|
|
1826
2749
|
if (live && !this.createBroker)
|
|
1827
2750
|
throw new HarnessError("web_denied", "Live tasks require the operator-approved broker; no network fallback is allowed.");
|
|
@@ -1829,12 +2752,12 @@ class SandboxWorker {
|
|
|
1829
2752
|
const created = dockerJson(await this.engine.call("POST", `/containers/${record2.containerId}/exec`, spec, signal), 201);
|
|
1830
2753
|
if (!/^[a-f0-9]{64}$/.test(created?.Id ?? ""))
|
|
1831
2754
|
throw new HarnessError("sandbox_protocol", "The engine did not return an isolated exec identity.");
|
|
1832
|
-
const inspect = () => this.engine.call("GET", `/exec/${created.Id}/json`, undefined, signal).then((response) => dockerJson(response));
|
|
2755
|
+
const inspect = () => this.engine.call("GET", `/exec/${created.Id}/json`, undefined, signal, undefined, Math.max(65536, Buffer.byteLength(canonicalJson(spec)) * 2 + 8192)).then((response) => dockerJson(response));
|
|
1833
2756
|
const initial = await inspect();
|
|
1834
2757
|
if (initial.ID !== created.Id || initial.ContainerID !== record2.containerId || initial.Running !== false || initial.OpenStdin !== true || initial.OpenStdout !== true || initial.OpenStderr !== true || initial.ProcessConfig?.privileged !== false || initial.ProcessConfig?.tty !== false || initial.ProcessConfig?.user !== spec.User || initial.ProcessConfig?.entrypoint !== spec.Cmd[0] || canonicalJson(initial.ProcessConfig?.arguments) !== canonicalJson(spec.Cmd.slice(1)))
|
|
1835
2758
|
throw new HarnessError("sandbox_policy", "The engine did not retain the fixed isolated task command.");
|
|
1836
2759
|
await this.journal.assertTaskRunning(record2);
|
|
1837
|
-
const broker = live ? await this.createBroker(task, signal,
|
|
2760
|
+
const broker = live ? await this.createBroker(task, signal, callId) : undefined;
|
|
1838
2761
|
let refusal;
|
|
1839
2762
|
const exchange = broker ? async (request) => {
|
|
1840
2763
|
try {
|
|
@@ -1848,21 +2771,26 @@ class SandboxWorker {
|
|
|
1848
2771
|
}
|
|
1849
2772
|
} : undefined;
|
|
1850
2773
|
await this.journal.assertTaskRunning(record2);
|
|
1851
|
-
|
|
2774
|
+
let failed = true;
|
|
1852
2775
|
try {
|
|
1853
|
-
|
|
2776
|
+
const session = await this.engine.startExec(created.Id, task.input, signal, taskTimeoutMs(task.tool), exchange, Math.max(65536, taskResultLimit(task.tool) + 16384));
|
|
2777
|
+
const result = await this.collectIsolated(record2, task, created.Id, session, inspect, signal, broker);
|
|
2778
|
+
failed = false;
|
|
2779
|
+
return result;
|
|
1854
2780
|
} catch (error) {
|
|
1855
2781
|
if (broker?.inFlight())
|
|
1856
2782
|
throw new HarnessError("web_uncertain", "A live request was in flight when the task failed; inspect the journal before any retry.");
|
|
1857
2783
|
if (refusal !== undefined)
|
|
1858
2784
|
throw refusal;
|
|
1859
2785
|
throw error;
|
|
2786
|
+
} finally {
|
|
2787
|
+
await broker?.close?.(failed);
|
|
1860
2788
|
}
|
|
1861
2789
|
}
|
|
1862
2790
|
async collectIsolated(record2, task, execId, session, inspect, signal, broker) {
|
|
1863
2791
|
const created = { Id: execId };
|
|
1864
2792
|
const output = dockerLogs(await session.result);
|
|
1865
|
-
if (Buffer.byteLength(output) >
|
|
2793
|
+
if (Buffer.byteLength(output) > taskResultLimit(task.tool))
|
|
1866
2794
|
throw new HarnessError("sandbox_output_limit", "The isolated result exceeds its size limit.");
|
|
1867
2795
|
while (true) {
|
|
1868
2796
|
const status = await inspect();
|
|
@@ -1887,4 +2815,4 @@ class SandboxWorker {
|
|
|
1887
2815
|
}
|
|
1888
2816
|
}
|
|
1889
2817
|
|
|
1890
|
-
export {
|
|
2818
|
+
export { counted, ran, scanned, ToolRegistry, JOERN_QUERY_ID, JoernProfileSchema, joernCandidate, EcosystemSchema, MAX_DATABASE_BYTES, SecretScanProfileSchema, DependencyScanProfileSchema, SOURCE_SCANNER_LABEL, ScannerImageProfilesSchema, SCANNER_FINDINGS_MAX, ScannerStatusSchema, PDF_BYTES_LIMIT, PDF_RESULT_LIMIT, REPO_SEARCH_FILES, REPO_SEARCH_MATCHES, REPO_EXCERPT, DockerEngine, dockerJson, sandboxConfig, ACTIVE_SCAN_TASK_TIMEOUT_MS, ACTIVE_SCAN_TOOLS, ActiveScannersSchema, scannerEndpoint, validateActiveScanners, SCANNER_REQUEST_STRIP_HEADERS, scannerRequestAllowed, sqlmapCandidate, PORT_CONNECT_TIMEOUT_MS, PORT_SCAN_TASK_TIMEOUT_MS, PortScanProfileSchema, PortOutcomeSchema, portDestination, JOERN_QUERY, packageJavaScript, PackageImportObservationSchema, PackageImportResultSchema, SandboxReceiptSchema, taskRuntimeDigest, checkSandbox, sandboxFromEnvironment, SandboxWorker };
|