@mettlelabs/mettle 0.0.0-beta.1-linux-arm64 → 0.0.0-beta.2-darwin-arm64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +503 -8
- package/bin/mettle +2 -2
- package/harness/audit-investigation.mjs +16 -16
- package/harness/chunk-3wt8a8fq.mjs +1716 -0
- package/harness/{chunk-13tgmdey.mjs → chunk-49ytr8d5.mjs} +1 -1
- package/harness/chunk-4vx34w5h.mjs +2357 -0
- package/harness/{chunk-xqyc4ca2.mjs → chunk-9dg1btk1.mjs} +1220 -292
- package/harness/chunk-c7dx5mdd.mjs +10807 -0
- package/harness/{chunk-msg872ea.mjs → chunk-h1td5kq9.mjs} +97 -9
- package/harness/{chunk-4jtnt9k5.mjs → chunk-jp69h6sj.mjs} +1 -1
- package/harness/chunk-rd80vvc5.mjs +22047 -0
- package/harness/{chunk-5cvb1jnb.mjs → chunk-w0qwc5z2.mjs} +2 -2
- package/harness/issue-api-token.mjs +2 -2
- package/harness/launcher.mjs +318 -18
- package/harness/sandbox.mjs +8 -7
- package/harness/server.mjs +22883 -1891
- package/package.json +2 -5
- package/harness/chunk-2gt4cy7x.mjs +0 -2290
- package/harness/chunk-32y07kkp.mjs +0 -3747
- package/harness/chunk-7bg2gnv2.mjs +0 -7540
|
@@ -0,0 +1,2357 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CONTRACT_VERSION,
|
|
3
|
+
SEAM_VERSION,
|
|
4
|
+
IdSchema,
|
|
5
|
+
ChangeContextSchema,
|
|
6
|
+
RunConfigSchema,
|
|
7
|
+
ArtifactRefSchema,
|
|
8
|
+
WebIdentitySchema,
|
|
9
|
+
EndpointInputSchema,
|
|
10
|
+
EndpointLeadSchema,
|
|
11
|
+
ApiDescriptionSchema,
|
|
12
|
+
WebSurfaceSchema,
|
|
13
|
+
ApiDescriptionReferenceSchema,
|
|
14
|
+
InvestigationDepthSchema,
|
|
15
|
+
InvestigationPlanBindingSchema,
|
|
16
|
+
packageIdentity,
|
|
17
|
+
SourceReadPolicySchema,
|
|
18
|
+
SOURCE_ANALYSIS_VERSION,
|
|
19
|
+
SOURCE_PARSER_ID,
|
|
20
|
+
EventSchema,
|
|
21
|
+
TargetIdentitySchema,
|
|
22
|
+
VerifyRequestSchema,
|
|
23
|
+
WebsiteFamilySchema,
|
|
24
|
+
caseSteps,
|
|
25
|
+
InvestigationModelSchema,
|
|
26
|
+
SourceDeploymentBindingSchema,
|
|
27
|
+
PROVIDER_KEYS,
|
|
28
|
+
isLoopback,
|
|
29
|
+
providerFixture,
|
|
30
|
+
providerKey,
|
|
31
|
+
catalogIdentities,
|
|
32
|
+
offeredIdentities,
|
|
33
|
+
HarnessError,
|
|
34
|
+
canonicalJson,
|
|
35
|
+
sha256,
|
|
36
|
+
digest,
|
|
37
|
+
endpointInput,
|
|
38
|
+
requestInputs,
|
|
39
|
+
endpointShape,
|
|
40
|
+
endpointIdentity,
|
|
41
|
+
matchesEndpointTemplate,
|
|
42
|
+
apiDescription,
|
|
43
|
+
captureResponseText,
|
|
44
|
+
captureEvidence,
|
|
45
|
+
candidateRevision,
|
|
46
|
+
admitIndependentVerification,
|
|
47
|
+
packageSourceMatches,
|
|
48
|
+
sourceRoute,
|
|
49
|
+
routeTarget,
|
|
50
|
+
validateOracleVerdict
|
|
51
|
+
} from "./chunk-rd80vvc5.mjs";
|
|
52
|
+
import {
|
|
53
|
+
CommitSchema,
|
|
54
|
+
RepositoryConfigSchema,
|
|
55
|
+
hasRepositorySource,
|
|
56
|
+
RepositoryTestProfileSchema,
|
|
57
|
+
BashProfilesSchema,
|
|
58
|
+
ExperimentPolicySchema,
|
|
59
|
+
SemgrepProfileSchema,
|
|
60
|
+
WebCaptureSchema,
|
|
61
|
+
PermissionPolicySchema,
|
|
62
|
+
GVISOR_RUNTIME,
|
|
63
|
+
SandboxConfigSchema,
|
|
64
|
+
sandboxRuntime,
|
|
65
|
+
sandboxAllowsExecution,
|
|
66
|
+
TaskNameSchema,
|
|
67
|
+
SCANNER_CAPABILITIES,
|
|
68
|
+
privateFile,
|
|
69
|
+
SOURCE_TOTAL_BYTES,
|
|
70
|
+
SOURCE_FILES_MAX,
|
|
71
|
+
executionPlan,
|
|
72
|
+
workStrategy,
|
|
73
|
+
sourceManifest,
|
|
74
|
+
withinHttpScope,
|
|
75
|
+
PublicAddressSchema,
|
|
76
|
+
RegisteredAddressSchema,
|
|
77
|
+
LiveWebSchema,
|
|
78
|
+
validateLiveWeb,
|
|
79
|
+
OwnershipProofSchema,
|
|
80
|
+
ownsHost,
|
|
81
|
+
ownershipToken,
|
|
82
|
+
WEBSITE_PASSIVE_METHODS,
|
|
83
|
+
WebScopeSchema,
|
|
84
|
+
validateRegisteredWebTarget,
|
|
85
|
+
validateWebTarget,
|
|
86
|
+
webRequestAllowed,
|
|
87
|
+
webExplicitAddress,
|
|
88
|
+
WorkJournal,
|
|
89
|
+
ownsVerificationJob,
|
|
90
|
+
WEBSITE_DISCOVERY_TOOLS,
|
|
91
|
+
isWebsiteCoverageTool,
|
|
92
|
+
counterModelProfile,
|
|
93
|
+
liveWebScope,
|
|
94
|
+
authorizationIdentities,
|
|
95
|
+
authorizationObservations,
|
|
96
|
+
authorizationAssessment,
|
|
97
|
+
WebsiteWorkPolicySchema,
|
|
98
|
+
WebsiteWorklist,
|
|
99
|
+
PostgresJournal
|
|
100
|
+
} from "./chunk-c7dx5mdd.mjs";
|
|
101
|
+
import {
|
|
102
|
+
RunAuthoritySchema
|
|
103
|
+
} from "./chunk-zvkxtj82.mjs";
|
|
104
|
+
import {
|
|
105
|
+
FoundryClient,
|
|
106
|
+
openAIPricing,
|
|
107
|
+
OpenAIClient,
|
|
108
|
+
byteInputBound,
|
|
109
|
+
imageInputBound
|
|
110
|
+
} from "./chunk-3wt8a8fq.mjs";
|
|
111
|
+
import {
|
|
112
|
+
JoernProfileSchema,
|
|
113
|
+
joernCandidate,
|
|
114
|
+
SecretScanProfileSchema,
|
|
115
|
+
DependencyScanProfileSchema,
|
|
116
|
+
ActiveScannersSchema,
|
|
117
|
+
validateActiveScanners,
|
|
118
|
+
sqlmapCandidate,
|
|
119
|
+
PortScanProfileSchema,
|
|
120
|
+
packageJavaScript
|
|
121
|
+
} from "./chunk-9dg1btk1.mjs";
|
|
122
|
+
import {
|
|
123
|
+
exports_iso
|
|
124
|
+
} from "./chunk-b71gpkxg.mjs";
|
|
125
|
+
import {
|
|
126
|
+
NEVER,
|
|
127
|
+
string,
|
|
128
|
+
uuid,
|
|
129
|
+
url,
|
|
130
|
+
number,
|
|
131
|
+
boolean,
|
|
132
|
+
array,
|
|
133
|
+
object,
|
|
134
|
+
strictObject,
|
|
135
|
+
discriminatedUnion,
|
|
136
|
+
record,
|
|
137
|
+
_enum,
|
|
138
|
+
literal
|
|
139
|
+
} from "./chunk-1sygmp11.mjs";
|
|
140
|
+
|
|
141
|
+
// apps/harness/src/runtime/models.ts
|
|
142
|
+
var INPUT_BOUND_REVISION = "development-byte-bound-v1";
|
|
143
|
+
var FIRST_TOKEN_MS = 30000;
|
|
144
|
+
var MS_PER_OUTPUT_TOKEN = 25;
|
|
145
|
+
function deadline(maxOutputTokens) {
|
|
146
|
+
return FIRST_TOKEN_MS + maxOutputTokens * MS_PER_OUTPUT_TOKEN;
|
|
147
|
+
}
|
|
148
|
+
var OPENAI = Object.freeze({
|
|
149
|
+
"gpt-5.5": {
|
|
150
|
+
snapshot: "gpt-5.5-2026-04-23",
|
|
151
|
+
effort: "medium",
|
|
152
|
+
rates: {
|
|
153
|
+
revision: "openai-gpt-5.5-standard-2026-09-29",
|
|
154
|
+
longInputTokens: 272000,
|
|
155
|
+
standard: {
|
|
156
|
+
inputNanousdPerToken: 5000,
|
|
157
|
+
cachedInputNanousdPerToken: 500,
|
|
158
|
+
outputNanousdPerToken: 30000
|
|
159
|
+
},
|
|
160
|
+
long: {
|
|
161
|
+
inputNanousdPerToken: 1e4,
|
|
162
|
+
cachedInputNanousdPerToken: 1000,
|
|
163
|
+
outputNanousdPerToken: 45000
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
function openAIModel(identity) {
|
|
169
|
+
const model = { ...identity, ...OPENAI[identity.model] };
|
|
170
|
+
const pricing = openAIPricing(model.rates);
|
|
171
|
+
return Object.freeze({
|
|
172
|
+
model: model.model,
|
|
173
|
+
provider: "openai",
|
|
174
|
+
id: `openai:responses:${model.model}:${model.effort}:${pricing.revision}`,
|
|
175
|
+
responseModels: Object.freeze([model.model, model.snapshot]),
|
|
176
|
+
contextTokens: model.contextTokens,
|
|
177
|
+
maxOutputTokens: model.maxOutputTokens,
|
|
178
|
+
timeoutMs: deadline(model.maxOutputTokens),
|
|
179
|
+
imageTokens: model.imageTokens,
|
|
180
|
+
inputBoundRevision: `${INPUT_BOUND_REVISION}-image-${model.imageTokens}`,
|
|
181
|
+
pricing,
|
|
182
|
+
client: (apiKey, options) => new OpenAIClient({
|
|
183
|
+
apiKey,
|
|
184
|
+
model: model.model,
|
|
185
|
+
snapshot: model.snapshot,
|
|
186
|
+
effort: model.effort,
|
|
187
|
+
rates: model.rates,
|
|
188
|
+
...options.fetch ? { fetch: options.fetch } : {}
|
|
189
|
+
})
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
function foundryModel(identity) {
|
|
193
|
+
const { endpoint, deployment } = identity;
|
|
194
|
+
return Object.freeze({
|
|
195
|
+
maxOutputTokens: identity.maxOutputTokens,
|
|
196
|
+
timeoutMs: deadline(identity.maxOutputTokens),
|
|
197
|
+
inputBoundRevision: INPUT_BOUND_REVISION,
|
|
198
|
+
model: identity.model,
|
|
199
|
+
provider: "foundry",
|
|
200
|
+
id: `foundry:${new URL(endpoint).hostname}:${deployment.name}:${deployment.model}`,
|
|
201
|
+
responseModels: Object.freeze([
|
|
202
|
+
...new Set([deployment.model, deployment.name])
|
|
203
|
+
]),
|
|
204
|
+
contextTokens: identity.contextTokens,
|
|
205
|
+
client: (apiKey, options) => new FoundryClient({
|
|
206
|
+
apiKey,
|
|
207
|
+
model: identity.model,
|
|
208
|
+
deployment: deployment.name,
|
|
209
|
+
endpoint,
|
|
210
|
+
...options.fetch ? { fetch: options.fetch } : {}
|
|
211
|
+
})
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
function catalogModels(environment) {
|
|
215
|
+
return catalogIdentities(environment).map((identity) => identity.provider === "foundry" ? foundryModel(identity) : openAIModel(identity));
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// apps/harness/src/runtime/live-profile.ts
|
|
219
|
+
function defaultModel(environment) {
|
|
220
|
+
return offeredIdentities(environment)[0]?.model ?? catalogModels(environment)[0].model;
|
|
221
|
+
}
|
|
222
|
+
function liveRoute(environment = process.env, model = defaultModel(environment)) {
|
|
223
|
+
const route = catalogModels(environment).find((entry) => entry.model === model);
|
|
224
|
+
if (!route)
|
|
225
|
+
throw new HarnessError("unsupported_profile", "The requested live model is not supported.");
|
|
226
|
+
return route;
|
|
227
|
+
}
|
|
228
|
+
function liveClient(environment = process.env, options = {}) {
|
|
229
|
+
const route = liveRoute(environment, options.model);
|
|
230
|
+
const key = providerKey(environment, route.provider);
|
|
231
|
+
if (!key)
|
|
232
|
+
throw new HarnessError("model_credential_missing", `${PROVIDER_KEYS[route.provider]} is required for ${route.model}.`);
|
|
233
|
+
const fixture = options.fetch ?? providerFixture(environment);
|
|
234
|
+
return route.client(key, {
|
|
235
|
+
...options,
|
|
236
|
+
...fixture ? { fetch: fixture } : {}
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
function refuseDispatch() {
|
|
240
|
+
throw new HarnessError("model_dispatch_refused", "This command makes no model calls.");
|
|
241
|
+
}
|
|
242
|
+
var NO_DISPATCH = Object.freeze({
|
|
243
|
+
seamVersion: SEAM_VERSION.model,
|
|
244
|
+
complete: async () => refuseDispatch(),
|
|
245
|
+
stream: () => refuseDispatch()
|
|
246
|
+
});
|
|
247
|
+
function liveCounterProfile(environment = process.env, client, model = liveRoute(environment).model) {
|
|
248
|
+
const route = liveRoute(environment, model);
|
|
249
|
+
return counterModelProfile({
|
|
250
|
+
route: {
|
|
251
|
+
id: route.id,
|
|
252
|
+
provider: route.provider,
|
|
253
|
+
model: route.model,
|
|
254
|
+
responseModels: route.responseModels,
|
|
255
|
+
sampling: [],
|
|
256
|
+
vision: route.imageTokens !== undefined,
|
|
257
|
+
client: client ?? liveClient(environment, { model: route.model })
|
|
258
|
+
},
|
|
259
|
+
inputBound: route.imageTokens === undefined ? byteInputBound : imageInputBound(route.imageTokens),
|
|
260
|
+
inputBoundRevision: route.inputBoundRevision,
|
|
261
|
+
maxOutputTokens: route.maxOutputTokens,
|
|
262
|
+
contextTokens: route.contextTokens,
|
|
263
|
+
timeoutMs: route.timeoutMs,
|
|
264
|
+
pricing: route.pricing
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
function liveProfiles(environment = process.env, model = defaultModel(environment)) {
|
|
268
|
+
const primary = liveCounterProfile(environment, undefined, model);
|
|
269
|
+
return [
|
|
270
|
+
primary,
|
|
271
|
+
...offeredIdentities(environment).filter((identity) => identity.model !== primary.route.model).map((identity) => liveCounterProfile(environment, undefined, identity.model))
|
|
272
|
+
];
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// apps/harness/src/storage/investigation-journal.ts
|
|
276
|
+
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
277
|
+
|
|
278
|
+
// apps/harness/src/storage/adapters.ts
|
|
279
|
+
import { randomUUID } from "node:crypto";
|
|
280
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
281
|
+
var endpoint = strictObject({
|
|
282
|
+
url: url().max(2048).refine((value) => {
|
|
283
|
+
const url2 = new URL(value);
|
|
284
|
+
return ["http:", "https:"].includes(url2.protocol) && withinHttpScope(url2, url2);
|
|
285
|
+
}),
|
|
286
|
+
address: RegisteredAddressSchema
|
|
287
|
+
});
|
|
288
|
+
var credential = strictObject({
|
|
289
|
+
file: string().startsWith("/"),
|
|
290
|
+
sha256: string().regex(/^[a-f0-9]{64}$/)
|
|
291
|
+
});
|
|
292
|
+
var bytes = string().min(1).max(10924).refine((value) => Buffer.from(value, "base64").toString("base64") === value && Buffer.from(value, "base64").length <= 8192);
|
|
293
|
+
var ProtocolProfileSchema = discriminatedUnion("transport", [
|
|
294
|
+
strictObject({
|
|
295
|
+
id: IdSchema,
|
|
296
|
+
transport: literal("raw-http"),
|
|
297
|
+
endpoint,
|
|
298
|
+
control: bytes,
|
|
299
|
+
probe: bytes
|
|
300
|
+
}),
|
|
301
|
+
strictObject({
|
|
302
|
+
id: IdSchema,
|
|
303
|
+
transport: literal("websocket"),
|
|
304
|
+
endpoint,
|
|
305
|
+
control: array(bytes).min(1).max(8),
|
|
306
|
+
probe: array(bytes).min(1).max(8),
|
|
307
|
+
receiveMessages: number().int().min(1).max(8),
|
|
308
|
+
binary: boolean()
|
|
309
|
+
})
|
|
310
|
+
]).refine((value) => {
|
|
311
|
+
if (value.transport !== "websocket" || value.binary)
|
|
312
|
+
return true;
|
|
313
|
+
try {
|
|
314
|
+
for (const encoded of [...value.control, ...value.probe])
|
|
315
|
+
new TextDecoder("utf8", { fatal: true }).decode(Buffer.from(encoded, "base64"));
|
|
316
|
+
return true;
|
|
317
|
+
} catch {
|
|
318
|
+
return false;
|
|
319
|
+
}
|
|
320
|
+
}, "WebSocket text recipes must contain valid UTF-8.");
|
|
321
|
+
var AgentTargetProfileSchema = strictObject({
|
|
322
|
+
id: IdSchema,
|
|
323
|
+
transport: _enum(["http-json", "mcp"]),
|
|
324
|
+
endpoint,
|
|
325
|
+
credential: credential.optional(),
|
|
326
|
+
inputField: string().regex(/^[a-zA-Z][a-zA-Z0-9_]{0,63}$/).refine((name) => !["constructor", "prototype", "__proto__", "trialId"].includes(name)),
|
|
327
|
+
toolName: string().regex(/^[a-zA-Z0-9_.-]{1,128}$/).optional(),
|
|
328
|
+
observer: strictObject({
|
|
329
|
+
endpoint,
|
|
330
|
+
credential,
|
|
331
|
+
resetUrl: url(),
|
|
332
|
+
observeUrl: url(),
|
|
333
|
+
independent: literal(true)
|
|
334
|
+
})
|
|
335
|
+
}).superRefine((value, context) => {
|
|
336
|
+
const origin = new URL(value.observer.endpoint.url);
|
|
337
|
+
if (origin.origin === new URL(value.endpoint.url).origin || value.transport === "mcp" !== Boolean(value.toolName) || ![value.observer.resetUrl, value.observer.observeUrl].every((url2) => withinHttpScope(origin, new URL(url2))))
|
|
338
|
+
context.addIssue({
|
|
339
|
+
code: "custom",
|
|
340
|
+
message: "The observer must have its own origin and bounded reset/observe paths; MCP requires an approved tool name."
|
|
341
|
+
});
|
|
342
|
+
});
|
|
343
|
+
var AdapterPolicySchema = strictObject({
|
|
344
|
+
approvedBy: string().min(1).max(256),
|
|
345
|
+
expiresAt: exports_iso.datetime(),
|
|
346
|
+
buildIdentityRef: string().min(1).max(2048),
|
|
347
|
+
timeoutMs: number().int().min(100).max(15000),
|
|
348
|
+
internalScope: boolean().default(false),
|
|
349
|
+
requestsPerSecond: number().int().min(1).max(5).default(1),
|
|
350
|
+
maxRequests: number().int().min(1).max(1000),
|
|
351
|
+
maxTotalBytes: number().int().min(65536).max(134217728),
|
|
352
|
+
concurrency: number().int().min(1).max(8),
|
|
353
|
+
protocols: array(ProtocolProfileSchema).max(16),
|
|
354
|
+
agents: array(AgentTargetProfileSchema).max(16)
|
|
355
|
+
}).refine((value) => {
|
|
356
|
+
const ids = [...value.protocols, ...value.agents].map((profile) => profile.id);
|
|
357
|
+
return ids.length > 0 && new Set(ids).size === ids.length;
|
|
358
|
+
}).superRefine((value, context) => {
|
|
359
|
+
if (value.internalScope)
|
|
360
|
+
return;
|
|
361
|
+
const endpoints = [
|
|
362
|
+
...value.protocols.map((profile) => profile.endpoint),
|
|
363
|
+
...value.agents.flatMap((profile) => [
|
|
364
|
+
profile.endpoint,
|
|
365
|
+
profile.observer.endpoint
|
|
366
|
+
])
|
|
367
|
+
];
|
|
368
|
+
if (endpoints.some((entry) => !PublicAddressSchema.safeParse(entry.address).success))
|
|
369
|
+
context.addIssue({
|
|
370
|
+
code: "custom",
|
|
371
|
+
message: "Private adapter or observer addresses require explicit internalScope approval."
|
|
372
|
+
});
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
class AdapterJournal {
|
|
376
|
+
journal;
|
|
377
|
+
lease;
|
|
378
|
+
callId;
|
|
379
|
+
policy;
|
|
380
|
+
constructor(journal, lease, callId, policy) {
|
|
381
|
+
this.journal = journal;
|
|
382
|
+
this.lease = lease;
|
|
383
|
+
this.callId = callId;
|
|
384
|
+
this.policy = policy;
|
|
385
|
+
}
|
|
386
|
+
async reserve(descriptor, signal) {
|
|
387
|
+
for (;; ) {
|
|
388
|
+
signal.throwIfAborted();
|
|
389
|
+
const reservation = await this.journal.fenced(this.lease, async ({ client }) => {
|
|
390
|
+
signal.throwIfAborted();
|
|
391
|
+
const action = await client.query("SELECT 1 FROM harness_mastra.actions WHERE run_id=$1 AND call_id=$2 AND tool IN ('protocol_run','agent_target') AND state='dispatched' AND child_id IS NOT DISTINCT FROM $3", [this.lease.runId, this.callId, this.lease.child?.id ?? null]);
|
|
392
|
+
const usage = await client.query("SELECT count(*)::int AS requests,count(*) FILTER(WHERE w.state='dispatched')::int AS active,COALESCE(sum(w.bytes),0)::float8 AS bytes,clock_timestamp()>=$2::timestamptz AS expired,GREATEST(0,EXTRACT(EPOCH FROM (max(w.dispatched_at)+($3::double precision * interval '1 millisecond')-clock_timestamp()))*1000)::double precision AS wait_ms FROM harness_mastra.web_requests w JOIN harness_mastra.actions a ON a.run_id=w.run_id AND a.call_id=w.call_id WHERE w.run_id=$1 AND a.tool IN ('protocol_run','agent_target')", [
|
|
393
|
+
this.lease.runId,
|
|
394
|
+
this.policy.expiresAt,
|
|
395
|
+
1000 / this.policy.requestsPerSecond
|
|
396
|
+
]);
|
|
397
|
+
const used = usage.rows[0];
|
|
398
|
+
if (!action.rowCount || used.expired || used.requests >= this.policy.maxRequests || used.bytes + 65536 > this.policy.maxTotalBytes)
|
|
399
|
+
throw new HarnessError("adapter_denied", "Adapter dispatch requires its owning action, active grant and remaining request, concurrency and byte budgets.");
|
|
400
|
+
if (used.active >= this.policy.concurrency || used.wait_ms > 0)
|
|
401
|
+
return {
|
|
402
|
+
waitMs: Math.max(50, Math.ceil(used.wait_ms)),
|
|
403
|
+
id: undefined
|
|
404
|
+
};
|
|
405
|
+
signal.throwIfAborted();
|
|
406
|
+
const id = randomUUID();
|
|
407
|
+
await client.query("INSERT INTO harness_mastra.web_requests(id,run_id,call_id,request_digest,policy_digest,bytes,state) VALUES($1,$2,$3,$4,$5,65536,'dispatched')", [
|
|
408
|
+
id,
|
|
409
|
+
this.lease.runId,
|
|
410
|
+
this.callId,
|
|
411
|
+
digest(descriptor),
|
|
412
|
+
digest(this.policy)
|
|
413
|
+
]);
|
|
414
|
+
return { id, waitMs: 0 };
|
|
415
|
+
});
|
|
416
|
+
if (reservation.id)
|
|
417
|
+
return reservation.id;
|
|
418
|
+
await sleep(reservation.waitMs, undefined, { signal });
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
async complete(id, bytes, observation) {
|
|
422
|
+
if (!Number.isInteger(bytes) || bytes < 0 || bytes > 65536)
|
|
423
|
+
throw new HarnessError("adapter_uncertain", "Adapter response exceeded its reserved bound.");
|
|
424
|
+
const ref = await this.journal.captureActionEvidence(this.lease, this.callId, {
|
|
425
|
+
kind: "adapter-observation",
|
|
426
|
+
requestId: id,
|
|
427
|
+
policyDigest: digest(this.policy),
|
|
428
|
+
observation
|
|
429
|
+
});
|
|
430
|
+
await this.journal.fenced(this.lease, async ({ client }) => {
|
|
431
|
+
const result = await client.query("UPDATE harness_mastra.web_requests SET state='completed',bytes=$4,response_digest=$5,response_ref=$6 WHERE id=$1 AND run_id=$2 AND call_id=$3 AND state='dispatched' AND policy_digest=$7", [
|
|
432
|
+
id,
|
|
433
|
+
this.lease.runId,
|
|
434
|
+
this.callId,
|
|
435
|
+
bytes,
|
|
436
|
+
digest(observation),
|
|
437
|
+
ref,
|
|
438
|
+
digest(this.policy)
|
|
439
|
+
]);
|
|
440
|
+
if (result.rowCount !== 1)
|
|
441
|
+
throw new HarnessError("adapter_uncertain", "Adapter evidence cannot settle a different or completed reservation.");
|
|
442
|
+
await this.journal.writeWebProgress(client, this.lease.runId);
|
|
443
|
+
}, "settlement");
|
|
444
|
+
return ref;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// apps/harness/src/storage/assessment-operations.ts
|
|
449
|
+
var VerificationPolicySchema = strictObject({
|
|
450
|
+
authority: strictObject({
|
|
451
|
+
kind: literal("gyms_oracle"),
|
|
452
|
+
id: string().min(1).max(128)
|
|
453
|
+
}),
|
|
454
|
+
policy: strictObject({
|
|
455
|
+
id: string().min(1).max(128),
|
|
456
|
+
revision: string().min(1).max(128),
|
|
457
|
+
claims: array(_enum(["vulnerability", "dependency_exploitability"])).min(1).max(2),
|
|
458
|
+
requiresExpiry: boolean()
|
|
459
|
+
})
|
|
460
|
+
});
|
|
461
|
+
var VerificationIntentSchema = strictObject({
|
|
462
|
+
request: VerifyRequestSchema,
|
|
463
|
+
context: VerificationPolicySchema,
|
|
464
|
+
approvalDigest: string().regex(/^[a-f0-9]{64}$/)
|
|
465
|
+
});
|
|
466
|
+
|
|
467
|
+
// apps/harness/src/storage/callbacks.ts
|
|
468
|
+
import { randomBytes } from "node:crypto";
|
|
469
|
+
var CallbackPolicySchema = strictObject({
|
|
470
|
+
publicOrigin: url().max(2048).refine((value) => {
|
|
471
|
+
const url2 = new URL(value);
|
|
472
|
+
return ["http:", "https:"].includes(url2.protocol) && url2.origin === value && !url2.username && !url2.password;
|
|
473
|
+
}),
|
|
474
|
+
ttlMs: number().int().min(1000).max(600000),
|
|
475
|
+
maxRegistrations: number().int().min(1).max(32)
|
|
476
|
+
});
|
|
477
|
+
var CallbackOpenSchema = strictObject({
|
|
478
|
+
hypothesis: string().min(1).max(512)
|
|
479
|
+
});
|
|
480
|
+
var CallbackReadSchema = strictObject({
|
|
481
|
+
registrationId: string().min(1).max(256)
|
|
482
|
+
});
|
|
483
|
+
var CallbackEventSchema = strictObject({
|
|
484
|
+
transport: literal("http"),
|
|
485
|
+
method: string().min(1).max(16),
|
|
486
|
+
requestDigest: string().regex(/^[a-f0-9]{64}$/),
|
|
487
|
+
peerDigest: string().regex(/^[a-f0-9]{64}$/),
|
|
488
|
+
bytes: number().int().min(0).max(4096)
|
|
489
|
+
});
|
|
490
|
+
var CallbackReadResultSchema = strictObject({
|
|
491
|
+
registrationId: string(),
|
|
492
|
+
closed: boolean(),
|
|
493
|
+
expiresAt: exports_iso.datetime(),
|
|
494
|
+
controlCount: number().int().nonnegative(),
|
|
495
|
+
probeCount: number().int().nonnegative(),
|
|
496
|
+
omitted: number().int().nonnegative(),
|
|
497
|
+
events: array(CallbackEventSchema.extend({
|
|
498
|
+
arm: _enum(["control", "probe"]),
|
|
499
|
+
receivedAt: exports_iso.datetime()
|
|
500
|
+
})).max(32),
|
|
501
|
+
assertion: _enum(["pending", "observed", "not_observed", "inconclusive"]),
|
|
502
|
+
verification: literal("not-independent")
|
|
503
|
+
});
|
|
504
|
+
|
|
505
|
+
class CallbackJournal {
|
|
506
|
+
journal;
|
|
507
|
+
lease;
|
|
508
|
+
policy;
|
|
509
|
+
constructor(journal, lease, policy) {
|
|
510
|
+
this.journal = journal;
|
|
511
|
+
this.lease = lease;
|
|
512
|
+
this.policy = policy;
|
|
513
|
+
}
|
|
514
|
+
async open(input, callId) {
|
|
515
|
+
const args = CallbackOpenSchema.parse(input);
|
|
516
|
+
const control = randomBytes(32).toString("hex"), probe = randomBytes(32).toString("hex");
|
|
517
|
+
return this.journal.fenced(this.lease, async ({ client }) => {
|
|
518
|
+
const action = await client.query("SELECT 1 FROM harness_mastra.actions WHERE run_id=$1 AND call_id=$2 AND tool='callback_open' AND args_digest=$3 AND state='dispatched' AND child_id IS NOT DISTINCT FROM $4", [this.lease.runId, callId, digest(args), this.lease.child?.id ?? null]);
|
|
519
|
+
const count = await client.query("SELECT count(*)::integer AS count FROM harness_mastra.callback_registrations WHERE run_id=$1", [this.lease.runId]);
|
|
520
|
+
if (!action.rowCount || count.rows[0].count >= this.policy.maxRegistrations)
|
|
521
|
+
throw new HarnessError("callback_denied", "Callback registration requires a dispatched action and remaining registration budget.");
|
|
522
|
+
const result = await client.query("INSERT INTO harness_mastra.callback_registrations(run_id,call_id,policy_digest,control_hash,probe_hash,expires_at) VALUES($1,$2,$3,$4,$5,clock_timestamp()+$6*interval '1 millisecond') RETURNING expires_at", [
|
|
523
|
+
this.lease.runId,
|
|
524
|
+
callId,
|
|
525
|
+
digest(this.policy),
|
|
526
|
+
sha256(control),
|
|
527
|
+
sha256(probe),
|
|
528
|
+
this.policy.ttlMs
|
|
529
|
+
]);
|
|
530
|
+
return {
|
|
531
|
+
registrationId: callId,
|
|
532
|
+
controlUrl: `${this.policy.publicOrigin}/callbacks/${control}`,
|
|
533
|
+
probeUrl: `${this.policy.publicOrigin}/callbacks/${probe}`,
|
|
534
|
+
expiresAt: result.rows[0].expires_at.toISOString(),
|
|
535
|
+
verification: "not-independent"
|
|
536
|
+
};
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
async read(input) {
|
|
540
|
+
const { registrationId } = CallbackReadSchema.parse(input);
|
|
541
|
+
return this.journal.fenced(this.lease, async ({ client }) => {
|
|
542
|
+
const result = await client.query("SELECT *,clock_timestamp()>=expires_at AS closed FROM harness_mastra.callback_registrations WHERE run_id=$1 AND call_id=$2 AND policy_digest=$3", [this.lease.runId, registrationId, digest(this.policy)]);
|
|
543
|
+
const row = result.rows[0];
|
|
544
|
+
if (!row)
|
|
545
|
+
throw new HarnessError("callback_denied", "Callback evidence belongs to a different run or policy.");
|
|
546
|
+
return CallbackReadResultSchema.parse({
|
|
547
|
+
registrationId,
|
|
548
|
+
closed: row.closed,
|
|
549
|
+
expiresAt: row.expires_at.toISOString(),
|
|
550
|
+
controlCount: Number(row.control_count),
|
|
551
|
+
probeCount: Number(row.probe_count),
|
|
552
|
+
omitted: Number(row.omitted),
|
|
553
|
+
events: row.events,
|
|
554
|
+
assertion: callbackAssertion(row.closed, Number(row.control_count), Number(row.probe_count), Number(row.omitted)),
|
|
555
|
+
verification: "not-independent"
|
|
556
|
+
});
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
function callbackAssertion(closed, control, probe, omitted) {
|
|
561
|
+
return !closed ? "pending" : omitted || control > 0 ? "inconclusive" : probe > 0 ? "observed" : "not_observed";
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// apps/harness/src/storage/stage-analysis.ts
|
|
565
|
+
var StageAnalysisSchema = strictObject({
|
|
566
|
+
kind: literal("stage-analysis"),
|
|
567
|
+
role: string().min(1).max(128),
|
|
568
|
+
text: string().max(8192),
|
|
569
|
+
truncated: boolean(),
|
|
570
|
+
completedActions: number().int().nonnegative(),
|
|
571
|
+
failedActions: number().int().nonnegative()
|
|
572
|
+
});
|
|
573
|
+
function stagedWebsite(grant) {
|
|
574
|
+
return grant.mode === "pentest" && Boolean(grant.config.website && grant.stages);
|
|
575
|
+
}
|
|
576
|
+
async function stageAnalyses(journal, lease) {
|
|
577
|
+
return journal.fenced(lease, async ({ client }) => {
|
|
578
|
+
const rows = await client.query("SELECT job_key,CASE WHEN result->>'kind'='stage-analysis' THEN result ELSE result->'analysis' END AS result FROM harness_mastra.child_jobs WHERE run_id=$1 AND state='completed' AND (result->>'kind'='stage-analysis' OR result->'analysis'->>'kind'='stage-analysis') ORDER BY job_key", [lease.runId]);
|
|
579
|
+
return rows.rows.map((row) => ({
|
|
580
|
+
jobKey: row.job_key,
|
|
581
|
+
analysis: StageAnalysisSchema.parse(row.result)
|
|
582
|
+
}));
|
|
583
|
+
});
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
// apps/harness/src/storage/capability-plan.ts
|
|
587
|
+
var SOURCE = [
|
|
588
|
+
"repo_inventory",
|
|
589
|
+
"repo_context",
|
|
590
|
+
"repo_read",
|
|
591
|
+
"repo_search",
|
|
592
|
+
"change_inspect",
|
|
593
|
+
"change_impact",
|
|
594
|
+
"semgrep_scan",
|
|
595
|
+
"secret_scan",
|
|
596
|
+
"dependency_scan",
|
|
597
|
+
"joern_confirm",
|
|
598
|
+
"patch_propose",
|
|
599
|
+
"test_run",
|
|
600
|
+
"package_import_probe"
|
|
601
|
+
];
|
|
602
|
+
var WEB = [
|
|
603
|
+
"protocol_run",
|
|
604
|
+
"agent_target",
|
|
605
|
+
"callback_open",
|
|
606
|
+
"callback_read",
|
|
607
|
+
"execution_assert",
|
|
608
|
+
"environment_check",
|
|
609
|
+
"web_inventory",
|
|
610
|
+
"site_recon_save",
|
|
611
|
+
"investigation_read",
|
|
612
|
+
"web_discover",
|
|
613
|
+
"web_request",
|
|
614
|
+
"web_browse",
|
|
615
|
+
"web_case",
|
|
616
|
+
"http_request",
|
|
617
|
+
"session_health",
|
|
618
|
+
"browser_workflow",
|
|
619
|
+
"identity_catalog",
|
|
620
|
+
"policy_read",
|
|
621
|
+
"nuclei_scan",
|
|
622
|
+
"port_scan",
|
|
623
|
+
"zap_scan",
|
|
624
|
+
"sqli_confirm",
|
|
625
|
+
"web_capture_read",
|
|
626
|
+
"web_capture_compare"
|
|
627
|
+
];
|
|
628
|
+
var SHARED = [
|
|
629
|
+
"coverage_record",
|
|
630
|
+
"evidence_catalog",
|
|
631
|
+
"evidence_inspect",
|
|
632
|
+
"engagement_describe",
|
|
633
|
+
"finding_propose",
|
|
634
|
+
"bash_run",
|
|
635
|
+
"experiment_run"
|
|
636
|
+
];
|
|
637
|
+
function environmentRequirements(grant) {
|
|
638
|
+
if (grant.mode !== "pentest" || !grant.sandbox)
|
|
639
|
+
return;
|
|
640
|
+
const browserVersion = grant.config.website?.browserVersion ?? grant.config.live?.browser?.playwrightVersion;
|
|
641
|
+
const scanners = [
|
|
642
|
+
...grant.config.portScan ? [{ tool: "port_scan", version: grant.config.portScan.version }] : [],
|
|
643
|
+
...new Map((grant.config.scanners?.profiles ?? []).map((entry) => [
|
|
644
|
+
entry.tool,
|
|
645
|
+
{ tool: entry.tool, version: entry.version }
|
|
646
|
+
])).values()
|
|
647
|
+
];
|
|
648
|
+
return browserVersion || scanners.length ? { ...browserVersion ? { browserVersion } : {}, scanners } : undefined;
|
|
649
|
+
}
|
|
650
|
+
function capabilityPlan(grant) {
|
|
651
|
+
const bindings = new Map;
|
|
652
|
+
const bind = (names, id, value, version = null, rules) => {
|
|
653
|
+
if (value === undefined || value === false)
|
|
654
|
+
return;
|
|
655
|
+
for (const name of names)
|
|
656
|
+
bindings.set(name, [
|
|
657
|
+
...bindings.get(name) ?? [],
|
|
658
|
+
{
|
|
659
|
+
id,
|
|
660
|
+
digest: digest(value),
|
|
661
|
+
version,
|
|
662
|
+
ruleDigest: rules === undefined ? null : digest(rules)
|
|
663
|
+
}
|
|
664
|
+
]);
|
|
665
|
+
};
|
|
666
|
+
const source = grant.mode === "codereview" ? grant.config : grant.config.sourceAssistance;
|
|
667
|
+
if (source) {
|
|
668
|
+
const review = grant.mode === "codereview" ? grant.config : undefined;
|
|
669
|
+
if (review?.repository.package?.ecosystem === "npm" && grant.sandbox && sandboxAllowsExecution(grant.sandbox, "package_import_probe"))
|
|
670
|
+
bind(["package_import_probe"], "npm-import-descriptors-v1", review.repository, "node24");
|
|
671
|
+
if (grant.mode === "codereview")
|
|
672
|
+
bind(["coverage_record"], "source-work", grant.stages);
|
|
673
|
+
bind(["repo_inventory", "repo_context", "repo_read", "repo_search"], "repository", source.repository);
|
|
674
|
+
const change = review?.change;
|
|
675
|
+
bind(["change_inspect", "change_impact"], "change", change);
|
|
676
|
+
bind(["repo_context", ...change ? ["change_impact"] : []], "source-analysis", { parser: SOURCE_PARSER_ID, strategy: SOURCE_ANALYSIS_VERSION }, SOURCE_ANALYSIS_VERSION);
|
|
677
|
+
bind(["patch_propose"], "patch-authority", source.allowPatchProposals);
|
|
678
|
+
bind(["semgrep_scan"], "semgrep", review?.semgrep, review?.semgrep?.version, review?.semgrep?.rules);
|
|
679
|
+
bind(["secret_scan"], "gitleaks", review?.secretScan, review?.secretScan?.version);
|
|
680
|
+
bind(["joern_confirm"], "joern", review?.joern, review?.joern?.version);
|
|
681
|
+
bind(["dependency_scan"], "osv", review?.dependencyScan, review?.dependencyScan?.version, review?.dependencyScan?.database);
|
|
682
|
+
for (const profile of source.testProfiles ?? [])
|
|
683
|
+
bind(["test_run"], profile.id, profile, "node24");
|
|
684
|
+
}
|
|
685
|
+
if (grant.mode === "pentest") {
|
|
686
|
+
const config = grant.config;
|
|
687
|
+
bind(["port_scan"], "naabu", config.portScan, config.portScan?.version);
|
|
688
|
+
bind(["callback_open", "callback_read"], "callback-collector", config.callbacks, "http-v1");
|
|
689
|
+
for (const profile of config.adapters?.protocols ?? [])
|
|
690
|
+
bind(["protocol_run"], profile.id, profile, "wire-v1");
|
|
691
|
+
for (const profile of config.adapters?.agents ?? [])
|
|
692
|
+
bind(["agent_target"], profile.id, profile, profile.transport === "mcp" ? "2025-11-25" : "http-json-v1");
|
|
693
|
+
bind(["environment_check"], "execution-prerequisites", environmentRequirements(grant));
|
|
694
|
+
bind(["web_discover", "web_request"], "website", config.website);
|
|
695
|
+
if (stagedWebsite(grant)) {
|
|
696
|
+
bind(["site_recon_save"], "site-recon-v2", config.website, "1.0.0");
|
|
697
|
+
bind(["investigation_read"], "stage-agents-v1", grant.stages, "1.0.0");
|
|
698
|
+
}
|
|
699
|
+
bind(["web_inventory"], "surface-discovery", config.website || config.live || config.apiDescriptions ? {
|
|
700
|
+
strategy: "web-surface-v1",
|
|
701
|
+
website: config.website ?? null,
|
|
702
|
+
live: config.live ?? null,
|
|
703
|
+
descriptions: config.apiDescriptions ?? []
|
|
704
|
+
} : undefined, "1.0.0");
|
|
705
|
+
if (config.website && config.live?.sessions.some((session) => session.capabilities?.includes("website")))
|
|
706
|
+
bind([
|
|
707
|
+
"web_discover",
|
|
708
|
+
"web_request",
|
|
709
|
+
...config.website?.browserVersion ? ["web_browse"] : []
|
|
710
|
+
], "website-sessions", config.live);
|
|
711
|
+
if (config.website?.browserVersion)
|
|
712
|
+
bind(["web_browse", "execution_assert"], "browser", config.website, config.website.browserVersion);
|
|
713
|
+
if (config.captures.length)
|
|
714
|
+
bind(["web_capture_read", "web_capture_compare"], "saved-captures", config.captures);
|
|
715
|
+
bind(["http_request", "session_health"], "live-sessions", config.live);
|
|
716
|
+
bind(["web_case"], "executable-cases", config.live?.testCases);
|
|
717
|
+
bind(["browser_workflow"], "live-browser", config.live?.browser, config.live?.browser?.playwrightVersion);
|
|
718
|
+
bind(["identity_catalog", "policy_read"], "identity-policy", config.permissionPolicy);
|
|
719
|
+
for (const profile of config.scanners?.profiles ?? [])
|
|
720
|
+
bind([profile.tool], profile.id, profile, profile.version, profile.tool === "nuclei_scan" ? profile.template : profile.tool === "zap_scan" ? profile.rules : { parameter: profile.parameter, technique: profile.technique });
|
|
721
|
+
if (config.website)
|
|
722
|
+
bind(["coverage_record"], "website-work", grant.stages?.websiteWork);
|
|
723
|
+
}
|
|
724
|
+
for (const profile of grant.config.bashProfiles ?? [])
|
|
725
|
+
bind(["bash_run"], profile.id, profile, "bash-offline-v1", profile.script);
|
|
726
|
+
bind(["experiment_run"], "custom-experiments", grant.config.experiments, "offline-v1");
|
|
727
|
+
for (const name of SHARED.filter((name) => !["bash_run", "experiment_run"].includes(name)))
|
|
728
|
+
bindings.set(name, []);
|
|
729
|
+
const names = [
|
|
730
|
+
...SOURCE,
|
|
731
|
+
...WEB.filter((name) => name !== "investigation_read" || stagedWebsite(grant)),
|
|
732
|
+
...SHARED
|
|
733
|
+
];
|
|
734
|
+
return names.sort().map((name) => {
|
|
735
|
+
const applicable = grant.mode === "codereview" ? !WEB.includes(name) : !SOURCE.includes(name) || Boolean(source);
|
|
736
|
+
const configured = bindings.has(name);
|
|
737
|
+
const isolated = TaskNameSchema.options.some((task) => task === name) && Boolean(grant.sandbox);
|
|
738
|
+
return {
|
|
739
|
+
name,
|
|
740
|
+
applicable,
|
|
741
|
+
configured,
|
|
742
|
+
reason: !applicable ? `The ${grant.mode} subject does not support this capability.` : !configured ? "No admitted profile or required input is configured for this capability." : "The admitted configuration permits this capability; preflight and actual execution are recorded separately.",
|
|
743
|
+
execution: isolated ? {
|
|
744
|
+
location: grant.sandbox.location ?? "local",
|
|
745
|
+
runtime: sandboxRuntime(grant.sandbox),
|
|
746
|
+
image: grant.sandbox.image,
|
|
747
|
+
engineId: grant.sandbox.engineId ?? null
|
|
748
|
+
} : {
|
|
749
|
+
location: "backend-host",
|
|
750
|
+
runtime: ["web_case", "protocol_run", "agent_target"].includes(name) ? "host-broker" : "host-read",
|
|
751
|
+
image: null,
|
|
752
|
+
engineId: null
|
|
753
|
+
},
|
|
754
|
+
profiles: bindings.get(name) ?? [],
|
|
755
|
+
requiredChecks: [
|
|
756
|
+
"registry",
|
|
757
|
+
...isolated ? ["sandbox"] : [],
|
|
758
|
+
...[
|
|
759
|
+
"http_request",
|
|
760
|
+
"session_health",
|
|
761
|
+
"browser_workflow",
|
|
762
|
+
"web_case"
|
|
763
|
+
].includes(name) ? ["sessions"] : []
|
|
764
|
+
]
|
|
765
|
+
};
|
|
766
|
+
});
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
// apps/harness/src/storage/pentest-config.ts
|
|
770
|
+
function pentestConfig(environment = process.env, bundled = true) {
|
|
771
|
+
return Object.freeze({
|
|
772
|
+
ownershipTestMode: !bundled && (environment.HARNESS_API_AUTH ?? "file") === "file" && isLoopback(environment.HARNESS_API_HOST ?? "127.0.0.1")
|
|
773
|
+
});
|
|
774
|
+
}
|
|
775
|
+
var PENTEST_CONFIG = pentestConfig();
|
|
776
|
+
|
|
777
|
+
// apps/harness/src/storage/site-recon.ts
|
|
778
|
+
var SITE_RECON_JOB = "recon:site-discovery:v1";
|
|
779
|
+
var SITE_RECON_SCAN = Object.freeze({
|
|
780
|
+
callId: "recon:site-context:v1",
|
|
781
|
+
tool: "site_recon_save",
|
|
782
|
+
args: {}
|
|
783
|
+
});
|
|
784
|
+
var SiteReconNarrativeSchema = strictObject({
|
|
785
|
+
text: string().max(32768),
|
|
786
|
+
truncated: boolean(),
|
|
787
|
+
specialists: array(StageAnalysisSchema).length(3)
|
|
788
|
+
});
|
|
789
|
+
var SectionNameSchema = _enum([
|
|
790
|
+
"notes",
|
|
791
|
+
"surface",
|
|
792
|
+
"endpoints",
|
|
793
|
+
"omissions",
|
|
794
|
+
"observations"
|
|
795
|
+
]);
|
|
796
|
+
var SiteReconFileSchema = strictObject({
|
|
797
|
+
kind: literal("site-recon"),
|
|
798
|
+
version: literal("1.0.0"),
|
|
799
|
+
filename: literal("site-recon.json"),
|
|
800
|
+
runId: uuid(),
|
|
801
|
+
target: url().max(2048),
|
|
802
|
+
targetRevision: string().min(1).max(2048),
|
|
803
|
+
scope: WebScopeSchema,
|
|
804
|
+
summary: string().max(1024),
|
|
805
|
+
narrativeTruncated: boolean(),
|
|
806
|
+
endpoints: number().int().nonnegative(),
|
|
807
|
+
observedEndpoints: number().int().nonnegative(),
|
|
808
|
+
omitted: number().int().nonnegative(),
|
|
809
|
+
sections: array(strictObject({
|
|
810
|
+
name: SectionNameSchema,
|
|
811
|
+
artifactRef: ArtifactRefSchema,
|
|
812
|
+
parts: number().int().positive(),
|
|
813
|
+
records: number().int().nonnegative()
|
|
814
|
+
})).length(5),
|
|
815
|
+
coverage: literal("discovery-only"),
|
|
816
|
+
interpretation: literal("Discovery snapshot before Hunt. Agent notes are unverified analysis, not findings or authorization. Follow section artifact references and nextArtifactRef links for the full retained data; cite original observations for security claims. Later observations remain in web_inventory.")
|
|
817
|
+
});
|
|
818
|
+
async function captureSiteRecon(journal, lease, callId, signal) {
|
|
819
|
+
const grant = journal.grant;
|
|
820
|
+
if (lease.child || grant.mode !== "pentest" || !grant.config.website || !grant.stages || callId !== SITE_RECON_SCAN.callId)
|
|
821
|
+
throw new HarnessError("tool_denied", "Only the website Recon coordinator can save the shared site reference.");
|
|
822
|
+
const narrative = await journal.fenced(lease, async ({ client }) => {
|
|
823
|
+
const result = await client.query("SELECT result FROM harness_mastra.child_jobs WHERE run_id=$1 AND job_key=$2 AND state='completed'", [lease.runId, SITE_RECON_JOB]);
|
|
824
|
+
if (!result.rows.length)
|
|
825
|
+
throw new HarnessError("tool_denied", "Detailed site discovery has no committed result.");
|
|
826
|
+
return SiteReconNarrativeSchema.parse(result.rows[0].result);
|
|
827
|
+
});
|
|
828
|
+
const surface = await journal.reportWeb(lease.runId);
|
|
829
|
+
const { endpoints, omissions, ...metadata } = surface;
|
|
830
|
+
const observations = (await journal.actions(lease.runId)).filter((action) => action.state === "completed" && action.artifactRef && [
|
|
831
|
+
...WEBSITE_DISCOVERY_TOOLS,
|
|
832
|
+
"http_request",
|
|
833
|
+
"session_health",
|
|
834
|
+
"browser_workflow"
|
|
835
|
+
].includes(action.tool)).map(({ callId, tool, artifactRef }) => ({ callId, tool, artifactRef }));
|
|
836
|
+
const sections = [];
|
|
837
|
+
const content = [
|
|
838
|
+
["notes", [narrative]],
|
|
839
|
+
["surface", [metadata]],
|
|
840
|
+
["endpoints", endpoints],
|
|
841
|
+
["omissions", omissions],
|
|
842
|
+
["observations", observations]
|
|
843
|
+
];
|
|
844
|
+
for (const [name, records] of content) {
|
|
845
|
+
const text = `[${records.map((record) => captureResponseText(canonicalJson(record))).join(",")}]`;
|
|
846
|
+
const characters = Array.from(text);
|
|
847
|
+
const parts = Math.ceil(characters.length / 8192);
|
|
848
|
+
let nextArtifactRef = null;
|
|
849
|
+
for (let index = parts - 1;index >= 0; index--) {
|
|
850
|
+
if (signal.aborted)
|
|
851
|
+
throw new HarnessError("stop_requested", "Site reference capture was cancelled.");
|
|
852
|
+
nextArtifactRef = await journal.captureActionEvidence(lease, callId, {
|
|
853
|
+
kind: "site-recon-part",
|
|
854
|
+
filename: `site-recon-${name}-${index + 1}.json`,
|
|
855
|
+
section: name,
|
|
856
|
+
index,
|
|
857
|
+
format: "json-fragment",
|
|
858
|
+
text: characters.slice(index * 8192, (index + 1) * 8192).join(""),
|
|
859
|
+
nextArtifactRef
|
|
860
|
+
});
|
|
861
|
+
}
|
|
862
|
+
sections.push({
|
|
863
|
+
name,
|
|
864
|
+
artifactRef: nextArtifactRef,
|
|
865
|
+
parts,
|
|
866
|
+
records: records.length
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
return SiteReconFileSchema.parse({
|
|
870
|
+
kind: "site-recon",
|
|
871
|
+
version: "1.0.0",
|
|
872
|
+
filename: "site-recon.json",
|
|
873
|
+
runId: lease.runId,
|
|
874
|
+
target: grant.target,
|
|
875
|
+
targetRevision: surface.targetRevision,
|
|
876
|
+
scope: grant.config.website,
|
|
877
|
+
summary: captureResponseText(narrative.text).slice(0, 1024),
|
|
878
|
+
narrativeTruncated: narrative.truncated,
|
|
879
|
+
endpoints: endpoints.length,
|
|
880
|
+
observedEndpoints: endpoints.filter((endpoint) => endpoint.state === "observed").length,
|
|
881
|
+
omitted: surface.omitted,
|
|
882
|
+
sections,
|
|
883
|
+
coverage: "discovery-only",
|
|
884
|
+
interpretation: "Discovery snapshot before Hunt. Agent notes are unverified analysis, not findings or authorization. Follow section artifact references and nextArtifactRef links for the full retained data; cite original observations for security claims. Later observations remain in web_inventory."
|
|
885
|
+
});
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
// apps/harness/src/storage/tool-calls.ts
|
|
889
|
+
function completedToolCalls(runId, actions, excludedTools = []) {
|
|
890
|
+
const byCall = new Map(actions.map((action) => [action.callId, action]));
|
|
891
|
+
return async (requestedRun, callId) => {
|
|
892
|
+
const action = byCall.get(callId);
|
|
893
|
+
return requestedRun === runId && action?.state === "completed" && action.artifactRef && !excludedTools.includes(action.tool) ? { callId, tool: action.tool, artifactRef: action.artifactRef } : null;
|
|
894
|
+
};
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
// apps/harness/src/storage/web-scanner-plan.ts
|
|
898
|
+
var WEB_SCANNER_STRATEGY = "surface-scanners-v1";
|
|
899
|
+
function webScannerCall(targetRevision, profile) {
|
|
900
|
+
return {
|
|
901
|
+
callId: `surface-scan:${digest({ strategy: WEB_SCANNER_STRATEGY, targetRevision, profile })}`,
|
|
902
|
+
tool: profile.tool,
|
|
903
|
+
args: { profileId: profile.id }
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
function webScannerPlan(target, targetRevision, endpoints, config, scope, live) {
|
|
907
|
+
return config.profiles.map((profile) => {
|
|
908
|
+
const method = profile.tool === "sqli_confirm" ? profile.request?.method ?? "GET" : "GET";
|
|
909
|
+
const urls = profile.tool === "nuclei_scan" ? profile.template.paths.map((path) => new URL(path, profile.url).href) : [profile.url];
|
|
910
|
+
const matches = urls.map((url) => endpoints.flatMap((endpoint) => endpoint.shape.method === method && !endpoint.shape.operation && (profile.tool !== "sqli_confirm" || !profile.request || endpoint.shape.contentType === profile.request.contentType) ? endpoint.samples.flatMap((sample) => sample.url === url && sample.source === "observation" && sample.evidenceRef && sample.status !== null && sample.status >= 200 && sample.status < 300 && (sample.identity?.id ?? null) === (profile.identity ?? null) ? [{ endpoint, sample }] : []) : [])[0]);
|
|
911
|
+
const matched = matches.flatMap((match) => match ? [match] : []);
|
|
912
|
+
const blocked = urls.some((url) => !webRequestAllowed(target, { url, method }, scope)) ? "The profile contains a request outside the shared website scope." : profile.identity && !live?.sessions.some((session) => session.id === profile.identity && session.capabilities?.includes("scanners")) ? "The profile requires an explicitly admitted scanner session." : matched.length !== urls.length ? `Every exact profile URL needs a successful observed ${method} for its configured identity and body format; hints, other methods, login failures and other identities do not qualify.` : profile.tool === "sqli_confirm" && !matched[0].endpoint.shape.inputs.some((input) => input.location === (profile.request ? "body" : "query") && input.name === `${profile.request?.contentType === "application/json" ? "/" : ""}${profile.parameter}`) ? "The approved SQLMap parameter was not observed in its granted input location." : profile.tool === "zap_scan" && profile.rules.includes(40012) && !matched[0].endpoint.shape.inputs.some((input) => input.location === "query" && new URL(profile.url).searchParams.has(input.name)) ? "The GET-only ZAP reflected-input preset requires an observed query input; body/form coverage needs a separately supported adapter." : undefined;
|
|
913
|
+
return {
|
|
914
|
+
strategyVersion: WEB_SCANNER_STRATEGY,
|
|
915
|
+
profileId: profile.id,
|
|
916
|
+
profileDigest: digest(profile),
|
|
917
|
+
tool: profile.tool,
|
|
918
|
+
version: profile.version,
|
|
919
|
+
callId: profile.tool === "sqli_confirm" ? null : webScannerCall(targetRevision, profile).callId,
|
|
920
|
+
selection: blocked ? "blocked" : profile.tool === "sqli_confirm" ? "deferred" : "ready",
|
|
921
|
+
reason: blocked ?? (profile.tool === "sqli_confirm" ? "An exact pending CWE-89 candidate is required; SQLMap runs in verification, never from a discovery lead alone." : "Observed exact requests match this operator-owned profile. Execution still requires current admission, preflight and approval."),
|
|
922
|
+
endpointIds: [...new Set(matched.map(({ endpoint }) => endpoint.id))],
|
|
923
|
+
evidenceRefs: [
|
|
924
|
+
...new Set(matched.map(({ sample }) => sample.evidenceRef))
|
|
925
|
+
]
|
|
926
|
+
};
|
|
927
|
+
});
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
// apps/harness/src/storage/web-surface.ts
|
|
931
|
+
var ResponseSchema = object({
|
|
932
|
+
kind: literal("web-response"),
|
|
933
|
+
request: object({
|
|
934
|
+
url: url().max(2048),
|
|
935
|
+
method: string(),
|
|
936
|
+
body: string().optional(),
|
|
937
|
+
headers: record(string(), string()).optional(),
|
|
938
|
+
inputNames: array(EndpointInputSchema).max(64).optional()
|
|
939
|
+
}),
|
|
940
|
+
status: number().int().min(100).max(599),
|
|
941
|
+
bodyComplete: boolean().optional(),
|
|
942
|
+
identity: WebIdentitySchema.optional()
|
|
943
|
+
});
|
|
944
|
+
var LeadSchema = object({
|
|
945
|
+
url: string().min(1).max(2048),
|
|
946
|
+
method: EndpointLeadSchema.shape.method,
|
|
947
|
+
kind: EndpointLeadSchema.shape.kind.optional(),
|
|
948
|
+
inputs: EndpointLeadSchema.shape.inputs.optional(),
|
|
949
|
+
parameters: array(string()).max(64).optional(),
|
|
950
|
+
contentType: EndpointLeadSchema.shape.contentType.optional(),
|
|
951
|
+
operation: EndpointLeadSchema.shape.operation.optional(),
|
|
952
|
+
authentication: EndpointLeadSchema.shape.authentication.optional(),
|
|
953
|
+
stateChanging: boolean().optional(),
|
|
954
|
+
limitations: EndpointLeadSchema.shape.limitations.optional()
|
|
955
|
+
});
|
|
956
|
+
async function webSurface(journal, runId, config, client = journal.pool) {
|
|
957
|
+
await journal.read(runId, client);
|
|
958
|
+
const endpoints = new Map;
|
|
959
|
+
const omissions = [];
|
|
960
|
+
let omitted = 0;
|
|
961
|
+
const omission = (source, url2, reason, evidenceRef = null) => {
|
|
962
|
+
omitted++;
|
|
963
|
+
if (omissions.length < 500)
|
|
964
|
+
omissions.push({
|
|
965
|
+
source: source.slice(0, 256),
|
|
966
|
+
url: url2.slice(0, 2048),
|
|
967
|
+
reason: reason.slice(0, 512),
|
|
968
|
+
evidenceRef
|
|
969
|
+
});
|
|
970
|
+
};
|
|
971
|
+
const descriptions = config.descriptions ?? [];
|
|
972
|
+
const declared = descriptions.flatMap((description) => description.routes.map((route) => ({ route, description })));
|
|
973
|
+
const add = (value, source, sourceId, evidenceRef = null, status = null, identity) => {
|
|
974
|
+
const parsed = LeadSchema.safeParse(value);
|
|
975
|
+
if (!parsed.success) {
|
|
976
|
+
omission(sourceId, "", "Malformed or unsupported endpoint lead.", evidenceRef);
|
|
977
|
+
return;
|
|
978
|
+
}
|
|
979
|
+
const point = parsed.data;
|
|
980
|
+
let normalized;
|
|
981
|
+
let declaration;
|
|
982
|
+
try {
|
|
983
|
+
if (source === "observation")
|
|
984
|
+
declaration = declared.find(({ route }) => route.method === point.method && route.operation === (point.operation ?? null) && route.contentType === (point.contentType ?? null) && matchesEndpointTemplate(route.url, point.url));
|
|
985
|
+
const queryInputs = requestInputs({
|
|
986
|
+
url: point.url,
|
|
987
|
+
method: point.method
|
|
988
|
+
}).inputs;
|
|
989
|
+
const inputs = new Map([
|
|
990
|
+
...queryInputs,
|
|
991
|
+
...(point.parameters ?? []).flatMap((key) => endpointInput("query", key) ?? []),
|
|
992
|
+
...declaration?.route.inputs.filter((entry) => entry.location === "path") ?? [],
|
|
993
|
+
...point.inputs ?? []
|
|
994
|
+
].map((entry) => [`${entry.location}:${entry.name}`, entry]));
|
|
995
|
+
const declaredUrl = source === "operator" ? point.url : declaration?.route.url;
|
|
996
|
+
normalized = endpointShape({
|
|
997
|
+
...point,
|
|
998
|
+
inputs: [...inputs.values()],
|
|
999
|
+
contentType: point.contentType ?? null,
|
|
1000
|
+
...declaredUrl ? {
|
|
1001
|
+
declaredPath: new URL(declaredUrl).pathname.replace(/%7B/gi, "{").replace(/%7D/gi, "}")
|
|
1002
|
+
} : {}
|
|
1003
|
+
});
|
|
1004
|
+
} catch {
|
|
1005
|
+
normalized = undefined;
|
|
1006
|
+
}
|
|
1007
|
+
if (!normalized) {
|
|
1008
|
+
omission(sourceId, point.url, "URL, method or input shape is unsupported or exceeds the retained bound.", evidenceRef);
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
const { shape, confidence } = normalized;
|
|
1012
|
+
const observedInputs = shape.inputs.filter(({ location }) => location === "header" || location === "cookie");
|
|
1013
|
+
shape.inputs = shape.inputs.filter(({ location }) => location !== "header" && location !== "cookie");
|
|
1014
|
+
const permitted = config.scope && webRequestAllowed(config.target, { url: point.url, method: point.method }, config.scope);
|
|
1015
|
+
const inaccessible = status !== null && [401, 403, 404, 405].includes(status);
|
|
1016
|
+
const state = !permitted ? "ignored" : source === "operator" ? "supplied" : source === "lead" ? "lead" : inaccessible ? "inaccessible" : "observed";
|
|
1017
|
+
const reason = !permitted ? "Origin, path or method is outside the admitted website scope; discovery does not grant dispatch authority." : source === "operator" ? "Pinned operator-supplied API declaration; successful concrete requests have not been established by this declaration." : source === "lead" ? "Discovery lead without a matching committed concrete response." : inaccessible ? `Concrete request returned HTTP ${status}; this response does not establish access to the route.` : `Concrete request and HTTP ${status} response retained; this is discovery evidence, not vulnerability coverage.`;
|
|
1018
|
+
const id = endpointIdentity(shape);
|
|
1019
|
+
const sample = {
|
|
1020
|
+
url: point.url,
|
|
1021
|
+
evidenceRef,
|
|
1022
|
+
status,
|
|
1023
|
+
source,
|
|
1024
|
+
sourceId,
|
|
1025
|
+
...observedInputs.length ? { inputs: observedInputs } : {},
|
|
1026
|
+
...identity ? { identity } : {}
|
|
1027
|
+
};
|
|
1028
|
+
const existing = endpoints.get(id);
|
|
1029
|
+
const gaps = [
|
|
1030
|
+
...point.limitations ?? [],
|
|
1031
|
+
...declaration?.route.limitations ?? [],
|
|
1032
|
+
...confidence === "heuristic" ? [
|
|
1033
|
+
"Numeric or identifier-like path segments were grouped heuristically; concrete samples retain the original paths."
|
|
1034
|
+
] : []
|
|
1035
|
+
];
|
|
1036
|
+
if (existing) {
|
|
1037
|
+
if (!existing.samples.some((entry) => entry.url === sample.url && entry.evidenceRef === sample.evidenceRef && entry.sourceId === sample.sourceId)) {
|
|
1038
|
+
if (existing.samples.length < 32)
|
|
1039
|
+
existing.samples.push(sample);
|
|
1040
|
+
else {
|
|
1041
|
+
existing.omittedSamples++;
|
|
1042
|
+
if (sample.source === "observation" && !existing.samples.some((entry) => entry.source === "observation"))
|
|
1043
|
+
existing.samples[31] = sample;
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
const rank = {
|
|
1047
|
+
ignored: 0,
|
|
1048
|
+
unsupported: 0,
|
|
1049
|
+
lead: 1,
|
|
1050
|
+
supplied: 2,
|
|
1051
|
+
inaccessible: 3,
|
|
1052
|
+
observed: 4
|
|
1053
|
+
};
|
|
1054
|
+
if (rank[state] > rank[existing.state]) {
|
|
1055
|
+
existing.state = state;
|
|
1056
|
+
existing.reason = reason;
|
|
1057
|
+
}
|
|
1058
|
+
if (source === "operator") {
|
|
1059
|
+
if (existing.authentication !== "unknown" && existing.authentication !== point.authentication)
|
|
1060
|
+
gaps.push("Supplied descriptions disagree about authentication; inspect their pinned definitions and establish runtime controls.");
|
|
1061
|
+
existing.authentication = point.authentication ?? "unknown";
|
|
1062
|
+
existing.shape = shape;
|
|
1063
|
+
existing.shapeConfidence = confidence;
|
|
1064
|
+
existing.stateChanging = point.stateChanging ?? existing.stateChanging;
|
|
1065
|
+
}
|
|
1066
|
+
existing.limitations = [
|
|
1067
|
+
...new Set([...existing.limitations, ...gaps])
|
|
1068
|
+
].slice(0, 32);
|
|
1069
|
+
return;
|
|
1070
|
+
}
|
|
1071
|
+
if (endpoints.size >= 1000) {
|
|
1072
|
+
omission(sourceId, point.url, "Additional endpoint shapes exceed the 1,000-shape surface bound.", evidenceRef);
|
|
1073
|
+
return;
|
|
1074
|
+
}
|
|
1075
|
+
endpoints.set(id, {
|
|
1076
|
+
id,
|
|
1077
|
+
shape,
|
|
1078
|
+
shapeConfidence: confidence,
|
|
1079
|
+
state,
|
|
1080
|
+
reason,
|
|
1081
|
+
authentication: point.authentication ?? declaration?.route.authentication ?? "unknown",
|
|
1082
|
+
stateChanging: point.stateChanging ?? !["GET", "HEAD", "OPTIONS"].includes(point.method),
|
|
1083
|
+
samples: [sample],
|
|
1084
|
+
omittedSamples: 0,
|
|
1085
|
+
limitations: [...new Set(gaps)].slice(0, 32)
|
|
1086
|
+
});
|
|
1087
|
+
};
|
|
1088
|
+
add({ url: config.target, method: "GET", kind: "page" }, "lead", "target");
|
|
1089
|
+
for (const { route, description } of declared)
|
|
1090
|
+
add(route, "operator", description.id);
|
|
1091
|
+
for (const description of descriptions) {
|
|
1092
|
+
omitted += description.omitted;
|
|
1093
|
+
if (description.omitted && omissions.length < 500)
|
|
1094
|
+
omissions.push({
|
|
1095
|
+
source: description.id,
|
|
1096
|
+
url: "",
|
|
1097
|
+
reason: `${description.omitted} API routes were omitted during bounded offline intake; see the description limitations.`,
|
|
1098
|
+
evidenceRef: null
|
|
1099
|
+
});
|
|
1100
|
+
}
|
|
1101
|
+
const actions = (await journal.actions(runId, client)).filter((action) => action.artifactRef && action.state === "completed" && isWebsiteCoverageTool(action.tool) && action.tool !== "coverage_record");
|
|
1102
|
+
const responseRefs = new Map;
|
|
1103
|
+
const artifacts = await Promise.all(actions.map(async (action) => ({
|
|
1104
|
+
action,
|
|
1105
|
+
value: JSON.parse((await journal.artifact(runId, action.artifactRef, client)).bytes)
|
|
1106
|
+
})));
|
|
1107
|
+
for (const { action, value } of artifacts) {
|
|
1108
|
+
for (const point of [...value.leads ?? [], ...value.endpoints ?? []])
|
|
1109
|
+
add(point, "lead", action.callId, action.artifactRef);
|
|
1110
|
+
for (const blocked of value.blocked ?? [])
|
|
1111
|
+
omission(action.callId, String(blocked.url ?? ""), `Browser route not dispatched: ${String(blocked.reason ?? "unknown")}`, action.artifactRef);
|
|
1112
|
+
if (Number.isSafeInteger(value.omitted) && value.omitted > 0) {
|
|
1113
|
+
omitted += value.omitted;
|
|
1114
|
+
if (omissions.length < 500)
|
|
1115
|
+
omissions.push({
|
|
1116
|
+
source: action.callId,
|
|
1117
|
+
url: "",
|
|
1118
|
+
reason: `${value.omitted} discoveries were omitted by the tool's input/output or traversal bound.`,
|
|
1119
|
+
evidenceRef: action.artifactRef
|
|
1120
|
+
});
|
|
1121
|
+
}
|
|
1122
|
+
for (const valueRef of value.network?.responseRefs ?? []) {
|
|
1123
|
+
const ref = ArtifactRefSchema.safeParse(valueRef);
|
|
1124
|
+
if (ref.success)
|
|
1125
|
+
responseRefs.set(ref.data, action.callId);
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
for (const testCase of config.live?.testCases ?? []) {
|
|
1129
|
+
const request = config.live.requests.find((entry) => entry.url === testCase.endpoint.url && entry.method === testCase.endpoint.method);
|
|
1130
|
+
add({
|
|
1131
|
+
...testCase.endpoint,
|
|
1132
|
+
kind: testCase.endpoint.operation ? "graphql" : "response",
|
|
1133
|
+
...request ? requestInputs({
|
|
1134
|
+
...request,
|
|
1135
|
+
headers: {
|
|
1136
|
+
...request.headers,
|
|
1137
|
+
...request.contentType ? { "content-type": request.contentType } : {}
|
|
1138
|
+
}
|
|
1139
|
+
}) : {},
|
|
1140
|
+
authentication: "required",
|
|
1141
|
+
stateChanging: !["GET", "HEAD", "OPTIONS"].includes(testCase.endpoint.method),
|
|
1142
|
+
limitations: [
|
|
1143
|
+
"Operator-approved executable case; effect, control and cleanup results have not yet been established."
|
|
1144
|
+
]
|
|
1145
|
+
}, "operator", `case:${testCase.id}`);
|
|
1146
|
+
}
|
|
1147
|
+
const captures = await client.query("SELECT response_ref,call_id FROM harness_mastra.web_requests WHERE run_id=$1 AND state='completed' AND response_ref IS NOT NULL ORDER BY dispatched_at,id", [runId]);
|
|
1148
|
+
for (const capture of captures.rows)
|
|
1149
|
+
responseRefs.set(capture.response_ref, capture.call_id);
|
|
1150
|
+
const responses = await Promise.all([...responseRefs].map(async ([ref, callId]) => ({
|
|
1151
|
+
ref,
|
|
1152
|
+
callId,
|
|
1153
|
+
response: ResponseSchema.parse(JSON.parse((await journal.artifact(runId, ref, client)).bytes))
|
|
1154
|
+
})));
|
|
1155
|
+
for (const { ref, callId, response } of responses) {
|
|
1156
|
+
if (response.bodyComplete === false)
|
|
1157
|
+
omission(callId, response.request.url, "The response exceeded its admitted byte bound; only a body prefix was retained and the remaining content was not inspected.", ref);
|
|
1158
|
+
const details = requestInputs(response.request);
|
|
1159
|
+
for (const operation of details.operations?.length ? details.operations : [undefined])
|
|
1160
|
+
add({
|
|
1161
|
+
...response.request,
|
|
1162
|
+
...details,
|
|
1163
|
+
kind: "response",
|
|
1164
|
+
...operation ? { operation } : {}
|
|
1165
|
+
}, "observation", callId, ref, response.status, response.identity);
|
|
1166
|
+
omitted += details.omitted;
|
|
1167
|
+
}
|
|
1168
|
+
const authorization = config.live?.comparisons && config.permissionPolicy ? await authorizationObservations(journal, runId, undefined, client).then((observations) => config.live.comparisons.map((definition) => authorizationAssessment(definition, config.live, config.permissionPolicy, observations))) : undefined;
|
|
1169
|
+
return WebSurfaceSchema.parse({
|
|
1170
|
+
version: "1.0.0",
|
|
1171
|
+
strategyVersion: "web-surface-v1",
|
|
1172
|
+
runId,
|
|
1173
|
+
target: config.target,
|
|
1174
|
+
targetRevision: config.targetRevision,
|
|
1175
|
+
endpoints: [...endpoints.values()].sort((a, b) => a.shape.origin.localeCompare(b.shape.origin) || a.shape.path.localeCompare(b.shape.path) || a.shape.method.localeCompare(b.shape.method) || a.id.localeCompare(b.id)),
|
|
1176
|
+
descriptions: descriptions.map(({ routes: _routes, ...description }) => description),
|
|
1177
|
+
...authorization ? { authorization } : {},
|
|
1178
|
+
...config.scanners && config.scope ? {
|
|
1179
|
+
scannerPlan: webScannerPlan(config.target, config.targetRevision, [...endpoints.values()], config.scanners, config.scope, config.live)
|
|
1180
|
+
} : {},
|
|
1181
|
+
omissions,
|
|
1182
|
+
omitted,
|
|
1183
|
+
interpretation: "Observed requests, supplied API descriptions and discovery leads are distinct. Surface discovery is not vulnerability coverage or proof of route completeness."
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
// apps/harness/src/storage/investigation-journal.ts
|
|
1188
|
+
var GRANT_MAX_BYTES = 262144;
|
|
1189
|
+
var GRANT_MAX_LABEL = `${GRANT_MAX_BYTES / 1024} KiB`;
|
|
1190
|
+
var sourceAssistance = strictObject({
|
|
1191
|
+
repository: RepositoryConfigSchema,
|
|
1192
|
+
binding: SourceDeploymentBindingSchema,
|
|
1193
|
+
allowPatchProposals: literal(true).optional(),
|
|
1194
|
+
testProfiles: array(RepositoryTestProfileSchema).min(1).max(8).optional()
|
|
1195
|
+
});
|
|
1196
|
+
var AgentAConfigSchema = strictObject({
|
|
1197
|
+
buildIdentityRef: string().min(1).max(2048),
|
|
1198
|
+
callbacks: CallbackPolicySchema.optional(),
|
|
1199
|
+
adapters: AdapterPolicySchema.optional(),
|
|
1200
|
+
captures: array(WebCaptureSchema).max(16),
|
|
1201
|
+
permissionPolicy: PermissionPolicySchema.optional(),
|
|
1202
|
+
live: LiveWebSchema.optional(),
|
|
1203
|
+
website: WebScopeSchema.optional(),
|
|
1204
|
+
websiteFamilies: array(WebsiteFamilySchema).min(1).max(10).refine((values) => new Set(values).size === values.length).optional(),
|
|
1205
|
+
apiDescriptions: array(ApiDescriptionSchema).max(4).optional(),
|
|
1206
|
+
scanners: ActiveScannersSchema.optional(),
|
|
1207
|
+
portScan: PortScanProfileSchema.optional(),
|
|
1208
|
+
sourceAssistance: sourceAssistance.optional(),
|
|
1209
|
+
ownershipProof: OwnershipProofSchema.optional(),
|
|
1210
|
+
ownershipProofs: array(OwnershipProofSchema).max(8).optional(),
|
|
1211
|
+
bashProfiles: BashProfilesSchema.optional(),
|
|
1212
|
+
experiments: ExperimentPolicySchema.optional()
|
|
1213
|
+
});
|
|
1214
|
+
var AgentBConfigSchema = strictObject({
|
|
1215
|
+
repository: RepositoryConfigSchema,
|
|
1216
|
+
change: ChangeContextSchema.extend({
|
|
1217
|
+
baseRevision: CommitSchema,
|
|
1218
|
+
headRevision: CommitSchema
|
|
1219
|
+
}).optional(),
|
|
1220
|
+
allowPatchProposals: literal(true).optional(),
|
|
1221
|
+
testProfiles: array(RepositoryTestProfileSchema).min(1).max(8).optional(),
|
|
1222
|
+
semgrep: SemgrepProfileSchema.optional(),
|
|
1223
|
+
secretScan: SecretScanProfileSchema.optional(),
|
|
1224
|
+
dependencyScan: DependencyScanProfileSchema.optional(),
|
|
1225
|
+
joern: JoernProfileSchema.optional(),
|
|
1226
|
+
bashProfiles: BashProfilesSchema.optional(),
|
|
1227
|
+
experiments: ExperimentPolicySchema.optional()
|
|
1228
|
+
});
|
|
1229
|
+
var GymsVerifierSchema = strictObject({
|
|
1230
|
+
origin: url(),
|
|
1231
|
+
credentialFile: string().startsWith("/"),
|
|
1232
|
+
credentialSha256: string().regex(/^[a-f0-9]{64}$/),
|
|
1233
|
+
targetIdentity: TargetIdentitySchema,
|
|
1234
|
+
rangeEndpoint: url(),
|
|
1235
|
+
context: VerificationPolicySchema,
|
|
1236
|
+
pocInputs: array(string().max(2048)).max(16),
|
|
1237
|
+
proofDirectory: string().startsWith("/"),
|
|
1238
|
+
timeoutMs: number().int().min(100).max(120000)
|
|
1239
|
+
});
|
|
1240
|
+
var StagesSchema = strictObject({
|
|
1241
|
+
maxWorkers: number().int().min(1).max(8),
|
|
1242
|
+
maxWorkItems: number().int().min(1).max(1000).optional(),
|
|
1243
|
+
huntPartitions: number().int().min(1).max(8).default(1),
|
|
1244
|
+
websiteWork: WebsiteWorkPolicySchema.optional(),
|
|
1245
|
+
goal: strictObject({
|
|
1246
|
+
objective: string().min(1).max(16384)
|
|
1247
|
+
}).optional()
|
|
1248
|
+
});
|
|
1249
|
+
var InvestigationGrantSchema = discriminatedUnion("mode", [
|
|
1250
|
+
strictObject({
|
|
1251
|
+
mode: literal("pentest"),
|
|
1252
|
+
plan: InvestigationPlanBindingSchema.optional(),
|
|
1253
|
+
model: InvestigationModelSchema.optional(),
|
|
1254
|
+
boundary: _enum([
|
|
1255
|
+
"captured-evidence-only",
|
|
1256
|
+
"gvisor-captured-evidence",
|
|
1257
|
+
"gvisor-brokered-web",
|
|
1258
|
+
"docker-captured-evidence",
|
|
1259
|
+
"docker-brokered-web"
|
|
1260
|
+
]),
|
|
1261
|
+
sandbox: SandboxConfigSchema.optional(),
|
|
1262
|
+
target: url().max(2048),
|
|
1263
|
+
stages: StagesSchema.optional(),
|
|
1264
|
+
config: AgentAConfigSchema,
|
|
1265
|
+
verifiers: array(GymsVerifierSchema).min(1).max(8).optional()
|
|
1266
|
+
}),
|
|
1267
|
+
strictObject({
|
|
1268
|
+
mode: literal("codereview"),
|
|
1269
|
+
plan: InvestigationPlanBindingSchema.optional(),
|
|
1270
|
+
model: InvestigationModelSchema.optional(),
|
|
1271
|
+
boundary: _enum([
|
|
1272
|
+
"trusted-mirror-read-only",
|
|
1273
|
+
"gvisor-repository",
|
|
1274
|
+
"docker-repository"
|
|
1275
|
+
]),
|
|
1276
|
+
sandbox: SandboxConfigSchema.optional(),
|
|
1277
|
+
target: string().min(1).max(2048),
|
|
1278
|
+
stages: StagesSchema.optional(),
|
|
1279
|
+
config: AgentBConfigSchema,
|
|
1280
|
+
verifiers: array(GymsVerifierSchema).min(1).max(8).optional()
|
|
1281
|
+
})
|
|
1282
|
+
]);
|
|
1283
|
+
async function loadInvestigationGrant(path) {
|
|
1284
|
+
try {
|
|
1285
|
+
return investigationGrant(JSON.parse(await privateFile(path, GRANT_MAX_BYTES)));
|
|
1286
|
+
} catch {
|
|
1287
|
+
throw new HarnessError("investigation_config", `Use a private, operator-owned UTF-8 JSON investigation grant (maximum ${GRANT_MAX_LABEL}). No supplied contents are logged.`);
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
function deepFreeze(value) {
|
|
1291
|
+
if (value && typeof value === "object")
|
|
1292
|
+
for (const entry of Object.values(value))
|
|
1293
|
+
deepFreeze(entry);
|
|
1294
|
+
return Object.freeze(value);
|
|
1295
|
+
}
|
|
1296
|
+
var admittedGrants = new WeakSet;
|
|
1297
|
+
function investigationGrant(input) {
|
|
1298
|
+
if (typeof input === "object" && input && admittedGrants.has(input))
|
|
1299
|
+
return input;
|
|
1300
|
+
const grant = InvestigationGrantSchema.parse(input);
|
|
1301
|
+
if (grant.plan && !grant.stages)
|
|
1302
|
+
throw new HarnessError("investigation_config", "Versioned plans require the admitted five-stage execution path.");
|
|
1303
|
+
const route = grantRoute(grant);
|
|
1304
|
+
if (grant.mode === "pentest" !== (route.kind === "web"))
|
|
1305
|
+
throw new HarnessError("investigation_scope", "The grant mode must match its target: codereview for a GitHub repository or pull request, pentest for a site.");
|
|
1306
|
+
if (grant.verifiers && (grant.verifiers.length > 1 && !grant.stages || new Set(grant.verifiers.map((verifier) => `${verifier.targetIdentity.rangeId}/${verifier.targetIdentity.instanceId}`)).size !== grant.verifiers.length || new Set(grant.verifiers.map((verifier) => verifier.rangeEndpoint)).size !== grant.verifiers.length))
|
|
1307
|
+
throw new HarnessError("investigation_config", "More than one verifier requires staged execution, and every verifier needs a distinct explicitly granted target instance and endpoint.");
|
|
1308
|
+
const isolated = grant.boundary.startsWith("gvisor-") || grant.boundary.startsWith("docker-");
|
|
1309
|
+
if (isolated !== (grant.sandbox !== undefined) || grant.sandbox && grant.boundary.startsWith("gvisor-") !== (sandboxRuntime(grant.sandbox) === GVISOR_RUNTIME))
|
|
1310
|
+
throw new HarnessError("investigation_config", "The investigation boundary must match its pinned sandbox image, Unix socket and selected runtime; runtime or host fallback is forbidden.");
|
|
1311
|
+
if ((grant.config.bashProfiles || grant.config.experiments) && (!grant.sandbox || !sandboxAllowsExecution(grant.sandbox, "bash_run")))
|
|
1312
|
+
throw new HarnessError("investigation_scope", "Bash profiles and custom experiments require an explicitly granted gVisor boundary.");
|
|
1313
|
+
if (grant.mode === "pentest") {
|
|
1314
|
+
const target = new URL(grant.target);
|
|
1315
|
+
const live = grant.config.live;
|
|
1316
|
+
const scanners = grant.config.scanners;
|
|
1317
|
+
const website = grant.config.website;
|
|
1318
|
+
const ports = grant.config.portScan;
|
|
1319
|
+
if (ports && (ports.host !== target.hostname || !grant.sandbox || website && (!website.addresses.includes(ports.address) || Date.parse(ports.expiresAt) > Date.parse(website.expiresAt))))
|
|
1320
|
+
throw new HarnessError("investigation_scope", "TCP discovery requires an explicitly approved target host, pinned address, expiry and sandbox.");
|
|
1321
|
+
if (grant.config.adapters && (website?.environment !== "registered" || grant.config.adapters.buildIdentityRef !== grant.config.buildIdentityRef || [
|
|
1322
|
+
...grant.config.adapters.protocols,
|
|
1323
|
+
...grant.config.adapters.agents
|
|
1324
|
+
].some((entry) => !withinHttpScope(new URL(grant.target), new URL(entry.endpoint.url)))))
|
|
1325
|
+
throw new HarnessError("investigation_scope", "Adapters require the registered disposable build and target endpoints inside its admitted URL.");
|
|
1326
|
+
if (website) {
|
|
1327
|
+
if (website.ownershipTestMode && !PENTEST_CONFIG.ownershipTestMode)
|
|
1328
|
+
throw new HarnessError("investigation_scope", "This backend does not admit ownership test mode. Prepare a new engagement with verified ownership or passive methods.");
|
|
1329
|
+
const proof = grant.config.ownershipProof;
|
|
1330
|
+
const proofs = [
|
|
1331
|
+
...proof ? [proof] : [],
|
|
1332
|
+
...grant.config.ownershipProofs ?? []
|
|
1333
|
+
];
|
|
1334
|
+
if (website.ownershipTestMode && proofs.length)
|
|
1335
|
+
throw new HarnessError("investigation_scope", "Ownership test mode cannot also claim verified ownership.");
|
|
1336
|
+
if (website.autoScope && !proofs.some((entry) => entry.method === "dns-txt" && entry.apex === website.autoScope.apex))
|
|
1337
|
+
throw new HarnessError("investigation_scope", "Automatic origins require an operator-approved profile and a recorded DNS proof for the exact apex.");
|
|
1338
|
+
if (website.environment === "live" && !website.ownershipTestMode && website.scopes?.some((scope) => scope.methods.some((method) => !WEBSITE_PASSIVE_METHODS.includes(method)) && !proofs.some((entry) => ownsHost(entry, new URL(scope.url).hostname))))
|
|
1339
|
+
throw new HarnessError("investigation_scope", "Every active website scope requires its own host or apex ownership proof.");
|
|
1340
|
+
const validateTarget = (url2) => website.environment === "registered" ? validateRegisteredWebTarget(url2, website) : validateWebTarget(url2);
|
|
1341
|
+
validateTarget(grant.target);
|
|
1342
|
+
if (website.registration && website.registration.identity.targetRevision !== grant.config.buildIdentityRef)
|
|
1343
|
+
throw new HarnessError("investigation_scope", "The registered deployment revision must match the run's build identity.");
|
|
1344
|
+
if (website.scopes) {
|
|
1345
|
+
for (const scope of website.scopes)
|
|
1346
|
+
validateTarget(scope.url);
|
|
1347
|
+
if (website.scopes[0]?.url !== grant.target || new Set(website.scopes.map((scope) => scope.url)).size !== website.scopes.length)
|
|
1348
|
+
throw new HarnessError("investigation_scope", "Website scopes must start with the target and be unique.");
|
|
1349
|
+
}
|
|
1350
|
+
if (scanners?.endpoints.some((endpoint) => !(website.scopes ?? [
|
|
1351
|
+
{ url: grant.target, addresses: website.addresses }
|
|
1352
|
+
]).some((scope) => new URL(scope.url).origin === endpoint.origin && scope.addresses[0] === endpoint.address)))
|
|
1353
|
+
throw new HarnessError("investigation_scope", "Scanner endpoints must use the website grant’s pinned origin and address.");
|
|
1354
|
+
if (live && [
|
|
1355
|
+
...live.requests,
|
|
1356
|
+
...live.sessions.map((session) => session.healthRequest)
|
|
1357
|
+
].some((request) => webExplicitAddress(website, grant.target, request) !== live.address || request.contentType !== undefined && !website.requestHeaders?.includes("content-type") || Object.keys(request.headers ?? {}).some((name) => !website.requestHeaders?.includes(name))))
|
|
1358
|
+
throw new HarnessError("investigation_scope", "Live session requests must share the website's admitted paths, methods, headers and pinned address.");
|
|
1359
|
+
}
|
|
1360
|
+
if (scanners)
|
|
1361
|
+
validateActiveScanners(scanners, grant.target, grant.config.buildIdentityRef);
|
|
1362
|
+
if (grant.config.ownershipProof && grant.config.ownershipProof.host !== target.hostname)
|
|
1363
|
+
throw new HarnessError("investigation_scope", "A recorded ownership proof must name the target host.");
|
|
1364
|
+
if ((grant.boundary === "gvisor-brokered-web" || grant.boundary === "docker-brokered-web") !== Boolean(live || scanners || website || ports) || !live && !scanners && !website && !ports && !grant.config.captures.length)
|
|
1365
|
+
throw new HarnessError("investigation_scope", "Live web access requires an explicit brokered container boundary; offline runs require captures.");
|
|
1366
|
+
if (live) {
|
|
1367
|
+
try {
|
|
1368
|
+
validateLiveWeb(live, grant.target, website?.environment === "registered");
|
|
1369
|
+
} catch {
|
|
1370
|
+
throw new HarnessError("investigation_scope", "The live grant contains unbound requests, sessions or network scope.");
|
|
1371
|
+
}
|
|
1372
|
+
if (!grant.config.permissionPolicy || live.sessions.some((session) => !grant.config.permissionPolicy.identities.some((identity) => identity.id === session.id && identity.role === session.expected.role && identity.tenant === session.expected.tenant)))
|
|
1373
|
+
throw new HarnessError("investigation_scope", "Live sessions must bind the approved permission policy identities.");
|
|
1374
|
+
}
|
|
1375
|
+
if (live && grant.config.permissionPolicy)
|
|
1376
|
+
for (const comparison of live.comparisons ?? []) {
|
|
1377
|
+
authorizationIdentities(comparison, live, grant.config.permissionPolicy);
|
|
1378
|
+
const request = live.requests.find((entry) => entry.id === comparison.requestId);
|
|
1379
|
+
if (!["GET", "HEAD"].includes(request.method))
|
|
1380
|
+
throw new HarnessError("investigation_scope", "Authorization pair fixtures currently require read-only requests; state-changing flows need an admitted cleanup workflow.");
|
|
1381
|
+
}
|
|
1382
|
+
for (const testCase of live?.testCases ?? [])
|
|
1383
|
+
if (!website || testCase.disposable.buildIdentityRef !== grant.config.buildIdentityRef || [...testCase.control.steps, ...testCase.test.steps].some((group) => group.length > website.concurrency))
|
|
1384
|
+
throw new HarnessError("investigation_scope", "Executable cases require the exact disposable build and sufficient admitted concurrency.");
|
|
1385
|
+
for (const profile of scanners?.profiles ?? [])
|
|
1386
|
+
if (profile.identity && (!live || !website || !live.sessions.some((session) => session.id === profile.identity && session.capabilities?.includes("scanners")) || new URL(profile.url).origin !== target.origin))
|
|
1387
|
+
throw new HarnessError("investigation_scope", "Authenticated scanners require a matching explicit session capability and a shared website policy.");
|
|
1388
|
+
const ids = new Set;
|
|
1389
|
+
const policy = grant.config.permissionPolicy;
|
|
1390
|
+
if (policy && (policy.target !== grant.target || policy.buildIdentityRef !== grant.config.buildIdentityRef || grant.config.captures.some((capture) => !policy.identities.some((identity) => identity.id === capture.identity))))
|
|
1391
|
+
throw new HarnessError("investigation_scope", "Expected permissions must bind this exact target, build and capture identities.");
|
|
1392
|
+
for (const capture of grant.config.captures) {
|
|
1393
|
+
const url2 = new URL(capture.url);
|
|
1394
|
+
if (!withinHttpScope(target, url2) || ids.has(capture.id))
|
|
1395
|
+
throw new HarnessError("investigation_scope", "Captures must have unique IDs and remain under the exact granted origin and path.");
|
|
1396
|
+
ids.add(capture.id);
|
|
1397
|
+
}
|
|
1398
|
+
} else if (route.kind === "local") {
|
|
1399
|
+
if (route.target !== grant.config.repository.repository || grant.config.change)
|
|
1400
|
+
throw new HarnessError("investigation_scope", "A local review must name the pinned directory and carries no change.");
|
|
1401
|
+
} else if (route.kind === "package") {
|
|
1402
|
+
const source = grant.config.repository;
|
|
1403
|
+
if (!packageSourceMatches(route, source.package) || source.repository !== packageIdentity(source.package) || source.commit !== source.package.sha256 || !source.snapshotFile || source.gitDirectory || source.retrieval || grant.config.change)
|
|
1404
|
+
throw new HarnessError("investigation_scope", "A package review must bind the requested package and version to its verified published archive snapshot.");
|
|
1405
|
+
} else {
|
|
1406
|
+
if (route.kind === "web" || route.repository !== grant.config.repository.repository)
|
|
1407
|
+
throw new HarnessError("investigation_scope", "Repository target must name the pinned repository: a GitHub repository or pull request URL for it.");
|
|
1408
|
+
const change = grant.config.change;
|
|
1409
|
+
const pullRequestId = route.kind === "pull_request" ? route.pullRequestId : undefined;
|
|
1410
|
+
if ((pullRequestId !== undefined || change) && (change?.repository !== grant.config.repository.repository || change.pullRequestId !== pullRequestId || change.headRevision !== grant.config.repository.commit))
|
|
1411
|
+
throw new HarnessError("investigation_scope", "The approved change must bind this exact repository, PR identity and head revision.");
|
|
1412
|
+
}
|
|
1413
|
+
const source = grant.mode === "codereview" ? grant.config : grant.config.sourceAssistance;
|
|
1414
|
+
if (grant.mode === "pentest" && source) {
|
|
1415
|
+
const binding = grant.config.sourceAssistance.binding;
|
|
1416
|
+
if (binding.repository !== source.repository.repository || binding.revision !== source.repository.commit || binding.buildIdentityRef !== grant.config.buildIdentityRef || !grant.config.website?.registration || digest(binding.targetIdentity) !== digest(grant.config.website.registration.identity))
|
|
1417
|
+
throw new HarnessError("investigation_scope", "Source assistance requires the approved repository, source revision and exact registered deployment identity. An operator mapping is not independent deployment attestation.");
|
|
1418
|
+
}
|
|
1419
|
+
if (source) {
|
|
1420
|
+
const repository = source.repository;
|
|
1421
|
+
if (repository.package && route.kind !== "package")
|
|
1422
|
+
throw new HarnessError("investigation_scope", "Published package provenance requires a matching package target.");
|
|
1423
|
+
if (!hasRepositorySource(repository))
|
|
1424
|
+
throw new HarnessError("investigation_scope", "A repository requires exactly one trusted mirror or digest-bound source snapshot.");
|
|
1425
|
+
}
|
|
1426
|
+
if (source?.testProfiles) {
|
|
1427
|
+
const profiles = source.testProfiles;
|
|
1428
|
+
if (!grant.sandbox || !sandboxAllowsExecution(grant.sandbox, "test_run") || new Set(profiles.map((profile) => profile.id)).size !== profiles.length || profiles.some((profile) => !source.repository.paths.includes(profile.entrypoint)))
|
|
1429
|
+
throw new HarnessError("investigation_scope", "Test profiles require gVisor isolation, unique IDs and explicitly approved entrypoints.");
|
|
1430
|
+
}
|
|
1431
|
+
if (grant.mode === "codereview" && (grant.config.semgrep || grant.config.secretScan || grant.config.joern || grant.config.dependencyScan) && !grant.sandbox)
|
|
1432
|
+
throw new HarnessError("investigation_scope", "Source scanners require an explicitly admitted container boundary.");
|
|
1433
|
+
if (grant.mode === "codereview" && grant.config.joern && !grant.stages)
|
|
1434
|
+
throw new HarnessError("investigation_scope", "Joern requires a staged candidate verification job.");
|
|
1435
|
+
if (grant.mode === "codereview" && grant.config.dependencyScan?.paths.some((path) => !grant.config.repository.paths.includes(path)))
|
|
1436
|
+
throw new HarnessError("investigation_scope", "Dependency scanner paths must belong to the approved repository snapshot.");
|
|
1437
|
+
deepFreeze(grant);
|
|
1438
|
+
admittedGrants.add(grant);
|
|
1439
|
+
return grant;
|
|
1440
|
+
}
|
|
1441
|
+
function grantRoute(grant) {
|
|
1442
|
+
if (grant.mode === "pentest")
|
|
1443
|
+
return routeTarget(grant.target);
|
|
1444
|
+
if (grant.config.repository.package || /^(npm|pypi|cargo|oci):/.test(grant.target))
|
|
1445
|
+
return routeTarget(grant.target);
|
|
1446
|
+
if (grant.target.startsWith("/"))
|
|
1447
|
+
return routeTarget(grant.target);
|
|
1448
|
+
const { repository } = grant.config.repository;
|
|
1449
|
+
const pullRequestId = grant.config.change?.pullRequestId;
|
|
1450
|
+
if (grant.target === repository)
|
|
1451
|
+
return sourceRoute(repository, pullRequestId);
|
|
1452
|
+
const route = routeTarget(grant.target);
|
|
1453
|
+
return route.kind === "repository" && pullRequestId !== undefined ? sourceRoute(route.repository, pullRequestId) : route;
|
|
1454
|
+
}
|
|
1455
|
+
function grantCapabilities(grant, route) {
|
|
1456
|
+
const scanners = grant.mode === "codereview" ? SCANNER_CAPABILITIES.flatMap(({ profile, capability }) => grant.config[profile] ? [capability] : []) : [];
|
|
1457
|
+
return {
|
|
1458
|
+
defaults: [route.capabilities[0], ...scanners],
|
|
1459
|
+
accepted: [
|
|
1460
|
+
...route.capabilities,
|
|
1461
|
+
...grant.mode === "codereview" && grant.config.change ? ["pull_requests"] : [],
|
|
1462
|
+
...scanners
|
|
1463
|
+
]
|
|
1464
|
+
};
|
|
1465
|
+
}
|
|
1466
|
+
function investigationScope(grant) {
|
|
1467
|
+
return grant.mode === "pentest" && grant.config.website?.scopes ? grant.config.website.scopes.map((scope) => scope.url) : [grant.target];
|
|
1468
|
+
}
|
|
1469
|
+
function investigationRunConfig(grant, profile) {
|
|
1470
|
+
const route = grantRoute(grant);
|
|
1471
|
+
return RunConfigSchema.parse({
|
|
1472
|
+
contractVersion: CONTRACT_VERSION,
|
|
1473
|
+
model: profile.route.model,
|
|
1474
|
+
capabilities: grantCapabilities(grant, route).defaults,
|
|
1475
|
+
targetKind: route.targetKind,
|
|
1476
|
+
scope: investigationScope(grant),
|
|
1477
|
+
...grant.mode === "codereview" && grant.config.change ? { change: grant.config.change } : {}
|
|
1478
|
+
});
|
|
1479
|
+
}
|
|
1480
|
+
function validateInvestigationRun(target, input, profile, grant) {
|
|
1481
|
+
const route = grantRoute(grant);
|
|
1482
|
+
const granted = grantCapabilities(grant, route);
|
|
1483
|
+
const change = grant.mode === "codereview" ? grant.config.change : undefined;
|
|
1484
|
+
const requested = RunConfigSchema.parse(input);
|
|
1485
|
+
const capabilities = requested.capabilities ?? granted.defaults;
|
|
1486
|
+
const config = RunConfigSchema.parse({
|
|
1487
|
+
...requested,
|
|
1488
|
+
targetKind: requested.targetKind ?? route.targetKind,
|
|
1489
|
+
capabilities,
|
|
1490
|
+
scope: requested.scope ?? investigationScope(grant),
|
|
1491
|
+
...requested.change === undefined && change ? { change } : {}
|
|
1492
|
+
});
|
|
1493
|
+
if (grant.model && (grant.model.id !== profile.route.model || grant.model.provider !== profile.route.provider))
|
|
1494
|
+
throw new HarnessError("scope_denied", "The model and provider must match the operator's grant.");
|
|
1495
|
+
if (target !== grant.target || config.model !== profile.route.model || config.targetKind !== route.targetKind || capabilities.some((capability) => !granted.accepted.includes(capability)))
|
|
1496
|
+
throw new HarnessError("unsupported_profile", "The run must match its explicitly configured investigation grant, boundary and model. Ungranted tools are not enabled.");
|
|
1497
|
+
if (canonicalJson(config.scope ?? null) !== canonicalJson(investigationScope(grant)))
|
|
1498
|
+
throw new HarnessError("scope_denied", "The investigation requires its exact configured scope.");
|
|
1499
|
+
if (canonicalJson(config.change ?? null) !== canonicalJson(grant.mode === "codereview" ? grant.config.change ?? null : null))
|
|
1500
|
+
throw new HarnessError("scope_denied", "PR identity, base and head must match the immutable review grant.");
|
|
1501
|
+
return config;
|
|
1502
|
+
}
|
|
1503
|
+
function admitInvestigationModel(profile, grant) {
|
|
1504
|
+
if (profile.pricing && !grant.model)
|
|
1505
|
+
throw new HarnessError("scope_denied", "New paid investigations require a grant naming the permitted model and provider. Retain the original grant for recovery of existing runs.");
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
class InvestigationJournal extends PostgresJournal {
|
|
1509
|
+
profile;
|
|
1510
|
+
runtimeDigest;
|
|
1511
|
+
reportScope = "assessment";
|
|
1512
|
+
grant;
|
|
1513
|
+
identity;
|
|
1514
|
+
constructor(connection, profile, grant, authority, evidenceStore, runtimeDigest) {
|
|
1515
|
+
super(connection, profile, authority, evidenceStore);
|
|
1516
|
+
this.profile = profile;
|
|
1517
|
+
this.runtimeDigest = runtimeDigest;
|
|
1518
|
+
this.grant = investigationGrant(grant);
|
|
1519
|
+
this.identity = canonicalJson(this.grant);
|
|
1520
|
+
if (this.grant.sandbox && !runtimeDigest)
|
|
1521
|
+
throw new HarnessError("investigation_config", "An isolated grant must be given the sandbox runtime digest that binds its task policy.");
|
|
1522
|
+
}
|
|
1523
|
+
get runtimeVersion() {
|
|
1524
|
+
const isolation = this.grant.sandbox ? `/${this.runtimeDigest(this.grant.sandbox.image, sandboxRuntime(this.grant.sandbox))}` : "";
|
|
1525
|
+
const scan = this.prescans.length ? "/prescan-v2" : "";
|
|
1526
|
+
const recon = stagedWebsite(this.grant) ? "/site-recon-v2/stage-agents-v1/website-checks-v1" : "";
|
|
1527
|
+
const changes = this.grant.mode === "codereview" && this.grant.config.change && this.grant.plan ? "/change-review-v1" : "";
|
|
1528
|
+
return `${super.runtimeVersion}/investigator-v1/${this.grant.boundary}:${digest(this.identity)}${isolation}${scan}${recon}${changes}`;
|
|
1529
|
+
}
|
|
1530
|
+
get workerLimit() {
|
|
1531
|
+
return this.grant.stages?.maxWorkers ?? 0;
|
|
1532
|
+
}
|
|
1533
|
+
get reportPlan() {
|
|
1534
|
+
return executionPlan(this.grant.plan, this.grant.mode === "pentest" ? this.grant.config.websiteFamilies : undefined);
|
|
1535
|
+
}
|
|
1536
|
+
get reportRepository() {
|
|
1537
|
+
return this.grant.mode === "codereview" ? this.grant.config.repository : this.grant.config.sourceAssistance?.repository;
|
|
1538
|
+
}
|
|
1539
|
+
get reportSourceBinding() {
|
|
1540
|
+
return this.grant.mode === "pentest" ? this.grant.config.sourceAssistance?.binding : undefined;
|
|
1541
|
+
}
|
|
1542
|
+
get work() {
|
|
1543
|
+
return new WorkJournal(this, {
|
|
1544
|
+
scopeDigest: digest(this.grant),
|
|
1545
|
+
targetRevision: this.grant.mode === "codereview" ? this.grant.config.repository.commit : this.grant.config.buildIdentityRef,
|
|
1546
|
+
strategyVersion: workStrategy(this.grant.plan),
|
|
1547
|
+
dispatchLimit: Math.min(this.grant.stages?.websiteWork?.maxItems ?? this.grant.stages?.maxWorkItems ?? 1000, this.reportPlan?.maxWorkItems ?? 1000)
|
|
1548
|
+
}, this.grant);
|
|
1549
|
+
}
|
|
1550
|
+
get websiteWorklist() {
|
|
1551
|
+
const grant = this.grant;
|
|
1552
|
+
return stagedWebsite(grant) && grant.stages.websiteWork ? new WebsiteWorklist(this, grant.target, grant.config.website, {
|
|
1553
|
+
...grant.stages.websiteWork,
|
|
1554
|
+
maxItems: Math.min(grant.stages.websiteWork.maxItems, this.reportPlan?.maxWorkItems ?? 1000),
|
|
1555
|
+
maxRounds: Math.min(grant.stages.websiteWork.maxRounds, this.reportPlan?.web.maxRounds ?? 12)
|
|
1556
|
+
}, grant.config.websiteFamilies ?? this.reportPlan?.web.classes, grant.config.live, grant.config.permissionPolicy) : undefined;
|
|
1557
|
+
}
|
|
1558
|
+
get prescans() {
|
|
1559
|
+
if (this.grant.mode === "pentest")
|
|
1560
|
+
return this.grant.config.website ? [
|
|
1561
|
+
{
|
|
1562
|
+
callId: `prescan:web_discover:${digest(this.identity)}`,
|
|
1563
|
+
tool: "web_discover",
|
|
1564
|
+
args: {}
|
|
1565
|
+
}
|
|
1566
|
+
] : [];
|
|
1567
|
+
const config = this.grant.config;
|
|
1568
|
+
return [
|
|
1569
|
+
...config.secretScan ? ["secret_scan"] : [],
|
|
1570
|
+
...config.dependencyScan ? ["dependency_scan"] : [],
|
|
1571
|
+
...config.semgrep ? ["semgrep_scan"] : []
|
|
1572
|
+
].map((tool) => ({
|
|
1573
|
+
callId: `prescan:${tool}:${digest(this.identity)}`,
|
|
1574
|
+
tool,
|
|
1575
|
+
args: {}
|
|
1576
|
+
}));
|
|
1577
|
+
}
|
|
1578
|
+
get packageImportScans() {
|
|
1579
|
+
const grant = this.grant;
|
|
1580
|
+
if (grant.mode !== "codereview" || grant.config.repository.package?.ecosystem !== "npm" || !grant.sandbox || !sandboxAllowsExecution(grant.sandbox, "package_import_probe"))
|
|
1581
|
+
return [];
|
|
1582
|
+
return grant.config.repository.paths.filter(packageJavaScript).sort().map((path) => ({
|
|
1583
|
+
callId: `prescan:package_import_probe:${digest({ identity: this.identity, path })}`,
|
|
1584
|
+
tool: "package_import_probe",
|
|
1585
|
+
args: { path }
|
|
1586
|
+
}));
|
|
1587
|
+
}
|
|
1588
|
+
sourceChangeScan(path) {
|
|
1589
|
+
return {
|
|
1590
|
+
callId: `prescan:change_inspect:${digest({ identity: this.identity, path })}`,
|
|
1591
|
+
tool: "change_inspect",
|
|
1592
|
+
args: { paths: [path] }
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
get sourceChangeScans() {
|
|
1596
|
+
const grant = this.grant;
|
|
1597
|
+
if (grant.mode !== "codereview" || !grant.config.change || !grant.plan)
|
|
1598
|
+
return [];
|
|
1599
|
+
return [...grant.config.repository.paths].sort().map((path) => this.sourceChangeScan(path));
|
|
1600
|
+
}
|
|
1601
|
+
async sourceContextScan(runId, client = this.pool) {
|
|
1602
|
+
const grant = this.grant;
|
|
1603
|
+
const plan = this.reportPlan;
|
|
1604
|
+
if (grant.mode !== "codereview" || !plan || grant.config.change)
|
|
1605
|
+
return;
|
|
1606
|
+
const saved = await sourceManifest(client, runId);
|
|
1607
|
+
if (!saved)
|
|
1608
|
+
return;
|
|
1609
|
+
const manifest = saved.manifest;
|
|
1610
|
+
if (manifest.repository !== grant.config.repository.repository || manifest.headRevision !== grant.config.repository.commit || saved.authorityDigest !== digest(grant.config.repository))
|
|
1611
|
+
throw new HarnessError("evidence_integrity", "Source planning requires the admitted immutable inventory.");
|
|
1612
|
+
const head = manifest.revisions.find((revision) => revision.commit === manifest.headRevision);
|
|
1613
|
+
const paths = head.entries.filter((entry) => entry.admitted && ["javascript", "typescript"].includes(entry.language)).sort((a, b) => Number(/(route|controller|handler|middleware|auth)/i.test(b.path)) - Number(/(route|controller|handler|middleware|auth)/i.test(a.path)) || a.path.localeCompare(b.path)).slice(0, Math.min(8, plan.source.indexFiles)).map((entry) => entry.path);
|
|
1614
|
+
return {
|
|
1615
|
+
scan: paths.length ? {
|
|
1616
|
+
callId: `prescan:repo_context:${digest({ identity: this.identity, plan })}`,
|
|
1617
|
+
tool: "repo_context",
|
|
1618
|
+
args: {
|
|
1619
|
+
paths,
|
|
1620
|
+
depth: plan.source.indexDepth,
|
|
1621
|
+
maxFiles: plan.source.indexFiles
|
|
1622
|
+
}
|
|
1623
|
+
} : undefined
|
|
1624
|
+
};
|
|
1625
|
+
}
|
|
1626
|
+
get websiteChecks() {
|
|
1627
|
+
const grant = this.grant;
|
|
1628
|
+
if (!stagedWebsite(grant))
|
|
1629
|
+
return [];
|
|
1630
|
+
return [
|
|
1631
|
+
...grant.config.website.browserVersion ? [{ tool: "web_browse", args: { url: grant.target, actions: [] } }] : [],
|
|
1632
|
+
...(grant.config.live?.sessions ?? []).map((session) => ({
|
|
1633
|
+
tool: "session_health",
|
|
1634
|
+
args: { identity: session.id }
|
|
1635
|
+
}))
|
|
1636
|
+
].map((scan) => ({
|
|
1637
|
+
...scan,
|
|
1638
|
+
callId: `website-check:${scan.tool}:${digest({ identity: this.identity, args: scan.args })}`
|
|
1639
|
+
}));
|
|
1640
|
+
}
|
|
1641
|
+
get portScans() {
|
|
1642
|
+
const grant = this.grant;
|
|
1643
|
+
return grant.mode === "pentest" && grant.config.portScan ? [
|
|
1644
|
+
{
|
|
1645
|
+
callId: `prescan:port_scan:${digest(grant.config.portScan)}`,
|
|
1646
|
+
tool: "port_scan",
|
|
1647
|
+
args: {}
|
|
1648
|
+
}
|
|
1649
|
+
] : [];
|
|
1650
|
+
}
|
|
1651
|
+
get websiteScans() {
|
|
1652
|
+
const grant = this.grant;
|
|
1653
|
+
return stagedWebsite(grant) ? (grant.config.scanners?.profiles ?? []).filter((profile) => profile.tool !== "sqli_confirm").map((profile) => webScannerCall(grant.config.buildIdentityRef, profile)) : [];
|
|
1654
|
+
}
|
|
1655
|
+
verificationScans(finding) {
|
|
1656
|
+
if (this.grant.mode === "codereview" && this.grant.stages && this.grant.config.joern) {
|
|
1657
|
+
const repository = this.grant.config.repository;
|
|
1658
|
+
const trace = joernCandidate(finding, {
|
|
1659
|
+
repository: repository.repository,
|
|
1660
|
+
revision: repository.commit,
|
|
1661
|
+
paths: repository.paths
|
|
1662
|
+
});
|
|
1663
|
+
return trace ? [
|
|
1664
|
+
{
|
|
1665
|
+
callId: `verify-joern:${digest({ findingId: finding.id, profile: this.grant.config.joern, trace })}`,
|
|
1666
|
+
tool: "joern_confirm",
|
|
1667
|
+
args: { findingId: finding.id }
|
|
1668
|
+
}
|
|
1669
|
+
] : [];
|
|
1670
|
+
}
|
|
1671
|
+
if (this.grant.mode !== "pentest" || !this.grant.stages)
|
|
1672
|
+
return [];
|
|
1673
|
+
return (this.grant.config.scanners?.profiles ?? []).filter((profile) => profile.tool === "sqli_confirm" && sqlmapCandidate(finding, profile)).map((profile) => ({
|
|
1674
|
+
callId: `verify-sqli:${digest({ findingId: finding.id, profile })}`,
|
|
1675
|
+
tool: "sqli_confirm",
|
|
1676
|
+
args: { profileId: profile.id, findingId: finding.id }
|
|
1677
|
+
}));
|
|
1678
|
+
}
|
|
1679
|
+
async assertActionProposal(client, runId, callId, tool, argsDigest, args, childId) {
|
|
1680
|
+
if (!childId && stagedWebsite(this.grant) && callId === SITE_RECON_SCAN.callId && tool === SITE_RECON_SCAN.tool && argsDigest === digest(SITE_RECON_SCAN.args)) {
|
|
1681
|
+
const completed = await client.query("SELECT 1 FROM harness_mastra.child_jobs WHERE run_id=$1 AND job_key=$2 AND state='completed'", [runId, SITE_RECON_JOB]);
|
|
1682
|
+
if (completed.rowCount)
|
|
1683
|
+
return;
|
|
1684
|
+
}
|
|
1685
|
+
if (!childId && callId === "environment-check-v1" && tool === "environment_check" && argsDigest === digest({}) && environmentRequirements(this.grant))
|
|
1686
|
+
return;
|
|
1687
|
+
if (!childId && [
|
|
1688
|
+
...this.prescans,
|
|
1689
|
+
...this.packageImportScans,
|
|
1690
|
+
...this.portScans,
|
|
1691
|
+
...this.websiteChecks,
|
|
1692
|
+
...this.websiteScans
|
|
1693
|
+
].some((scan) => callId === scan.callId && tool === scan.tool && argsDigest === digest(scan.args)))
|
|
1694
|
+
return;
|
|
1695
|
+
if (!childId && tool === "change_inspect" && this.grant.mode === "codereview" && this.grant.config.change && this.grant.plan) {
|
|
1696
|
+
const selected = object({ paths: array(string()).length(1) }).safeParse(args);
|
|
1697
|
+
if (selected.success && this.grant.config.repository.paths.includes(selected.data.paths[0])) {
|
|
1698
|
+
const scan = this.sourceChangeScan(selected.data.paths[0]);
|
|
1699
|
+
if (callId === scan.callId && argsDigest === digest(scan.args))
|
|
1700
|
+
return;
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
if (!childId && tool === "repo_context") {
|
|
1704
|
+
const planned = await this.sourceContextScan(runId, client);
|
|
1705
|
+
if (planned?.scan && planned.scan.callId === callId && argsDigest === digest(planned.scan.args))
|
|
1706
|
+
return;
|
|
1707
|
+
}
|
|
1708
|
+
const candidate = object({ findingId: string() }).safeParse(args);
|
|
1709
|
+
if (["sqli_confirm", "joern_confirm"].includes(tool) && childId && candidate.success) {
|
|
1710
|
+
const records = await client.query("SELECT f.finding,j.job_key FROM harness_mastra.child_jobs j JOIN harness_mastra.findings f ON f.run_id=j.run_id AND f.id=$3 WHERE j.run_id=$1 AND j.id=$2 AND f.status='candidate' AND f.result='pending'", [runId, childId, candidate.data.findingId]);
|
|
1711
|
+
const row = records.rows[0];
|
|
1712
|
+
if (row && ownsVerificationJob(row.job_key, row.finding.id) && this.verificationScans(row.finding).some((scan) => scan.callId === callId && scan.tool === tool && digest(scan.args) === argsDigest))
|
|
1713
|
+
return;
|
|
1714
|
+
}
|
|
1715
|
+
if (tool === "joern_confirm")
|
|
1716
|
+
throw new HarnessError("scanner_denied", "Joern runs only once in its admitted candidate verification job.");
|
|
1717
|
+
await super.assertActionProposal(client, runId, callId, tool, argsDigest, args, childId);
|
|
1718
|
+
}
|
|
1719
|
+
async assertInvestigation(lease, context) {
|
|
1720
|
+
const run = await this.read(lease.runId);
|
|
1721
|
+
if (context.profile.runtimeVersion !== this.profile.runtimeVersion || run.chat || context.mode !== this.grant.mode || canonicalJson(context.config) !== canonicalJson(this.grant.config) || canonicalJson(context.capabilities) !== canonicalJson(run.config.capabilities))
|
|
1722
|
+
throw new HarnessError("investigation_identity", "The execution context must match its durable grant, model route and accounting.");
|
|
1723
|
+
await this.assertRecoverable(lease);
|
|
1724
|
+
}
|
|
1725
|
+
validateRun(target, input) {
|
|
1726
|
+
return validateInvestigationRun(target, input, this.profile, this.grant);
|
|
1727
|
+
}
|
|
1728
|
+
admitRun() {
|
|
1729
|
+
admitInvestigationModel(this.profile, this.grant);
|
|
1730
|
+
}
|
|
1731
|
+
parseEvent(input) {
|
|
1732
|
+
return EventSchema.parse(input);
|
|
1733
|
+
}
|
|
1734
|
+
describeRun(target, config) {
|
|
1735
|
+
return {
|
|
1736
|
+
target,
|
|
1737
|
+
mode: this.grant.mode,
|
|
1738
|
+
capabilities: config.capabilities
|
|
1739
|
+
};
|
|
1740
|
+
}
|
|
1741
|
+
async appendInvestigationSummary(lease, summary) {
|
|
1742
|
+
await this.fenced(lease, async ({ client }) => {
|
|
1743
|
+
const previous = await client.query("SELECT 1 FROM harness_mastra.events WHERE run_id=$1 AND payload->>'type'='step' AND payload->>'stepId'='investigation-summary-v1'", [lease.runId]);
|
|
1744
|
+
if (!previous.rowCount)
|
|
1745
|
+
await this.append(client, lease.runId, {
|
|
1746
|
+
type: "step",
|
|
1747
|
+
stepId: "investigation-summary-v1",
|
|
1748
|
+
summary
|
|
1749
|
+
});
|
|
1750
|
+
});
|
|
1751
|
+
}
|
|
1752
|
+
async recordedExternal(client, runId, kind, key, requestDigest) {
|
|
1753
|
+
const previous = (await client.query("SELECT * FROM harness_mastra.external_operations WHERE run_id=$1 AND kind=$2 AND operation_key=$3", [runId, kind, key])).rows[0];
|
|
1754
|
+
if (!previous)
|
|
1755
|
+
return;
|
|
1756
|
+
if (previous.request_digest !== requestDigest)
|
|
1757
|
+
throw new HarnessError("operation_conflict", "A recorded external operation cannot be replaced with a different intent.");
|
|
1758
|
+
if (previous.state !== "completed")
|
|
1759
|
+
throw new HarnessError("uncertain_execution", "The external operation may already have executed. Reconcile it before attempting any replacement; no retry was sent.");
|
|
1760
|
+
return {
|
|
1761
|
+
runId,
|
|
1762
|
+
kind,
|
|
1763
|
+
key,
|
|
1764
|
+
owner: previous.owner,
|
|
1765
|
+
requestDigest,
|
|
1766
|
+
artifactRef: previous.artifact_ref
|
|
1767
|
+
};
|
|
1768
|
+
}
|
|
1769
|
+
async completedExternal(runId, kind, key, intent) {
|
|
1770
|
+
await this.read(runId);
|
|
1771
|
+
const recorded = await this.recordedExternal(this.pool, runId, kind, key, digest(intent));
|
|
1772
|
+
return recorded?.artifactRef;
|
|
1773
|
+
}
|
|
1774
|
+
async reserveExternal(runId, kind, key, intent, lease) {
|
|
1775
|
+
if (!key || key.length > 256 || Buffer.byteLength(canonicalJson(intent)) > 65536 || captureEvidence(intent).bytes !== canonicalJson(intent))
|
|
1776
|
+
throw new HarnessError("operation_denied", "Invalid bounded external operation intent.");
|
|
1777
|
+
return this.transaction(async (client) => {
|
|
1778
|
+
const row = await this.locked(client, runId);
|
|
1779
|
+
const run = this.record(row);
|
|
1780
|
+
if (lease)
|
|
1781
|
+
await this.checkDispatch(client, row, lease);
|
|
1782
|
+
else if (run.terminal?.outcome !== "completed" || !run.terminal.cleanupOk)
|
|
1783
|
+
throw new HarnessError("operation_denied", "External assessment stages require a completed run with verified cleanup.");
|
|
1784
|
+
const requestDigest = digest(intent);
|
|
1785
|
+
const recorded = await this.recordedExternal(client, runId, kind, key, requestDigest);
|
|
1786
|
+
if (recorded)
|
|
1787
|
+
return recorded;
|
|
1788
|
+
const parsed = kind === "gyms" ? VerificationIntentSchema.parse(intent) : undefined;
|
|
1789
|
+
const unresolved = await client.query("SELECT e.kind,e.intent,c.state,c.parent_epoch,c.lease_until>clock_timestamp() AS live FROM harness_mastra.external_operations e LEFT JOIN harness_mastra.child_jobs c ON c.id=e.child_id AND c.run_id=e.run_id WHERE e.run_id=$1 AND e.state='dispatched'", [runId]);
|
|
1790
|
+
const parallel = parsed && lease?.child && this.grant.verifiers?.some((verifier) => canonicalJson(verifier.targetIdentity) === canonicalJson(TargetIdentitySchema.strip().parse(parsed.request.identity)));
|
|
1791
|
+
if (unresolved.rows.some((operation) => !parallel || operation.kind !== "gyms" || operation.state !== "running" || operation.parent_epoch !== lease?.epoch || !operation.live))
|
|
1792
|
+
throw new HarnessError("uncertain_execution", "Another external operation is unresolved; a new ID cannot bypass it.");
|
|
1793
|
+
if (parsed) {
|
|
1794
|
+
const candidate = (await this.findings(runId, client)).find((entry) => entry.finding.id === parsed.request.candidate.id);
|
|
1795
|
+
if (!candidate || !["pending", "needs_human"].includes(candidate.result) || parsed.request.identity.runId !== runId || key !== `${parsed.request.candidate.id}/${parsed.request.identity.attemptId}` || candidateRevision(candidate.finding) !== candidateRevision(parsed.request.candidate))
|
|
1796
|
+
throw new HarnessError("verification_denied", "Only the exact committed undecided candidate can be verified.");
|
|
1797
|
+
}
|
|
1798
|
+
const identity = parsed?.request.identity;
|
|
1799
|
+
const slot = identity ? digest({ rangeId: identity.rangeId, instanceId: identity.instanceId }) : null;
|
|
1800
|
+
if (slot) {
|
|
1801
|
+
await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [
|
|
1802
|
+
slot
|
|
1803
|
+
]);
|
|
1804
|
+
const occupied = await client.query("SELECT 1 FROM harness_mastra.external_operations WHERE verification_slot=$1 AND state='dispatched' LIMIT 1", [slot]);
|
|
1805
|
+
if (occupied.rowCount)
|
|
1806
|
+
throw new HarnessError("uncertain_execution", "This verification instance has an unresolved operation.");
|
|
1807
|
+
}
|
|
1808
|
+
const owner = randomUUID2();
|
|
1809
|
+
await client.query("INSERT INTO harness_mastra.external_operations(run_id,kind,operation_key,owner,request_digest,intent,state,child_id,verification_slot) VALUES ($1,$2,$3,$4,$5,$6,'dispatched',$7,$8)", [
|
|
1810
|
+
runId,
|
|
1811
|
+
kind,
|
|
1812
|
+
key,
|
|
1813
|
+
owner,
|
|
1814
|
+
requestDigest,
|
|
1815
|
+
intent,
|
|
1816
|
+
lease?.child?.id ?? null,
|
|
1817
|
+
slot
|
|
1818
|
+
]);
|
|
1819
|
+
return { runId, kind, key, owner, requestDigest };
|
|
1820
|
+
});
|
|
1821
|
+
}
|
|
1822
|
+
async dispatchedExternal(client, operation, lease) {
|
|
1823
|
+
const stored = (await client.query("SELECT intent FROM harness_mastra.external_operations WHERE run_id=$1 AND kind=$2 AND operation_key=$3 AND owner=$4 AND request_digest=$5 AND state='dispatched' AND ($6::uuid IS NULL OR child_id=$6)", [
|
|
1824
|
+
operation.runId,
|
|
1825
|
+
operation.kind,
|
|
1826
|
+
operation.key,
|
|
1827
|
+
operation.owner,
|
|
1828
|
+
operation.requestDigest,
|
|
1829
|
+
lease?.child?.id ?? null
|
|
1830
|
+
])).rows[0];
|
|
1831
|
+
if (!stored)
|
|
1832
|
+
throw new HarnessError("operation_denied", "No matching dispatched external operation exists.");
|
|
1833
|
+
return stored.intent;
|
|
1834
|
+
}
|
|
1835
|
+
async completeExternal(operation, value, verification, lease) {
|
|
1836
|
+
const result = captureEvidence(value);
|
|
1837
|
+
if (Buffer.byteLength(result.bytes) > 65536 || result.bytes !== canonicalJson(value))
|
|
1838
|
+
throw new HarnessError("operation_denied", "External operation result exceeds the capture bound or is not already redacted.");
|
|
1839
|
+
if (operation.kind !== "gyms" && verification)
|
|
1840
|
+
throw new HarnessError("operation_denied", "Delivery cannot confirm a finding.");
|
|
1841
|
+
const { run, authority, intent } = await this.transaction(async (client) => {
|
|
1842
|
+
const row = await this.locked(client, operation.runId);
|
|
1843
|
+
if (lease)
|
|
1844
|
+
await this.checkOwner(client, row, lease);
|
|
1845
|
+
return {
|
|
1846
|
+
run: this.record(row),
|
|
1847
|
+
authority: row.authority ?? null,
|
|
1848
|
+
intent: await this.dispatchedExternal(client, operation, lease)
|
|
1849
|
+
};
|
|
1850
|
+
});
|
|
1851
|
+
let admitted;
|
|
1852
|
+
let inconclusive;
|
|
1853
|
+
if (operation.kind === "gyms") {
|
|
1854
|
+
const parsed = VerificationIntentSchema.parse(intent);
|
|
1855
|
+
const verdict = validateOracleVerdict(parsed.request, value);
|
|
1856
|
+
if (verdict.artifactRef || verification) {
|
|
1857
|
+
if (!verification || Buffer.byteLength(verification.proof) > 65536 || captureEvidence(JSON.parse(verification.proof)).bytes !== verification.proof || verdict.artifactRef !== `sha256:${sha256(verification.proof)}`)
|
|
1858
|
+
throw new HarnessError("verification_denied", "The protected verifier proof must be exact canonical redacted evidence bound to its verdict.");
|
|
1859
|
+
}
|
|
1860
|
+
if (verdict.result === "needs_human" && verification?.finding)
|
|
1861
|
+
throw new HarnessError("verification_denied", "Inconclusive reproduction cannot supply a decided finding.");
|
|
1862
|
+
if (verdict.result === "needs_human" && verification)
|
|
1863
|
+
inconclusive = verdict;
|
|
1864
|
+
if (verdict.result !== "needs_human") {
|
|
1865
|
+
if (!verification?.finding)
|
|
1866
|
+
throw new HarnessError("verification_denied", "Decided verification requires its independently admitted finding.");
|
|
1867
|
+
admitted = await admitIndependentVerification(parsed.request.candidate, {
|
|
1868
|
+
result: verdict.result,
|
|
1869
|
+
method: verdict.method,
|
|
1870
|
+
artifactRef: verdict.artifactRef,
|
|
1871
|
+
receipt: verdict.receipt
|
|
1872
|
+
}, {
|
|
1873
|
+
...parsed.context,
|
|
1874
|
+
runId: run.id,
|
|
1875
|
+
capabilities: run.config.capabilities ?? [],
|
|
1876
|
+
now: new Date,
|
|
1877
|
+
getToolCall: completedToolCalls(run.id, await this.actions(run.id)),
|
|
1878
|
+
getArtifact: async (runId, ref) => runId === run.id && ref === verdict.artifactRef ? { bytes: verification.proof, redacted: true } : this.artifact(runId, ref)
|
|
1879
|
+
});
|
|
1880
|
+
if (canonicalJson(admitted) !== canonicalJson(verification.finding))
|
|
1881
|
+
throw new HarnessError("verification_denied", "The requested update differs from the admitted verification.");
|
|
1882
|
+
const current = (await this.findings(run.id)).find((entry) => entry.finding.id === admitted.id);
|
|
1883
|
+
if (!current || !["pending", "needs_human"].includes(current.result) || candidateRevision(current.finding) !== candidateRevision(admitted))
|
|
1884
|
+
throw new HarnessError("verification_denied", "The candidate was decided concurrently; no verdict was overwritten.");
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
const capture = await this.storeCapture(run.id, result.bytes, authority);
|
|
1888
|
+
const proofCapture = verification ? await this.storeCapture(run.id, verification.proof, authority) : undefined;
|
|
1889
|
+
await this.transaction(async (client) => {
|
|
1890
|
+
const row = await this.locked(client, run.id);
|
|
1891
|
+
if (lease)
|
|
1892
|
+
await this.checkOwner(client, row, lease);
|
|
1893
|
+
const completed = await client.query("UPDATE harness_mastra.external_operations SET state='completed',artifact_ref=$6,completed_at=clock_timestamp() WHERE run_id=$1 AND kind=$2 AND operation_key=$3 AND owner=$4 AND request_digest=$5 AND state='dispatched' AND ($7::uuid IS NULL OR child_id=$7)", [
|
|
1894
|
+
run.id,
|
|
1895
|
+
operation.kind,
|
|
1896
|
+
operation.key,
|
|
1897
|
+
operation.owner,
|
|
1898
|
+
operation.requestDigest,
|
|
1899
|
+
capture.ref,
|
|
1900
|
+
lease?.child?.id ?? null
|
|
1901
|
+
]);
|
|
1902
|
+
if (completed.rowCount !== 1)
|
|
1903
|
+
throw new HarnessError("operation_denied", "External completion has no matching dispatch.");
|
|
1904
|
+
await this.insertArtifact(client, run.id, capture);
|
|
1905
|
+
if (proofCapture)
|
|
1906
|
+
await this.insertArtifact(client, run.id, proofCapture);
|
|
1907
|
+
if (inconclusive && proofCapture) {
|
|
1908
|
+
const current = (await this.findings(run.id, client)).find((entry) => entry.finding.id === inconclusive.identity.candidateId);
|
|
1909
|
+
if (!current || !["pending", "needs_human"].includes(current.result) || candidateRevision(current.finding) !== inconclusive.identity.candidateRevision)
|
|
1910
|
+
throw new HarnessError("verification_denied", "The candidate changed during inconclusive reproduction.");
|
|
1911
|
+
if (current.result === "pending")
|
|
1912
|
+
await this.humanSettlement(client, run.id, current, proofCapture.ref);
|
|
1913
|
+
else {
|
|
1914
|
+
const finding = {
|
|
1915
|
+
...current.finding,
|
|
1916
|
+
verification: {
|
|
1917
|
+
result: "needs_human",
|
|
1918
|
+
artifactRef: proofCapture.ref
|
|
1919
|
+
}
|
|
1920
|
+
};
|
|
1921
|
+
await client.query("UPDATE harness_mastra.findings SET finding=$3 WHERE run_id=$1 AND id=$2", [run.id, finding.id, finding]);
|
|
1922
|
+
await this.append(client, run.id, {
|
|
1923
|
+
type: "finding.update",
|
|
1924
|
+
findingId: finding.id,
|
|
1925
|
+
status: "candidate",
|
|
1926
|
+
verification: finding.verification
|
|
1927
|
+
});
|
|
1928
|
+
}
|
|
1929
|
+
}
|
|
1930
|
+
if (admitted && proofCapture) {
|
|
1931
|
+
const updated = await client.query("UPDATE harness_mastra.findings SET status=$3,result=$4,finding=$5 WHERE run_id=$1 AND id=$2 AND candidate_revision=$6 AND result IN ('pending','needs_human')", [
|
|
1932
|
+
run.id,
|
|
1933
|
+
admitted.id,
|
|
1934
|
+
admitted.status,
|
|
1935
|
+
admitted.verification.result,
|
|
1936
|
+
admitted,
|
|
1937
|
+
candidateRevision(admitted)
|
|
1938
|
+
]);
|
|
1939
|
+
if (updated.rowCount !== 1)
|
|
1940
|
+
throw new HarnessError("verification_denied", "The candidate was decided concurrently; no verdict was overwritten.");
|
|
1941
|
+
await this.append(client, run.id, {
|
|
1942
|
+
type: "finding.update",
|
|
1943
|
+
findingId: admitted.id,
|
|
1944
|
+
status: admitted.status,
|
|
1945
|
+
verification: admitted.verification
|
|
1946
|
+
});
|
|
1947
|
+
}
|
|
1948
|
+
});
|
|
1949
|
+
return capture.ref;
|
|
1950
|
+
}
|
|
1951
|
+
async reportVerification(runId) {
|
|
1952
|
+
await this.read(runId);
|
|
1953
|
+
const result = await this.pool.query("SELECT intent,completed_at FROM harness_mastra.external_operations WHERE run_id=$1 AND kind='gyms' AND state='completed' ORDER BY completed_at", [runId]);
|
|
1954
|
+
const policies = new Map;
|
|
1955
|
+
for (const row of result.rows) {
|
|
1956
|
+
const { context } = VerificationIntentSchema.parse(row.intent);
|
|
1957
|
+
policies.set(digest(context), context);
|
|
1958
|
+
}
|
|
1959
|
+
return {
|
|
1960
|
+
policies: [...policies.values()],
|
|
1961
|
+
verifiedAt: result.rows.at(-1)?.completed_at
|
|
1962
|
+
};
|
|
1963
|
+
}
|
|
1964
|
+
async reportWorklist(runId, surface) {
|
|
1965
|
+
const work = await this.websiteWorklist?.read(runId, surface);
|
|
1966
|
+
return work ? {
|
|
1967
|
+
items: work.items,
|
|
1968
|
+
discoveryOmissions: work.omitted,
|
|
1969
|
+
pendingLimit: work.pendingLimit
|
|
1970
|
+
} : undefined;
|
|
1971
|
+
}
|
|
1972
|
+
async reportWeb(runId, client = this.pool) {
|
|
1973
|
+
const grant = this.grant;
|
|
1974
|
+
return grant.mode === "pentest" && (grant.config.website || grant.config.live || grant.config.apiDescriptions) ? webSurface(this, runId, {
|
|
1975
|
+
target: grant.target,
|
|
1976
|
+
targetRevision: grant.config.buildIdentityRef,
|
|
1977
|
+
scope: grant.config.website ?? (grant.config.live ? liveWebScope(grant.config.live, grant.target) : undefined),
|
|
1978
|
+
descriptions: grant.config.apiDescriptions,
|
|
1979
|
+
live: grant.config.live,
|
|
1980
|
+
permissionPolicy: grant.config.permissionPolicy,
|
|
1981
|
+
scanners: grant.config.website && grant.stages ? grant.config.scanners : undefined
|
|
1982
|
+
}, client) : undefined;
|
|
1983
|
+
}
|
|
1984
|
+
async verificationCoverage(runId) {
|
|
1985
|
+
await this.read(runId);
|
|
1986
|
+
const verified = await this.pool.query("SELECT intent,artifact_ref FROM harness_mastra.external_operations WHERE run_id=$1 AND kind='gyms' AND state='completed'", [runId]);
|
|
1987
|
+
const verifiedIds = new Set;
|
|
1988
|
+
const verdicts = await Promise.all(verified.rows.map(async (row) => {
|
|
1989
|
+
const { request } = VerificationIntentSchema.parse(row.intent);
|
|
1990
|
+
return {
|
|
1991
|
+
request,
|
|
1992
|
+
verdict: validateOracleVerdict(request, JSON.parse((await this.artifact(runId, row.artifact_ref)).bytes))
|
|
1993
|
+
};
|
|
1994
|
+
}));
|
|
1995
|
+
for (const { request, verdict } of verdicts) {
|
|
1996
|
+
if (verdict.result !== "needs_human")
|
|
1997
|
+
verifiedIds.add(request.candidate.id);
|
|
1998
|
+
}
|
|
1999
|
+
const byClass = new Map;
|
|
2000
|
+
for (const record of await this.findings(runId)) {
|
|
2001
|
+
const cwe = record.finding.cwe ?? "unclassified";
|
|
2002
|
+
const entry = byClass.get(cwe) ?? {
|
|
2003
|
+
candidates: 0,
|
|
2004
|
+
confirmed: 0,
|
|
2005
|
+
independentlyVerified: 0
|
|
2006
|
+
};
|
|
2007
|
+
entry.candidates++;
|
|
2008
|
+
if (record.status === "confirmed")
|
|
2009
|
+
entry.confirmed++;
|
|
2010
|
+
if (verifiedIds.has(record.finding.id))
|
|
2011
|
+
entry.independentlyVerified++;
|
|
2012
|
+
byClass.set(cwe, entry);
|
|
2013
|
+
}
|
|
2014
|
+
return [...byClass.entries()].map(([cwe, entry]) => ({ cwe, ...entry })).sort((a, b) => a.cwe.localeCompare(b.cwe));
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
// apps/harness/src/storage/website-setup.ts
|
|
2019
|
+
var WEBSITE_PRESETS = {
|
|
2020
|
+
quick: {
|
|
2021
|
+
durationMs: 600000,
|
|
2022
|
+
maxRequests: 100,
|
|
2023
|
+
maxPages: 5,
|
|
2024
|
+
maxDepth: 1,
|
|
2025
|
+
maxTotalBytes: 8388608
|
|
2026
|
+
},
|
|
2027
|
+
standard: {
|
|
2028
|
+
durationMs: 900000,
|
|
2029
|
+
maxRequests: 500,
|
|
2030
|
+
maxPages: 10,
|
|
2031
|
+
maxDepth: 2,
|
|
2032
|
+
maxTotalBytes: 33554432
|
|
2033
|
+
},
|
|
2034
|
+
deep: {
|
|
2035
|
+
durationMs: 3600000,
|
|
2036
|
+
maxRequests: 2000,
|
|
2037
|
+
maxPages: 40,
|
|
2038
|
+
maxDepth: 5,
|
|
2039
|
+
maxTotalBytes: 134217728
|
|
2040
|
+
}
|
|
2041
|
+
};
|
|
2042
|
+
function websiteLimits(policy, request) {
|
|
2043
|
+
const limits = {
|
|
2044
|
+
...request.trafficPreset ? WEBSITE_PRESETS[request.trafficPreset] : {},
|
|
2045
|
+
...request.limits
|
|
2046
|
+
};
|
|
2047
|
+
const selected = { ...policy };
|
|
2048
|
+
for (const key of [
|
|
2049
|
+
"maxRequests",
|
|
2050
|
+
"maxTotalBytes",
|
|
2051
|
+
"maxPages",
|
|
2052
|
+
"maxDepth",
|
|
2053
|
+
"requestsPerSecond",
|
|
2054
|
+
"concurrency"
|
|
2055
|
+
]) {
|
|
2056
|
+
const value = limits[key];
|
|
2057
|
+
if (value !== undefined)
|
|
2058
|
+
selected[key] = Math.min(policy[key], value);
|
|
2059
|
+
}
|
|
2060
|
+
return selected;
|
|
2061
|
+
}
|
|
2062
|
+
function websiteSetup(profile) {
|
|
2063
|
+
if (profile.mode !== "pentest")
|
|
2064
|
+
return;
|
|
2065
|
+
return {
|
|
2066
|
+
profileId: profile.id,
|
|
2067
|
+
identities: profile.grant.config.live?.sessions.map((entry) => entry.id) ?? [],
|
|
2068
|
+
apiDescriptions: [
|
|
2069
|
+
...new Set([
|
|
2070
|
+
...profile.apiDescriptions?.map((entry) => entry.id) ?? [],
|
|
2071
|
+
...profile.grant.config.apiDescriptions?.map((entry) => entry.id) ?? []
|
|
2072
|
+
])
|
|
2073
|
+
],
|
|
2074
|
+
scanners: profile.grant.config.scanners?.profiles.map((entry) => entry.id) ?? [],
|
|
2075
|
+
verifiers: profile.grant.verifiers?.map((entry) => `${entry.targetIdentity.rangeId}/${entry.targetIdentity.instanceId}`) ?? [],
|
|
2076
|
+
...profile.grant.config.websiteFamilies ? { families: profile.grant.config.websiteFamilies } : {}
|
|
2077
|
+
};
|
|
2078
|
+
}
|
|
2079
|
+
function selectWebsiteProfile(request, source) {
|
|
2080
|
+
if (source.mode !== "pentest")
|
|
2081
|
+
throw new HarnessError("preparation_denied", "Website setup requires an approved pentest profile.");
|
|
2082
|
+
const profile = structuredClone(source);
|
|
2083
|
+
const select = (values, ids, id) => {
|
|
2084
|
+
if (ids === undefined)
|
|
2085
|
+
return [...values];
|
|
2086
|
+
if (new Set(ids).size !== ids.length || ids.some((key) => !values.some((value) => id(value) === key)))
|
|
2087
|
+
throw new HarnessError("preparation_denied", "Every selected capability must name a unique entry in the approved profile.");
|
|
2088
|
+
return values.filter((value) => ids.includes(id(value)));
|
|
2089
|
+
};
|
|
2090
|
+
const config = profile.grant.config;
|
|
2091
|
+
if (request.families) {
|
|
2092
|
+
if (config.websiteFamilies && request.families.some((family) => !config.websiteFamilies.includes(family)))
|
|
2093
|
+
throw new HarnessError("preparation_denied", "Selected families must remain within the approved profile.");
|
|
2094
|
+
config.websiteFamilies = [...request.families];
|
|
2095
|
+
}
|
|
2096
|
+
if (request.trafficPreset || request.limits) {
|
|
2097
|
+
if (!config.website)
|
|
2098
|
+
throw new HarnessError("preparation_denied", "Traffic selection requires an approved website policy.");
|
|
2099
|
+
config.website = websiteLimits(config.website, request);
|
|
2100
|
+
}
|
|
2101
|
+
if (config.live) {
|
|
2102
|
+
const sessions = select(config.live.sessions, request.identities, (entry) => entry.id);
|
|
2103
|
+
if (sessions.length) {
|
|
2104
|
+
const ids = new Set(sessions.map((entry) => entry.id));
|
|
2105
|
+
const comparisons = config.live.comparisons?.filter((entry) => (!config.websiteFamilies || config.websiteFamilies.includes("authorization")) && [entry.controlRuleId, entry.comparisonRuleId].every((id) => config.permissionPolicy?.rules.some((rule) => rule.id === id && ids.has(rule.identity))));
|
|
2106
|
+
config.live = {
|
|
2107
|
+
...config.live,
|
|
2108
|
+
sessions,
|
|
2109
|
+
...comparisons?.length ? { comparisons } : {}
|
|
2110
|
+
};
|
|
2111
|
+
if (!comparisons?.length)
|
|
2112
|
+
delete config.live.comparisons;
|
|
2113
|
+
const testCases = config.live.testCases?.filter((entry) => (!config.websiteFamilies || config.websiteFamilies.includes(entry.family)) && caseSteps(entry).every((step) => ids.has(step.identity)));
|
|
2114
|
+
if (testCases?.length)
|
|
2115
|
+
config.live.testCases = testCases;
|
|
2116
|
+
else
|
|
2117
|
+
delete config.live.testCases;
|
|
2118
|
+
} else
|
|
2119
|
+
delete config.live;
|
|
2120
|
+
} else
|
|
2121
|
+
select([], request.identities, String);
|
|
2122
|
+
if (config.scanners) {
|
|
2123
|
+
const profiles = select(config.scanners.profiles, request.scanners, (entry) => entry.id);
|
|
2124
|
+
if (profiles.length)
|
|
2125
|
+
config.scanners = { ...config.scanners, profiles };
|
|
2126
|
+
else
|
|
2127
|
+
delete config.scanners;
|
|
2128
|
+
} else
|
|
2129
|
+
select([], request.scanners, String);
|
|
2130
|
+
const descriptions = websiteSetup(profile).apiDescriptions;
|
|
2131
|
+
select(descriptions, request.apiDescriptions, String);
|
|
2132
|
+
if (request.apiDescriptions) {
|
|
2133
|
+
if (profile.apiDescriptions) {
|
|
2134
|
+
profile.apiDescriptions = profile.apiDescriptions.filter((entry) => request.apiDescriptions.includes(entry.id));
|
|
2135
|
+
if (!profile.apiDescriptions.length)
|
|
2136
|
+
delete profile.apiDescriptions;
|
|
2137
|
+
}
|
|
2138
|
+
if (config.apiDescriptions)
|
|
2139
|
+
config.apiDescriptions = config.apiDescriptions.filter((entry) => request.apiDescriptions.includes(entry.id));
|
|
2140
|
+
}
|
|
2141
|
+
const verifiers = select(profile.grant.verifiers ?? [], request.verifiers, (entry) => `${entry.targetIdentity.rangeId}/${entry.targetIdentity.instanceId}`);
|
|
2142
|
+
if (verifiers.length)
|
|
2143
|
+
profile.grant.verifiers = verifiers;
|
|
2144
|
+
else
|
|
2145
|
+
delete profile.grant.verifiers;
|
|
2146
|
+
return profile;
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
// apps/harness/src/storage/investigation-profiles.ts
|
|
2150
|
+
var source = InvestigationGrantSchema.options[1];
|
|
2151
|
+
var base = {
|
|
2152
|
+
id: IdSchema,
|
|
2153
|
+
authority: RunAuthoritySchema,
|
|
2154
|
+
expiresAt: exports_iso.datetime(),
|
|
2155
|
+
models: array(InvestigationModelSchema).min(1).max(1024),
|
|
2156
|
+
stages: source.shape.stages.unwrap(),
|
|
2157
|
+
defaultDepth: InvestigationDepthSchema.optional()
|
|
2158
|
+
};
|
|
2159
|
+
var InvestigationProfileSchema = discriminatedUnion("mode", [
|
|
2160
|
+
strictObject({
|
|
2161
|
+
...base,
|
|
2162
|
+
mode: literal("codereview"),
|
|
2163
|
+
sandbox: SandboxConfigSchema,
|
|
2164
|
+
workspaceRoot: string().startsWith("/"),
|
|
2165
|
+
localRoot: string().startsWith("/").max(256).optional(),
|
|
2166
|
+
maxFiles: number().int().min(1).max(SOURCE_FILES_MAX),
|
|
2167
|
+
maxSourceBytes: number().int().min(1).max(SOURCE_TOTAL_BYTES),
|
|
2168
|
+
sourceExpansion: SourceReadPolicySchema.optional(),
|
|
2169
|
+
githubTokenFile: string().startsWith("/").optional(),
|
|
2170
|
+
allowPatchProposals: literal(true).optional(),
|
|
2171
|
+
testProfiles: source.shape.config.shape.testProfiles,
|
|
2172
|
+
verifiers: source.shape.verifiers,
|
|
2173
|
+
experiments: ExperimentPolicySchema.optional(),
|
|
2174
|
+
secretScan: SecretScanProfileSchema.optional(),
|
|
2175
|
+
dependencyScan: DependencyScanProfileSchema.omit({
|
|
2176
|
+
paths: true
|
|
2177
|
+
}).optional(),
|
|
2178
|
+
semgrep: SemgrepProfileSchema.optional(),
|
|
2179
|
+
joern: JoernProfileSchema.optional()
|
|
2180
|
+
}),
|
|
2181
|
+
strictObject({
|
|
2182
|
+
...base,
|
|
2183
|
+
mode: literal("pentest"),
|
|
2184
|
+
grant: InvestigationGrantSchema.options[0],
|
|
2185
|
+
apiDescriptions: array(ApiDescriptionReferenceSchema).min(1).max(4).optional()
|
|
2186
|
+
})
|
|
2187
|
+
]);
|
|
2188
|
+
async function loadReviewVerifiers(path) {
|
|
2189
|
+
try {
|
|
2190
|
+
return source.shape.verifiers.unwrap().parse(JSON.parse(await privateFile(path, 1048576)));
|
|
2191
|
+
} catch {
|
|
2192
|
+
throw new HarnessError("investigation_config", "HARNESS_REVIEW_VERIFIERS_FILE must name a private, operator-owned JSON array of one to eight pinned verifiers (maximum 1 MiB). No supplied contents are logged.");
|
|
2193
|
+
}
|
|
2194
|
+
}
|
|
2195
|
+
var InvestigationProfilesSchema = array(InvestigationProfileSchema).min(1).max(64).refine((profiles) => new Set(profiles.map((profile) => profile.id)).size === profiles.length).transform((profiles, context) => {
|
|
2196
|
+
try {
|
|
2197
|
+
return profiles.map(routedProfile);
|
|
2198
|
+
} catch {
|
|
2199
|
+
context.addIssue({
|
|
2200
|
+
code: "custom",
|
|
2201
|
+
message: "Every pentest grant URL must route to an http(s) site."
|
|
2202
|
+
});
|
|
2203
|
+
return NEVER;
|
|
2204
|
+
}
|
|
2205
|
+
});
|
|
2206
|
+
function routedProfile(profile) {
|
|
2207
|
+
if (profile.mode === "codereview")
|
|
2208
|
+
return profile;
|
|
2209
|
+
const { grant } = profile;
|
|
2210
|
+
const { permissionPolicy, website } = grant.config;
|
|
2211
|
+
return {
|
|
2212
|
+
...profile,
|
|
2213
|
+
grant: {
|
|
2214
|
+
...grant,
|
|
2215
|
+
target: siteHref(grant.target),
|
|
2216
|
+
config: {
|
|
2217
|
+
...grant.config,
|
|
2218
|
+
...permissionPolicy ? {
|
|
2219
|
+
permissionPolicy: {
|
|
2220
|
+
...permissionPolicy,
|
|
2221
|
+
target: siteHref(permissionPolicy.target)
|
|
2222
|
+
}
|
|
2223
|
+
} : {},
|
|
2224
|
+
...website?.scopes ? {
|
|
2225
|
+
website: {
|
|
2226
|
+
...website,
|
|
2227
|
+
scopes: website.scopes.map((scope) => ({
|
|
2228
|
+
...scope,
|
|
2229
|
+
url: siteHref(scope.url)
|
|
2230
|
+
}))
|
|
2231
|
+
}
|
|
2232
|
+
} : {}
|
|
2233
|
+
}
|
|
2234
|
+
}
|
|
2235
|
+
};
|
|
2236
|
+
}
|
|
2237
|
+
function siteHref(url) {
|
|
2238
|
+
const route = routeTarget(url);
|
|
2239
|
+
if (route.kind !== "web")
|
|
2240
|
+
throw new HarnessError("investigation_config", "A pentest grant URL must name an http(s) site.");
|
|
2241
|
+
return route.target;
|
|
2242
|
+
}
|
|
2243
|
+
async function loadInvestigationProfiles(path) {
|
|
2244
|
+
let profiles;
|
|
2245
|
+
try {
|
|
2246
|
+
profiles = InvestigationProfilesSchema.parse(JSON.parse(await privateFile(path, 1048576)));
|
|
2247
|
+
} catch {
|
|
2248
|
+
throw new HarnessError("investigation_config", "Use a private, operator-owned UTF-8 JSON array of investigation profiles (maximum 1 MiB). No supplied contents are logged.");
|
|
2249
|
+
}
|
|
2250
|
+
for (const profile of profiles)
|
|
2251
|
+
if (profile.mode === "codereview" && profile.testProfiles) {
|
|
2252
|
+
const tests = profile.testProfiles;
|
|
2253
|
+
if (!sandboxAllowsExecution(profile.sandbox, "test_run") || new Set(tests.map((test) => test.id)).size !== tests.length || tests.length > profile.maxFiles)
|
|
2254
|
+
throw new HarnessError("investigation_config", "Test profiles require gVisor isolation, unique IDs and no more entrypoints than maxFiles.");
|
|
2255
|
+
}
|
|
2256
|
+
return profiles;
|
|
2257
|
+
}
|
|
2258
|
+
function admitPreparation(profile, authority, request, now = Date.now()) {
|
|
2259
|
+
const target = routeTarget(request.target);
|
|
2260
|
+
const permitted = digest(profile.authority) === digest(authority) && profile.id === request.profileId && Date.parse(profile.expiresAt) > now && profile.models.some((model) => model.id === request.model) && (!request.setup || profile.mode === "pentest") && (profile.mode === "codereview" ? profile.localRoot ? target.kind === "local" && target.target === profile.localRoot && !profile.sourceExpansion : target.kind === "repository" || target.kind === "pull_request" || target.kind === "package" : target.kind === "web" && !request.paths && profile.grant.target === target.target);
|
|
2261
|
+
if (!permitted)
|
|
2262
|
+
throw new HarnessError("preparation_denied", "The requested target or model is outside the operator's current investigation profile.");
|
|
2263
|
+
const proofs = profile.mode === "pentest" ? [
|
|
2264
|
+
...profile.grant.config.ownershipProof ? [profile.grant.config.ownershipProof] : [],
|
|
2265
|
+
...profile.grant.config.ownershipProofs ?? []
|
|
2266
|
+
] : [];
|
|
2267
|
+
for (const proof of proofs)
|
|
2268
|
+
if (proof.token !== ownershipToken(authority, proof.apex ?? proof.host) || Date.parse(proof.expiresAt) <= now)
|
|
2269
|
+
throw new HarnessError("preparation_denied", "The recorded ownership proof is not bound to this operator and host.");
|
|
2270
|
+
}
|
|
2271
|
+
function automaticWebsiteProfile(profiles, authority, target, model, now = Date.now()) {
|
|
2272
|
+
const matches = profiles.filter((profile) => profile.mode === "pentest" && profile.grant.config.website && profile.grant.target === target && digest(profile.authority) === digest(authority) && Date.parse(profile.expiresAt) > now && profile.models.some((entry) => entry.id === model.id && entry.provider === model.provider));
|
|
2273
|
+
if (matches.length > 1)
|
|
2274
|
+
throw new HarnessError("preparation_denied", `More than one approved website profile matches. Choose a profileId: ${matches.map((profile) => profile.id).join(", ")}.`);
|
|
2275
|
+
const selected = matches[0];
|
|
2276
|
+
if (selected)
|
|
2277
|
+
admitPreparation(selected, authority, { profileId: selected.id, target, model: model.id }, now);
|
|
2278
|
+
return selected;
|
|
2279
|
+
}
|
|
2280
|
+
async function preparedWebsiteGrant(profile, request) {
|
|
2281
|
+
if (request.setup)
|
|
2282
|
+
profile = selectWebsiteProfile(request.setup, profile);
|
|
2283
|
+
const grant = InvestigationGrantSchema.options[0].parse(profileGrant(profile, request));
|
|
2284
|
+
if (!profile.apiDescriptions)
|
|
2285
|
+
return grant;
|
|
2286
|
+
const descriptions = [
|
|
2287
|
+
...grant.config.apiDescriptions ?? [],
|
|
2288
|
+
...await Promise.all(profile.apiDescriptions.map(async (reference) => {
|
|
2289
|
+
const description = apiDescription(reference, await privateFile(reference.documentFile, 1048576));
|
|
2290
|
+
if (reference.source) {
|
|
2291
|
+
const { originalFile, ...source } = reference.source;
|
|
2292
|
+
if (sha256(await privateFile(originalFile, 1048576)) !== source.sha256)
|
|
2293
|
+
throw new HarnessError("preparation_api", "The original API description differs from its pinned source digest.");
|
|
2294
|
+
description.source = source;
|
|
2295
|
+
}
|
|
2296
|
+
return description;
|
|
2297
|
+
}))
|
|
2298
|
+
];
|
|
2299
|
+
if (new Set(descriptions.map((entry) => entry.id)).size !== descriptions.length)
|
|
2300
|
+
throw new HarnessError("preparation_api", "API descriptions must have unique operator-owned IDs.");
|
|
2301
|
+
return investigationGrant({
|
|
2302
|
+
...grant,
|
|
2303
|
+
config: { ...grant.config, apiDescriptions: descriptions }
|
|
2304
|
+
});
|
|
2305
|
+
}
|
|
2306
|
+
function profileGrant(profile, request, source) {
|
|
2307
|
+
const model = profile.models.find((entry) => entry.id === request.model);
|
|
2308
|
+
const depth = request.depth ?? profile.defaultDepth;
|
|
2309
|
+
const shared = {
|
|
2310
|
+
model,
|
|
2311
|
+
stages: profile.stages,
|
|
2312
|
+
...depth ? { plan: { version: "1.0.0", depth } } : {}
|
|
2313
|
+
};
|
|
2314
|
+
if (profile.mode === "pentest")
|
|
2315
|
+
return investigationGrant({ ...profile.grant, ...shared });
|
|
2316
|
+
if (!source)
|
|
2317
|
+
throw new HarnessError("preparation_source", "An immutable repository snapshot is required.");
|
|
2318
|
+
const { retrieval: _retrieval, ...repository } = source.repository;
|
|
2319
|
+
const sourceExpansion = routeTarget(request.target).kind === "package" ? undefined : profile.sourceExpansion;
|
|
2320
|
+
return investigationGrant({
|
|
2321
|
+
mode: "codereview",
|
|
2322
|
+
target: request.target,
|
|
2323
|
+
...shared,
|
|
2324
|
+
sandbox: profile.sandbox,
|
|
2325
|
+
...profile.verifiers ? { verifiers: profile.verifiers } : {},
|
|
2326
|
+
boundary: sandboxRuntime(profile.sandbox) === "runc" ? "docker-repository" : "gvisor-repository",
|
|
2327
|
+
config: {
|
|
2328
|
+
repository: {
|
|
2329
|
+
...repository,
|
|
2330
|
+
...sourceExpansion ? {
|
|
2331
|
+
retrieval: {
|
|
2332
|
+
policy: sourceExpansion,
|
|
2333
|
+
expiresAt: profile.expiresAt,
|
|
2334
|
+
...profile.githubTokenFile ? { githubTokenFile: profile.githubTokenFile } : {}
|
|
2335
|
+
}
|
|
2336
|
+
} : {}
|
|
2337
|
+
},
|
|
2338
|
+
...source.change ? { change: source.change } : {},
|
|
2339
|
+
...profile.allowPatchProposals ? { allowPatchProposals: true } : {},
|
|
2340
|
+
...profile.testProfiles?.some((test) => repository.paths.includes(test.entrypoint)) ? {
|
|
2341
|
+
testProfiles: profile.testProfiles.filter((test) => repository.paths.includes(test.entrypoint))
|
|
2342
|
+
} : {},
|
|
2343
|
+
...profile.experiments ? { experiments: profile.experiments } : {},
|
|
2344
|
+
...profile.secretScan ? { secretScan: profile.secretScan } : {},
|
|
2345
|
+
...profile.semgrep ? { semgrep: profile.semgrep } : {},
|
|
2346
|
+
...profile.joern ? { joern: profile.joern } : {},
|
|
2347
|
+
...profile.dependencyScan && source.dependencyPaths.length ? {
|
|
2348
|
+
dependencyScan: {
|
|
2349
|
+
...profile.dependencyScan,
|
|
2350
|
+
paths: source.dependencyPaths
|
|
2351
|
+
}
|
|
2352
|
+
} : {}
|
|
2353
|
+
}
|
|
2354
|
+
});
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2357
|
+
export { liveRoute, liveProfiles, AdapterJournal, CallbackOpenSchema, CallbackReadSchema, CallbackReadResultSchema, CallbackJournal, StageAnalysisSchema, stagedWebsite, stageAnalyses, environmentRequirements, capabilityPlan, PENTEST_CONFIG, SITE_RECON_JOB, SITE_RECON_SCAN, SiteReconFileSchema, captureSiteRecon, completedToolCalls, AgentAConfigSchema, AgentBConfigSchema, GymsVerifierSchema, loadInvestigationGrant, investigationGrant, investigationScope, investigationRunConfig, validateInvestigationRun, admitInvestigationModel, InvestigationJournal, WEBSITE_PRESETS, websiteLimits, websiteSetup, selectWebsiteProfile, InvestigationProfileSchema, loadReviewVerifiers, InvestigationProfilesSchema, loadInvestigationProfiles, admitPreparation, automaticWebsiteProfile, preparedWebsiteGrant, profileGrant };
|