maka-agent 0.2.0-dev.4.20260830 → 0.2.0-dev.6.20260830
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/eval-bundle-path.js +1 -1
- package/dist/pi-tui-runner.js +3 -1
- package/dist/runtime-host-onboarding.js +10 -1
- package/dist/runtime-host-tui-context.js +2 -6
- package/native/runtime-host-peer/prebuilds/win32-x64/maka_runtime_host_peer.node +0 -0
- package/node_modules/@maka/eval/dist/external-subject.js +1 -1
- package/node_modules/@maka/eval/dist/harbor-external-subject.js +2 -2
- package/package.json +1 -2
- package/packages/eval/dist/attempt-store.js +0 -138
- package/packages/eval/dist/cli.js +0 -119
- package/packages/eval/dist/experiment-directory.js +0 -59
- package/packages/eval/dist/experiment.js +0 -49
- package/packages/eval/dist/external-subject.js +0 -488
- package/packages/eval/dist/harbor-external-subject.js +0 -929
- package/packages/eval/dist/harbor-maka-subject.js +0 -83
- package/packages/eval/dist/harness-environment.js +0 -80
- package/packages/eval/dist/harness-executor.js +0 -1013
- package/packages/eval/dist/index.js +0 -28
- package/packages/eval/dist/install-preflight.js +0 -201
- package/packages/eval/dist/maka-artifacts.js +0 -126
- package/packages/eval/dist/maka-runtime-policy.js +0 -42
- package/packages/eval/dist/maka-subject.js +0 -268
- package/packages/eval/dist/metering-checkpoint.js +0 -80
- package/packages/eval/dist/provider-admission.js +0 -39
- package/packages/eval/dist/provider-metering.js +0 -67
- package/packages/eval/dist/provider-web-tool-surface.js +0 -74
- package/packages/eval/dist/relay-result-frame.js +0 -38
- package/packages/eval/dist/result.js +0 -94
- package/packages/eval/dist/runner.js +0 -290
- package/packages/eval/dist/spec.js +0 -135
- package/packages/eval/dist/toolchain-verification.js +0 -130
- package/packages/eval/harbor/deepseek-codex-models.json +0 -69
- package/packages/eval/harbor/deepseek-harness-profile/cordis.patch.yml +0 -140
- package/packages/eval/harbor/deepseek-harness-profile/cordis.yml +0 -18
- package/packages/eval/harbor/deepseek-harness-profile/package.json +0 -10
- package/packages/eval/harbor/docker-compose-egress-proxy.yaml +0 -57
- package/packages/eval/harbor/egress-proxy/Dockerfile +0 -25
- package/packages/eval/harbor/egress-proxy/entrypoint.sh +0 -62
- package/packages/eval/harbor/egress-proxy/network-policy +0 -148
- package/packages/eval/harbor/egress_filter.py +0 -435
- package/packages/eval/harbor/eval_framework.py +0 -40
- package/packages/eval/harbor/relay_agent.py +0 -686
- package/packages/eval/harbor/run_trial.py +0 -223
- package/packages/eval/package.json +0 -20
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
#!/bin/sh
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
-
# or more contributor license agreements. See the NOTICE file
|
|
4
|
-
# distributed with this work for additional information
|
|
5
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
6
|
-
# to you under the Apache License, Version 2.0 (the
|
|
7
|
-
# "License"); you may not use this file except in compliance
|
|
8
|
-
# with the License. You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing,
|
|
13
|
-
# software distributed under the License is distributed on an
|
|
14
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
-
# KIND, either express or implied. See the License for the
|
|
16
|
-
# specific language governing permissions and limitations
|
|
17
|
-
# under the License.
|
|
18
|
-
|
|
19
|
-
set -eu
|
|
20
|
-
|
|
21
|
-
STATE_DIR=/opt/maka-egress-state
|
|
22
|
-
CERT_DIR=/opt/maka-egress
|
|
23
|
-
|
|
24
|
-
mkdir -p "$STATE_DIR" "$CERT_DIR"
|
|
25
|
-
# Create the audit log if this is the first start. Truncating here would erase
|
|
26
|
-
# hits already written if the proxy restarts mid-trial, and the trial would
|
|
27
|
-
# look like an honest zero-hit run.
|
|
28
|
-
touch "$STATE_DIR/hits.jsonl"
|
|
29
|
-
|
|
30
|
-
# confdir holds the CA private key and this cell's audit log, so it stays
|
|
31
|
-
# container-private and only the certificate reaches the volume the subject
|
|
32
|
-
# mounts. Creating the CA here instead of letting mitmdump create it on startup
|
|
33
|
-
# publishes the certificate before the proxy port opens, so the health gate
|
|
34
|
-
# cannot release `main` against a missing or left-over certificate.
|
|
35
|
-
python -c 'import sys
|
|
36
|
-
from mitmproxy.certs import CertStore
|
|
37
|
-
CertStore.from_store(sys.argv[1], "mitmproxy", 2048)' "$STATE_DIR"
|
|
38
|
-
# Publish by rename so a restart cannot expose a truncated certificate to a
|
|
39
|
-
# subject that is already running against the previous one.
|
|
40
|
-
cp "$STATE_DIR/mitmproxy-ca-cert.pem" "$CERT_DIR/.mitmproxy-ca-cert.pem"
|
|
41
|
-
mv "$CERT_DIR/.mitmproxy-ca-cert.pem" "$CERT_DIR/mitmproxy-ca-cert.pem"
|
|
42
|
-
|
|
43
|
-
# The subject only needs this address so HTTPS_PROXY can keep using the
|
|
44
|
-
# service hostname after the namespace policy refuses Docker DNS.
|
|
45
|
-
ip="$(getent ahostsv4 "$(hostname)" | awk 'NR == 1 { print $1 }')"
|
|
46
|
-
case "$ip" in
|
|
47
|
-
"" | 127.*)
|
|
48
|
-
echo "could not publish Eval egress proxy IPv4" >&2
|
|
49
|
-
exit 1
|
|
50
|
-
;;
|
|
51
|
-
esac
|
|
52
|
-
printf '%s\n' "$ip" > "$CERT_DIR/.proxy-ipv4"
|
|
53
|
-
mv "$CERT_DIR/.proxy-ipv4" "$CERT_DIR/proxy-ipv4"
|
|
54
|
-
|
|
55
|
-
exec mitmdump \
|
|
56
|
-
--quiet \
|
|
57
|
-
--listen-host 0.0.0.0 \
|
|
58
|
-
--listen-port 8080 \
|
|
59
|
-
--set block_global=false \
|
|
60
|
-
--set rawtcp=false \
|
|
61
|
-
--set confdir="$STATE_DIR" \
|
|
62
|
-
--scripts /opt/maka-eval/egress_filter.py
|
|
@@ -1,148 +0,0 @@
|
|
|
1
|
-
#!/bin/sh
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
-
# or more contributor license agreements. See the NOTICE file
|
|
4
|
-
# distributed with this work for additional information
|
|
5
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
6
|
-
# to you under the Apache License, Version 2.0 (the
|
|
7
|
-
# "License"); you may not use this file except in compliance
|
|
8
|
-
# with the License. You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing,
|
|
13
|
-
# software distributed under the License is distributed on an
|
|
14
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
-
# KIND, either express or implied. See the License for the
|
|
16
|
-
# specific language governing permissions and limitations
|
|
17
|
-
# under the License.
|
|
18
|
-
|
|
19
|
-
set -eu
|
|
20
|
-
|
|
21
|
-
# gost's allowlist is never written here: the sidecar entrypoint empties it at
|
|
22
|
-
# start and this policy only ever forwards through the audited proxy, so gost
|
|
23
|
-
# forwards nothing on its own. That also removes the packet-mark exemption gost
|
|
24
|
-
# needs when it does forward — the sidecar shares the subject's network
|
|
25
|
-
# namespace, so a mark the sidecar can set is one the subject can set too.
|
|
26
|
-
GOST_PORT=12345
|
|
27
|
-
NFTABLES_RULESET_NAME=gost_egress
|
|
28
|
-
PROXY_HOST=maka-eval-mitmproxy
|
|
29
|
-
PROXY_PORT=8080
|
|
30
|
-
|
|
31
|
-
remove_nftables() {
|
|
32
|
-
nft delete table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || true
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
PROXY_IPV4_PATH=/opt/maka-egress/proxy-ipv4
|
|
36
|
-
|
|
37
|
-
resolve_proxy_ip() {
|
|
38
|
-
# The proxy publishes this file before it is healthy. Reading it does not
|
|
39
|
-
# depend on Docker DNS, so deny-all then allow still has an address.
|
|
40
|
-
if [ ! -s "$PROXY_IPV4_PATH" ]; then
|
|
41
|
-
return
|
|
42
|
-
fi
|
|
43
|
-
ip=$(tr -d ' \t\r\n' < "$PROXY_IPV4_PATH")
|
|
44
|
-
case "$ip" in
|
|
45
|
-
"" | *.*.*.*.* | *[!0-9.]*)
|
|
46
|
-
return
|
|
47
|
-
;;
|
|
48
|
-
esac
|
|
49
|
-
# Same four-octet program as relay_agent.IPV4_OCTET_AWK.
|
|
50
|
-
if ! printf '%s\n' "$ip" | awk '
|
|
51
|
-
BEGIN { FS = "." }
|
|
52
|
-
NF != 4 { exit 1 }
|
|
53
|
-
{
|
|
54
|
-
for (i = 1; i <= 4; i++) {
|
|
55
|
-
if ($i !~ /^(0|[1-9][0-9]*)$/ || $i + 0 > 255) exit 1
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
'; then
|
|
59
|
-
return
|
|
60
|
-
fi
|
|
61
|
-
printf '%s\n' "$ip"
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
# The only difference between the two modes this policy installs is whether the
|
|
65
|
-
# audited proxy is exempt, so they are one ruleset with one optional pair of
|
|
66
|
-
# rules rather than two rulesets that have to be kept in step.
|
|
67
|
-
apply_ruleset() {
|
|
68
|
-
proxy_ip="${1:-}"
|
|
69
|
-
nft --file - <<EOF
|
|
70
|
-
add table inet $NFTABLES_RULESET_NAME
|
|
71
|
-
flush table inet $NFTABLES_RULESET_NAME
|
|
72
|
-
|
|
73
|
-
table inet $NFTABLES_RULESET_NAME {
|
|
74
|
-
chain output {
|
|
75
|
-
type nat hook output priority dstnat; policy accept;
|
|
76
|
-
|
|
77
|
-
fib daddr type local return
|
|
78
|
-
${proxy_ip:+ ip daddr $proxy_ip tcp dport $PROXY_PORT return}
|
|
79
|
-
meta l4proto tcp redirect to :$GOST_PORT
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
chain egress {
|
|
83
|
-
type filter hook output priority filter; policy accept;
|
|
84
|
-
|
|
85
|
-
# Docker DNATs 127.0.0.11:53 onto another local port before this hook.
|
|
86
|
-
# Matching only dport 53 here misses, and the local exemption below
|
|
87
|
-
# would then accept the rewritten packet.
|
|
88
|
-
ip daddr 127.0.0.11 reject
|
|
89
|
-
fib daddr type local accept
|
|
90
|
-
${proxy_ip:+ ip daddr $proxy_ip tcp dport $PROXY_PORT accept}
|
|
91
|
-
# Rejecting everything else rather than only non-TCP: the nat chain above
|
|
92
|
-
# rewrote every TCP destination this policy admits to a local one, so what
|
|
93
|
-
# reaches here still carrying a public destination is traffic the nat hook
|
|
94
|
-
# never saw — a connection conntrack established before the policy existed.
|
|
95
|
-
reject
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
EOF
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
setup_proxy_only() {
|
|
102
|
-
proxy_ip="$(resolve_proxy_ip)"
|
|
103
|
-
if [ -z "$proxy_ip" ]; then
|
|
104
|
-
echo "could not resolve Eval egress proxy" >&2
|
|
105
|
-
exit 1
|
|
106
|
-
fi
|
|
107
|
-
apply_ruleset "$proxy_ip"
|
|
108
|
-
echo "ok: Eval proxy-only egress applied ($proxy_ip:$PROXY_PORT)"
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
case "${1:-}" in
|
|
112
|
-
allow-all)
|
|
113
|
-
remove_nftables
|
|
114
|
-
echo "ok: allow all egress"
|
|
115
|
-
;;
|
|
116
|
-
deny-all)
|
|
117
|
-
# A phase that asked for no network gets none. Answering it with the
|
|
118
|
-
# proxy-only ruleset would hand it a route this override invented, which
|
|
119
|
-
# Harbor's own deny-all does not grant either.
|
|
120
|
-
apply_ruleset
|
|
121
|
-
echo "ok: Eval no-network egress applied"
|
|
122
|
-
;;
|
|
123
|
-
allow)
|
|
124
|
-
shift
|
|
125
|
-
if [ "$#" -ne 1 ] || [ "$1" != "$PROXY_HOST" ]; then
|
|
126
|
-
echo "Eval egress policy accepts only $PROXY_HOST" >&2
|
|
127
|
-
exit 2
|
|
128
|
-
fi
|
|
129
|
-
setup_proxy_only
|
|
130
|
-
;;
|
|
131
|
-
show)
|
|
132
|
-
if ! rules="$(nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null)"; then
|
|
133
|
-
echo "mode: allow-all"
|
|
134
|
-
elif printf '%s' "$rules" | grep -q "dport $PROXY_PORT"; then
|
|
135
|
-
echo "mode: Eval proxy-only"
|
|
136
|
-
echo "proxy: $PROXY_HOST:$PROXY_PORT"
|
|
137
|
-
else
|
|
138
|
-
echo "mode: Eval no-network"
|
|
139
|
-
fi
|
|
140
|
-
;;
|
|
141
|
-
rules)
|
|
142
|
-
nft list table inet "$NFTABLES_RULESET_NAME" 2>/dev/null || echo "mode: allow-all"
|
|
143
|
-
;;
|
|
144
|
-
*)
|
|
145
|
-
echo "Usage: network-policy {show|allow-all|deny-all|allow|rules}" >&2
|
|
146
|
-
exit 2
|
|
147
|
-
;;
|
|
148
|
-
esac
|
|
@@ -1,435 +0,0 @@
|
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
-
# or more contributor license agreements. See the NOTICE file
|
|
3
|
-
# distributed with this work for additional information
|
|
4
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
-
# to you under the Apache License, Version 2.0 (the
|
|
6
|
-
# "License"); you may not use this file except in compliance
|
|
7
|
-
# with the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing,
|
|
12
|
-
# software distributed under the License is distributed on an
|
|
13
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
-
# KIND, either express or implied. See the License for the
|
|
15
|
-
# specific language governing permissions and limitations
|
|
16
|
-
# under the License.
|
|
17
|
-
|
|
18
|
-
"""Fail-closed URL contamination filter for Eval subject egress."""
|
|
19
|
-
|
|
20
|
-
from __future__ import annotations
|
|
21
|
-
|
|
22
|
-
import json
|
|
23
|
-
import os
|
|
24
|
-
import re
|
|
25
|
-
import time
|
|
26
|
-
from pathlib import Path
|
|
27
|
-
from urllib.parse import unquote, urlsplit
|
|
28
|
-
|
|
29
|
-
PINNED_REVISION = "d49e28f1e4ddd13d289e85a5f312a66750951932"
|
|
30
|
-
MAX_DECODE_PASSES = 4
|
|
31
|
-
MAX_AUDIT_BYTES = 1024 * 1024
|
|
32
|
-
AUDIT_PATH = Path(
|
|
33
|
-
os.environ.get("MAKA_EVAL_EGRESS_AUDIT", "/opt/maka-egress-state/hits.jsonl")
|
|
34
|
-
)
|
|
35
|
-
PERCENT_ESCAPE = re.compile(r"%(?![0-9a-fA-F]{2})")
|
|
36
|
-
TERMINAL_BENCH = re.compile(r"terminal[-_.%/+\s]*bench", re.IGNORECASE)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def contamination_rule(raw_url: str) -> tuple[str, str, str] | None:
|
|
40
|
-
normalized = normalize_url(raw_url)
|
|
41
|
-
url = urlsplit(normalized)
|
|
42
|
-
host = (url.hostname or "").lower().rstrip(".")
|
|
43
|
-
path_query = f"{url.path}?{url.query}" if url.query else url.path
|
|
44
|
-
lowered = path_query.lower()
|
|
45
|
-
|
|
46
|
-
# Search the host and the path separately. A benchmark name in the hostname
|
|
47
|
-
# is a contamination surface, and searching the two fields joined would let
|
|
48
|
-
# a rule match across their boundary.
|
|
49
|
-
def anywhere(needle: str) -> bool:
|
|
50
|
-
return needle in host or needle in lowered
|
|
51
|
-
|
|
52
|
-
if host == "r.jina.ai":
|
|
53
|
-
inner = unquote(url.path.lstrip("/"))
|
|
54
|
-
if inner.startswith(("http://", "https://")):
|
|
55
|
-
nested = contamination_rule(inner)
|
|
56
|
-
if nested:
|
|
57
|
-
return (f"jina_recursive:{nested[0]}", host, path_query)
|
|
58
|
-
|
|
59
|
-
if anywhere(PINNED_REVISION):
|
|
60
|
-
return ("pinned_revision", host, path_query)
|
|
61
|
-
if host == "tbench.ai" or host.endswith(".tbench.ai"):
|
|
62
|
-
return ("tbench_domain", host, path_query)
|
|
63
|
-
if host == "hub.harborframework.com" and "/tasks/terminal-bench" in lowered:
|
|
64
|
-
return ("harbor_task_registry", host, path_query)
|
|
65
|
-
if benchmark_repository(host, lowered):
|
|
66
|
-
return ("benchmark_repository", host, path_query)
|
|
67
|
-
if public_trajectory_repository(host, lowered):
|
|
68
|
-
return ("public_trajectory", host, path_query)
|
|
69
|
-
if anywhere("patches-terminalbench-"):
|
|
70
|
-
return ("known_patch_artifact", host, path_query)
|
|
71
|
-
if TERMINAL_BENCH.search(host) or TERMINAL_BENCH.search(lowered):
|
|
72
|
-
return ("terminal_bench_url", host, path_query)
|
|
73
|
-
return None
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
def normalize_url(raw_url: str) -> str:
|
|
77
|
-
value = raw_url.strip()
|
|
78
|
-
if not value:
|
|
79
|
-
raise ValueError("empty URL")
|
|
80
|
-
for _ in range(MAX_DECODE_PASSES):
|
|
81
|
-
if PERCENT_ESCAPE.search(value):
|
|
82
|
-
raise ValueError("malformed percent escape")
|
|
83
|
-
decoded = unquote(value)
|
|
84
|
-
if decoded == value:
|
|
85
|
-
break
|
|
86
|
-
value = decoded
|
|
87
|
-
else:
|
|
88
|
-
raise ValueError("URL exceeded decode limit")
|
|
89
|
-
parsed = urlsplit(value)
|
|
90
|
-
if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname:
|
|
91
|
-
raise ValueError("unsupported URL")
|
|
92
|
-
return value
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
def benchmark_repository(host: str, path_query: str) -> bool:
|
|
96
|
-
repositories = (
|
|
97
|
-
"harbor-framework/terminal-bench",
|
|
98
|
-
"terminal-benchmarks/terminal-bench",
|
|
99
|
-
"tbench-ai/terminal-bench",
|
|
100
|
-
)
|
|
101
|
-
return host in {
|
|
102
|
-
"github.com",
|
|
103
|
-
"api.github.com",
|
|
104
|
-
"raw.githubusercontent.com",
|
|
105
|
-
"codeload.github.com",
|
|
106
|
-
} and any(repository in path_query for repository in repositories)
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
def public_trajectory_repository(host: str, path_query: str) -> bool:
|
|
110
|
-
return (
|
|
111
|
-
host in {"github.com", "api.github.com", "raw.githubusercontent.com", "huggingface.co"}
|
|
112
|
-
and "hqeric/maka-eval-trajectories" in path_query
|
|
113
|
-
)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
try:
|
|
117
|
-
from mitmproxy import http
|
|
118
|
-
except ImportError:
|
|
119
|
-
http = None
|
|
120
|
-
|
|
121
|
-
try:
|
|
122
|
-
from mitmproxy.proxy import commands as proxy_commands
|
|
123
|
-
from mitmproxy.proxy.layer import Layer
|
|
124
|
-
from mitmproxy.proxy.layers import ClientTLSLayer, ServerTLSLayer
|
|
125
|
-
from mitmproxy.proxy.layers.tcp import TCPLayer
|
|
126
|
-
except ImportError:
|
|
127
|
-
proxy_commands = None
|
|
128
|
-
Layer = object
|
|
129
|
-
ClientTLSLayer = None
|
|
130
|
-
ServerTLSLayer = None
|
|
131
|
-
TCPLayer = None
|
|
132
|
-
|
|
133
|
-
try:
|
|
134
|
-
from mitmproxy.net.tls import starts_like_tls_record
|
|
135
|
-
except ImportError:
|
|
136
|
-
def starts_like_tls_record(data: bytes) -> bool:
|
|
137
|
-
return len(data) >= 3 and data[0] == 0x16 and data[1] == 0x03
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
def configure(updated: object) -> None:
|
|
141
|
-
# HTTP 101 upgrades construct TCPLayer inside the HTTP layer without
|
|
142
|
-
# another next_layer hook. rawtcp=false makes that path CloseConnection
|
|
143
|
-
# itself; WebSocket upgrades stay on the websocket layer.
|
|
144
|
-
try:
|
|
145
|
-
from mitmproxy import ctx
|
|
146
|
-
except ImportError:
|
|
147
|
-
return
|
|
148
|
-
if getattr(ctx.options, "rawtcp", False):
|
|
149
|
-
ctx.options.rawtcp = False
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
def request(flow: object) -> None:
|
|
153
|
-
apply_http_policy(flow, flow.request.pretty_url)
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
def response(flow: object) -> None:
|
|
157
|
-
response = getattr(flow, "response", None)
|
|
158
|
-
if getattr(response, "status_code", None) != 101:
|
|
159
|
-
return
|
|
160
|
-
# mitmproxy 12.2.3 sets flow.websocket before HttpResponseHook only when
|
|
161
|
-
# the 101 is a real WebSocket upgrade (Upgrade + version 13 + option on).
|
|
162
|
-
# A websocket Upgrade header alone still falls through to CloseConnection
|
|
163
|
-
# under rawtcp=false and must be audited.
|
|
164
|
-
if getattr(flow, "websocket", None) is not None:
|
|
165
|
-
return
|
|
166
|
-
record_raw_tunnel(flow)
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
def http_connect(flow: object) -> None:
|
|
170
|
-
try:
|
|
171
|
-
raw_url = connect_target_url(flow)
|
|
172
|
-
except Exception:
|
|
173
|
-
raw_url = ""
|
|
174
|
-
apply_http_policy(flow, raw_url)
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
def tcp_start(flow: object) -> None:
|
|
178
|
-
# Last resort if a TCPLayer is still admitted (tcp_hosts / ignore).
|
|
179
|
-
# CONNECT raw is closed by next_layer → CloseRawLayer; HTTP 101 raw is
|
|
180
|
-
# closed by rawtcp=false. Neither of those paths starts a TCPLayer.
|
|
181
|
-
record_raw_tunnel(flow)
|
|
182
|
-
kill_flow(flow)
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
def tcp_message(flow: object) -> None:
|
|
186
|
-
messages = getattr(flow, "messages", None)
|
|
187
|
-
if messages:
|
|
188
|
-
messages[-1].content = b""
|
|
189
|
-
kill_flow(flow)
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
def next_layer(nextlayer: object) -> None:
|
|
193
|
-
# Script addons run before the built-in classifier assigns layer. If we
|
|
194
|
-
# set CloseRawLayer here, NextLayer leaves it in place. Waiting for
|
|
195
|
-
# isinstance(..., TCPLayer) never fires on the production CONNECT path.
|
|
196
|
-
current = getattr(nextlayer, "layer", None)
|
|
197
|
-
context = getattr(nextlayer, "context", None)
|
|
198
|
-
if current is None:
|
|
199
|
-
data_client = _next_layer_bytes(nextlayer, "data_client")
|
|
200
|
-
if _is_fragmented_tls_record_prefix(data_client):
|
|
201
|
-
if ClientTLSLayer is None or ServerTLSLayer is None:
|
|
202
|
-
return
|
|
203
|
-
server_tls = ServerTLSLayer(context)
|
|
204
|
-
server_tls.child_layer = ClientTLSLayer(context)
|
|
205
|
-
nextlayer.layer = server_tls
|
|
206
|
-
return
|
|
207
|
-
if not looks_like_raw_tcp(nextlayer):
|
|
208
|
-
return
|
|
209
|
-
record_raw_tunnel(context)
|
|
210
|
-
nextlayer.layer = CloseRawLayer(context)
|
|
211
|
-
return
|
|
212
|
-
if TCPLayer is None or not isinstance(current, TCPLayer):
|
|
213
|
-
return
|
|
214
|
-
record_raw_tunnel(context)
|
|
215
|
-
closer = CloseRawLayer(context)
|
|
216
|
-
replace_layer(context, current, closer)
|
|
217
|
-
nextlayer.layer = closer
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
def apply_http_policy(flow: object, raw_url: str) -> None:
|
|
221
|
-
if http is None:
|
|
222
|
-
raise RuntimeError("mitmproxy is required to run the Eval egress filter")
|
|
223
|
-
try:
|
|
224
|
-
matched = contamination_rule(raw_url)
|
|
225
|
-
if not matched:
|
|
226
|
-
return
|
|
227
|
-
rule_id, host, normalized_path = matched
|
|
228
|
-
append_audit(rule_id, host, normalized_path)
|
|
229
|
-
flow.response = blocked_response(rule_id)
|
|
230
|
-
except Exception as error:
|
|
231
|
-
flow.response = http.Response.make(
|
|
232
|
-
503,
|
|
233
|
-
b"Eval egress policy could not classify this request.\n",
|
|
234
|
-
{
|
|
235
|
-
"Content-Type": "text/plain; charset=utf-8",
|
|
236
|
-
"X-Maka-Eval-Egress-Rule": "policy_error",
|
|
237
|
-
},
|
|
238
|
-
)
|
|
239
|
-
try:
|
|
240
|
-
append_audit("policy_error", "", type(error).__name__)
|
|
241
|
-
except Exception:
|
|
242
|
-
pass
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
def connect_target_url(flow: object) -> str:
|
|
246
|
-
request = flow.request
|
|
247
|
-
host = (getattr(request, "pretty_host", None) or getattr(request, "host", "") or "").strip()
|
|
248
|
-
if not host:
|
|
249
|
-
raise ValueError("empty CONNECT host")
|
|
250
|
-
if ":" in host and not host.startswith("["):
|
|
251
|
-
host = f"[{host}]"
|
|
252
|
-
port = getattr(request, "port", None)
|
|
253
|
-
if port in (None, 443):
|
|
254
|
-
return f"https://{host}/"
|
|
255
|
-
if port == 80:
|
|
256
|
-
return f"http://{host}/"
|
|
257
|
-
return f"https://{host}:{port}/"
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
def looks_like_raw_tcp(nextlayer: object) -> bool:
|
|
261
|
-
"""Close only when the bytes cannot still become HTTP or TLS.
|
|
262
|
-
|
|
263
|
-
Script next_layer runs before mitmproxy 12.2.3's classifier. Copying its
|
|
264
|
-
`probably_no_http` test here would treat `GET` / `GET ` as raw and assign
|
|
265
|
-
CloseRawLayer while the request line is still arriving. With rawtcp=false
|
|
266
|
-
the built-in path would have kept waiting for HttpLayer.
|
|
267
|
-
"""
|
|
268
|
-
data_client = _next_layer_bytes(nextlayer, "data_client")
|
|
269
|
-
data_server = _next_layer_bytes(nextlayer, "data_server")
|
|
270
|
-
if _could_start_tls_record(data_client):
|
|
271
|
-
return False
|
|
272
|
-
if not data_client and not data_server:
|
|
273
|
-
return False
|
|
274
|
-
if data_server or data_client.startswith(b"SSH"):
|
|
275
|
-
return True
|
|
276
|
-
return not _still_could_be_http(data_client)
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
def _could_start_tls_record(data: bytes) -> bool:
|
|
280
|
-
"""Keep a fragmented ClientHello undecided until its 3-byte prefix exists."""
|
|
281
|
-
if starts_like_tls_record(data):
|
|
282
|
-
return True
|
|
283
|
-
return _is_fragmented_tls_record_prefix(data)
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
def _is_fragmented_tls_record_prefix(data: bytes) -> bool:
|
|
287
|
-
return 0 < len(data) < 3 and b"\x16\x03".startswith(data)
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
def _still_could_be_http(data: bytes) -> bool:
|
|
291
|
-
first_line, newline, _rest = data.partition(b"\n")
|
|
292
|
-
line = first_line.rstrip(b"\r")
|
|
293
|
-
method, space, _remainder = line.partition(b" ")
|
|
294
|
-
if not method.isascii() or not method.isalpha():
|
|
295
|
-
return False
|
|
296
|
-
if newline and not space:
|
|
297
|
-
return False
|
|
298
|
-
return True
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
def _next_layer_bytes(nextlayer: object, name: str) -> bytes:
|
|
302
|
-
getter = getattr(nextlayer, name, None)
|
|
303
|
-
if not callable(getter):
|
|
304
|
-
return b""
|
|
305
|
-
try:
|
|
306
|
-
data = getter()
|
|
307
|
-
except Exception:
|
|
308
|
-
return b""
|
|
309
|
-
return bytes(data) if isinstance(data, (bytes, bytearray)) else b""
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
def tcp_peer(flow: object) -> tuple[str, str]:
|
|
313
|
-
server = getattr(flow, "server_conn", None) or getattr(flow, "server", None)
|
|
314
|
-
address = getattr(server, "address", None) if server is not None else None
|
|
315
|
-
if isinstance(address, (tuple, list)) and address:
|
|
316
|
-
host = str(address[0])[:255]
|
|
317
|
-
port = address[1] if len(address) > 1 else ""
|
|
318
|
-
return host, f":{port}" if port != "" else ""
|
|
319
|
-
return "", ""
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
def record_raw_tunnel(flow: object) -> None:
|
|
323
|
-
host, path = tcp_peer(flow)
|
|
324
|
-
try:
|
|
325
|
-
append_audit("raw_tunnel", host, path)
|
|
326
|
-
except Exception:
|
|
327
|
-
pass
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
def kill_flow(flow: object) -> None:
|
|
331
|
-
kill = getattr(flow, "kill", None)
|
|
332
|
-
if callable(kill) and getattr(flow, "killable", True):
|
|
333
|
-
try:
|
|
334
|
-
kill()
|
|
335
|
-
except Exception:
|
|
336
|
-
pass
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
def replace_layer(context: object, current: object, closer: object) -> None:
|
|
340
|
-
layers = getattr(context, "layers", None)
|
|
341
|
-
if not isinstance(layers, list):
|
|
342
|
-
return
|
|
343
|
-
try:
|
|
344
|
-
index = layers.index(current)
|
|
345
|
-
except ValueError:
|
|
346
|
-
index = len(layers)
|
|
347
|
-
if closer in layers:
|
|
348
|
-
layers.remove(closer)
|
|
349
|
-
if current in layers:
|
|
350
|
-
layers.remove(current)
|
|
351
|
-
layers.insert(min(index, len(layers)), closer)
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
class CloseRawLayer(Layer):
|
|
355
|
-
def __init__(self, context: object) -> None:
|
|
356
|
-
if Layer is object:
|
|
357
|
-
self.context = context
|
|
358
|
-
return
|
|
359
|
-
if getattr(context, "layers", None) is None:
|
|
360
|
-
context.layers = []
|
|
361
|
-
if getattr(context, "options", None) is None:
|
|
362
|
-
context.options = type("Options", (), {"proxy_debug": False})()
|
|
363
|
-
super().__init__(context)
|
|
364
|
-
|
|
365
|
-
def handle_event(self, event: object):
|
|
366
|
-
if proxy_commands is None:
|
|
367
|
-
return
|
|
368
|
-
yield
|
|
369
|
-
for name in ("client", "server"):
|
|
370
|
-
connection = getattr(self.context, name, None)
|
|
371
|
-
if connection is not None:
|
|
372
|
-
yield proxy_commands.CloseConnection(connection)
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
def blocked_response(rule_id: str):
|
|
376
|
-
return http.Response.make(
|
|
377
|
-
451,
|
|
378
|
-
b"Benchmark source or public solution access is blocked during evaluation.\n",
|
|
379
|
-
{
|
|
380
|
-
"Content-Type": "text/plain; charset=utf-8",
|
|
381
|
-
"X-Maka-Eval-Egress-Rule": rule_id,
|
|
382
|
-
},
|
|
383
|
-
)
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
def append_audit(rule_id: str, host: str, normalized_path: str) -> None:
|
|
387
|
-
AUDIT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
388
|
-
if AUDIT_PATH.exists() and AUDIT_PATH.stat().st_size >= MAX_AUDIT_BYTES:
|
|
389
|
-
write_truncation_marker()
|
|
390
|
-
return
|
|
391
|
-
record = {
|
|
392
|
-
"ts": int(time.time() * 1000),
|
|
393
|
-
"ruleId": rule_id,
|
|
394
|
-
"host": host[:255],
|
|
395
|
-
"normalizedPath": normalized_path[:4096],
|
|
396
|
-
}
|
|
397
|
-
with AUDIT_PATH.open("a", encoding="utf-8") as stream:
|
|
398
|
-
stream.write(json.dumps(record, ensure_ascii=True, separators=(",", ":")) + "\n")
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
def write_truncation_marker() -> None:
|
|
402
|
-
if audit_already_truncated():
|
|
403
|
-
return
|
|
404
|
-
record = {
|
|
405
|
-
"ts": int(time.time() * 1000),
|
|
406
|
-
"ruleId": "audit_truncated",
|
|
407
|
-
"host": "",
|
|
408
|
-
"normalizedPath": "",
|
|
409
|
-
}
|
|
410
|
-
prefix = ""
|
|
411
|
-
if AUDIT_PATH.exists() and AUDIT_PATH.stat().st_size > 0:
|
|
412
|
-
with AUDIT_PATH.open("rb") as stream:
|
|
413
|
-
stream.seek(-1, os.SEEK_END)
|
|
414
|
-
if stream.read(1) != b"\n":
|
|
415
|
-
prefix = "\n"
|
|
416
|
-
with AUDIT_PATH.open("a", encoding="utf-8") as stream:
|
|
417
|
-
stream.write(prefix + json.dumps(record, ensure_ascii=True, separators=(",", ":")) + "\n")
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
def audit_already_truncated() -> bool:
|
|
421
|
-
if not AUDIT_PATH.exists():
|
|
422
|
-
return False
|
|
423
|
-
with AUDIT_PATH.open("rb") as stream:
|
|
424
|
-
stream.seek(0, os.SEEK_END)
|
|
425
|
-
size = stream.tell()
|
|
426
|
-
stream.seek(max(0, size - 4096))
|
|
427
|
-
tail = stream.read().decode("utf-8", errors="ignore")
|
|
428
|
-
lines = [line for line in tail.splitlines() if line.strip()]
|
|
429
|
-
if not lines:
|
|
430
|
-
return False
|
|
431
|
-
try:
|
|
432
|
-
parsed = json.loads(lines[-1])
|
|
433
|
-
except json.JSONDecodeError:
|
|
434
|
-
return False
|
|
435
|
-
return isinstance(parsed, dict) and parsed.get("ruleId") == "audit_truncated"
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
-
# or more contributor license agreements. See the NOTICE file
|
|
3
|
-
# distributed with this work for additional information
|
|
4
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
-
# to you under the Apache License, Version 2.0 (the
|
|
6
|
-
# "License"); you may not use this file except in compliance
|
|
7
|
-
# with the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing,
|
|
12
|
-
# software distributed under the License is distributed on an
|
|
13
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
-
# KIND, either express or implied. See the License for the
|
|
15
|
-
# specific language governing permissions and limitations
|
|
16
|
-
# under the License.
|
|
17
|
-
|
|
18
|
-
"""Authoritative Eval harness framework selection.
|
|
19
|
-
|
|
20
|
-
`run_trial.py` validates the argv framework and installs it here before Harbor,
|
|
21
|
-
Pier, or the shared relay can import. The relay must not read the environment.
|
|
22
|
-
"""
|
|
23
|
-
|
|
24
|
-
from __future__ import annotations
|
|
25
|
-
|
|
26
|
-
_FRAMEWORKS = frozenset({"harbor", "pier"})
|
|
27
|
-
_framework: str | None = None
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def install(framework: str) -> None:
|
|
31
|
-
if framework not in _FRAMEWORKS:
|
|
32
|
-
raise RuntimeError("framework must be harbor or pier")
|
|
33
|
-
global _framework
|
|
34
|
-
_framework = framework
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def selected() -> str:
|
|
38
|
-
if _framework not in _FRAMEWORKS:
|
|
39
|
-
raise RuntimeError("Eval framework selection is not installed")
|
|
40
|
-
return _framework
|