@ccoalm/ccl-skills 0.15.4 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/marketplace/plugins/ccl-skills/skills/code-review/SKILL.md +15 -15
- package/dist/assets/marketplace/plugins/ccl-skills/skills/code-review/references/staged-review-contract.md +70 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/code-review/scripts/codex_review.sh +286 -40
- package/dist/assets/marketplace/plugins/ccl-skills/skills/code-review/scripts/review_gate.py +177 -89
- package/dist/assets/marketplace/plugins/ccl-skills/skills/code-review/scripts/test_cli_review_wrappers.sh +396 -25
- package/dist/assets/marketplace/plugins/ccl-skills/skills/code-review/scripts/test_review_gate.sh +236 -1
- package/dist/assets/marketplace/plugins/ccl-skills/skills/product-rd-workflow/SKILL.md +1 -1
- package/dist/assets/marketplace/plugins/ccl-skills/skills/product-rd-workflow/references/pre-final-continuation-gate.md +3 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/references/source-register.md +20 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/review_ledger_binding.py +56 -5
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_ai_coding_implementation_gates.sh +6 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/skill-extraction-workflow/scripts/test_review_ledger_binding.sh +68 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/testing-strategy/references/ci-fixtures-and-flake-control.md +14 -0
- package/dist/assets/marketplace/plugins/ccl-skills/skills/testing-strategy/references/scenario-testing.md +1 -1
- package/dist/assets/release.json +17 -17
- package/package.json +1 -1
|
@@ -698,18 +698,31 @@ if [ "${1:-}" = mcp ] && [ "${2:-}" = list ]; then
|
|
|
698
698
|
fi
|
|
699
699
|
python3 - "$@" <<'PY_MCP_LIST'
|
|
700
700
|
import json, os, sys, tomllib
|
|
701
|
+
from pathlib import Path
|
|
701
702
|
|
|
702
703
|
arguments = iter(sys.argv[1:])
|
|
703
704
|
servers = {}
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
if
|
|
712
|
-
|
|
705
|
+
plugin_sourced = set()
|
|
706
|
+
# Inherited servers come from the home the CLI was actually handed, never from
|
|
707
|
+
# the test's own environment: whether the reviewer sees a user's server is
|
|
708
|
+
# exactly the question, and a stub that injects one regardless of the home
|
|
709
|
+
# could not tell a private home from the user's.
|
|
710
|
+
home = Path(os.environ.get("CODEX_HOME", ""))
|
|
711
|
+
home_config = home / "config.toml" if os.environ.get("CODEX_HOME") else None
|
|
712
|
+
if home_config is not None and home_config.exists():
|
|
713
|
+
try:
|
|
714
|
+
home_data = tomllib.loads(home_config.read_text(encoding="utf-8"))
|
|
715
|
+
except (OSError, UnicodeError, tomllib.TOMLDecodeError):
|
|
716
|
+
home_data = {}
|
|
717
|
+
for inherited_name, inherited in (home_data.get("mcp_servers") or {}).items():
|
|
718
|
+
if isinstance(inherited, dict):
|
|
719
|
+
servers[inherited_name] = dict(inherited)
|
|
720
|
+
servers[inherited_name].setdefault("enabled", True)
|
|
721
|
+
# A plugin contributes its server outside `mcp_servers`, which is why an
|
|
722
|
+
# override under that table cannot reach it.
|
|
723
|
+
if os.environ.get("CODEX_HOME") and (home / "plugins" / "provided.json").exists():
|
|
724
|
+
servers["plugin_provided"] = {"command": "/bin/false", "args": [], "enabled": True}
|
|
725
|
+
plugin_sourced.add("plugin_provided")
|
|
713
726
|
for argument in arguments:
|
|
714
727
|
if argument in {"-c", "--config"}:
|
|
715
728
|
# Codex splits the override path separately from its TOML value;
|
|
@@ -724,18 +737,76 @@ for argument in arguments:
|
|
|
724
737
|
else:
|
|
725
738
|
continue
|
|
726
739
|
for name, settings in configured_servers.items():
|
|
740
|
+
if name in plugin_sourced:
|
|
741
|
+
# A plugin contributes its server outside `mcp_servers`, so an
|
|
742
|
+
# override under that table builds an entry with no transport
|
|
743
|
+
# and the CLI refuses to load the configuration at all.
|
|
744
|
+
sys.stderr.write(
|
|
745
|
+
"Error: failed to load bootstrap configuration\n\n"
|
|
746
|
+
"Caused by:\n invalid transport\n"
|
|
747
|
+
" in `mcp_servers." + name + "`\n")
|
|
748
|
+
sys.exit(1)
|
|
727
749
|
servers.setdefault(name, {}).update(settings)
|
|
728
|
-
|
|
750
|
+
rows = [
|
|
729
751
|
{"name": name, "enabled": server.get("enabled", True),
|
|
730
752
|
"transport": {"type": "stdio", "command": server["command"],
|
|
731
753
|
"args": server.get("args", [])}}
|
|
732
754
|
for name, server in servers.items()
|
|
733
|
-
]
|
|
755
|
+
]
|
|
756
|
+
if os.environ.get("STUB_MALFORMED_DISABLED_ROW"):
|
|
757
|
+
# The malformed row is disabled: a check that filters before validating
|
|
758
|
+
# would never look at it.
|
|
759
|
+
bad = {"missing": {}, "empty": {"name": ""}, "nonstring": {"name": 7}}[
|
|
760
|
+
os.environ["STUB_MALFORMED_DISABLED_ROW"]]
|
|
761
|
+
rows.append({**bad, "enabled": False,
|
|
762
|
+
"transport": {"type": "stdio", "command": "/bin/false", "args": []}})
|
|
763
|
+
if os.environ.get("STUB_DUPLICATE_PACKET_ROW") == "1":
|
|
764
|
+
# A JSON array can carry the same name twice; a dict of servers cannot.
|
|
765
|
+
rows.append({"name": "code_review_packet", "enabled": False,
|
|
766
|
+
"transport": {"type": "stdio", "command": "/bin/false", "args": []}})
|
|
767
|
+
print(json.dumps(rows))
|
|
734
768
|
PY_MCP_LIST
|
|
735
769
|
exit $?
|
|
736
770
|
fi
|
|
737
771
|
touch "$state/codex_invoked"
|
|
738
772
|
printf '%s' "$0" >"$state/codex_argv0"
|
|
773
|
+
printf '%s' "${CODEX_HOME:-}" >"$state/codex_home"
|
|
774
|
+
if [ "${STUB_REPLACE_AUTH_LINK:-}" = 1 ] && [ -n "${CODEX_HOME:-}" ]; then
|
|
775
|
+
rm -f "$CODEX_HOME/auth.json"
|
|
776
|
+
printf '%s\n' '{"tokens":{"access":"rotated"}}' >"$CODEX_HOME/auth.json"
|
|
777
|
+
fi
|
|
778
|
+
python3 - "${CODEX_HOME:-}" "$state/codex_home_shape" <<'PY_HOME_SHAPE'
|
|
779
|
+
import json, os, sys, tomllib
|
|
780
|
+
from pathlib import Path
|
|
781
|
+
|
|
782
|
+
# The wrapper deletes its run directory on exit, so the private home can only
|
|
783
|
+
# be inspected from inside the run.
|
|
784
|
+
home = Path(sys.argv[1]) if sys.argv[1] else None
|
|
785
|
+
shape = {"home": sys.argv[1], "config_keys": [], "has_mcp_servers": None,
|
|
786
|
+
"has_plugins": None, "auth_link": None}
|
|
787
|
+
if home is not None:
|
|
788
|
+
config = home / "config.toml"
|
|
789
|
+
if config.exists():
|
|
790
|
+
try:
|
|
791
|
+
data = tomllib.loads(config.read_text(encoding="utf-8"))
|
|
792
|
+
except (OSError, UnicodeError, tomllib.TOMLDecodeError):
|
|
793
|
+
data = {"__unreadable__": True}
|
|
794
|
+
shape["config_keys"] = sorted(data)
|
|
795
|
+
shape["has_mcp_servers"] = "mcp_servers" in data
|
|
796
|
+
shape["model"] = data.get("model")
|
|
797
|
+
shape["model_reasoning_effort"] = data.get("model_reasoning_effort")
|
|
798
|
+
providers = data.get("model_providers")
|
|
799
|
+
shape["provider_keys"] = sorted(providers) if isinstance(providers, dict) else None
|
|
800
|
+
else:
|
|
801
|
+
shape["has_mcp_servers"] = False
|
|
802
|
+
shape["has_plugins"] = (home / "plugins").exists()
|
|
803
|
+
auth = home / "auth.json"
|
|
804
|
+
if auth.is_symlink():
|
|
805
|
+
shape["auth_link"] = os.readlink(auth)
|
|
806
|
+
elif auth.exists():
|
|
807
|
+
shape["auth_link"] = "__regular_file__"
|
|
808
|
+
Path(sys.argv[2]).write_text(json.dumps(shape), encoding="utf-8")
|
|
809
|
+
PY_HOME_SHAPE
|
|
739
810
|
printf '%s' "${CMUX_CODEX_HOOKS_DISABLED:-}" >"$state/codex_cmux_hooks_disabled"
|
|
740
811
|
last_message=""
|
|
741
812
|
has_model=no
|
|
@@ -802,6 +873,76 @@ if [ "$behavior" = "no_events" ]; then
|
|
|
802
873
|
fi
|
|
803
874
|
printf '%s\n' '{"type":"thread.started","thread_id":"test-thread"}'
|
|
804
875
|
printf '%s\n' '{"type":"turn.started"}'
|
|
876
|
+
# Transport failures whose only account of themselves is the event stream. The
|
|
877
|
+
# CLI reports these on stdout, so a wrapper that classifies from stderr alone
|
|
878
|
+
# cannot see them.
|
|
879
|
+
if [ "$behavior" = "usage_limit_event" ]; then
|
|
880
|
+
printf '%s\n' '{"type":"error","message":"You'"'"'ve hit your usage limit. Visit https://example.invalid/settings/usage?A1B2C3QUERYSECRET to purchase more credits."}'
|
|
881
|
+
printf '%s\n' '{"type":"turn.failed","error":{"message":"You'"'"'ve hit your usage limit."}}'
|
|
882
|
+
exit 1
|
|
883
|
+
fi
|
|
884
|
+
if [ "$behavior" = "usage_limit_turn_failed" ]; then
|
|
885
|
+
printf '%s\n' '{"type":"turn.failed","error":{"message":"request rejected: rate limit exceeded for this account"}}'
|
|
886
|
+
exit 1
|
|
887
|
+
fi
|
|
888
|
+
if [ "$behavior" = "auth_event" ]; then
|
|
889
|
+
printf '%s\n' '{"type":"error","message":"unauthorized: the stored credential was rejected"}'
|
|
890
|
+
exit 1
|
|
891
|
+
fi
|
|
892
|
+
# The packet is untrusted, and the model quotes it. Quota vocabulary reaching
|
|
893
|
+
# the classifier from here would let a reviewed diff choose its own reviewer.
|
|
894
|
+
if [ "$behavior" = "quota_in_model_output" ]; then
|
|
895
|
+
printf '%s\n' '{"type":"item.completed","item":{"type":"agent_message","text":"the diff mentions 429 rate limit and quota handling CODEXLEAKMARKER7f3a"}}'
|
|
896
|
+
printf '%s\n' '{"type":"item.completed","item":{"type":"error","message":"429 quota rate limit CODEXLEAKMARKER7f3a"}}'
|
|
897
|
+
printf '%s\n' 'stub failed for an unrelated reason' >&2
|
|
898
|
+
exit 1
|
|
899
|
+
fi
|
|
900
|
+
if [ "$behavior" = "sensitive_streams" ]; then
|
|
901
|
+
# The credential-shaped values are assembled at runtime: written whole they
|
|
902
|
+
# would put a real `sk-` token shape in the repository, which the credential
|
|
903
|
+
# scanner in validate-skill.sh refuses -- correctly.
|
|
904
|
+
#
|
|
905
|
+
# They travel on the EVENT stream, because that is the stream the receipt
|
|
906
|
+
# reads. A fixture that put them on stderr would leave every redaction
|
|
907
|
+
# assertion below vacuously green.
|
|
908
|
+
python3 - "$HOME" 'livetoken00000000000000' 'urlsecretvalue' <<'PY_SENSITIVE'
|
|
909
|
+
import json, sys
|
|
910
|
+
# Both credential shapes are assembled here rather than written literally. A
|
|
911
|
+
# `sk-` token is refused by validate-skill.sh, and a credentialed URL trips the
|
|
912
|
+
# review gate's own egress tripwire on every later review of this repository --
|
|
913
|
+
# both scanners behaving correctly on a fixture that only looks real.
|
|
914
|
+
# The password carries a literal "@": a userinfo rule that stops at the first
|
|
915
|
+
# one leaves the tail of the password in the receipt.
|
|
916
|
+
# The host carries no dot: with one, the userinfo tail plus host reads as an
|
|
917
|
+
# email address, and a reviewer quoting the finding puts that shape into a
|
|
918
|
+
# receipt the public-sanitization gate then refuses.
|
|
919
|
+
credentialed_url = "https://proxyuser:p" + "@" + "ss" + sys.argv[3] + "@" + "localhost/path"
|
|
920
|
+
print(json.dumps({"type": "error", "message": (
|
|
921
|
+
"failed while reading " + sys.argv[1] + "/private/thing"
|
|
922
|
+
" sk-" + sys.argv[2] + " token=supersecretvalue"
|
|
923
|
+
" access_token=accesssecretvalue client_secret=clientsecretvalue"
|
|
924
|
+
' {"refresh_token": "refreshsecretvalue"}'
|
|
925
|
+
" session=sessionsecretvalue cookie=cookiesecretvalue auth=authsecretvalue"
|
|
926
|
+
" code=codesecretvalue bearer=bearersecretvalue sid=sidsecretvalue"
|
|
927
|
+
" " + credentialed_url +
|
|
928
|
+
' quoted="quotedsecretvalue more of it"'
|
|
929
|
+
)}))
|
|
930
|
+
PY_SENSITIVE
|
|
931
|
+
printf 'STDERRONLYMARKER5z should never reach the receipt\n' >&2
|
|
932
|
+
exit 1
|
|
933
|
+
fi
|
|
934
|
+
if [ "$behavior" = "silent_failure" ]; then
|
|
935
|
+
exit 1
|
|
936
|
+
fi
|
|
937
|
+
if [ "$behavior" = "stderr_only" ]; then
|
|
938
|
+
python3 -c 'print("startup noise line. " * 60)' >&2
|
|
939
|
+
printf 'the real failure is here STDERRTAILMARKER9x\n' >&2
|
|
940
|
+
exit 1
|
|
941
|
+
fi
|
|
942
|
+
if [ "$behavior" = "long_error_event" ]; then
|
|
943
|
+
python3 -c 'import json;print(json.dumps({"type":"error","message":"E"*4000}))'
|
|
944
|
+
exit 1
|
|
945
|
+
fi
|
|
805
946
|
case "$behavior" in
|
|
806
947
|
packet_read|packet_search|packet_tampered) printf '%s\n' '{"status":"passed","concern_results":[{"concern":"correctness","conclusion":"Independently checked correctness against the frozen candidate."}],"findings":[]}' >"$last_message" ;;
|
|
807
948
|
pass|shell_disable_required|missing_shell_feature|removed_shell_feature|ignored_shell_disable|mcp_capability_missing|inherited_mcp|inherited_mcp_dot|inherited_mcp_space|inherited_mcp_quote|skills_budget_warning|skills_budget_warning_after_concern|hook_trust_warning|hook_trust_warning_started|hook_trust_warning_repeated_after_concern|unknown_error_valid_result) printf '%s\n' '{"status":"passed","concern_results":[{"concern":"correctness","conclusion":"Independently checked correctness against the frozen candidate."}],"findings":[]}' >"$last_message" ;;
|
|
@@ -1802,16 +1943,29 @@ out="$(run_codex packet_tampered)"; rc=$?
|
|
|
1802
1943
|
check "Codex rejects altered packet tool bytes through wrapper and parser" \
|
|
1803
1944
|
'[ "$rc" = 2 ] && [ "$(field reason_code "$out")" = binding_mismatch ] && [ "$(field cascade_eligible "$out")" = False ]'
|
|
1804
1945
|
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1946
|
+
# A user's own server, however its name is spelled, must not reach the reviewer
|
|
1947
|
+
# and must not be touched: the wrapper neither disables it nor names it.
|
|
1948
|
+
inherited_index=0
|
|
1949
|
+
for inherited_mcp_name in 'unrelated' 'unrelated.name' 'unrelated name' 'unrelated"name'; do
|
|
1950
|
+
inherited_index=$((inherited_index + 1))
|
|
1951
|
+
inherited_source="$WORK/codex-inherited-$inherited_index"
|
|
1952
|
+
mkdir -p "$inherited_source"
|
|
1953
|
+
python3 - "$inherited_source/config.toml" "$inherited_mcp_name" <<'PY_INHERITED_SOURCE'
|
|
1954
|
+
import json, sys
|
|
1955
|
+
from pathlib import Path
|
|
1956
|
+
|
|
1957
|
+
Path(sys.argv[1]).write_text(
|
|
1958
|
+
"[mcp_servers]\n"
|
|
1959
|
+
+ json.dumps(sys.argv[2])
|
|
1960
|
+
+ ' = { command = "/bin/false", args = [] }\n',
|
|
1961
|
+
encoding="utf-8",
|
|
1962
|
+
)
|
|
1963
|
+
PY_INHERITED_SOURCE
|
|
1964
|
+
printf '%s\n' '{"tokens":{"access":"seeded"}}' >"$inherited_source/auth.json"
|
|
1965
|
+
chmod 0600 "$inherited_source/auth.json"
|
|
1812
1966
|
rm -f "$WORK/state/codex_invoked" "$WORK/state/codex_configs"
|
|
1813
|
-
out="$(run_codex "$
|
|
1814
|
-
|
|
1967
|
+
out="$(run_codex pass claude "$inherited_source")"; rc=$?
|
|
1968
|
+
inherited_mcp_untouched="$(python3 - "$inherited_mcp_name" "$WORK/state/codex_configs" <<'PY_MCP_UNTOUCHED'
|
|
1815
1969
|
import sys, tomllib
|
|
1816
1970
|
from pathlib import Path
|
|
1817
1971
|
|
|
@@ -1821,15 +1975,140 @@ for override in path.read_text().splitlines() if path.exists() else []:
|
|
|
1821
1975
|
key, value = override.split("=", 1)
|
|
1822
1976
|
if key.strip() == "mcp_servers":
|
|
1823
1977
|
tables.append(tomllib.loads("servers=" + value)["servers"])
|
|
1978
|
+
elif key.strip().startswith("mcp_servers."):
|
|
1979
|
+
tables.append({key.strip().split(".")[1]: {}})
|
|
1824
1980
|
print(len(tables) == 1
|
|
1825
|
-
and
|
|
1826
|
-
and
|
|
1827
|
-
|
|
1981
|
+
and sys.argv[1] not in tables[0]
|
|
1982
|
+
and list(tables[0]) == ["code_review_packet"])
|
|
1983
|
+
PY_MCP_UNTOUCHED
|
|
1828
1984
|
)"
|
|
1829
|
-
check "Codex
|
|
1830
|
-
'[ "$rc" = 0 ] && [ "$(field status "$out")" = passed ] && [ -e "$WORK/state/codex_invoked" ] && [ "$
|
|
1985
|
+
check "Codex never reaches or names the user's own MCP server ($inherited_mcp_name)" \
|
|
1986
|
+
'[ "$rc" = 0 ] && [ "$(field status "$out")" = passed ] && [ -e "$WORK/state/codex_invoked" ] && [ "$inherited_mcp_untouched" = True ]'
|
|
1831
1987
|
done
|
|
1832
1988
|
|
|
1989
|
+
# The shape that used to dead-end the preflight: a server contributed outside
|
|
1990
|
+
# `mcp_servers`, which no override under that table can reach.
|
|
1991
|
+
plugin_source="$WORK/codex-plugin-source"
|
|
1992
|
+
mkdir -p "$plugin_source/plugins"
|
|
1993
|
+
printf '%s\n' '{"server":"plugin_provided"}' >"$plugin_source/plugins/provided.json"
|
|
1994
|
+
printf '%s\n' '{"tokens":{"access":"seeded"}}' >"$plugin_source/auth.json"
|
|
1995
|
+
chmod 0600 "$plugin_source/auth.json"
|
|
1996
|
+
rm -f "$WORK/state/codex_invoked" "$WORK/state/codex_configs"
|
|
1997
|
+
out="$(run_codex pass claude "$plugin_source")"; rc=$?
|
|
1998
|
+
check "Codex reviews on a host whose plugin contributes an MCP server" \
|
|
1999
|
+
'[ "$rc" = 0 ] && [ "$(field status "$out")" = passed ] && [ -e "$WORK/state/codex_invoked" ]'
|
|
2000
|
+
|
|
2001
|
+
# A canned verdict cannot prove the repair: a CLI that accepts the configuration
|
|
2002
|
+
# and still refuses read_packet would pass every assertion above. This one
|
|
2003
|
+
# replays the real packet server's bytes through the parser on that same host.
|
|
2004
|
+
rm -f "$WORK/state/codex_invoked" "$WORK/state/codex_packet_call"
|
|
2005
|
+
out="$(run_codex packet_read claude "$plugin_source")"; rc=$?
|
|
2006
|
+
check "Codex completes a real frozen packet read on a plugin-contributing host" \
|
|
2007
|
+
'[ "$rc" = 0 ] && [ "$(field status "$out")" = passed ] && [ "$(cat "$WORK/state/codex_packet_call")" = read_packet ]'
|
|
2008
|
+
|
|
2009
|
+
rm -f "$WORK/state/codex_invoked" "$WORK/state/codex_home"
|
|
2010
|
+
mkdir -p "$WORK/codex-foreign-source"
|
|
2011
|
+
printf '%s\n' 'profile = "review"' '' '[profiles.review]' 'model = "seeded-model"' 'model_reasoning_effort = "xhigh"' \
|
|
2012
|
+
'' '[model_providers."proxy.v1"]' 'name = "line one\\nline two"' '' '[mcp_servers.foreign]' 'command = "/bin/false"' 'args = []' \
|
|
2013
|
+
>"$WORK/codex-foreign-source/config.toml"
|
|
2014
|
+
printf '%s\n' '{"tokens":{"access":"seeded"}}' >"$WORK/codex-foreign-source/auth.json"
|
|
2015
|
+
chmod 0600 "$WORK/codex-foreign-source/auth.json"
|
|
2016
|
+
out="$(run_codex pass claude "$WORK/codex-foreign-source")"; rc=$?
|
|
2017
|
+
private_home_shape="$(python3 - "$WORK/state/codex_home_shape" "$WORK/codex-foreign-source" <<'PY_PRIVATE_HOME'
|
|
2018
|
+
import json, os, sys
|
|
2019
|
+
from pathlib import Path
|
|
2020
|
+
|
|
2021
|
+
|
|
2022
|
+
def normalized(value):
|
|
2023
|
+
# TMPDIR here can carry a trailing separator, so the recorded and expected
|
|
2024
|
+
# paths differ as strings while naming the same file.
|
|
2025
|
+
return Path(os.path.normpath(str(value))) if value else None
|
|
2026
|
+
|
|
2027
|
+
|
|
2028
|
+
path = Path(sys.argv[1])
|
|
2029
|
+
if not path.exists():
|
|
2030
|
+
print("no-run-observed"); raise SystemExit
|
|
2031
|
+
shape = json.loads(path.read_text(encoding="utf-8"))
|
|
2032
|
+
source = normalized(sys.argv[2])
|
|
2033
|
+
home = normalized(shape.get("home"))
|
|
2034
|
+
if home is None or home == source or source in home.parents:
|
|
2035
|
+
print("shares-user-home"); raise SystemExit
|
|
2036
|
+
if shape.get("has_mcp_servers") is not False or shape.get("has_plugins") is not False:
|
|
2037
|
+
print("inherited-servers-present"); raise SystemExit
|
|
2038
|
+
if shape.get("model") != "seeded-model" or shape.get("model_reasoning_effort") != "xhigh":
|
|
2039
|
+
print("model-preference-lost"); raise SystemExit
|
|
2040
|
+
if shape.get("provider_keys") != ["proxy.v1"]:
|
|
2041
|
+
print("provider-key-mangled"); raise SystemExit
|
|
2042
|
+
if normalized(shape.get("auth_link")) != source / "auth.json":
|
|
2043
|
+
print("credential-not-linked"); raise SystemExit
|
|
2044
|
+
print("private")
|
|
2045
|
+
PY_PRIVATE_HOME
|
|
2046
|
+
)"
|
|
2047
|
+
check "Codex reviews from a private home that never carried the user's MCP servers" \
|
|
2048
|
+
'[ "$rc" = 0 ] && [ "$(field status "$out")" = passed ] && [ "$private_home_shape" = private ]'
|
|
2049
|
+
|
|
2050
|
+
# The link back to the user's credential is what keeps a rotated token in the
|
|
2051
|
+
# user's own file; a CLI that replaced it with a regular file would leave the
|
|
2052
|
+
# credential inside this run directory instead. Prove the post-run check fires.
|
|
2053
|
+
rm -f "$WORK/state/codex_invoked"
|
|
2054
|
+
# A host that names a profile it does not define must not quietly review on a
|
|
2055
|
+
# different model; the wrapper refuses instead of falling through.
|
|
2056
|
+
mkdir -p "$WORK/codex-broken-profile"
|
|
2057
|
+
printf '%s\n' 'profile = "missing"' 'model = "top-level-model"' >"$WORK/codex-broken-profile/config.toml"
|
|
2058
|
+
printf '%s\n' '{"tokens":{"access":"seeded"}}' >"$WORK/codex-broken-profile/auth.json"
|
|
2059
|
+
chmod 0600 "$WORK/codex-broken-profile/auth.json"
|
|
2060
|
+
rm -f "$WORK/state/codex_invoked"
|
|
2061
|
+
out="$(run_codex pass claude "$WORK/codex-broken-profile")"; rc=$?
|
|
2062
|
+
check "Codex refuses a selected profile it cannot resolve instead of substituting a model" \
|
|
2063
|
+
'[ "$rc" = 2 ] && [ "$(field reason "$out")" = codex_home_preferences_unreadable ] && [ ! -e "$WORK/state/codex_invoked" ]'
|
|
2064
|
+
|
|
2065
|
+
# A malformed row that happens to be disabled must still fail the preflight;
|
|
2066
|
+
# filtering before validating would step over it.
|
|
2067
|
+
for malformed in missing empty nonstring; do
|
|
2068
|
+
export STUB_MALFORMED_DISABLED_ROW="$malformed"
|
|
2069
|
+
rm -f "$WORK/state/codex_invoked"
|
|
2070
|
+
out="$(run_codex pass)"; rc=$?
|
|
2071
|
+
unset STUB_MALFORMED_DISABLED_ROW
|
|
2072
|
+
check "Codex refuses a malformed disabled MCP row ($malformed name)" \
|
|
2073
|
+
'[ "$rc" = 2 ] && [ "$(field reason "$out")" = codex_packet_tools_unavailable ] && [ ! -e "$WORK/state/codex_invoked" ]'
|
|
2074
|
+
done
|
|
2075
|
+
|
|
2076
|
+
# The enabled-set check alone would accept one correctly bound row beside a
|
|
2077
|
+
# disabled duplicate of the same name, so the uniqueness claim needs its own case.
|
|
2078
|
+
export STUB_DUPLICATE_PACKET_ROW=1
|
|
2079
|
+
rm -f "$WORK/state/codex_invoked"
|
|
2080
|
+
out="$(run_codex pass)"; rc=$?
|
|
2081
|
+
unset STUB_DUPLICATE_PACKET_ROW
|
|
2082
|
+
check "Codex refuses a duplicated packet-server row even when the duplicate is disabled" \
|
|
2083
|
+
'[ "$rc" = 2 ] && [ "$(field reason "$out")" = codex_packet_tools_unavailable ] && [ ! -e "$WORK/state/codex_invoked" ]'
|
|
2084
|
+
|
|
2085
|
+
export STUB_REPLACE_AUTH_LINK=1
|
|
2086
|
+
out="$(run_codex pass claude "$WORK/codex-foreign-source")"; rc=$?
|
|
2087
|
+
unset STUB_REPLACE_AUTH_LINK
|
|
2088
|
+
check "Codex refuses when the run replaced the linked credential with a file" \
|
|
2089
|
+
'[ "$rc" = 2 ] && [ "$(field reason "$out")" = codex_runtime_home_credential_moved ] && [ "$(field reason_code "$out")" = binding_mismatch ] && [ "$(field cascade_eligible "$out")" = False ]'
|
|
2090
|
+
|
|
2091
|
+
rm -f "$WORK/state/codex_invoked" "$WORK/state/codex_configs"
|
|
2092
|
+
out="$(run_codex pass)"; rc=$?
|
|
2093
|
+
packet_approval_mode="$(python3 - "$WORK/state/codex_configs" <<'PY_MCP_APPROVAL'
|
|
2094
|
+
import sys, tomllib
|
|
2095
|
+
from pathlib import Path
|
|
2096
|
+
|
|
2097
|
+
path = Path(sys.argv[1])
|
|
2098
|
+
servers = {}
|
|
2099
|
+
for override in path.read_text().splitlines() if path.exists() else []:
|
|
2100
|
+
key, value = override.split("=", 1)
|
|
2101
|
+
if key.strip() == "mcp_servers":
|
|
2102
|
+
servers = tomllib.loads("servers=" + value)["servers"]
|
|
2103
|
+
packet = servers.get("code_review_packet", {})
|
|
2104
|
+
print(packet.get("default_tools_approval_mode") == "approve"
|
|
2105
|
+
and all(other.get("default_tools_approval_mode") is None
|
|
2106
|
+
for name, other in servers.items() if name != "code_review_packet"))
|
|
2107
|
+
PY_MCP_APPROVAL
|
|
2108
|
+
)"
|
|
2109
|
+
check "Codex declares the packet server auto-approved without widening the sandbox" \
|
|
2110
|
+
'[ "$rc" = 0 ] && [ "$(field status "$out")" = passed ] && [ "$packet_approval_mode" = True ] && [ "$(cat "$WORK/state/codex_read_only")" = yes ]'
|
|
2111
|
+
|
|
1833
2112
|
rm -f "$WORK/state/codex_invoked" "$WORK/state/codex_help_invoked"
|
|
1834
2113
|
probe_started=$SECONDS
|
|
1835
2114
|
out="$(run_codex help_hang)"; rc=$?
|
|
@@ -1977,6 +2256,98 @@ out="$(run_codex signal_exit)"; rc=$?
|
|
|
1977
2256
|
check "Codex process signals are terminal operator interrupts" \
|
|
1978
2257
|
'[ "$rc" = 2 ] && [ "$(field reason_code "$out")" = operator_interrupt ] && [ "$(field cascade_eligible "$out")" = False ]'
|
|
1979
2258
|
|
|
2259
|
+
# The CLI reports supply and credential failures as structured events on stdout,
|
|
2260
|
+
# not on stderr. Classifying from stderr alone turns a cascade-eligible quota
|
|
2261
|
+
# into a terminal unknown failure and stops the whole reviewer lane.
|
|
2262
|
+
out="$(run_codex usage_limit_event)"; rc=$?
|
|
2263
|
+
check "Codex reads a usage limit reported only on the event stream" \
|
|
2264
|
+
'[ "$rc" = 2 ] && [ "$(field reason_code "$out")" = quota ] && [ "$(field cascade_eligible "$out")" = True ]'
|
|
2265
|
+
|
|
2266
|
+
out="$(run_codex usage_limit_turn_failed)"; rc=$?
|
|
2267
|
+
check "Codex reads a rate limit carried by a failed turn" \
|
|
2268
|
+
'[ "$rc" = 2 ] && [ "$(field reason_code "$out")" = quota ] && [ "$(field cascade_eligible "$out")" = True ]'
|
|
2269
|
+
|
|
2270
|
+
out="$(run_codex auth_event)"; rc=$?
|
|
2271
|
+
check "Codex reads a credential rejection reported only on the event stream" \
|
|
2272
|
+
'[ "$rc" = 2 ] && [ "$(field reason_code "$out")" = provider_unavailable ] && [ "$(field cascade_eligible "$out")" = True ]'
|
|
2273
|
+
|
|
2274
|
+
# Only the transport's own top-level errors classify. Model-authored content
|
|
2275
|
+
# arrives nested under `item`, and the packet it quotes is untrusted.
|
|
2276
|
+
out="$(run_codex quota_in_model_output)"; rc=$?
|
|
2277
|
+
check "Codex refuses to classify from packet-derived model output" \
|
|
2278
|
+
'[ "$rc" = 2 ] && [ "$(field reason_code "$out")" = unknown_client_failure ] && [ "$(field cascade_eligible "$out")" = False ]'
|
|
2279
|
+
|
|
2280
|
+
|
|
2281
|
+
|
|
2282
|
+
|
|
2283
|
+
# The one invariant that replaces every redaction row: no input can influence
|
|
2284
|
+
# the receipt's text, because there is none derived from the run. The fixture
|
|
2285
|
+
# below carries home paths, several credential-assignment shapes, URL userinfo
|
|
2286
|
+
# with a separator in the password, and a stderr-only marker; none of it can
|
|
2287
|
+
# reach the receipt, and the check is equality with a constant rather than the
|
|
2288
|
+
# absence of a list of shapes.
|
|
2289
|
+
DIAG_CONSTANT="the transport output for this failure is in transport_run_dir"
|
|
2290
|
+
out="$(run_codex sensitive_streams)"; rc=$?
|
|
2291
|
+
diag="$(field transport_diagnostic "$out")"
|
|
2292
|
+
check "Codex records a receipt text no input can influence" \
|
|
2293
|
+
'[ "$rc" = 2 ] && [ "$diag" = "$DIAG_CONSTANT" ]'
|
|
2294
|
+
|
|
2295
|
+
out="$(run_codex silent_failure)"; rc=$?
|
|
2296
|
+
check "Codex records the same text when the transport says nothing" \
|
|
2297
|
+
'[ "$rc" = 2 ] && [ "$(field transport_diagnostic "$out")" = "$DIAG_CONSTANT" ]'
|
|
2298
|
+
|
|
2299
|
+
out="$(run_codex stderr_only)"; rc=$?
|
|
2300
|
+
diag="$(field transport_diagnostic "$out")"
|
|
2301
|
+
check "Codex does not quote stderr into the receipt" \
|
|
2302
|
+
'[ "$diag" = "$DIAG_CONSTANT" ]'
|
|
2303
|
+
|
|
2304
|
+
# The streams have to survive the failure. Deleting them with the run directory
|
|
2305
|
+
# is the defect this whole round started from.
|
|
2306
|
+
run_dir="$(field transport_run_dir "$out")"
|
|
2307
|
+
# The wrapper records a path under $HOME with $HOME replaced, so a committed
|
|
2308
|
+
# receipt carries no username. Expand it before testing the directory, or this
|
|
2309
|
+
# row false-REDs on any host whose TMPDIR sits under $HOME.
|
|
2310
|
+
case "$run_dir" in "~/"*) run_dir="$HOME/${run_dir#\~/}" ;; esac
|
|
2311
|
+
check "Codex preserves the run directory on a transport failure and names it" \
|
|
2312
|
+
'[ -n "$run_dir" ] && [ -d "$run_dir" ] && [ -s "$run_dir/stderr.log" ] && grep -q STDERRTAILMARKER9x "$run_dir/stderr.log"'
|
|
2313
|
+
|
|
2314
|
+
check "Codex keeps the preserved run directory private" \
|
|
2315
|
+
'[ "$(dir_mode "$run_dir")" = 700 ]'
|
|
2316
|
+
|
|
2317
|
+
# The eliding compares physical paths, so a home spelled with a trailing slash
|
|
2318
|
+
# -- or through a symlink -- still elides. Comparing the literal `$HOME` string
|
|
2319
|
+
# would put the username in a committed receipt on exactly those hosts.
|
|
2320
|
+
# Invoked directly rather than through run_codex: that helper pins TMPDIR, and
|
|
2321
|
+
# this row needs a TMPDIR that sits under the home it is testing.
|
|
2322
|
+
mkdir -p "$WORK/fakehome/tmp"
|
|
2323
|
+
out="$(STUB_BEHAVIOR=stderr_only REVIEW_WRAPPER_TEST_STATE="$WORK/state" \
|
|
2324
|
+
PATH="$WORK/bin:$PATH" TMPDIR="$WORK/fakehome/tmp" HOME="$WORK/fakehome/" \
|
|
2325
|
+
CODEX_HOME="$WORK/codex-source" TEST_DIFF_PATH="$WORK/diff.patch" \
|
|
2326
|
+
TEST_PROFILE_PATH="$WORK/review-profile.json" \
|
|
2327
|
+
bash "$DIR/codex_review.sh" --implementer-family claude \
|
|
2328
|
+
--diff-file "$WORK/diff.patch" --review-profile-file "$WORK/review-profile.json" \
|
|
2329
|
+
--mode review --timeout 30)"; rc=$?
|
|
2330
|
+
run_dir="$(field transport_run_dir "$out")"
|
|
2331
|
+
# HOME=/ is real in root and arbitrary-uid containers. Treated as a home
|
|
2332
|
+
# spelling it would rewrite every separator in the excerpt.
|
|
2333
|
+
out2="$(STUB_BEHAVIOR=sensitive_streams REVIEW_WRAPPER_TEST_STATE="$WORK/state" \
|
|
2334
|
+
PATH="$WORK/bin:$PATH" TMPDIR="$WORK/tmp" HOME=/ \
|
|
2335
|
+
CODEX_HOME="$WORK/codex-source" TEST_DIFF_PATH="$WORK/diff.patch" \
|
|
2336
|
+
TEST_PROFILE_PATH="$WORK/review-profile.json" \
|
|
2337
|
+
bash "$DIR/codex_review.sh" --implementer-family claude \
|
|
2338
|
+
--diff-file "$WORK/diff.patch" --review-profile-file "$WORK/review-profile.json" \
|
|
2339
|
+
--mode review --timeout 30)"
|
|
2340
|
+
dir2="$(field transport_run_dir "$out2")"
|
|
2341
|
+
check "Codex does not treat a separator-only home as a path to elide" \
|
|
2342
|
+
'case "$dir2" in "~"*) false ;; /*) [ -d "$dir2" ] ;; *) false ;; esac'
|
|
2343
|
+
|
|
2344
|
+
check "Codex elides a home path spelled with a trailing slash" \
|
|
2345
|
+
'case "$run_dir" in "~/"*) case "$run_dir" in *fakehome*) false ;; *) true ;; esac ;; *) false ;; esac'
|
|
2346
|
+
|
|
2347
|
+
out="$(run_codex pass)"; rc=$?
|
|
2348
|
+
check "Codex adds no diagnostic field to a successful review" \
|
|
2349
|
+
'[ "$rc" = 0 ] && json_lacks_key "$out" transport_diagnostic && json_lacks_key "$out" transport_run_dir'
|
|
2350
|
+
|
|
1980
2351
|
echo '----'
|
|
1981
2352
|
if [ "$fails" -eq 0 ]; then
|
|
1982
2353
|
echo cli_review_wrapper_tests_ok
|