@teevee98/synapse-hub 4.1.4 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/rules/git-push-protocol.md +2 -2
- package/.agents/skills/01-web-automation/auth-session-and-cookie-manager/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/bats-testing-patterns/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/clean-pom-crafter/SKILL.md +8 -3
- package/.agents/skills/01-web-automation/e2e-testing-patterns/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/katalon-cli-runner/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/lam-script-generator/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/mobile-appium-som/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/BaseScreen.java +2 -2
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/LoginScreen.java +2 -2
- package/.agents/skills/01-web-automation/mobile-appium-som/examples/TC_MobileLogin.java +2 -2
- package/.agents/skills/01-web-automation/playwright-live-interaction/SKILL.md +7 -2
- package/.agents/skills/01-web-automation/playwright-webapp-testing/SKILL.md +6 -1
- package/.agents/skills/01-web-automation/web-agent-gym-benchmarking/SKILL.md +6 -1
- package/.agents/skills/02-test-design/architecture-decision-records/SKILL.md +6 -1
- package/.agents/skills/02-test-design/gherkin-to-suite/SKILL.md +6 -1
- package/.agents/skills/02-test-design/rbt-manual-testing/SKILL.md +6 -1
- package/.agents/skills/02-test-design/review-spec/SKILL.md +4 -1
- package/.agents/skills/02-test-design/tot-exploratory-testing/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/api-network-mocking-interception/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/api-spec-to-e2e-matrix/SKILL.md +6 -1
- package/.agents/skills/03-api-and-contracts/consumer-driven-contract-testing/SKILL.md +7 -2
- package/.agents/skills/04-performance-and-load/backend-load-and-stress-patterns/SKILL.md +7 -2
- package/.agents/skills/04-performance-and-load/lean-impact-scoreboard/SKILL.md +4 -1
- package/.agents/skills/04-performance-and-load/web-vitals-and-performance-profiler/SKILL.md +6 -1
- package/.agents/skills/05-security-and-compliance/security-dast-sentinel/SKILL.md +6 -1
- package/.agents/skills/05-security-and-compliance/skillspector-security-guard/SKILL.md +6 -1
- package/.agents/skills/05-security-and-compliance/wcag-audit-patterns/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/agentless-qa-repair/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/antigravity-test-sentinel/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/failure-autopsy-and-healing/SKILL.md +7 -2
- package/.agents/skills/06-healing-and-resilience/flaky-test-sentinel-and-quarantine/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/self-healing-locator/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/systematic-debugging/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/test-driven-development/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/verification-before-completion/SKILL.md +6 -1
- package/.agents/skills/06-healing-and-resilience/visual-regression-sentinel/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/db-state-verifier-and-seeder/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/file-and-report-validator/SKILL.md +6 -1
- package/.agents/skills/07-data-and-state/synthetic-test-data-generator/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/code-refactoring-refactor-clean/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/graphify-impact-analyzer/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/lean-code-ladder/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/repo-quality-audit/SKILL.md +4 -1
- package/.agents/skills/08-code-quality/senior-code-review/SKILL.md +4 -1
- package/.agents/skills/08-code-quality/surgical-coding-principles/SKILL.md +6 -1
- package/.agents/skills/08-code-quality/technical-debt-harvester/SKILL.md +4 -1
- package/.agents/skills/09-ci-and-devops/github-actions-templates/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/actor-qa-swarm/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/agent-orchestration-multi-agent-optimize/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/event-stream-qa-orchestrator/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/persistent-planning-files/SKILL.md +6 -1
- package/.agents/skills/10-agent-orchestration/qa-memory-os/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/meta-prompt-crafter/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/mipro-prompt-compiler/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/qa-aci-interface/SKILL.md +6 -1
- package/.agents/skills/11-ai-and-prompting/token-and-context-optimizer/SKILL.md +6 -1
- package/AGENTS.md +7 -1
- package/LICENSE +5 -5
- package/NOTICE +1 -1
- package/README.en.md +4 -4
- package/README.md +4 -4
- package/agentskills.json +1 -1
- package/docs/ARCHITECTURE.en.md +2 -2
- package/docs/ARCHITECTURE.md +11 -2
- package/docs/CLI_REFERENCE.en.md +6 -4
- package/docs/CLI_REFERENCE.md +1 -1
- package/docs/CONTRIBUTING.en.md +1 -1
- package/docs/CONTRIBUTING.md +2 -2
- package/docs/TOOL_CATALOG.md +6 -2
- package/docs/research/agent_skill_training_analysis.md +1 -1
- package/package.json +1 -1
- package/plugin.json +1 -1
- package/skills_manifest.json +320 -55
- package/synapse.config.json +1 -1
- package/templates/ui/index.html +2 -2
- package/tools/TOOL_CATALOG.json +63 -3
- package/tools/__init__.py +1 -1
- package/tools/api_fuzzer.py +179 -118
- package/tools/author_integrity.py +121 -11
- package/tools/benchmark_swarm_context.py +119 -18
- package/tools/break_ui_generator.py +233 -0
- package/tools/browser_helpers.js +2 -2
- package/tools/check_version_sync.py +18 -50
- package/tools/cli.py +109 -18
- package/tools/core/bm25_retriever.py +218 -0
- package/tools/core/i18n.py +4 -4
- package/tools/core/provider_usage_adapter.py +317 -0
- package/tools/core/token_estimator.py +42 -14
- package/tools/count_tokens.py +5 -0
- package/tools/coverage_reporter.py +245 -0
- package/tools/dashboard_generator.py +6 -6
- package/tools/db_sandbox.py +146 -4
- package/tools/distill_skill.py +191 -0
- package/tools/git_pre_commit_guard.py +12 -0
- package/tools/humanize_linter.py +3 -0
- package/tools/mcp_docx_server.py +1 -1
- package/tools/mcp_excel_server.py +1 -1
- package/tools/mcp_server.py +75 -15
- package/tools/micro_mutator.py +84 -14
- package/tools/pentest_scope_guard.py +399 -56
- package/tools/perf/benchmark_perf_runner.py +42 -0
- package/tools/rules_router.py +401 -41
- package/tools/security_scanner.py +80 -0
- package/tools/senior_code_review.py +14 -0
- package/tools/skill_canonical_digest.py +244 -0
- package/tools/skill_profiles.py +77 -7
- package/tools/slide_deck_exporter.py +477 -0
- package/tools/sync_skills_manifest.py +166 -0
- package/tools/task_memory_manager.py +260 -0
- package/tools/ui_server.py +1 -1
- package/tools/update_sentinel.py +3 -3
- package/tools/validate_host_interop.py +485 -0
- package/tools/validate_skills_schema.py +282 -0
- package/tools/verify_all.py +572 -42
- package/tools/welcome_page.py +2 -2
package/tools/mcp_server.py
CHANGED
|
@@ -14,6 +14,7 @@ import os
|
|
|
14
14
|
import sys
|
|
15
15
|
import json
|
|
16
16
|
import re
|
|
17
|
+
import tempfile
|
|
17
18
|
from pathlib import Path
|
|
18
19
|
from typing import Dict, Any, Optional, Set
|
|
19
20
|
|
|
@@ -49,7 +50,7 @@ import gherkin_suite
|
|
|
49
50
|
import api_fuzzer
|
|
50
51
|
|
|
51
52
|
SERVER_NAME = "enterprise-agent-qa"
|
|
52
|
-
SERVER_VERSION = "4.
|
|
53
|
+
SERVER_VERSION = "4.2.0"
|
|
53
54
|
|
|
54
55
|
MCP_TOOLS = [
|
|
55
56
|
{
|
|
@@ -214,7 +215,8 @@ MCP_TOOLS = [
|
|
|
214
215
|
"action": {"type": "string", "enum": ["snapshot", "restore", "audit_and_purge"], "description": "Sandbox operation to execute."},
|
|
215
216
|
"db_path": {"type": "string", "description": "Path to database file (optional for SQLite)."},
|
|
216
217
|
"snapshot_id": {"type": "string", "description": "Snapshot identifier (default: pristine_baseline)."},
|
|
217
|
-
"dry_run": {"type": "boolean", "description": "For audit_and_purge: preview
|
|
218
|
+
"dry_run": {"type": "boolean", "default": True, "description": "For restore and audit_and_purge: preview actions without modifying database (default: true)."},
|
|
219
|
+
"consent": {"type": "boolean", "default": False, "description": "Explicit consent acknowledgement for destructive state modification when dry_run=False."}
|
|
218
220
|
},
|
|
219
221
|
"required": ["action"]
|
|
220
222
|
}
|
|
@@ -262,8 +264,9 @@ MCP_TOOLS = [
|
|
|
262
264
|
"properties": {
|
|
263
265
|
"base_url": {"type": "string", "description": "Base URL of target API (non-production allowlist enforced)."},
|
|
264
266
|
"spec_path": {"type": "string", "description": "Path or URL to OpenAPI/Swagger JSON spec."},
|
|
265
|
-
"mode": {"type": "string", "enum": ["active", "passive"], "description": "active fuzzing or passive offline spec audit."},
|
|
266
|
-
"dry_run": {"type": "boolean", "description": "Deterministic offline mock (default true for safety)."}
|
|
267
|
+
"mode": {"type": "string", "enum": ["active", "passive"], "default": "passive", "description": "active fuzzing or passive offline spec audit (default: passive)."},
|
|
268
|
+
"dry_run": {"type": "boolean", "default": True, "description": "Deterministic offline mock (default true for safety)."},
|
|
269
|
+
"consent": {"type": "boolean", "default": False, "description": "Explicit consent acknowledgement for active mutations when mode=active and dry_run=False."}
|
|
267
270
|
},
|
|
268
271
|
"required": ["base_url"]
|
|
269
272
|
}
|
|
@@ -295,8 +298,12 @@ def _validate_safe_path(filepath: str, allowed_root: Optional[str] = None, allow
|
|
|
295
298
|
root_norm = os.path.normcase(str(root_path))
|
|
296
299
|
target_norm = os.path.normcase(str(target_path))
|
|
297
300
|
|
|
298
|
-
# Path must reside within the allowed root
|
|
299
|
-
|
|
301
|
+
# Path must reside within the allowed root OR within the OS temporary directory
|
|
302
|
+
temp_dir_norm = os.path.normcase(str(Path(tempfile.gettempdir()).resolve()))
|
|
303
|
+
is_temp_path = target_norm == temp_dir_norm or target_norm.startswith(temp_dir_norm + os.sep)
|
|
304
|
+
is_in_root = (target_norm == root_norm or target_norm.startswith(root_norm + os.sep))
|
|
305
|
+
|
|
306
|
+
if not is_in_root and not is_temp_path:
|
|
300
307
|
raise PermissionError(f"Access denied: path '{filepath}' resolves outside allowed project root '{root_path}'")
|
|
301
308
|
|
|
302
309
|
# Reject explicit sensitive configuration / secret files and private keys
|
|
@@ -334,8 +341,16 @@ def _validate_safe_path(filepath: str, allowed_root: Optional[str] = None, allow
|
|
|
334
341
|
except OSError:
|
|
335
342
|
sensitive_prefixes.append(Path(win_val))
|
|
336
343
|
|
|
344
|
+
temp_dir_norm = os.path.normcase(str(Path(tempfile.gettempdir()).resolve()))
|
|
345
|
+
is_temp_path = target_norm == temp_dir_norm or target_norm.startswith(temp_dir_norm + os.sep)
|
|
346
|
+
|
|
337
347
|
for sp in sensitive_prefixes:
|
|
338
348
|
sp_norm = os.path.normcase(str(sp))
|
|
349
|
+
if is_temp_path and sys.platform == "win32" and sp_norm in (
|
|
350
|
+
os.path.normcase(os.environ.get("LOCALAPPDATA", "")),
|
|
351
|
+
os.path.normcase(os.environ.get("APPDATA", "")),
|
|
352
|
+
):
|
|
353
|
+
continue
|
|
339
354
|
if target_norm == sp_norm or target_norm.startswith(sp_norm + os.sep):
|
|
340
355
|
raise PermissionError(f"Access denied: path references restricted system directory '{sp}'")
|
|
341
356
|
|
|
@@ -504,15 +519,37 @@ def _handle_manage_db_sandbox(arguments: Dict[str, Any]) -> str:
|
|
|
504
519
|
safe_db = _validate_safe_path(raw_db) if raw_db else ""
|
|
505
520
|
raw_id = arguments.get("snapshot_id", "pristine_baseline")
|
|
506
521
|
safe_id = re.sub(r'[^a-zA-Z0-9_\-]', '', raw_id) or "pristine_baseline"
|
|
522
|
+
consent = bool(arguments.get("consent", False)) or (os.environ.get("PENTEST_CONSENT_ACKNOWLEDGED", "").strip().lower() in ("1", "true", "yes"))
|
|
507
523
|
|
|
508
524
|
sandbox = db_sandbox.DBSandbox(db_path=safe_db)
|
|
509
525
|
if action == "snapshot":
|
|
510
526
|
res = sandbox.create_snapshot(safe_id)
|
|
511
527
|
elif action == "restore":
|
|
512
|
-
|
|
528
|
+
dry_run = bool(arguments.get("dry_run", True))
|
|
529
|
+
if dry_run:
|
|
530
|
+
res = {
|
|
531
|
+
"status": "dry_run_preview",
|
|
532
|
+
"action": "restore",
|
|
533
|
+
"snapshot_id": safe_id,
|
|
534
|
+
"db_path": safe_db or sandbox.db_path,
|
|
535
|
+
"dry_run": True,
|
|
536
|
+
"impact_summary": f"DRY-RUN PREVIEW: Would restore database snapshot '{safe_id}' to '{safe_db or sandbox.db_path}'. Pass dry_run=False and consent=True to execute."
|
|
537
|
+
}
|
|
538
|
+
else:
|
|
539
|
+
if not consent:
|
|
540
|
+
raise PermissionError(
|
|
541
|
+
f"Active restore operation on database '{safe_db or sandbox.db_path}' requires explicit consent. "
|
|
542
|
+
f"Pass consent=True to confirm state modification."
|
|
543
|
+
)
|
|
544
|
+
res = sandbox.restore_snapshot(safe_id, consent=consent, dry_run=dry_run)
|
|
513
545
|
elif action == "audit_and_purge":
|
|
514
|
-
dry_run = bool(arguments.get("dry_run",
|
|
515
|
-
|
|
546
|
+
dry_run = bool(arguments.get("dry_run", True))
|
|
547
|
+
if not dry_run and not consent:
|
|
548
|
+
raise PermissionError(
|
|
549
|
+
f"Active purge operation on database '{safe_db or sandbox.db_path}' requires explicit consent. "
|
|
550
|
+
f"Pass consent=True to confirm record deletion."
|
|
551
|
+
)
|
|
552
|
+
res = sandbox.audit_and_purge_pollution(dry_run=dry_run, consent=consent)
|
|
516
553
|
else:
|
|
517
554
|
raise ValueError(f"Unknown sandbox action: {action}")
|
|
518
555
|
return json.dumps(res, indent=2)
|
|
@@ -556,14 +593,36 @@ def _handle_compile_gherkin_suite(arguments: Dict[str, Any]) -> str:
|
|
|
556
593
|
|
|
557
594
|
|
|
558
595
|
def _handle_fuzz_api(arguments: Dict[str, Any]) -> str:
|
|
559
|
-
mode = arguments.get("mode", "
|
|
560
|
-
dry_run = arguments.get("dry_run", True)
|
|
596
|
+
mode = arguments.get("mode", "passive")
|
|
597
|
+
dry_run = bool(arguments.get("dry_run", True))
|
|
598
|
+
consent = bool(arguments.get("consent", False)) or (os.environ.get("PENTEST_CONSENT_ACKNOWLEDGED", "").strip().lower() in ("1", "true", "yes"))
|
|
561
599
|
spec_path = arguments.get("spec_path")
|
|
562
|
-
|
|
563
|
-
|
|
600
|
+
base_url = arguments.get("base_url", "")
|
|
601
|
+
|
|
602
|
+
from pentest_scope_guard import PentestScopeGuard
|
|
603
|
+
guard = PentestScopeGuard(allow_internal=True)
|
|
604
|
+
ok, reason = guard.validate_target(base_url, actor="mcp_server.fuzz_api")
|
|
605
|
+
if not ok:
|
|
606
|
+
raise PermissionError(f"[PENTEST SCOPE GUARD - REJECTED] Target base_url '{base_url}' blocked: {reason}")
|
|
607
|
+
|
|
608
|
+
if spec_path:
|
|
609
|
+
if spec_path.startswith(("http://", "https://")):
|
|
610
|
+
spec_ok, spec_reason = guard.validate_target(spec_path, actor="mcp_server.fuzz_api.spec_url")
|
|
611
|
+
if not spec_ok:
|
|
612
|
+
raise PermissionError(f"[PENTEST SCOPE GUARD - REJECTED] spec_path URL '{spec_path}' blocked: {spec_reason}")
|
|
613
|
+
else:
|
|
614
|
+
spec_path = _validate_safe_path(spec_path)
|
|
615
|
+
|
|
616
|
+
if mode == "active" and not dry_run:
|
|
617
|
+
if not consent:
|
|
618
|
+
raise PermissionError(
|
|
619
|
+
"Active API fuzzing (dry_run=False) requires explicit consent. Set consent=True or run with dry_run=True / mode=passive."
|
|
620
|
+
)
|
|
621
|
+
|
|
564
622
|
fuzzer = api_fuzzer.ApiFuzzer(
|
|
565
|
-
base_url=
|
|
623
|
+
base_url=base_url,
|
|
566
624
|
spec_path_or_url=spec_path,
|
|
625
|
+
scope_guard=guard,
|
|
567
626
|
)
|
|
568
627
|
if mode == "passive":
|
|
569
628
|
scorecard = fuzzer.run_passive_scan()
|
|
@@ -573,13 +632,14 @@ def _handle_fuzz_api(arguments: Dict[str, Any]) -> str:
|
|
|
573
632
|
"severity_summary": scorecard["severity_summary"],
|
|
574
633
|
"findings": scorecard["findings"],
|
|
575
634
|
}, indent=2)
|
|
576
|
-
scorecard = fuzzer.run_fuzz_suite(dry_run=dry_run)
|
|
635
|
+
scorecard = fuzzer.run_fuzz_suite(dry_run=dry_run, consent=consent)
|
|
577
636
|
return json.dumps({
|
|
578
637
|
"mode": scorecard["mode"],
|
|
579
638
|
"total_mutations": scorecard["total_mutations"],
|
|
580
639
|
"passed": scorecard["passed"],
|
|
581
640
|
"failed": scorecard["failed"],
|
|
582
641
|
"severity_summary": scorecard["severity_summary"],
|
|
642
|
+
"dry_run": scorecard["dry_run"],
|
|
583
643
|
}, indent=2)
|
|
584
644
|
|
|
585
645
|
|
package/tools/micro_mutator.py
CHANGED
|
@@ -24,6 +24,7 @@ Implementation Note:
|
|
|
24
24
|
import os
|
|
25
25
|
import re
|
|
26
26
|
import sys
|
|
27
|
+
import glob
|
|
27
28
|
import json
|
|
28
29
|
import shlex
|
|
29
30
|
import html as html_lib
|
|
@@ -83,9 +84,23 @@ def resolve_scoped_test_cmd(scoped_template: str, target_file: str, repo_root: O
|
|
|
83
84
|
impacted = []
|
|
84
85
|
|
|
85
86
|
if not impacted:
|
|
86
|
-
# Fallback: if target is itself a test script, run on target
|
|
87
|
+
# Fallback 1: if target is itself a test script, run on target
|
|
87
88
|
if "tests/" in rel_path or "Scripts/" in rel_path or rel_path.endswith(("_test.py", ".spec.ts", ".test.ts", ".groovy")) or os.path.basename(rel_path).startswith("test_"):
|
|
88
89
|
impacted = [rel_path]
|
|
90
|
+
else:
|
|
91
|
+
# Fallback 2: convention-based test mapping for tools/<name>.py -> tests/test_<name>.py
|
|
92
|
+
base_mod = os.path.splitext(os.path.basename(rel_path))[0]
|
|
93
|
+
candidates = [
|
|
94
|
+
os.path.join("tests", f"test_{base_mod}.py"),
|
|
95
|
+
os.path.join("tests", f"{base_mod}_test.py"),
|
|
96
|
+
]
|
|
97
|
+
for cand in candidates:
|
|
98
|
+
cand_rel = cand.replace("\\", "/")
|
|
99
|
+
if os.path.isfile(os.path.join(root, cand_rel)):
|
|
100
|
+
impacted.append(cand_rel)
|
|
101
|
+
if not impacted:
|
|
102
|
+
for f in glob.glob(os.path.join(root, "tests", f"test_*{base_mod}*.py")):
|
|
103
|
+
impacted.append(os.path.relpath(f, root).replace("\\", "/"))
|
|
89
104
|
|
|
90
105
|
tests_str = " ".join(impacted) if impacted else ""
|
|
91
106
|
if "{test_files}" in scoped_template:
|
|
@@ -120,8 +135,8 @@ MUTATION_RULES = [
|
|
|
120
135
|
(r"\bor\b", "and", "Logical Inversion (or -> and)"),
|
|
121
136
|
(r"\s&&\s", " || ", "Logical AND Inversion (&& -> ||)"),
|
|
122
137
|
(r"\s\|\|\s", " && ", "Logical OR Inversion (|| -> &&)"),
|
|
123
|
-
(r"\s\bis\s", " is not ", "Identity Inversion (is -> is not)"),
|
|
124
|
-
(r"\s\bis
|
|
138
|
+
(r"\s\bis\s+(?!not\b)", " is not ", "Identity Inversion (is -> is not)"),
|
|
139
|
+
(r"\s\bis\s+not\s+", " is ", "Identity Inversion (is not -> is)"),
|
|
125
140
|
(r"\s\+\s", " - ", "Arithmetic Inversion (+ -> -)"),
|
|
126
141
|
(r"\s-\s", " + ", "Arithmetic Inversion (- -> +)"),
|
|
127
142
|
(r"\breturn\s+true\b", "return false", "Return Value Inversion (return true -> return false)"),
|
|
@@ -309,6 +324,13 @@ def generate_mutants_for_code(code: str, file_path: str = "") -> List[Dict[str,
|
|
|
309
324
|
mutated_lines[line_idx] = mutated_line
|
|
310
325
|
mutated_code = "".join(mutated_lines)
|
|
311
326
|
|
|
327
|
+
# Syntax guard for Python: ensure mutant produces valid Python syntax
|
|
328
|
+
if file_path.endswith((".py", ".pyw")):
|
|
329
|
+
try:
|
|
330
|
+
compile(mutated_code, file_path, "exec")
|
|
331
|
+
except SyntaxError:
|
|
332
|
+
continue
|
|
333
|
+
|
|
312
334
|
mutants.append({
|
|
313
335
|
"id": f"MUT-{mutant_id:04d}",
|
|
314
336
|
"file": file_path,
|
|
@@ -473,7 +495,14 @@ def evaluate_mutants_dynamically(file_path: str, code: str, mutants: List[Dict[s
|
|
|
473
495
|
else:
|
|
474
496
|
cmd_args = shlex.split(test_cmd, posix=True)
|
|
475
497
|
else:
|
|
476
|
-
cmd_args = test_cmd
|
|
498
|
+
cmd_args = list(test_cmd) if isinstance(test_cmd, (list, tuple)) else [str(test_cmd)]
|
|
499
|
+
|
|
500
|
+
# Guarantee sys.executable is used for running pytest or python
|
|
501
|
+
if cmd_args:
|
|
502
|
+
if cmd_args[0] == "pytest":
|
|
503
|
+
cmd_args = [sys.executable, "-m", "pytest"] + cmd_args[1:]
|
|
504
|
+
elif cmd_args[0] in ("python", "python3"):
|
|
505
|
+
cmd_args = [sys.executable] + cmd_args[1:]
|
|
477
506
|
use_shell = False
|
|
478
507
|
run_env = os.environ.copy()
|
|
479
508
|
run_env["PYTHONDONTWRITEBYTECODE"] = "1"
|
|
@@ -488,7 +517,7 @@ def evaluate_mutants_dynamically(file_path: str, code: str, mutants: List[Dict[s
|
|
|
488
517
|
cmd_args,
|
|
489
518
|
shell=use_shell,
|
|
490
519
|
capture_output=True,
|
|
491
|
-
timeout=
|
|
520
|
+
timeout=max(2.0, float(timeout)),
|
|
492
521
|
text=True,
|
|
493
522
|
env=run_env,
|
|
494
523
|
cwd=target_cwd
|
|
@@ -502,7 +531,7 @@ def evaluate_mutants_dynamically(file_path: str, code: str, mutants: List[Dict[s
|
|
|
502
531
|
except subprocess.TimeoutExpired:
|
|
503
532
|
for m in mutants:
|
|
504
533
|
m["status"] = "SURVIVED"
|
|
505
|
-
m["reason"] = f"Baseline execution error: timed out after {
|
|
534
|
+
m["reason"] = f"Baseline execution error: timed out after {float(timeout):.1f} seconds"
|
|
506
535
|
return mutants, 0.0
|
|
507
536
|
except Exception as e:
|
|
508
537
|
for m in mutants:
|
|
@@ -576,8 +605,11 @@ def evaluate_mutants_dynamically(file_path: str, code: str, mutants: List[Dict[s
|
|
|
576
605
|
with open(abs_path, "w", encoding="utf-8") as f:
|
|
577
606
|
f.write(code)
|
|
578
607
|
|
|
579
|
-
total_eval = valid_mutants_count
|
|
580
|
-
|
|
608
|
+
total_eval = valid_mutants_count
|
|
609
|
+
if total_eval == 0:
|
|
610
|
+
score = 0.0
|
|
611
|
+
else:
|
|
612
|
+
score = evaluate_mutation_score(killed_count, total_eval)
|
|
581
613
|
return mutants, score
|
|
582
614
|
|
|
583
615
|
|
|
@@ -704,12 +736,30 @@ def evaluate_target(
|
|
|
704
736
|
code = ""
|
|
705
737
|
|
|
706
738
|
killed_count = sum(1 for m in all_mutants if m["status"] == "KILLED")
|
|
707
|
-
|
|
739
|
+
if "DYNAMIC" in exec_mode:
|
|
740
|
+
valid_m_count = sum(1 for m in all_mutants if m["status"] in ("KILLED", "SURVIVED", "TIMED_OUT"))
|
|
741
|
+
score = evaluate_mutation_score(killed_count, valid_m_count) if valid_m_count > 0 else 0.0
|
|
742
|
+
else:
|
|
743
|
+
score = evaluate_mutation_score(killed_count, len(all_mutants))
|
|
708
744
|
|
|
709
745
|
killed = sum(1 for m in all_mutants if m["status"] == "KILLED")
|
|
710
746
|
survived = sum(1 for m in all_mutants if m["status"] == "SURVIVED")
|
|
747
|
+
timed_out = sum(1 for m in all_mutants if m["status"] == "TIMED_OUT")
|
|
748
|
+
invalid_syntax = sum(1 for m in all_mutants if m["status"] in ("INVALID_SYNTAX", "INVALID"))
|
|
749
|
+
valid_mutants = killed + survived + timed_out
|
|
711
750
|
no_mutants = (len(all_mutants) == 0)
|
|
712
|
-
|
|
751
|
+
no_valid_mutants = (valid_mutants == 0) if "DYNAMIC" in exec_mode else no_mutants
|
|
752
|
+
|
|
753
|
+
if "DYNAMIC" in exec_mode:
|
|
754
|
+
if no_valid_mutants:
|
|
755
|
+
passed = False
|
|
756
|
+
score = 0.0
|
|
757
|
+
else:
|
|
758
|
+
passed = (score >= min_score)
|
|
759
|
+
if survived > 0 and score < min_score:
|
|
760
|
+
passed = False
|
|
761
|
+
else:
|
|
762
|
+
passed = (score >= min_score) and not no_mutants
|
|
713
763
|
|
|
714
764
|
failed_files = {}
|
|
715
765
|
if not os.path.isfile(target) and min_file_score > 0.0:
|
|
@@ -721,13 +771,20 @@ def evaluate_target(
|
|
|
721
771
|
"target": target,
|
|
722
772
|
"mode": exec_mode,
|
|
723
773
|
"is_heuristic_static": (exec_mode == "STATIC"),
|
|
774
|
+
"measurement_kind": "dynamic_execution" if "DYNAMIC" in exec_mode else "static_ast_heuristic",
|
|
724
775
|
"total_mutants": len(all_mutants),
|
|
776
|
+
"valid_mutants": valid_mutants if "DYNAMIC" in exec_mode else len(all_mutants),
|
|
725
777
|
"killed": killed,
|
|
726
778
|
"survived": survived,
|
|
779
|
+
"timed_out": timed_out,
|
|
780
|
+
"invalid_syntax": invalid_syntax,
|
|
727
781
|
"mutation_score": score,
|
|
728
782
|
"static_assertion_sensitivity_score": score if exec_mode == "STATIC" else None,
|
|
783
|
+
"dynamic_mutation_score": score if "DYNAMIC" in exec_mode else None,
|
|
729
784
|
"runtime_mutation_kill_score": score if "DYNAMIC" in exec_mode else None,
|
|
785
|
+
"score_denominator": valid_mutants if "DYNAMIC" in exec_mode else len(all_mutants),
|
|
730
786
|
"no_mutants_generated": no_mutants,
|
|
787
|
+
"no_valid_mutants": no_valid_mutants,
|
|
731
788
|
"min_score": min_score,
|
|
732
789
|
"min_file_score": min_file_score,
|
|
733
790
|
"passed": passed,
|
|
@@ -789,21 +846,30 @@ def format_junit(report: Dict[str, Any]) -> str:
|
|
|
789
846
|
"""Formats mutation evaluation in standard JUnit XML format."""
|
|
790
847
|
total = max(1, report.get("total_mutants", 0))
|
|
791
848
|
survived = report.get("survived", 0)
|
|
849
|
+
timed_out = report.get("timed_out", 0)
|
|
850
|
+
invalid_syntax = report.get("invalid_syntax", 0)
|
|
792
851
|
lines = [
|
|
793
852
|
'<?xml version="1.0" encoding="utf-8"?>',
|
|
794
|
-
f'<testsuite name="synapse-micro-mutator" tests="{total}" failures="{survived}" errors="
|
|
853
|
+
f'<testsuite name="synapse-micro-mutator" tests="{total}" failures="{survived}" errors="{timed_out}" skipped="{invalid_syntax}">'
|
|
795
854
|
]
|
|
796
855
|
for m in report.get("mutants", []):
|
|
797
856
|
cname = (m.get("file") or report.get("target", "")).replace("/", ".").replace("\\", ".")
|
|
798
857
|
clean_desc = re.sub(r'[^a-zA-Z0-9_]', '_', m.get('description', 'mutant'))
|
|
799
858
|
tname = f"Mutant_L{m.get('line', 1)}_{clean_desc}"
|
|
800
859
|
lines.append(f' <testcase classname="{cname}" name="{tname}">')
|
|
801
|
-
|
|
860
|
+
status = m.get("status")
|
|
861
|
+
if status == "SURVIVED":
|
|
802
862
|
desc = html_lib.escape(m.get("description", ""))
|
|
803
863
|
orig = html_lib.escape(m.get("original_snippet", ""))
|
|
804
864
|
mut = html_lib.escape(m.get("mutated_snippet", ""))
|
|
805
865
|
reason = html_lib.escape(m.get("reason", "Downstream assertions failed to kill mutant"))
|
|
806
866
|
lines.append(f' <failure message="Mutant survived: {desc}" type="MUTANT_SURVIVED">Line {m.get("line", 1)}: {orig} -> {mut}. Reason: {reason}</failure>')
|
|
867
|
+
elif status == "TIMED_OUT":
|
|
868
|
+
reason = html_lib.escape(m.get("reason", "Test command execution timed out"))
|
|
869
|
+
lines.append(f' <error message="Mutant timed out" type="MUTANT_TIMED_OUT">{reason}</error>')
|
|
870
|
+
elif status in ("INVALID_SYNTAX", "INVALID"):
|
|
871
|
+
reason = html_lib.escape(m.get("reason", "Invalid syntax mutant discarded"))
|
|
872
|
+
lines.append(f' <skipped message="Discarded invalid mutant: {reason}"/>')
|
|
807
873
|
lines.append(' </testcase>')
|
|
808
874
|
if not report.get("mutants"):
|
|
809
875
|
lines.append(' <testcase classname="synapse.mutator" name="clean_run"/>')
|
|
@@ -890,7 +956,8 @@ def main():
|
|
|
890
956
|
mode_desc = f"LIVE DYNAMIC RUNNER (cmd: '{args.test_cmd}')"
|
|
891
957
|
else:
|
|
892
958
|
mode_desc = "STATIC HEURISTIC (fast heuristic assertion sensitivity scan; pass --test-cmd or --scoped-test-cmd for dynamic run)"
|
|
893
|
-
score_label = "Runtime Mutation
|
|
959
|
+
score_label = "Runtime Dynamic Mutation Score" if "DYNAMIC" in report.get("mode", "") else "Static Assertion Sensitivity Score"
|
|
960
|
+
denominator_label = f"(Denominator: {report.get('valid_mutants', report['total_mutants'])} valid mutants)" if "DYNAMIC" in report.get("mode", "") else f"(Denominator: {report['total_mutants']} syntactic expressions)"
|
|
894
961
|
|
|
895
962
|
print("\n======================================================================")
|
|
896
963
|
print(f" ASSERTION SENSITIVITY & SYNTACTIC MUTATION SENTINEL [{mode_tag}] ")
|
|
@@ -900,9 +967,12 @@ def main():
|
|
|
900
967
|
print(f" Mutants Injected : {report['total_mutants']}")
|
|
901
968
|
print(f" Mutants Caught/Killed : {report['killed']}")
|
|
902
969
|
print(f" Mutants Survived : {report['survived']}")
|
|
970
|
+
if "DYNAMIC" in report.get("mode", ""):
|
|
971
|
+
print(f" Mutants Timed Out : {report.get('timed_out', 0)} (inconclusive, not counted as killed)")
|
|
972
|
+
print(f" Invalid Mutants : {report.get('invalid_syntax', 0)} (discarded from denominator)")
|
|
903
973
|
if report.get("no_mutants_generated"):
|
|
904
974
|
print(" ⚠️ Notice : Zero syntactic mutants generated. Target lacks mutable logic expressions.")
|
|
905
|
-
print(f" {score_label:<
|
|
975
|
+
print(f" {score_label:<34}: {report['mutation_score']}% {denominator_label} (Required: >={report['min_score']}%)")
|
|
906
976
|
print("======================================================================\n")
|
|
907
977
|
|
|
908
978
|
for m in report['mutants'][:6]:
|