motionloom 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/CODE_OF_CONDUCT.md +19 -0
- package/CONTRIBUTING.md +65 -0
- package/README.md +187 -134
- package/ROADMAP.md +32 -0
- package/SECURITY.md +27 -0
- package/SKILL.md +33 -8
- package/SUPPORT.md +23 -0
- package/agent-card.json +21 -6
- package/bin/motionloom.mjs +23 -5
- package/docs/STATUS.md +33 -0
- package/docs/audits/2.1.0-deep-stress-evaluation.md +97 -0
- package/docs/audits/data/2.1.0-deep-stress-6900.json +329 -0
- package/docs/audits/data/deep-stress-latest.json +329 -0
- package/docs/audits/external-project-corpus-2026-08-13.md +26 -0
- package/docs/releases/2.1.0.md +23 -0
- package/docs/releases/npm-publish-from-workstation.md +6 -6
- package/package.json +52 -26
- package/references/intelligence-core.md +1 -1
- package/schemas/project-memory.schema.json +180 -0
- package/scripts/analyze.py +56 -0
- package/scripts/capture-runtime-telemetry.py +119 -0
- package/scripts/devlab.py +126 -0
- package/scripts/docs-audit.py +96 -0
- package/scripts/eval-intelligence.py +23 -0
- package/scripts/eval-projects.py +156 -0
- package/scripts/intelligence.py +106 -6
- package/scripts/pr.py +150 -0
- package/scripts/prepack-clean.mjs +37 -0
- package/scripts/project-memory.py +483 -0
- package/scripts/project_memory_loader.py +31 -0
- package/scripts/release-verify.py +52 -0
- package/scripts/render.py +65 -0
- package/scripts/report.py +44 -2
- package/scripts/review-hook.py +13 -2
- package/scripts/skill-doctor.py +12 -2
- package/scripts/to-dotlottie.mjs +26 -20
- package/src/core/analyzer.py +174 -25
- package/tests/evals/intelligence-cases.json +10 -0
- package/tests/evals/project-corpus.json +51 -0
- package/tests/scripts/run_tests.py +52 -1
- package/tests/scripts/test_project_memory.py +129 -0
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Cross-platform Dev Lab scene/task preparation and local serving helper."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import shutil
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
SAFE_NAME = re.compile(r"^[A-Za-z0-9._-]+$")
|
|
16
|
+
TASK_ARTIFACTS = (
|
|
17
|
+
"task.json", "browser-review.json", "review.json", "execution-report.json",
|
|
18
|
+
"handoff.json", "quality-report.json", "artifact-manifest.json",
|
|
19
|
+
"issue-register.json", "decision-log.jsonl", "project-memory.json",
|
|
20
|
+
"project-graph.json", "provenance.json", "capability-registry.json",
|
|
21
|
+
"motion-ir.json", "replay-bundle.json", "semantic-lint-report.json",
|
|
22
|
+
"continuity-report.json", "fix-plan.json", "browser-observation.md",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def fail(message: str) -> "NoReturn":
|
|
27
|
+
print(f"MotionLoom Dev Lab error: {message}", file=sys.stderr)
|
|
28
|
+
raise SystemExit(1)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def read_json(path: Path) -> dict:
|
|
32
|
+
try:
|
|
33
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
34
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
35
|
+
fail(f"cannot read JSON {path}: {exc}")
|
|
36
|
+
return value if isinstance(value, dict) else {}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def safe_name(value: str, label: str) -> None:
|
|
40
|
+
if not value or value in {".", ".."} or not SAFE_NAME.fullmatch(value):
|
|
41
|
+
fail(f"{label} contains unsafe path characters: {value!r}")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def inside(child: Path, parent: Path) -> bool:
|
|
45
|
+
try:
|
|
46
|
+
child.relative_to(parent)
|
|
47
|
+
return True
|
|
48
|
+
except ValueError:
|
|
49
|
+
return False
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def prepare_scene(root: Path, lab: Path, scene: str, task_dir: Path | None) -> tuple[Path, Path | None, str | None]:
|
|
53
|
+
safe_name(scene, "scene")
|
|
54
|
+
scene_dir = (root / "src" / "output" / scene).resolve()
|
|
55
|
+
if not scene_dir.is_dir():
|
|
56
|
+
fail(f"scene directory not found: {scene_dir}; render the scene first")
|
|
57
|
+
if not (scene_dir / "browser-review.json").is_file():
|
|
58
|
+
fail("browser-review.json is required; run review-hook prepare first")
|
|
59
|
+
lab_root = lab.resolve()
|
|
60
|
+
destination = (lab_root / "public" / "scenes" / scene).resolve()
|
|
61
|
+
if not inside(destination, (lab_root / "public" / "scenes").resolve()):
|
|
62
|
+
fail("scene destination escaped Dev Lab public directory")
|
|
63
|
+
if destination.exists():
|
|
64
|
+
shutil.rmtree(destination)
|
|
65
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
66
|
+
shutil.copytree(scene_dir, destination)
|
|
67
|
+
|
|
68
|
+
task_destination: Path | None = None
|
|
69
|
+
task_id: str | None = None
|
|
70
|
+
if task_dir is not None:
|
|
71
|
+
task_dir = task_dir.resolve()
|
|
72
|
+
if not inside(task_dir, root.resolve()):
|
|
73
|
+
fail("task bundle must be inside the MotionLoom repository")
|
|
74
|
+
task = read_json(task_dir / "task.json")
|
|
75
|
+
task_id = str(task.get("task_id") or "")
|
|
76
|
+
safe_name(task_id, "task_id")
|
|
77
|
+
task_destination = (lab_root / "public" / "tasks" / task_id).resolve()
|
|
78
|
+
if not inside(task_destination, (lab_root / "public" / "tasks").resolve()):
|
|
79
|
+
fail("task destination escaped Dev Lab public directory")
|
|
80
|
+
if task_destination.exists():
|
|
81
|
+
shutil.rmtree(task_destination)
|
|
82
|
+
task_destination.mkdir(parents=True, exist_ok=True)
|
|
83
|
+
for name in TASK_ARTIFACTS:
|
|
84
|
+
source = task_dir / name
|
|
85
|
+
if source.is_file():
|
|
86
|
+
shutil.copy2(source, task_destination / name)
|
|
87
|
+
return destination, task_destination, task_id
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def pnpm_executable() -> str:
|
|
91
|
+
candidate = "pnpm.cmd" if os.name == "nt" else "pnpm"
|
|
92
|
+
return shutil.which(candidate) or candidate
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def serve(lab: Path, port: int) -> int:
|
|
96
|
+
if not (lab / "public").is_dir():
|
|
97
|
+
fail(f"Dev Lab public directory not found: {lab / 'public'}")
|
|
98
|
+
if not (lab / "node_modules").is_dir():
|
|
99
|
+
print("== installing Dev Lab dependencies (first run) ==")
|
|
100
|
+
subprocess.run([pnpm_executable(), "install", "--silent"], cwd=lab, check=True)
|
|
101
|
+
print(f"== Dev Lab ready: http://localhost:{port}/ ==")
|
|
102
|
+
return subprocess.run([sys.executable, "-m", "http.server", str(port), "--directory", str(lab / "public")], check=False).returncode
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def main() -> int:
|
|
106
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
107
|
+
parser.add_argument("scene")
|
|
108
|
+
parser.add_argument("--task-dir")
|
|
109
|
+
parser.add_argument("--lab-dir")
|
|
110
|
+
parser.add_argument("--port", type=int, default=int(os.environ.get("PORT", "3300")))
|
|
111
|
+
parser.add_argument("--prepare-only", action="store_true")
|
|
112
|
+
args = parser.parse_args()
|
|
113
|
+
root = Path(__file__).resolve().parents[1]
|
|
114
|
+
lab = Path(args.lab_dir or os.environ.get("MOTIONLOOM_DEV_LAB") or root / "dev-lab").expanduser().resolve()
|
|
115
|
+
task_dir = Path(args.task_dir).expanduser().resolve() if args.task_dir else None
|
|
116
|
+
destination, task_destination, task_id = prepare_scene(root, lab, args.scene, task_dir)
|
|
117
|
+
print(f"== Dev Lab scene prepared: {destination} ==")
|
|
118
|
+
if task_destination:
|
|
119
|
+
print(f"== Dev Lab task bundle prepared: {task_destination} (task_id={task_id}) ==")
|
|
120
|
+
if args.prepare_only:
|
|
121
|
+
return 0
|
|
122
|
+
return serve(lab, args.port)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
if __name__ == "__main__":
|
|
126
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Validate public documentation, links and workflow safety without network access."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
ROOT = Path(sys.argv[1] if len(sys.argv) > 1 else Path(__file__).resolve().parents[1])
|
|
13
|
+
errors: list[str] = []
|
|
14
|
+
|
|
15
|
+
for markdown in sorted(ROOT.rglob("*.md")):
|
|
16
|
+
if any(part in {".git", "node_modules"} for part in markdown.parts):
|
|
17
|
+
continue
|
|
18
|
+
text = markdown.read_text(encoding="utf-8")
|
|
19
|
+
for target in re.findall(r"\[[^\]]+\]\(([^)]+)\)", text):
|
|
20
|
+
target = target.strip().split("#", 1)[0].split("?", 1)[0]
|
|
21
|
+
if not target or target.startswith(("http://", "https://", "mailto:", "#")):
|
|
22
|
+
continue
|
|
23
|
+
if not (markdown.parent / target).resolve().exists():
|
|
24
|
+
errors.append(f"{markdown.relative_to(ROOT)} -> missing {target}")
|
|
25
|
+
|
|
26
|
+
for relative in ["package.json", "agent-card.json", "project-context.example.json", "tests/evals/project-corpus.json"]:
|
|
27
|
+
path = ROOT / relative
|
|
28
|
+
try:
|
|
29
|
+
json.loads(path.read_text(encoding="utf-8"))
|
|
30
|
+
except Exception as exc:
|
|
31
|
+
errors.append(f"{relative}: invalid JSON: {exc}")
|
|
32
|
+
|
|
33
|
+
package = json.loads((ROOT / "package.json").read_text(encoding="utf-8"))
|
|
34
|
+
for required in ["author", "repository", "homepage", "bugs", "license", "engines", "files"]:
|
|
35
|
+
if not package.get(required):
|
|
36
|
+
errors.append(f"package.json: missing public metadata {required}")
|
|
37
|
+
if package.get("packageManager") != "pnpm@11.20.0":
|
|
38
|
+
errors.append("package.json: packageManager must pin pnpm@11.20.0")
|
|
39
|
+
|
|
40
|
+
readme = (ROOT / "README.md").read_text(encoding="utf-8")
|
|
41
|
+
for heading in ["Why MotionLoom", "Quick start", "Durable Project Memory", "Evidence, trust and review", "Documentation map"]:
|
|
42
|
+
if f"## {heading}" not in readme:
|
|
43
|
+
errors.append(f"README.md: missing heading {heading}")
|
|
44
|
+
|
|
45
|
+
workflow_dir = ROOT / ".github" / "workflows"
|
|
46
|
+
for workflow in sorted(workflow_dir.glob("*.yml")):
|
|
47
|
+
text = workflow.read_text(encoding="utf-8")
|
|
48
|
+
top_level_keys: dict[str, int] = {}
|
|
49
|
+
for line_number, line in enumerate(text.splitlines(), start=1):
|
|
50
|
+
match = re.match(r"^([A-Za-z_][A-Za-z0-9_-]*):(?:\s|$)", line)
|
|
51
|
+
if not match:
|
|
52
|
+
continue
|
|
53
|
+
key = match.group(1)
|
|
54
|
+
if key in top_level_keys:
|
|
55
|
+
errors.append(
|
|
56
|
+
f"{workflow.relative_to(ROOT)}: duplicate top-level key {key!r} "
|
|
57
|
+
f"at lines {top_level_keys[key]} and {line_number}"
|
|
58
|
+
)
|
|
59
|
+
else:
|
|
60
|
+
top_level_keys[key] = line_number
|
|
61
|
+
for required in ["name:", "on:", "jobs:", "permissions:"]:
|
|
62
|
+
if required not in text:
|
|
63
|
+
errors.append(f"{workflow.relative_to(ROOT)}: missing {required}")
|
|
64
|
+
if "pull_request:" in text and "secrets." in text:
|
|
65
|
+
errors.append(f"{workflow.relative_to(ROOT)}: secrets referenced in pull_request workflow")
|
|
66
|
+
|
|
67
|
+
release = (workflow_dir / "release.yml").read_text(encoding="utf-8")
|
|
68
|
+
for required in ["workflow_dispatch:", "environment: npm-release", "id-token: write", "release_version:", "scripts/release-verify.py"]:
|
|
69
|
+
if required not in release:
|
|
70
|
+
errors.append(f"release.yml: missing release safety control {required}")
|
|
71
|
+
|
|
72
|
+
devlab = (workflow_dir / "devlab.yml").read_text(encoding="utf-8")
|
|
73
|
+
for required in [
|
|
74
|
+
"cp -R src/output/browser-review-smoke dev-lab/public/scenes/browser-review-smoke",
|
|
75
|
+
"scenes/browser-review-smoke/manifest.json",
|
|
76
|
+
"scenes/browser-review-smoke/motion-spec.json",
|
|
77
|
+
"--diagnostics /tmp/motionloom-devlab-diagnostics",
|
|
78
|
+
"id: fixture",
|
|
79
|
+
"steps.fixture.outputs.task_id",
|
|
80
|
+
"steps.fixture.outputs.candidate_id",
|
|
81
|
+
'payload["expires_at"] = "2099-01-01T00:00:00Z"',
|
|
82
|
+
]:
|
|
83
|
+
if required not in devlab:
|
|
84
|
+
errors.append(f"devlab.yml: missing fixture/readiness control {required}")
|
|
85
|
+
|
|
86
|
+
if errors:
|
|
87
|
+
print("Documentation/workflow audit: FAIL")
|
|
88
|
+
print("\n".join(f"- {error}" for error in errors))
|
|
89
|
+
raise SystemExit(1)
|
|
90
|
+
|
|
91
|
+
print("Documentation/workflow audit: PASS")
|
|
92
|
+
print(f"markdown_files={len(list(ROOT.rglob('*.md')))}")
|
|
93
|
+
print(f"workflow_files={len(list(workflow_dir.glob('*.yml')))}")
|
|
94
|
+
print("internal_links=PASS")
|
|
95
|
+
print("json_metadata=PASS")
|
|
96
|
+
print("workflow_safety=PASS")
|
|
@@ -23,6 +23,7 @@ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
|
|
23
23
|
|
|
24
24
|
ROOT = Path(__file__).resolve().parents[1]
|
|
25
25
|
INTELLIGENCE = ROOT / "scripts/intelligence.py"
|
|
26
|
+
PROJECT_EVAL = ROOT / "scripts/eval-projects.py"
|
|
26
27
|
REPORT = ROOT / "scripts/report.py"
|
|
27
28
|
VERIFIER = ROOT / "scripts/evidence-verifier.py"
|
|
28
29
|
ATTESTATION = ROOT / "scripts/attestation.py"
|
|
@@ -134,6 +135,27 @@ def run_attestation_cases(root: Path, results: list[dict[str, object]]) -> None:
|
|
|
134
135
|
record(results, "p2-attestation-unknown-signer", unknown_result.returncode == 13, unknown_result.stdout + unknown_result.stderr)
|
|
135
136
|
|
|
136
137
|
|
|
138
|
+
def run_project_corpus_cases(root: Path, results: list[dict[str, object]]) -> None:
|
|
139
|
+
"""Keep first-party analyzer evidence separate from unavailable external evidence."""
|
|
140
|
+
first_party = invoke([
|
|
141
|
+
str(PROJECT_EVAL), "--workspace", str(ROOT), "--require-external", "0", "--allow-insufficient"
|
|
142
|
+
])
|
|
143
|
+
first_doc = json.loads(first_party.stdout) if first_party.stdout.strip().startswith("{") else {}
|
|
144
|
+
first_pass = any(item.get("id") == "motionloom-first-party" and item.get("status") == "pass" for item in first_doc.get("results", []))
|
|
145
|
+
record(results, "project-corpus-first-party-pass", first_party.returncode == 0 and first_doc.get("status") == "pass" and first_pass, first_party.stdout + first_party.stderr)
|
|
146
|
+
|
|
147
|
+
insufficient = invoke([
|
|
148
|
+
str(PROJECT_EVAL), "--workspace", str(ROOT), "--allow-insufficient"
|
|
149
|
+
])
|
|
150
|
+
insufficient_doc = json.loads(insufficient.stdout) if insufficient.stdout.strip().startswith("{") else {}
|
|
151
|
+
record(
|
|
152
|
+
results,
|
|
153
|
+
"project-corpus-insufficient-external-explicit",
|
|
154
|
+
insufficient.returncode == 0 and insufficient_doc.get("status") == "insufficient_evidence" and insufficient_doc.get("unavailable_external_projects") == 3,
|
|
155
|
+
insufficient.stdout + insufficient.stderr,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
137
159
|
def run_p1_cases(root: Path, task_dir: Path, results: list[dict[str, object]]) -> None:
|
|
138
160
|
"""Exercise P1 semantic, continuity and feedback contracts in isolated copies."""
|
|
139
161
|
lint_task = root / "p1-human-review"
|
|
@@ -364,6 +386,7 @@ def main() -> int:
|
|
|
364
386
|
run_performance_perceptual_cases(root, task_dir, results)
|
|
365
387
|
run_runtime_verifier_cases(root, results)
|
|
366
388
|
run_attestation_cases(root, results)
|
|
389
|
+
run_project_corpus_cases(root, results)
|
|
367
390
|
|
|
368
391
|
missing = expected - {str(item["id"]) for item in results}
|
|
369
392
|
for case_id in sorted(missing):
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Evaluate MotionLoom's project-aware analyzer against a labeled project corpus.
|
|
3
|
+
|
|
4
|
+
The manifest records provenance and expected signals, but this runner never
|
|
5
|
+
clones, installs or executes code from external projects. A checkout is only
|
|
6
|
+
considered available when the caller explicitly places it under --workspace.
|
|
7
|
+
Missing external projects produce ``insufficient_evidence`` rather than a
|
|
8
|
+
false pass. This keeps product-value claims separate from owned fixtures.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
import tempfile
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
22
|
+
DEFAULT_MANIFEST = ROOT / "tests/evals/project-corpus.json"
|
|
23
|
+
ANALYZER = ROOT / "scripts/analyze.py"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def invoke(project: Path, output: Path, args: argparse.Namespace) -> subprocess.CompletedProcess[str]:
|
|
27
|
+
command = [
|
|
28
|
+
sys.executable,
|
|
29
|
+
str(ANALYZER),
|
|
30
|
+
str(project),
|
|
31
|
+
"--output",
|
|
32
|
+
str(output),
|
|
33
|
+
"--max-files",
|
|
34
|
+
str(args.max_files),
|
|
35
|
+
"--max-bytes",
|
|
36
|
+
str(args.max_bytes),
|
|
37
|
+
"--max-seconds",
|
|
38
|
+
str(args.max_seconds),
|
|
39
|
+
]
|
|
40
|
+
for value in args.ignore_dir:
|
|
41
|
+
command.extend(["--ignore-dir", value])
|
|
42
|
+
for value in args.ignore_glob:
|
|
43
|
+
command.extend(["--ignore-glob", value])
|
|
44
|
+
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def evaluate_case(case: dict, workspace: Path, repository_root: Path, args: argparse.Namespace, temp: Path) -> dict:
|
|
48
|
+
relative = Path(str(case["local_path"]))
|
|
49
|
+
base = repository_root if case.get("scope") == "repository" else workspace
|
|
50
|
+
project = (base / relative).resolve()
|
|
51
|
+
try:
|
|
52
|
+
project.relative_to(base.resolve())
|
|
53
|
+
except ValueError:
|
|
54
|
+
return {"id": case["id"], "class": case.get("class"), "status": "fail", "detail": "local_path escapes evaluation root"}
|
|
55
|
+
if not project.is_dir():
|
|
56
|
+
return {
|
|
57
|
+
"id": case["id"],
|
|
58
|
+
"class": case.get("class"),
|
|
59
|
+
"status": "unavailable",
|
|
60
|
+
"external": bool(case.get("external")),
|
|
61
|
+
"source": case.get("source"),
|
|
62
|
+
"detail": f"checkout not present at {relative.as_posix()}",
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
output = temp / f"{case['id']}.json"
|
|
66
|
+
result = invoke(project, output, args)
|
|
67
|
+
if result.returncode != 0 or not output.is_file():
|
|
68
|
+
return {
|
|
69
|
+
"id": case["id"],
|
|
70
|
+
"class": case.get("class"),
|
|
71
|
+
"status": "fail",
|
|
72
|
+
"external": bool(case.get("external")),
|
|
73
|
+
"source": case.get("source"),
|
|
74
|
+
"detail": (result.stdout + result.stderr).strip()[-1000:],
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
context = json.loads(output.read_text(encoding="utf-8"))
|
|
78
|
+
expected = case.get("expected", {})
|
|
79
|
+
checks = []
|
|
80
|
+
if expected.get("name") is not None:
|
|
81
|
+
checks.append((context.get("name") == expected["name"], f"name={context.get('name')!r}"))
|
|
82
|
+
if expected.get("framework") is not None:
|
|
83
|
+
actual = context.get("stack", {}).get("framework")
|
|
84
|
+
checks.append((actual == expected["framework"], f"framework={actual!r}"))
|
|
85
|
+
if expected.get("scan_truncated") is not None:
|
|
86
|
+
checks.append((context.get("scan_truncated") is expected["scan_truncated"], f"scan_truncated={context.get('scan_truncated')!r}"))
|
|
87
|
+
passed = all(item[0] for item in checks)
|
|
88
|
+
return {
|
|
89
|
+
"id": case["id"],
|
|
90
|
+
"class": case.get("class"),
|
|
91
|
+
"status": "pass" if passed else "fail",
|
|
92
|
+
"external": bool(case.get("external")),
|
|
93
|
+
"source": case.get("source"),
|
|
94
|
+
"checks": [detail for _, detail in checks],
|
|
95
|
+
"scan": context.get("scan", {}),
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def main() -> int:
|
|
100
|
+
parser = argparse.ArgumentParser(description="Run a provenance-labeled MotionLoom project corpus evaluation.")
|
|
101
|
+
parser.add_argument("--manifest", default=str(DEFAULT_MANIFEST))
|
|
102
|
+
parser.add_argument("--workspace", default=str(ROOT), help="Explicit directory containing corpus checkouts")
|
|
103
|
+
parser.add_argument("--repository-root", default=str(ROOT), help="First-party repository root")
|
|
104
|
+
parser.add_argument("--output")
|
|
105
|
+
parser.add_argument("--require-external", type=int, help="Override manifest external-project requirement")
|
|
106
|
+
parser.add_argument("--allow-insufficient", action="store_true", help="Return success while reporting insufficient_evidence")
|
|
107
|
+
parser.add_argument("--max-files", type=int, default=2500)
|
|
108
|
+
parser.add_argument("--max-bytes", type=int, default=25_000_000)
|
|
109
|
+
parser.add_argument("--max-seconds", type=float, default=10.0)
|
|
110
|
+
parser.add_argument("--ignore-dir", action="append", default=[])
|
|
111
|
+
parser.add_argument("--ignore-glob", action="append", default=[])
|
|
112
|
+
args = parser.parse_args()
|
|
113
|
+
|
|
114
|
+
manifest_path = Path(args.manifest).expanduser().resolve()
|
|
115
|
+
workspace = Path(args.workspace).expanduser().resolve()
|
|
116
|
+
repository_root = Path(args.repository_root).expanduser().resolve()
|
|
117
|
+
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
118
|
+
cases = manifest.get("projects", [])
|
|
119
|
+
required_external = args.require_external if args.require_external is not None else int(manifest.get("required_external_projects", 0))
|
|
120
|
+
|
|
121
|
+
with tempfile.TemporaryDirectory(prefix="motionloom-project-eval-") as td:
|
|
122
|
+
results = [evaluate_case(case, workspace, repository_root, args, Path(td)) for case in cases]
|
|
123
|
+
available_external = sum(1 for item in results if item.get("external") and item.get("status") == "pass")
|
|
124
|
+
failed = [item for item in results if item["status"] == "fail"]
|
|
125
|
+
unavailable_external = [item for item in results if item.get("external") and item["status"] == "unavailable"]
|
|
126
|
+
if failed:
|
|
127
|
+
status = "fail"
|
|
128
|
+
elif available_external < required_external:
|
|
129
|
+
status = "insufficient_evidence"
|
|
130
|
+
else:
|
|
131
|
+
status = "pass"
|
|
132
|
+
|
|
133
|
+
report = {
|
|
134
|
+
"schema_version": "1.0",
|
|
135
|
+
"corpus_id": manifest.get("corpus_id"),
|
|
136
|
+
"status": status,
|
|
137
|
+
"workspace": str(workspace),
|
|
138
|
+
"required_external_projects": required_external,
|
|
139
|
+
"available_external_projects": available_external,
|
|
140
|
+
"unavailable_external_projects": len(unavailable_external),
|
|
141
|
+
"project_count": len(results),
|
|
142
|
+
"results": results,
|
|
143
|
+
}
|
|
144
|
+
payload = json.dumps(report, indent=2, ensure_ascii=False) + "\n"
|
|
145
|
+
if args.output:
|
|
146
|
+
output = Path(args.output).expanduser().resolve()
|
|
147
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
148
|
+
output.write_text(payload, encoding="utf-8")
|
|
149
|
+
print(payload, end="")
|
|
150
|
+
if status == "insufficient_evidence" and args.allow_insufficient:
|
|
151
|
+
return 0
|
|
152
|
+
return 0 if status == "pass" else 1
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
if __name__ == "__main__":
|
|
156
|
+
raise SystemExit(main())
|
package/scripts/intelligence.py
CHANGED
|
@@ -824,11 +824,54 @@ def semantic_lint_validate_data(report: dict[str, Any]) -> list[str]:
|
|
|
824
824
|
issues.append("schema_version must be 0.1")
|
|
825
825
|
if report.get("status") not in {"pass", "warn", "fail"}:
|
|
826
826
|
issues.append("status must be pass, warn or fail")
|
|
827
|
-
|
|
828
|
-
|
|
827
|
+
for field in ("report_id", "task_id", "scene"):
|
|
828
|
+
if not isinstance(report.get(field), str) or not report.get(field).strip():
|
|
829
|
+
issues.append(f"{field} must be a non-empty string")
|
|
830
|
+
ruleset = report.get("ruleset")
|
|
831
|
+
if not isinstance(ruleset, dict) or not isinstance(ruleset.get("id"), str) or not ruleset.get("id").strip() or not isinstance(ruleset.get("version"), str) or not ruleset.get("version").strip():
|
|
832
|
+
issues.append("ruleset must contain non-empty id and version")
|
|
833
|
+
summary = report.get("summary") if isinstance(report.get("summary"), dict) else {}
|
|
834
|
+
summary_required = {"total", "errors", "warnings", "infos", "blocking"}
|
|
835
|
+
issues.extend(f"summary missing {field}" for field in sorted(summary_required - set(summary)))
|
|
836
|
+
findings = report.get("findings") if isinstance(report.get("findings"), list) else []
|
|
837
|
+
if not isinstance(report.get("findings"), list):
|
|
838
|
+
issues.append("findings must be an array")
|
|
829
839
|
if summary.get("total") != len(findings):
|
|
830
840
|
issues.append("summary.total does not match findings length")
|
|
831
|
-
|
|
841
|
+
finding_required = {"id", "rule_id", "category", "severity", "confidence", "basis", "message", "evidence_refs", "affected_paths", "approval_blocking"}
|
|
842
|
+
categories = {"intent", "timing", "easing", "accessibility", "performance", "continuity", "anti_pattern"}
|
|
843
|
+
severities = {"info", "warning", "error"}
|
|
844
|
+
bases = {"deterministic", "runtime", "human", "heuristic"}
|
|
845
|
+
severity_counts = {"info": 0, "warning": 0, "error": 0}
|
|
846
|
+
blocking = 0
|
|
847
|
+
for index, item in enumerate(findings):
|
|
848
|
+
if not isinstance(item, dict):
|
|
849
|
+
issues.append(f"finding[{index}] must be an object")
|
|
850
|
+
continue
|
|
851
|
+
issues.extend(f"finding[{index}] missing {field}" for field in sorted(finding_required - set(item)))
|
|
852
|
+
if item.get("severity") not in severities:
|
|
853
|
+
issues.append(f"finding[{index}] severity is invalid")
|
|
854
|
+
else:
|
|
855
|
+
severity_counts[item["severity"]] += 1
|
|
856
|
+
if item.get("category") not in categories:
|
|
857
|
+
issues.append(f"finding[{index}] category is invalid")
|
|
858
|
+
if item.get("basis") not in bases:
|
|
859
|
+
issues.append(f"finding[{index}] basis is invalid")
|
|
860
|
+
if not isinstance(item.get("confidence"), (int, float)) or not 0 <= item.get("confidence", -1) <= 1:
|
|
861
|
+
issues.append(f"finding[{index}] confidence must be between 0 and 1")
|
|
862
|
+
if not isinstance(item.get("evidence_refs"), list) or not isinstance(item.get("affected_paths"), list):
|
|
863
|
+
issues.append(f"finding[{index}] evidence_refs and affected_paths must be arrays")
|
|
864
|
+
if item.get("approval_blocking") is True:
|
|
865
|
+
blocking += 1
|
|
866
|
+
for field in ("total", "errors", "warnings", "infos", "blocking"):
|
|
867
|
+
if not isinstance(summary.get(field), int) or summary.get(field) < 0:
|
|
868
|
+
issues.append(f"summary.{field} must be a non-negative integer")
|
|
869
|
+
if summary.get("errors") != severity_counts["error"]:
|
|
870
|
+
issues.append("summary.errors does not match findings")
|
|
871
|
+
if summary.get("warnings") != severity_counts["warning"]:
|
|
872
|
+
issues.append("summary.warnings does not match findings")
|
|
873
|
+
if summary.get("infos") != severity_counts["info"]:
|
|
874
|
+
issues.append("summary.infos does not match findings")
|
|
832
875
|
if summary.get("blocking") != blocking:
|
|
833
876
|
issues.append("summary.blocking does not match findings")
|
|
834
877
|
if report.get("status") == "fail" and blocking == 0:
|
|
@@ -1040,15 +1083,72 @@ def continuity_report_data(task_dirs: list[Path], project_id: str | None = None)
|
|
|
1040
1083
|
|
|
1041
1084
|
def continuity_validate_data(report: dict[str, Any]) -> list[str]:
|
|
1042
1085
|
issues: list[str] = []
|
|
1086
|
+
required = {"schema_version", "report_id", "project_id", "status", "scenes", "transitions", "summary", "generated_at"}
|
|
1087
|
+
issues.extend(f"missing {field}" for field in sorted(required - set(report)))
|
|
1043
1088
|
if report.get("schema_version") != "0.1":
|
|
1044
1089
|
issues.append("schema_version must be 0.1")
|
|
1045
|
-
|
|
1046
|
-
|
|
1090
|
+
if report.get("status") not in {"pass", "warn", "fail"}:
|
|
1091
|
+
issues.append("status must be pass, warn or fail")
|
|
1092
|
+
for field in ("report_id", "project_id"):
|
|
1093
|
+
if not isinstance(report.get(field), str) or not report.get(field).strip():
|
|
1094
|
+
issues.append(f"{field} must be a non-empty string")
|
|
1095
|
+
scenes = report.get("scenes") if isinstance(report.get("scenes"), list) else []
|
|
1096
|
+
transitions = report.get("transitions") if isinstance(report.get("transitions"), list) else []
|
|
1097
|
+
if not isinstance(report.get("scenes"), list):
|
|
1098
|
+
issues.append("scenes must be an array")
|
|
1099
|
+
if not isinstance(report.get("transitions"), list):
|
|
1100
|
+
issues.append("transitions must be an array")
|
|
1047
1101
|
if not scenes:
|
|
1048
1102
|
issues.append("continuity report requires at least one scene")
|
|
1049
1103
|
if len(transitions) != max(0, len(scenes) - 1):
|
|
1050
1104
|
issues.append("transition count must equal scene count minus one")
|
|
1051
|
-
|
|
1105
|
+
scene_names = set()
|
|
1106
|
+
for index, scene in enumerate(scenes):
|
|
1107
|
+
if not isinstance(scene, dict):
|
|
1108
|
+
issues.append(f"scene[{index}] must be an object")
|
|
1109
|
+
continue
|
|
1110
|
+
scene_required = {"scene", "order", "motion_ir_path", "motion_ir_sha256", "context_hash", "intent"}
|
|
1111
|
+
issues.extend(f"scene[{index}] missing {field}" for field in sorted(scene_required - set(scene)))
|
|
1112
|
+
name = scene.get("scene")
|
|
1113
|
+
if not isinstance(name, str) or not name.strip():
|
|
1114
|
+
issues.append(f"scene[{index}].scene must be a non-empty string")
|
|
1115
|
+
elif name in scene_names:
|
|
1116
|
+
issues.append(f"duplicate scene name: {name}")
|
|
1117
|
+
else:
|
|
1118
|
+
scene_names.add(name)
|
|
1119
|
+
if not isinstance(scene.get("order"), int) or scene.get("order", -1) < 0:
|
|
1120
|
+
issues.append(f"scene[{index}].order must be a non-negative integer")
|
|
1121
|
+
for field in ("motion_ir_path", "intent"):
|
|
1122
|
+
if not isinstance(scene.get(field), str) or not scene.get(field).strip():
|
|
1123
|
+
issues.append(f"scene[{index}].{field} must be a non-empty string")
|
|
1124
|
+
for field in ("motion_ir_sha256", "context_hash"):
|
|
1125
|
+
value = scene.get(field)
|
|
1126
|
+
if not isinstance(value, str) or len(value) != 64 or any(char not in "0123456789abcdef" for char in value):
|
|
1127
|
+
issues.append(f"scene[{index}].{field} must be a lowercase SHA-256 digest")
|
|
1128
|
+
for index, transition in enumerate(transitions):
|
|
1129
|
+
if not isinstance(transition, dict):
|
|
1130
|
+
issues.append(f"transition[{index}] must be an object")
|
|
1131
|
+
continue
|
|
1132
|
+
transition_required = {"id", "from_scene", "to_scene", "status", "checks"}
|
|
1133
|
+
issues.extend(f"transition[{index}] missing {field}" for field in sorted(transition_required - set(transition)))
|
|
1134
|
+
if transition.get("status") not in {"pass", "warn", "fail"}:
|
|
1135
|
+
issues.append(f"transition[{index}].status is invalid")
|
|
1136
|
+
if transition.get("from_scene") not in scene_names or transition.get("to_scene") not in scene_names:
|
|
1137
|
+
issues.append(f"transition[{index}] references unknown scene")
|
|
1138
|
+
if not isinstance(transition.get("checks"), list) or any(not isinstance(item, str) or not item.strip() for item in transition.get("checks", [])):
|
|
1139
|
+
issues.append(f"transition[{index}].checks must be a non-empty-string array")
|
|
1140
|
+
summary = report.get("summary") if isinstance(report.get("summary"), dict) else {}
|
|
1141
|
+
summary_required = {"scene_count", "transition_count", "errors", "warnings", "blocking"}
|
|
1142
|
+
issues.extend(f"summary missing {field}" for field in sorted(summary_required - set(summary)))
|
|
1143
|
+
if report.get("summary") is not None and not isinstance(report.get("summary"), dict):
|
|
1144
|
+
issues.append("summary must be an object")
|
|
1145
|
+
for field in ("scene_count", "transition_count", "errors", "warnings", "blocking"):
|
|
1146
|
+
minimum = 1 if field == "scene_count" else 0
|
|
1147
|
+
if not isinstance(summary.get(field), int) or summary.get(field) < minimum:
|
|
1148
|
+
issues.append(f"summary.{field} must be a valid non-negative count")
|
|
1149
|
+
if summary.get("transition_count") != len(transitions):
|
|
1150
|
+
issues.append("summary.transition_count does not match transitions")
|
|
1151
|
+
if summary.get("scene_count") != len(scenes):
|
|
1052
1152
|
issues.append("summary.scene_count does not match scenes")
|
|
1053
1153
|
return issues
|
|
1054
1154
|
|