@blxzer/cursor-trellis 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/README.md +45 -21
- package/dist/configurators/workflow.d.ts +1 -1
- package/dist/configurators/workflow.d.ts.map +1 -1
- package/dist/configurators/workflow.js +20 -9
- package/dist/configurators/workflow.js.map +1 -1
- package/dist/migrations/manifests/0.1.3.json +9 -0
- package/dist/migrations/manifests/0.1.4.json +9 -0
- package/dist/templates/cursor/rules/retrieval-routing.mdc +9 -7
- package/dist/templates/markdown/agents.md +1 -1
- package/dist/templates/markdown/index.d.ts +1 -0
- package/dist/templates/markdown/index.d.ts.map +1 -1
- package/dist/templates/markdown/index.js +1 -0
- package/dist/templates/markdown/index.js.map +1 -1
- package/dist/templates/markdown/prompts/run-semantic-slice-12.md.txt +13 -29
- package/dist/templates/markdown/spec/guides/code-reuse-thinking-guide.md.txt +0 -49
- package/dist/templates/markdown/spec/guides/cross-layer-thinking-guide.md.txt +0 -85
- package/dist/templates/markdown/spec/guides/cursor-semantic-compliance.md.txt +7 -21
- package/dist/templates/markdown/spec/guides/cursor-subagent-policy.md.txt +6 -10
- package/dist/templates/markdown/spec/guides/retrieval-daily-guide.md.txt +17 -18
- package/dist/templates/shared-hooks/inject-retrieval-plan.py +42 -23
- package/dist/templates/shared-hooks/inject-subagent-context.py +6 -2
- package/dist/templates/shared-hooks/session-start.py +17 -9
- package/dist/templates/trellis/index.d.ts +8 -4
- package/dist/templates/trellis/index.d.ts.map +1 -1
- package/dist/templates/trellis/index.js +13 -7
- package/dist/templates/trellis/index.js.map +1 -1
- package/dist/templates/trellis/scripts/common/codebase_retrieval_router.py +80 -31
- package/dist/templates/trellis/scripts/common/config.py +1 -1
- package/dist/templates/trellis/scripts/common/cursor_retrieval_env.py +92 -0
- package/dist/templates/trellis/scripts/common/parent_orchestration.py +27 -15
- package/dist/templates/trellis/scripts/common/retrieval_adapter_metadata.py +100 -9
- package/dist/templates/trellis/scripts/common/retrieval_agent_instructions.py +76 -31
- package/dist/templates/trellis/scripts/common/retrieval_tool_classification.py +18 -3
- package/dist/templates/trellis/scripts/common/semantic_plan_gate.py +19 -0
- package/dist/templates/trellis/scripts/common/smart_search_evidence.py +5 -2
- package/dist/templates/trellis/scripts/common/smart_search_resolve.py +28 -3
- package/dist/templates/trellis/scripts/common/task_gates.py +372 -73
- package/dist/templates/trellis/scripts/common/task_map.py +29 -1
- package/dist/templates/trellis/scripts/cursor_retrieval_probe.py +396 -0
- package/dist/templates/trellis/scripts/cursor_retrieval_probe_prompt.md +300 -0
- package/dist/templates/trellis/scripts/retrieval_probe_matrix_template.json +126 -0
- package/dist/templates/trellis/workflow.md +9 -53
- package/dist/utils/codebase-retrieval-router.d.ts +5 -0
- package/dist/utils/codebase-retrieval-router.d.ts.map +1 -1
- package/dist/utils/codebase-retrieval-router.js +48 -28
- package/dist/utils/codebase-retrieval-router.js.map +1 -1
- package/dist/utils/cursor-retrieval-env.d.ts +28 -0
- package/dist/utils/cursor-retrieval-env.d.ts.map +1 -0
- package/dist/utils/cursor-retrieval-env.js +89 -0
- package/dist/utils/cursor-retrieval-env.js.map +1 -0
- package/dist/utils/project-capabilities.d.ts.map +1 -1
- package/dist/utils/project-capabilities.js +22 -15
- package/dist/utils/project-capabilities.js.map +1 -1
- package/dist/utils/retrieval-agent-instructions.d.ts.map +1 -1
- package/dist/utils/retrieval-agent-instructions.js +37 -21
- package/dist/utils/retrieval-agent-instructions.js.map +1 -1
- package/dist/utils/retrieval-tool-classification.d.ts +2 -0
- package/dist/utils/retrieval-tool-classification.d.ts.map +1 -1
- package/dist/utils/retrieval-tool-classification.js +10 -2
- package/dist/utils/retrieval-tool-classification.js.map +1 -1
- package/dist/utils/semantic-plan-gate.d.ts +8 -0
- package/dist/utils/semantic-plan-gate.d.ts.map +1 -0
- package/dist/utils/semantic-plan-gate.js +42 -0
- package/dist/utils/semantic-plan-gate.js.map +1 -0
- package/package.json +5 -43
- package/bin/smart-search.js +0 -79
- package/dist/templates/trellis/scripts/aggregate_retrieval_telemetry.py +0 -342
- package/dist/templates/trellis/scripts/batch_plan_envelope.py +0 -105
- package/vendor/smart-search/LICENSE +0 -21
- package/vendor/smart-search/README.md +0 -412
- package/vendor/smart-search/README.zh-CN.md +0 -430
- package/vendor/smart-search/npm/bin/smart-search.js +0 -63
- package/vendor/smart-search/npm/scripts/postinstall.js +0 -87
- package/vendor/smart-search/npm/scripts/resolve-prerelease-version.js +0 -108
- package/vendor/smart-search/npm/scripts/set-package-version.js +0 -35
- package/vendor/smart-search/npm/scripts/sync-python-version.js +0 -22
- package/vendor/smart-search/npm/scripts/test-wrapper-repair.js +0 -99
- package/vendor/smart-search/npm/scripts/test.js +0 -85
- package/vendor/smart-search/package.json +0 -42
- package/vendor/smart-search/pyproject.toml +0 -37
- package/vendor/smart-search/skills/smart-search-cli/SKILL.md +0 -387
- package/vendor/smart-search/skills/smart-search-cli/agents/openai.yaml +0 -3
- package/vendor/smart-search/skills/smart-search-cli/examples/batch-search.md +0 -98
- package/vendor/smart-search/skills/smart-search-cli/examples/evidence-gathering.md +0 -89
- package/vendor/smart-search/skills/smart-search-cli/references/cli-contract.md +0 -337
- package/vendor/smart-search/src/smart_search/__init__.py +0 -1
- package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/SKILL.md +0 -387
- package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/agents/openai.yaml +0 -3
- package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/batch-search.md +0 -98
- package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/evidence-gathering.md +0 -89
- package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/references/cli-contract.md +0 -337
- package/vendor/smart-search/src/smart_search/cli.py +0 -2003
- package/vendor/smart-search/src/smart_search/config.py +0 -580
- package/vendor/smart-search/src/smart_search/logger.py +0 -43
- package/vendor/smart-search/src/smart_search/providers/__init__.py +0 -16
- package/vendor/smart-search/src/smart_search/providers/base.py +0 -41
- package/vendor/smart-search/src/smart_search/providers/context7.py +0 -141
- package/vendor/smart-search/src/smart_search/providers/exa.py +0 -206
- package/vendor/smart-search/src/smart_search/providers/jina.py +0 -136
- package/vendor/smart-search/src/smart_search/providers/openai_compatible.py +0 -370
- package/vendor/smart-search/src/smart_search/providers/zhipu.py +0 -143
- package/vendor/smart-search/src/smart_search/service.py +0 -2865
- package/vendor/smart-search/src/smart_search/sources.py +0 -429
- package/vendor/smart-search/src/smart_search/utils.py +0 -220
package/package.json
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@blxzer/cursor-trellis",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "AI capabilities grow like ivy — Trellis provides the structure to guide them along a disciplined path",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
7
7
|
"types": "./dist/index.d.ts",
|
|
8
8
|
"bin": {
|
|
9
9
|
"trellis": "./bin/trellis.js",
|
|
10
|
-
"tl": "./bin/trellis.js"
|
|
11
|
-
"smart-search": "./bin/smart-search.js"
|
|
10
|
+
"tl": "./bin/trellis.js"
|
|
12
11
|
},
|
|
13
12
|
"publishConfig": {
|
|
14
13
|
"access": "public"
|
|
@@ -28,6 +27,7 @@
|
|
|
28
27
|
"author": "Mindfold LLC",
|
|
29
28
|
"license": "AGPL-3.0-only",
|
|
30
29
|
"dependencies": {
|
|
30
|
+
"@blxzer/smart-search": "^0.1.0",
|
|
31
31
|
"chalk": "^5.3.0",
|
|
32
32
|
"commander": "^12.1.0",
|
|
33
33
|
"figlet": "^1.9.4",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"inquirer": "^9.3.7",
|
|
36
36
|
"undici": "^6.21.0",
|
|
37
37
|
"zod": "^4.4.2",
|
|
38
|
-
"@blxzer/cursor-trellis-core": "0.1
|
|
38
|
+
"@blxzer/cursor-trellis-core": "0.2.1"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@eslint/js": "^9.18.0",
|
|
@@ -59,43 +59,7 @@
|
|
|
59
59
|
"scripts/postinstall.js",
|
|
60
60
|
"README.md",
|
|
61
61
|
"CHANGELOG.md",
|
|
62
|
-
"LICENSE"
|
|
63
|
-
"vendor/smart-search/LICENSE",
|
|
64
|
-
"vendor/smart-search/README.md",
|
|
65
|
-
"vendor/smart-search/README.zh-CN.md",
|
|
66
|
-
"vendor/smart-search/npm/bin/smart-search.js",
|
|
67
|
-
"vendor/smart-search/npm/scripts/postinstall.js",
|
|
68
|
-
"vendor/smart-search/npm/scripts/resolve-prerelease-version.js",
|
|
69
|
-
"vendor/smart-search/npm/scripts/set-package-version.js",
|
|
70
|
-
"vendor/smart-search/npm/scripts/sync-python-version.js",
|
|
71
|
-
"vendor/smart-search/npm/scripts/test-wrapper-repair.js",
|
|
72
|
-
"vendor/smart-search/npm/scripts/test.js",
|
|
73
|
-
"vendor/smart-search/package.json",
|
|
74
|
-
"vendor/smart-search/pyproject.toml",
|
|
75
|
-
"vendor/smart-search/skills/smart-search-cli/SKILL.md",
|
|
76
|
-
"vendor/smart-search/skills/smart-search-cli/agents/openai.yaml",
|
|
77
|
-
"vendor/smart-search/skills/smart-search-cli/examples/batch-search.md",
|
|
78
|
-
"vendor/smart-search/skills/smart-search-cli/examples/evidence-gathering.md",
|
|
79
|
-
"vendor/smart-search/skills/smart-search-cli/references/cli-contract.md",
|
|
80
|
-
"vendor/smart-search/src/smart_search/__init__.py",
|
|
81
|
-
"vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/SKILL.md",
|
|
82
|
-
"vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/agents/openai.yaml",
|
|
83
|
-
"vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/batch-search.md",
|
|
84
|
-
"vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/evidence-gathering.md",
|
|
85
|
-
"vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/references/cli-contract.md",
|
|
86
|
-
"vendor/smart-search/src/smart_search/cli.py",
|
|
87
|
-
"vendor/smart-search/src/smart_search/config.py",
|
|
88
|
-
"vendor/smart-search/src/smart_search/logger.py",
|
|
89
|
-
"vendor/smart-search/src/smart_search/providers/__init__.py",
|
|
90
|
-
"vendor/smart-search/src/smart_search/providers/base.py",
|
|
91
|
-
"vendor/smart-search/src/smart_search/providers/context7.py",
|
|
92
|
-
"vendor/smart-search/src/smart_search/providers/exa.py",
|
|
93
|
-
"vendor/smart-search/src/smart_search/providers/jina.py",
|
|
94
|
-
"vendor/smart-search/src/smart_search/providers/openai_compatible.py",
|
|
95
|
-
"vendor/smart-search/src/smart_search/providers/zhipu.py",
|
|
96
|
-
"vendor/smart-search/src/smart_search/service.py",
|
|
97
|
-
"vendor/smart-search/src/smart_search/sources.py",
|
|
98
|
-
"vendor/smart-search/src/smart_search/utils.py"
|
|
62
|
+
"LICENSE"
|
|
99
63
|
],
|
|
100
64
|
"repository": {
|
|
101
65
|
"type": "git",
|
|
@@ -106,8 +70,6 @@
|
|
|
106
70
|
"clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"",
|
|
107
71
|
"copy-templates": "node scripts/copy-templates.js",
|
|
108
72
|
"postinstall": "node scripts/postinstall.js",
|
|
109
|
-
"sync:smart-search": "node scripts/sync-smart-search-vendor.js",
|
|
110
|
-
"check:smart-search": "node scripts/check-smart-search-vendor.js",
|
|
111
73
|
"check:pack-files": "node scripts/check-cli-pack-files.js",
|
|
112
74
|
"check:release-pack": "node scripts/check-release-pack-contents.js",
|
|
113
75
|
"check:router-copy-sync": "node scripts/check-router-copy-sync.js",
|
package/bin/smart-search.js
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
import { spawn, spawnSync } from "node:child_process";
|
|
4
|
-
import fs from "node:fs";
|
|
5
|
-
import path from "node:path";
|
|
6
|
-
import { fileURLToPath } from "node:url";
|
|
7
|
-
|
|
8
|
-
const packageRoot = path.resolve(
|
|
9
|
-
path.dirname(fileURLToPath(import.meta.url)),
|
|
10
|
-
"..",
|
|
11
|
-
);
|
|
12
|
-
const vendorRoot = path.join(packageRoot, "vendor", "smart-search");
|
|
13
|
-
const callerCwd = process.env.INIT_CWD || process.cwd();
|
|
14
|
-
const venvDir = path.join(vendorRoot, ".smart-search-python");
|
|
15
|
-
const pythonPath =
|
|
16
|
-
process.platform === "win32"
|
|
17
|
-
? path.join(venvDir, "Scripts", "python.exe")
|
|
18
|
-
: path.join(venvDir, "bin", "python");
|
|
19
|
-
const postinstall = path.join(vendorRoot, "npm", "scripts", "postinstall.js");
|
|
20
|
-
|
|
21
|
-
function fail(message, code = 5) {
|
|
22
|
-
console.error(message);
|
|
23
|
-
process.exit(code);
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
if (!fs.existsSync(path.join(vendorRoot, "pyproject.toml"))) {
|
|
27
|
-
fail(
|
|
28
|
-
"Trellis smart-search runtime is missing. Reinstall @blxzer/cursor-trellis or run the package build again.",
|
|
29
|
-
);
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
if (!fs.existsSync(pythonPath)) {
|
|
33
|
-
if (!fs.existsSync(postinstall)) {
|
|
34
|
-
fail(
|
|
35
|
-
`smart-search Python runtime is missing and repair script was not found: ${postinstall}`,
|
|
36
|
-
);
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
console.error("smart-search Python runtime is missing; attempting repair...");
|
|
40
|
-
const repaired = spawnSync(process.execPath, [postinstall], {
|
|
41
|
-
cwd: vendorRoot,
|
|
42
|
-
stdio: "inherit",
|
|
43
|
-
windowsHide: true,
|
|
44
|
-
});
|
|
45
|
-
if (repaired.error) {
|
|
46
|
-
fail(`smart-search runtime repair failed: ${repaired.error.message}`);
|
|
47
|
-
}
|
|
48
|
-
if (repaired.status !== 0 || !fs.existsSync(pythonPath)) {
|
|
49
|
-
console.error("Trellis could not find the smart-search Python runtime.");
|
|
50
|
-
console.error(`Expected: ${pythonPath}`);
|
|
51
|
-
console.error("Repair it by reinstalling Trellis or running:");
|
|
52
|
-
console.error(" node scripts/postinstall.js");
|
|
53
|
-
process.exit(repaired.status || 5);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
const child = spawn(pythonPath, ["-m", "smart_search.cli", ...process.argv.slice(2)], {
|
|
58
|
-
cwd: callerCwd,
|
|
59
|
-
stdio: "inherit",
|
|
60
|
-
env: {
|
|
61
|
-
...process.env,
|
|
62
|
-
SMART_SEARCH_PACKAGE_ROOT: vendorRoot,
|
|
63
|
-
PYTHONIOENCODING: process.env.PYTHONIOENCODING || "utf-8",
|
|
64
|
-
PYTHONUTF8: process.env.PYTHONUTF8 || "1",
|
|
65
|
-
},
|
|
66
|
-
windowsHide: true,
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
child.on("error", (error) => {
|
|
70
|
-
fail(`Failed to start smart-search: ${error.message}`);
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
child.on("close", (code, signal) => {
|
|
74
|
-
if (signal) {
|
|
75
|
-
process.kill(process.pid, signal);
|
|
76
|
-
return;
|
|
77
|
-
}
|
|
78
|
-
process.exit(code ?? 5);
|
|
79
|
-
});
|
|
@@ -1,342 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Aggregate per-query retrieval telemetry JSONL into summary metrics (schema v2)."""
|
|
3
|
-
|
|
4
|
-
from __future__ import annotations
|
|
5
|
-
|
|
6
|
-
import argparse
|
|
7
|
-
import json
|
|
8
|
-
import sys
|
|
9
|
-
from pathlib import Path
|
|
10
|
-
from typing import Any
|
|
11
|
-
|
|
12
|
-
_SCRIPT_DIR = Path(__file__).resolve().parent
|
|
13
|
-
if str(_SCRIPT_DIR) not in sys.path:
|
|
14
|
-
sys.path.insert(0, str(_SCRIPT_DIR))
|
|
15
|
-
|
|
16
|
-
from common.retrieval_tool_classification import ( # noqa: E402
|
|
17
|
-
classify_tool_calls,
|
|
18
|
-
semantic_routes_in_plan,
|
|
19
|
-
structural_routes_in_plan,
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
SCHEMA_VERSION = 2
|
|
23
|
-
|
|
24
|
-
SEMANTIC_OUTCOMES = (
|
|
25
|
-
"success",
|
|
26
|
-
"partial",
|
|
27
|
-
"resource_exhausted",
|
|
28
|
-
"timeout",
|
|
29
|
-
"not_configured",
|
|
30
|
-
"unavailable",
|
|
31
|
-
"not_run",
|
|
32
|
-
"unknown",
|
|
33
|
-
)
|
|
34
|
-
|
|
35
|
-
SEMANTIC_SKIP_REASONS = (
|
|
36
|
-
"rg_corrob_sufficient",
|
|
37
|
-
"rg_empty_semantic_required",
|
|
38
|
-
"trap_only_semantic_required",
|
|
39
|
-
"not_in_plan",
|
|
40
|
-
"not_applicable",
|
|
41
|
-
"adapter_unavailable",
|
|
42
|
-
"agent_stopped_early",
|
|
43
|
-
"unknown",
|
|
44
|
-
"manual_not_recorded",
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def _rate(count: int, total: int) -> float:
|
|
49
|
-
return 0.0 if total == 0 else count / total
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def _avg(values: list[float]) -> float:
|
|
53
|
-
return 0.0 if not values else sum(values) / len(values)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
def compute_compliance_score(record: dict[str, Any]) -> float:
|
|
57
|
-
earned = 0.0
|
|
58
|
-
possible = 0.0
|
|
59
|
-
|
|
60
|
-
if record.get("structural_in_plan"):
|
|
61
|
-
possible += 1.0
|
|
62
|
-
if record.get("codegraph_executed"):
|
|
63
|
-
earned += 1.0
|
|
64
|
-
|
|
65
|
-
if record.get("semantic_in_plan"):
|
|
66
|
-
possible += 1.0
|
|
67
|
-
if record.get("semantic_executed") or record.get("semantic_skip_reason") == "rg_corrob_sufficient":
|
|
68
|
-
earned += 1.0
|
|
69
|
-
|
|
70
|
-
possible += 1.0
|
|
71
|
-
if record.get("read_verification_done"):
|
|
72
|
-
earned += 1.0
|
|
73
|
-
|
|
74
|
-
if record.get("plan_block_in_prompt"):
|
|
75
|
-
possible += 0.25
|
|
76
|
-
if record.get("router_cli_invoked"):
|
|
77
|
-
earned += 0.25
|
|
78
|
-
|
|
79
|
-
return 1.0 if possible == 0 else earned / possible
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def migrate_record(raw: dict[str, Any]) -> dict[str, Any]:
|
|
83
|
-
routes = raw.get("routes") or raw.get("routes_in_plan") or []
|
|
84
|
-
if not isinstance(routes, list):
|
|
85
|
-
routes = []
|
|
86
|
-
tools = raw.get("tools_called") or []
|
|
87
|
-
if not isinstance(tools, list):
|
|
88
|
-
tools = []
|
|
89
|
-
platform = str(raw.get("platform", "cursor"))
|
|
90
|
-
classified = classify_tool_calls([str(t) for t in tools], platform=platform)
|
|
91
|
-
|
|
92
|
-
structural_in = raw.get("structural_in_plan")
|
|
93
|
-
if not isinstance(structural_in, bool):
|
|
94
|
-
structural_in = structural_routes_in_plan([str(r) for r in routes])
|
|
95
|
-
|
|
96
|
-
semantic_in = raw.get("semantic_in_plan")
|
|
97
|
-
if not isinstance(semantic_in, bool):
|
|
98
|
-
semantic_in = semantic_routes_in_plan([str(r) for r in routes])
|
|
99
|
-
|
|
100
|
-
record: dict[str, Any] = {
|
|
101
|
-
"schema_version": SCHEMA_VERSION,
|
|
102
|
-
"query_id": str(raw.get("query_id", "")),
|
|
103
|
-
"dataset": str(raw.get("dataset", "unknown")),
|
|
104
|
-
"query_text": str(raw.get("query_text", "")),
|
|
105
|
-
"run_id": str(raw.get("run_id", "")),
|
|
106
|
-
"platform": platform,
|
|
107
|
-
"semantic_in_plan": bool(semantic_in),
|
|
108
|
-
"semantic_order": raw.get("semantic_order")
|
|
109
|
-
if isinstance(raw.get("semantic_order"), int)
|
|
110
|
-
else None,
|
|
111
|
-
"structural_in_plan": bool(structural_in),
|
|
112
|
-
"codegraph_in_plan": bool(raw.get("codegraph_in_plan", structural_in)),
|
|
113
|
-
"fallback_hint_present": bool(raw.get("fallback_hint_present")),
|
|
114
|
-
"intents": list(raw.get("intents") or []),
|
|
115
|
-
"routes": [str(r) for r in routes],
|
|
116
|
-
"routes_in_plan": [str(r) for r in routes],
|
|
117
|
-
"project_file_count": raw.get("project_file_count")
|
|
118
|
-
if isinstance(raw.get("project_file_count"), int)
|
|
119
|
-
else None,
|
|
120
|
-
"tools_called": classified.tools_called,
|
|
121
|
-
"grep_count": raw.get("grep_count")
|
|
122
|
-
if isinstance(raw.get("grep_count"), int)
|
|
123
|
-
else classified.grep_count,
|
|
124
|
-
"read_count": raw.get("read_count")
|
|
125
|
-
if isinstance(raw.get("read_count"), int)
|
|
126
|
-
else classified.read_count,
|
|
127
|
-
"codegraph_attempted": bool(
|
|
128
|
-
raw.get("codegraph_attempted", classified.codegraph_attempted)
|
|
129
|
-
),
|
|
130
|
-
"codegraph_executed": bool(
|
|
131
|
-
raw.get("codegraph_executed", classified.codegraph_executed)
|
|
132
|
-
),
|
|
133
|
-
"router_cli_invoked": bool(
|
|
134
|
-
raw.get("router_cli_invoked", classified.router_cli_invoked)
|
|
135
|
-
),
|
|
136
|
-
"plan_block_in_prompt": bool(raw.get("plan_block_in_prompt")),
|
|
137
|
-
"read_verification_done": bool(
|
|
138
|
-
raw.get("read_verification_done", classified.read_count > 0)
|
|
139
|
-
),
|
|
140
|
-
"semantic_attempted": bool(
|
|
141
|
-
raw.get("semantic_attempted", classified.semantic_attempted)
|
|
142
|
-
),
|
|
143
|
-
"semantic_executed": bool(
|
|
144
|
-
raw.get("semantic_executed", classified.semantic_executed)
|
|
145
|
-
),
|
|
146
|
-
"semantic_outcome": str(raw.get("semantic_outcome", "unknown")),
|
|
147
|
-
"semantic_success": bool(raw.get("semantic_success")),
|
|
148
|
-
"semantic_skip_reason": raw.get("semantic_skip_reason"),
|
|
149
|
-
"rg_candidate_count": raw.get("rg_candidate_count")
|
|
150
|
-
if isinstance(raw.get("rg_candidate_count"), int)
|
|
151
|
-
else None,
|
|
152
|
-
"rg_corrob_status": str(raw.get("rg_corrob_status", "unknown")),
|
|
153
|
-
"trap_only": bool(raw.get("trap_only")),
|
|
154
|
-
"corroborated_files": list(raw.get("corroborated_files") or []),
|
|
155
|
-
"adapter_errors": list(raw.get("adapter_errors") or []),
|
|
156
|
-
"candidate_pool_recall": raw.get("candidate_pool_recall")
|
|
157
|
-
if isinstance(raw.get("candidate_pool_recall"), (int, float))
|
|
158
|
-
else None,
|
|
159
|
-
"final_top_k_recall": raw.get("final_top_k_recall")
|
|
160
|
-
if isinstance(raw.get("final_top_k_recall"), (int, float))
|
|
161
|
-
else None,
|
|
162
|
-
"answer_score": raw.get("answer_score")
|
|
163
|
-
if isinstance(raw.get("answer_score"), (int, float))
|
|
164
|
-
else None,
|
|
165
|
-
"compliance_score": raw.get("compliance_score")
|
|
166
|
-
if isinstance(raw.get("compliance_score"), (int, float))
|
|
167
|
-
else None,
|
|
168
|
-
"platform_semantic_executed": bool(
|
|
169
|
-
raw.get("platform_semantic_executed", classified.platform_semantic_executed)
|
|
170
|
-
),
|
|
171
|
-
"fast_context_count": int(raw.get("fast_context_count", classified.fast_context_count)),
|
|
172
|
-
"cursor_fast_context_misuse": bool(
|
|
173
|
-
raw.get("cursor_fast_context_misuse", classified.cursor_fast_context_misuse)
|
|
174
|
-
),
|
|
175
|
-
}
|
|
176
|
-
if record["compliance_score"] is None:
|
|
177
|
-
record["compliance_score"] = compute_compliance_score(record)
|
|
178
|
-
return record
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
def derive_metrics(records: list[dict[str, Any]]) -> dict[str, Any]:
|
|
182
|
-
total = len(records)
|
|
183
|
-
outcome_counts = {k: 0 for k in SEMANTIC_OUTCOMES}
|
|
184
|
-
skip_counts = {k: 0 for k in SEMANTIC_SKIP_REASONS}
|
|
185
|
-
|
|
186
|
-
semantic_plan = semantic_exec = semantic_attempt = semantic_success = 0
|
|
187
|
-
platform_sem_exec = fast_context_total = fast_context_misuse = 0
|
|
188
|
-
codegraph_plan = codegraph_exec = router_cli = plan_block = read_verify = 0
|
|
189
|
-
compliance_scores: list[float] = []
|
|
190
|
-
answer_scores: list[float] = []
|
|
191
|
-
pool_recalls: list[float] = []
|
|
192
|
-
topk_recalls: list[float] = []
|
|
193
|
-
|
|
194
|
-
for rec in records:
|
|
195
|
-
if rec.get("semantic_in_plan"):
|
|
196
|
-
semantic_plan += 1
|
|
197
|
-
if rec.get("semantic_executed"):
|
|
198
|
-
semantic_exec += 1
|
|
199
|
-
if rec.get("platform_semantic_executed"):
|
|
200
|
-
platform_sem_exec += 1
|
|
201
|
-
fc = rec.get("fast_context_count")
|
|
202
|
-
if isinstance(fc, int) and fc > 0:
|
|
203
|
-
fast_context_total += fc
|
|
204
|
-
if rec.get("cursor_fast_context_misuse"):
|
|
205
|
-
fast_context_misuse += 1
|
|
206
|
-
if rec.get("semantic_attempted"):
|
|
207
|
-
semantic_attempt += 1
|
|
208
|
-
if rec.get("semantic_success") and rec.get("semantic_outcome") == "success":
|
|
209
|
-
semantic_success += 1
|
|
210
|
-
if rec.get("codegraph_in_plan"):
|
|
211
|
-
codegraph_plan += 1
|
|
212
|
-
if rec.get("codegraph_executed"):
|
|
213
|
-
codegraph_exec += 1
|
|
214
|
-
if rec.get("router_cli_invoked"):
|
|
215
|
-
router_cli += 1
|
|
216
|
-
if rec.get("plan_block_in_prompt"):
|
|
217
|
-
plan_block += 1
|
|
218
|
-
if rec.get("read_verification_done"):
|
|
219
|
-
read_verify += 1
|
|
220
|
-
|
|
221
|
-
outcome = str(rec.get("semantic_outcome", "unknown"))
|
|
222
|
-
if outcome in outcome_counts:
|
|
223
|
-
outcome_counts[outcome] += 1
|
|
224
|
-
skip = rec.get("semantic_skip_reason")
|
|
225
|
-
if skip and skip in skip_counts:
|
|
226
|
-
skip_counts[skip] += 1
|
|
227
|
-
|
|
228
|
-
cs = rec.get("compliance_score")
|
|
229
|
-
if isinstance(cs, (int, float)):
|
|
230
|
-
compliance_scores.append(float(cs))
|
|
231
|
-
ans = rec.get("answer_score")
|
|
232
|
-
if isinstance(ans, (int, float)):
|
|
233
|
-
answer_scores.append(float(ans))
|
|
234
|
-
cpr = rec.get("candidate_pool_recall")
|
|
235
|
-
if isinstance(cpr, (int, float)):
|
|
236
|
-
pool_recalls.append(float(cpr))
|
|
237
|
-
ftr = rec.get("final_top_k_recall")
|
|
238
|
-
if isinstance(ftr, (int, float)):
|
|
239
|
-
topk_recalls.append(float(ftr))
|
|
240
|
-
|
|
241
|
-
avg_pool = _avg(pool_recalls)
|
|
242
|
-
avg_topk = _avg(topk_recalls)
|
|
243
|
-
recall_drop = 0.0 if avg_pool <= 0 else 1.0 - avg_topk / avg_pool
|
|
244
|
-
|
|
245
|
-
return {
|
|
246
|
-
"schema_version": SCHEMA_VERSION,
|
|
247
|
-
"total_queries": total,
|
|
248
|
-
"semantic_plan_count": semantic_plan,
|
|
249
|
-
"semantic_exec_count": semantic_exec,
|
|
250
|
-
"semantic_attempt_count": semantic_attempt,
|
|
251
|
-
"semantic_exec_success_count": semantic_success,
|
|
252
|
-
"semantic_plan_rate": _rate(semantic_plan, total),
|
|
253
|
-
"semantic_exec_rate": _rate(semantic_exec, total),
|
|
254
|
-
"platform_semantic_exec_count": platform_sem_exec,
|
|
255
|
-
"platform_semantic_exec_rate": _rate(platform_sem_exec, total),
|
|
256
|
-
"fast_context_invocation_total": fast_context_total,
|
|
257
|
-
"cursor_fast_context_misuse_count": fast_context_misuse,
|
|
258
|
-
"cursor_fast_context_misuse_rate": _rate(fast_context_misuse, total),
|
|
259
|
-
"semantic_attempt_rate": _rate(semantic_attempt, total),
|
|
260
|
-
"semantic_exec_success_rate": _rate(semantic_success, total),
|
|
261
|
-
"codegraph_plan_count": codegraph_plan,
|
|
262
|
-
"codegraph_exec_count": codegraph_exec,
|
|
263
|
-
"codegraph_plan_rate": _rate(codegraph_plan, total),
|
|
264
|
-
"codegraph_exec_rate": _rate(codegraph_exec, total),
|
|
265
|
-
"router_cli_count": router_cli,
|
|
266
|
-
"router_cli_rate": _rate(router_cli, total),
|
|
267
|
-
"plan_block_count": plan_block,
|
|
268
|
-
"plan_block_rate": _rate(plan_block, total),
|
|
269
|
-
"read_verification_count": read_verify,
|
|
270
|
-
"read_verification_rate": _rate(read_verify, total),
|
|
271
|
-
"avg_compliance_score": _avg(compliance_scores),
|
|
272
|
-
"avg_answer_score": _avg(answer_scores),
|
|
273
|
-
"semantic_outcome_counts": outcome_counts,
|
|
274
|
-
"semantic_skip_reason_counts": skip_counts,
|
|
275
|
-
"avg_candidate_pool_recall": avg_pool,
|
|
276
|
-
"avg_final_top_k_recall": avg_topk,
|
|
277
|
-
"recall_drop_rate": recall_drop,
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
def load_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
282
|
-
records: list[dict[str, Any]] = []
|
|
283
|
-
for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
|
284
|
-
stripped = line.strip()
|
|
285
|
-
if not stripped or stripped.startswith("#"):
|
|
286
|
-
continue
|
|
287
|
-
try:
|
|
288
|
-
raw = json.loads(stripped)
|
|
289
|
-
except json.JSONDecodeError as exc:
|
|
290
|
-
raise ValueError(f"{path}:{line_no}: invalid JSON: {exc}") from exc
|
|
291
|
-
if not isinstance(raw, dict):
|
|
292
|
-
raise ValueError(f"{path}:{line_no}: expected JSON object per line")
|
|
293
|
-
records.append(migrate_record(raw))
|
|
294
|
-
return records
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
def main() -> int:
|
|
298
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
299
|
-
parser.add_argument(
|
|
300
|
-
"jsonl",
|
|
301
|
-
type=Path,
|
|
302
|
-
help="Per-query telemetry JSONL (one object per line)",
|
|
303
|
-
)
|
|
304
|
-
parser.add_argument(
|
|
305
|
-
"--markdown",
|
|
306
|
-
action="store_true",
|
|
307
|
-
help="Print a short markdown summary for eval reports",
|
|
308
|
-
)
|
|
309
|
-
args = parser.parse_args()
|
|
310
|
-
|
|
311
|
-
if not args.jsonl.is_file():
|
|
312
|
-
print(f"error: file not found: {args.jsonl}", file=sys.stderr)
|
|
313
|
-
return 1
|
|
314
|
-
|
|
315
|
-
records = load_jsonl(args.jsonl)
|
|
316
|
-
metrics = derive_metrics(records)
|
|
317
|
-
payload = {"metrics": metrics, "record_count": len(records)}
|
|
318
|
-
|
|
319
|
-
if args.markdown:
|
|
320
|
-
m = metrics
|
|
321
|
-
print("## Retrieval telemetry (derived from JSONL)\n")
|
|
322
|
-
print("| Metric | Value |")
|
|
323
|
-
print("| --- | ---: |")
|
|
324
|
-
print(f"| total_queries | {m['total_queries']} |")
|
|
325
|
-
print(f"| semantic_plan_rate | {m['semantic_plan_rate']:.1%} |")
|
|
326
|
-
print(f"| semantic_exec_rate | {m['semantic_exec_rate']:.1%} |")
|
|
327
|
-
print(f"| platform_semantic_exec_rate | {m.get('platform_semantic_exec_rate', 0):.1%} |")
|
|
328
|
-
print(f"| cursor_fast_context_misuse_rate | {m.get('cursor_fast_context_misuse_rate', 0):.1%} |")
|
|
329
|
-
print(f"| codegraph_plan_rate | {m['codegraph_plan_rate']:.1%} |")
|
|
330
|
-
print(f"| codegraph_exec_rate | {m['codegraph_exec_rate']:.1%} |")
|
|
331
|
-
print(f"| router_cli_rate | {m['router_cli_rate']:.1%} |")
|
|
332
|
-
print(f"| avg_compliance_score | {m['avg_compliance_score']:.3f} |")
|
|
333
|
-
print(f"| avg_answer_score | {m['avg_answer_score']:.3f} |")
|
|
334
|
-
print("\nDo not hand-copy plan/exec rates; regenerate from JSONL with this script.")
|
|
335
|
-
else:
|
|
336
|
-
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
337
|
-
|
|
338
|
-
return 0
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
if __name__ == "__main__":
|
|
342
|
-
raise SystemExit(main())
|
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Batch route_codebase_retrieval for plan metrics (REC-06: platform-semantic on Cursor)."""
|
|
3
|
-
|
|
4
|
-
from __future__ import annotations
|
|
5
|
-
|
|
6
|
-
import argparse
|
|
7
|
-
import json
|
|
8
|
-
import sys
|
|
9
|
-
from pathlib import Path
|
|
10
|
-
|
|
11
|
-
_SCRIPT_DIR = Path(__file__).resolve().parent
|
|
12
|
-
if str(_SCRIPT_DIR) not in sys.path:
|
|
13
|
-
sys.path.insert(0, str(_SCRIPT_DIR))
|
|
14
|
-
|
|
15
|
-
from common.codebase_retrieval_router import route_codebase_retrieval # noqa: E402
|
|
16
|
-
from common.retrieval_tool_classification import ( # noqa: E402
|
|
17
|
-
platform_semantic_route_order,
|
|
18
|
-
semantic_routes_in_plan,
|
|
19
|
-
)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def load_queries(path: Path) -> list[dict]:
|
|
23
|
-
text = path.read_text(encoding="utf-8-sig")
|
|
24
|
-
rows: list[dict] = []
|
|
25
|
-
for line in text.splitlines():
|
|
26
|
-
stripped = line.strip()
|
|
27
|
-
if not stripped:
|
|
28
|
-
continue
|
|
29
|
-
rows.append(json.loads(stripped))
|
|
30
|
-
return rows
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def main(argv: list[str] | None = None) -> int:
|
|
34
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
35
|
-
parser.add_argument(
|
|
36
|
-
"--queries",
|
|
37
|
-
type=Path,
|
|
38
|
-
default=None,
|
|
39
|
-
help="JSONL query file (default: ./queries.jsonl under cwd)",
|
|
40
|
-
)
|
|
41
|
-
parser.add_argument("--pretty", action="store_true")
|
|
42
|
-
args = parser.parse_args(argv)
|
|
43
|
-
|
|
44
|
-
root = Path.cwd()
|
|
45
|
-
queries_path = args.queries or (root / "queries.jsonl")
|
|
46
|
-
if not queries_path.is_file():
|
|
47
|
-
print(f"error: queries file not found: {queries_path}", file=sys.stderr)
|
|
48
|
-
return 1
|
|
49
|
-
|
|
50
|
-
rows = load_queries(queries_path)
|
|
51
|
-
out: list[dict] = []
|
|
52
|
-
plan_sem = 0
|
|
53
|
-
platform_sem = 0
|
|
54
|
-
fast_context_sem = 0
|
|
55
|
-
|
|
56
|
-
for row in rows:
|
|
57
|
-
query = str(row.get("query", ""))
|
|
58
|
-
plan = route_codebase_retrieval(query)
|
|
59
|
-
routes = plan.get("routes") or []
|
|
60
|
-
route_ids = [str(r.get("id", "")) for r in routes if isinstance(r, dict)]
|
|
61
|
-
sem_in_plan = semantic_routes_in_plan(route_ids)
|
|
62
|
-
if sem_in_plan:
|
|
63
|
-
plan_sem += 1
|
|
64
|
-
if "platform-semantic" in route_ids:
|
|
65
|
-
platform_sem += 1
|
|
66
|
-
if "semantic-fast-context" in route_ids:
|
|
67
|
-
fast_context_sem += 1
|
|
68
|
-
|
|
69
|
-
out.append(
|
|
70
|
-
{
|
|
71
|
-
"id": row.get("id"),
|
|
72
|
-
"intents": [i["id"] for i in plan.get("intents", []) if isinstance(i, dict)],
|
|
73
|
-
"route_ids": route_ids,
|
|
74
|
-
"semantic_in_plan": sem_in_plan,
|
|
75
|
-
"platform_semantic_order": platform_semantic_route_order(
|
|
76
|
-
[r for r in routes if isinstance(r, dict)]
|
|
77
|
-
),
|
|
78
|
-
"fallback_rg_empty": any(
|
|
79
|
-
"corroborated" in str(f.get("when", ""))
|
|
80
|
-
for f in (plan.get("fallback") or [])
|
|
81
|
-
if isinstance(f, dict)
|
|
82
|
-
),
|
|
83
|
-
}
|
|
84
|
-
)
|
|
85
|
-
|
|
86
|
-
n = len(rows)
|
|
87
|
-
summary = {
|
|
88
|
-
"n": n,
|
|
89
|
-
"semantic_plan_rate": plan_sem / n if n else 0.0,
|
|
90
|
-
"platform_semantic_plan_rate": platform_sem / n if n else 0.0,
|
|
91
|
-
"semantic_fast_context_plan_rate": fast_context_sem / n if n else 0.0,
|
|
92
|
-
"fallback_hint_rate": sum(1 for x in out if x["fallback_rg_empty"]) / n if n else 0.0,
|
|
93
|
-
"note": (
|
|
94
|
-
"Plan rates only. semantic_exec_rate must come from session tool logs "
|
|
95
|
-
"and classify_tool_calls(platform=cursor)."
|
|
96
|
-
),
|
|
97
|
-
}
|
|
98
|
-
payload = {"summary": summary, "queries": out}
|
|
99
|
-
indent = 2 if args.pretty else None
|
|
100
|
-
print(json.dumps(payload, ensure_ascii=False, indent=indent))
|
|
101
|
-
return 0
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if __name__ == "__main__":
|
|
105
|
-
raise SystemExit(main())
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2025 GuDaStudio
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|