@blxzer/cursor-trellis 0.1.2 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/CHANGELOG.md +58 -0
  2. package/README.md +45 -21
  3. package/dist/configurators/workflow.d.ts +1 -1
  4. package/dist/configurators/workflow.d.ts.map +1 -1
  5. package/dist/configurators/workflow.js +20 -9
  6. package/dist/configurators/workflow.js.map +1 -1
  7. package/dist/migrations/manifests/0.1.3.json +9 -0
  8. package/dist/migrations/manifests/0.1.4.json +9 -0
  9. package/dist/templates/cursor/rules/retrieval-routing.mdc +9 -7
  10. package/dist/templates/markdown/agents.md +1 -1
  11. package/dist/templates/markdown/index.d.ts +1 -0
  12. package/dist/templates/markdown/index.d.ts.map +1 -1
  13. package/dist/templates/markdown/index.js +1 -0
  14. package/dist/templates/markdown/index.js.map +1 -1
  15. package/dist/templates/markdown/prompts/run-semantic-slice-12.md.txt +13 -29
  16. package/dist/templates/markdown/spec/guides/code-reuse-thinking-guide.md.txt +0 -49
  17. package/dist/templates/markdown/spec/guides/cross-layer-thinking-guide.md.txt +0 -85
  18. package/dist/templates/markdown/spec/guides/cursor-semantic-compliance.md.txt +7 -21
  19. package/dist/templates/markdown/spec/guides/cursor-subagent-policy.md.txt +6 -10
  20. package/dist/templates/markdown/spec/guides/retrieval-daily-guide.md.txt +17 -18
  21. package/dist/templates/shared-hooks/inject-retrieval-plan.py +42 -23
  22. package/dist/templates/shared-hooks/inject-subagent-context.py +6 -2
  23. package/dist/templates/shared-hooks/session-start.py +17 -9
  24. package/dist/templates/trellis/index.d.ts +8 -4
  25. package/dist/templates/trellis/index.d.ts.map +1 -1
  26. package/dist/templates/trellis/index.js +13 -7
  27. package/dist/templates/trellis/index.js.map +1 -1
  28. package/dist/templates/trellis/scripts/common/codebase_retrieval_router.py +80 -31
  29. package/dist/templates/trellis/scripts/common/config.py +1 -1
  30. package/dist/templates/trellis/scripts/common/cursor_retrieval_env.py +92 -0
  31. package/dist/templates/trellis/scripts/common/parent_orchestration.py +27 -15
  32. package/dist/templates/trellis/scripts/common/retrieval_adapter_metadata.py +100 -9
  33. package/dist/templates/trellis/scripts/common/retrieval_agent_instructions.py +76 -31
  34. package/dist/templates/trellis/scripts/common/retrieval_tool_classification.py +18 -3
  35. package/dist/templates/trellis/scripts/common/semantic_plan_gate.py +19 -0
  36. package/dist/templates/trellis/scripts/common/smart_search_evidence.py +5 -2
  37. package/dist/templates/trellis/scripts/common/smart_search_resolve.py +28 -3
  38. package/dist/templates/trellis/scripts/common/task_gates.py +372 -73
  39. package/dist/templates/trellis/scripts/common/task_map.py +29 -1
  40. package/dist/templates/trellis/scripts/cursor_retrieval_probe.py +396 -0
  41. package/dist/templates/trellis/scripts/cursor_retrieval_probe_prompt.md +300 -0
  42. package/dist/templates/trellis/scripts/retrieval_probe_matrix_template.json +126 -0
  43. package/dist/templates/trellis/workflow.md +9 -53
  44. package/dist/utils/codebase-retrieval-router.d.ts +5 -0
  45. package/dist/utils/codebase-retrieval-router.d.ts.map +1 -1
  46. package/dist/utils/codebase-retrieval-router.js +48 -28
  47. package/dist/utils/codebase-retrieval-router.js.map +1 -1
  48. package/dist/utils/cursor-retrieval-env.d.ts +28 -0
  49. package/dist/utils/cursor-retrieval-env.d.ts.map +1 -0
  50. package/dist/utils/cursor-retrieval-env.js +89 -0
  51. package/dist/utils/cursor-retrieval-env.js.map +1 -0
  52. package/dist/utils/project-capabilities.d.ts.map +1 -1
  53. package/dist/utils/project-capabilities.js +22 -15
  54. package/dist/utils/project-capabilities.js.map +1 -1
  55. package/dist/utils/retrieval-agent-instructions.d.ts.map +1 -1
  56. package/dist/utils/retrieval-agent-instructions.js +37 -21
  57. package/dist/utils/retrieval-agent-instructions.js.map +1 -1
  58. package/dist/utils/retrieval-tool-classification.d.ts +2 -0
  59. package/dist/utils/retrieval-tool-classification.d.ts.map +1 -1
  60. package/dist/utils/retrieval-tool-classification.js +10 -2
  61. package/dist/utils/retrieval-tool-classification.js.map +1 -1
  62. package/dist/utils/semantic-plan-gate.d.ts +8 -0
  63. package/dist/utils/semantic-plan-gate.d.ts.map +1 -0
  64. package/dist/utils/semantic-plan-gate.js +42 -0
  65. package/dist/utils/semantic-plan-gate.js.map +1 -0
  66. package/package.json +5 -43
  67. package/bin/smart-search.js +0 -79
  68. package/dist/templates/trellis/scripts/aggregate_retrieval_telemetry.py +0 -342
  69. package/dist/templates/trellis/scripts/batch_plan_envelope.py +0 -105
  70. package/vendor/smart-search/LICENSE +0 -21
  71. package/vendor/smart-search/README.md +0 -412
  72. package/vendor/smart-search/README.zh-CN.md +0 -430
  73. package/vendor/smart-search/npm/bin/smart-search.js +0 -63
  74. package/vendor/smart-search/npm/scripts/postinstall.js +0 -87
  75. package/vendor/smart-search/npm/scripts/resolve-prerelease-version.js +0 -108
  76. package/vendor/smart-search/npm/scripts/set-package-version.js +0 -35
  77. package/vendor/smart-search/npm/scripts/sync-python-version.js +0 -22
  78. package/vendor/smart-search/npm/scripts/test-wrapper-repair.js +0 -99
  79. package/vendor/smart-search/npm/scripts/test.js +0 -85
  80. package/vendor/smart-search/package.json +0 -42
  81. package/vendor/smart-search/pyproject.toml +0 -37
  82. package/vendor/smart-search/skills/smart-search-cli/SKILL.md +0 -387
  83. package/vendor/smart-search/skills/smart-search-cli/agents/openai.yaml +0 -3
  84. package/vendor/smart-search/skills/smart-search-cli/examples/batch-search.md +0 -98
  85. package/vendor/smart-search/skills/smart-search-cli/examples/evidence-gathering.md +0 -89
  86. package/vendor/smart-search/skills/smart-search-cli/references/cli-contract.md +0 -337
  87. package/vendor/smart-search/src/smart_search/__init__.py +0 -1
  88. package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/SKILL.md +0 -387
  89. package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/agents/openai.yaml +0 -3
  90. package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/batch-search.md +0 -98
  91. package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/evidence-gathering.md +0 -89
  92. package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/references/cli-contract.md +0 -337
  93. package/vendor/smart-search/src/smart_search/cli.py +0 -2003
  94. package/vendor/smart-search/src/smart_search/config.py +0 -580
  95. package/vendor/smart-search/src/smart_search/logger.py +0 -43
  96. package/vendor/smart-search/src/smart_search/providers/__init__.py +0 -16
  97. package/vendor/smart-search/src/smart_search/providers/base.py +0 -41
  98. package/vendor/smart-search/src/smart_search/providers/context7.py +0 -141
  99. package/vendor/smart-search/src/smart_search/providers/exa.py +0 -206
  100. package/vendor/smart-search/src/smart_search/providers/jina.py +0 -136
  101. package/vendor/smart-search/src/smart_search/providers/openai_compatible.py +0 -370
  102. package/vendor/smart-search/src/smart_search/providers/zhipu.py +0 -143
  103. package/vendor/smart-search/src/smart_search/service.py +0 -2865
  104. package/vendor/smart-search/src/smart_search/sources.py +0 -429
  105. package/vendor/smart-search/src/smart_search/utils.py +0 -220
package/package.json CHANGED
@@ -1,14 +1,13 @@
1
1
  {
2
2
  "name": "@blxzer/cursor-trellis",
3
- "version": "0.1.2",
3
+ "version": "0.2.1",
4
4
  "description": "AI capabilities grow like ivy — Trellis provides the structure to guide them along a disciplined path",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
7
7
  "types": "./dist/index.d.ts",
8
8
  "bin": {
9
9
  "trellis": "./bin/trellis.js",
10
- "tl": "./bin/trellis.js",
11
- "smart-search": "./bin/smart-search.js"
10
+ "tl": "./bin/trellis.js"
12
11
  },
13
12
  "publishConfig": {
14
13
  "access": "public"
@@ -28,6 +27,7 @@
28
27
  "author": "Mindfold LLC",
29
28
  "license": "AGPL-3.0-only",
30
29
  "dependencies": {
30
+ "@blxzer/smart-search": "^0.1.0",
31
31
  "chalk": "^5.3.0",
32
32
  "commander": "^12.1.0",
33
33
  "figlet": "^1.9.4",
@@ -35,7 +35,7 @@
35
35
  "inquirer": "^9.3.7",
36
36
  "undici": "^6.21.0",
37
37
  "zod": "^4.4.2",
38
- "@blxzer/cursor-trellis-core": "0.1.2"
38
+ "@blxzer/cursor-trellis-core": "0.2.1"
39
39
  },
40
40
  "devDependencies": {
41
41
  "@eslint/js": "^9.18.0",
@@ -59,43 +59,7 @@
59
59
  "scripts/postinstall.js",
60
60
  "README.md",
61
61
  "CHANGELOG.md",
62
- "LICENSE",
63
- "vendor/smart-search/LICENSE",
64
- "vendor/smart-search/README.md",
65
- "vendor/smart-search/README.zh-CN.md",
66
- "vendor/smart-search/npm/bin/smart-search.js",
67
- "vendor/smart-search/npm/scripts/postinstall.js",
68
- "vendor/smart-search/npm/scripts/resolve-prerelease-version.js",
69
- "vendor/smart-search/npm/scripts/set-package-version.js",
70
- "vendor/smart-search/npm/scripts/sync-python-version.js",
71
- "vendor/smart-search/npm/scripts/test-wrapper-repair.js",
72
- "vendor/smart-search/npm/scripts/test.js",
73
- "vendor/smart-search/package.json",
74
- "vendor/smart-search/pyproject.toml",
75
- "vendor/smart-search/skills/smart-search-cli/SKILL.md",
76
- "vendor/smart-search/skills/smart-search-cli/agents/openai.yaml",
77
- "vendor/smart-search/skills/smart-search-cli/examples/batch-search.md",
78
- "vendor/smart-search/skills/smart-search-cli/examples/evidence-gathering.md",
79
- "vendor/smart-search/skills/smart-search-cli/references/cli-contract.md",
80
- "vendor/smart-search/src/smart_search/__init__.py",
81
- "vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/SKILL.md",
82
- "vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/agents/openai.yaml",
83
- "vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/batch-search.md",
84
- "vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/evidence-gathering.md",
85
- "vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/references/cli-contract.md",
86
- "vendor/smart-search/src/smart_search/cli.py",
87
- "vendor/smart-search/src/smart_search/config.py",
88
- "vendor/smart-search/src/smart_search/logger.py",
89
- "vendor/smart-search/src/smart_search/providers/__init__.py",
90
- "vendor/smart-search/src/smart_search/providers/base.py",
91
- "vendor/smart-search/src/smart_search/providers/context7.py",
92
- "vendor/smart-search/src/smart_search/providers/exa.py",
93
- "vendor/smart-search/src/smart_search/providers/jina.py",
94
- "vendor/smart-search/src/smart_search/providers/openai_compatible.py",
95
- "vendor/smart-search/src/smart_search/providers/zhipu.py",
96
- "vendor/smart-search/src/smart_search/service.py",
97
- "vendor/smart-search/src/smart_search/sources.py",
98
- "vendor/smart-search/src/smart_search/utils.py"
62
+ "LICENSE"
99
63
  ],
100
64
  "repository": {
101
65
  "type": "git",
@@ -106,8 +70,6 @@
106
70
  "clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"",
107
71
  "copy-templates": "node scripts/copy-templates.js",
108
72
  "postinstall": "node scripts/postinstall.js",
109
- "sync:smart-search": "node scripts/sync-smart-search-vendor.js",
110
- "check:smart-search": "node scripts/check-smart-search-vendor.js",
111
73
  "check:pack-files": "node scripts/check-cli-pack-files.js",
112
74
  "check:release-pack": "node scripts/check-release-pack-contents.js",
113
75
  "check:router-copy-sync": "node scripts/check-router-copy-sync.js",
@@ -1,79 +0,0 @@
1
- #!/usr/bin/env node
2
-
3
- import { spawn, spawnSync } from "node:child_process";
4
- import fs from "node:fs";
5
- import path from "node:path";
6
- import { fileURLToPath } from "node:url";
7
-
8
- const packageRoot = path.resolve(
9
- path.dirname(fileURLToPath(import.meta.url)),
10
- "..",
11
- );
12
- const vendorRoot = path.join(packageRoot, "vendor", "smart-search");
13
- const callerCwd = process.env.INIT_CWD || process.cwd();
14
- const venvDir = path.join(vendorRoot, ".smart-search-python");
15
- const pythonPath =
16
- process.platform === "win32"
17
- ? path.join(venvDir, "Scripts", "python.exe")
18
- : path.join(venvDir, "bin", "python");
19
- const postinstall = path.join(vendorRoot, "npm", "scripts", "postinstall.js");
20
-
21
- function fail(message, code = 5) {
22
- console.error(message);
23
- process.exit(code);
24
- }
25
-
26
- if (!fs.existsSync(path.join(vendorRoot, "pyproject.toml"))) {
27
- fail(
28
- "Trellis smart-search runtime is missing. Reinstall @blxzer/cursor-trellis or run the package build again.",
29
- );
30
- }
31
-
32
- if (!fs.existsSync(pythonPath)) {
33
- if (!fs.existsSync(postinstall)) {
34
- fail(
35
- `smart-search Python runtime is missing and repair script was not found: ${postinstall}`,
36
- );
37
- }
38
-
39
- console.error("smart-search Python runtime is missing; attempting repair...");
40
- const repaired = spawnSync(process.execPath, [postinstall], {
41
- cwd: vendorRoot,
42
- stdio: "inherit",
43
- windowsHide: true,
44
- });
45
- if (repaired.error) {
46
- fail(`smart-search runtime repair failed: ${repaired.error.message}`);
47
- }
48
- if (repaired.status !== 0 || !fs.existsSync(pythonPath)) {
49
- console.error("Trellis could not find the smart-search Python runtime.");
50
- console.error(`Expected: ${pythonPath}`);
51
- console.error("Repair it by reinstalling Trellis or running:");
52
- console.error(" node scripts/postinstall.js");
53
- process.exit(repaired.status || 5);
54
- }
55
- }
56
-
57
- const child = spawn(pythonPath, ["-m", "smart_search.cli", ...process.argv.slice(2)], {
58
- cwd: callerCwd,
59
- stdio: "inherit",
60
- env: {
61
- ...process.env,
62
- SMART_SEARCH_PACKAGE_ROOT: vendorRoot,
63
- PYTHONIOENCODING: process.env.PYTHONIOENCODING || "utf-8",
64
- PYTHONUTF8: process.env.PYTHONUTF8 || "1",
65
- },
66
- windowsHide: true,
67
- });
68
-
69
- child.on("error", (error) => {
70
- fail(`Failed to start smart-search: ${error.message}`);
71
- });
72
-
73
- child.on("close", (code, signal) => {
74
- if (signal) {
75
- process.kill(process.pid, signal);
76
- return;
77
- }
78
- process.exit(code ?? 5);
79
- });
@@ -1,342 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Aggregate per-query retrieval telemetry JSONL into summary metrics (schema v2)."""
3
-
4
- from __future__ import annotations
5
-
6
- import argparse
7
- import json
8
- import sys
9
- from pathlib import Path
10
- from typing import Any
11
-
12
- _SCRIPT_DIR = Path(__file__).resolve().parent
13
- if str(_SCRIPT_DIR) not in sys.path:
14
- sys.path.insert(0, str(_SCRIPT_DIR))
15
-
16
- from common.retrieval_tool_classification import ( # noqa: E402
17
- classify_tool_calls,
18
- semantic_routes_in_plan,
19
- structural_routes_in_plan,
20
- )
21
-
22
- SCHEMA_VERSION = 2
23
-
24
- SEMANTIC_OUTCOMES = (
25
- "success",
26
- "partial",
27
- "resource_exhausted",
28
- "timeout",
29
- "not_configured",
30
- "unavailable",
31
- "not_run",
32
- "unknown",
33
- )
34
-
35
- SEMANTIC_SKIP_REASONS = (
36
- "rg_corrob_sufficient",
37
- "rg_empty_semantic_required",
38
- "trap_only_semantic_required",
39
- "not_in_plan",
40
- "not_applicable",
41
- "adapter_unavailable",
42
- "agent_stopped_early",
43
- "unknown",
44
- "manual_not_recorded",
45
- )
46
-
47
-
48
- def _rate(count: int, total: int) -> float:
49
- return 0.0 if total == 0 else count / total
50
-
51
-
52
- def _avg(values: list[float]) -> float:
53
- return 0.0 if not values else sum(values) / len(values)
54
-
55
-
56
- def compute_compliance_score(record: dict[str, Any]) -> float:
57
- earned = 0.0
58
- possible = 0.0
59
-
60
- if record.get("structural_in_plan"):
61
- possible += 1.0
62
- if record.get("codegraph_executed"):
63
- earned += 1.0
64
-
65
- if record.get("semantic_in_plan"):
66
- possible += 1.0
67
- if record.get("semantic_executed") or record.get("semantic_skip_reason") == "rg_corrob_sufficient":
68
- earned += 1.0
69
-
70
- possible += 1.0
71
- if record.get("read_verification_done"):
72
- earned += 1.0
73
-
74
- if record.get("plan_block_in_prompt"):
75
- possible += 0.25
76
- if record.get("router_cli_invoked"):
77
- earned += 0.25
78
-
79
- return 1.0 if possible == 0 else earned / possible
80
-
81
-
82
- def migrate_record(raw: dict[str, Any]) -> dict[str, Any]:
83
- routes = raw.get("routes") or raw.get("routes_in_plan") or []
84
- if not isinstance(routes, list):
85
- routes = []
86
- tools = raw.get("tools_called") or []
87
- if not isinstance(tools, list):
88
- tools = []
89
- platform = str(raw.get("platform", "cursor"))
90
- classified = classify_tool_calls([str(t) for t in tools], platform=platform)
91
-
92
- structural_in = raw.get("structural_in_plan")
93
- if not isinstance(structural_in, bool):
94
- structural_in = structural_routes_in_plan([str(r) for r in routes])
95
-
96
- semantic_in = raw.get("semantic_in_plan")
97
- if not isinstance(semantic_in, bool):
98
- semantic_in = semantic_routes_in_plan([str(r) for r in routes])
99
-
100
- record: dict[str, Any] = {
101
- "schema_version": SCHEMA_VERSION,
102
- "query_id": str(raw.get("query_id", "")),
103
- "dataset": str(raw.get("dataset", "unknown")),
104
- "query_text": str(raw.get("query_text", "")),
105
- "run_id": str(raw.get("run_id", "")),
106
- "platform": platform,
107
- "semantic_in_plan": bool(semantic_in),
108
- "semantic_order": raw.get("semantic_order")
109
- if isinstance(raw.get("semantic_order"), int)
110
- else None,
111
- "structural_in_plan": bool(structural_in),
112
- "codegraph_in_plan": bool(raw.get("codegraph_in_plan", structural_in)),
113
- "fallback_hint_present": bool(raw.get("fallback_hint_present")),
114
- "intents": list(raw.get("intents") or []),
115
- "routes": [str(r) for r in routes],
116
- "routes_in_plan": [str(r) for r in routes],
117
- "project_file_count": raw.get("project_file_count")
118
- if isinstance(raw.get("project_file_count"), int)
119
- else None,
120
- "tools_called": classified.tools_called,
121
- "grep_count": raw.get("grep_count")
122
- if isinstance(raw.get("grep_count"), int)
123
- else classified.grep_count,
124
- "read_count": raw.get("read_count")
125
- if isinstance(raw.get("read_count"), int)
126
- else classified.read_count,
127
- "codegraph_attempted": bool(
128
- raw.get("codegraph_attempted", classified.codegraph_attempted)
129
- ),
130
- "codegraph_executed": bool(
131
- raw.get("codegraph_executed", classified.codegraph_executed)
132
- ),
133
- "router_cli_invoked": bool(
134
- raw.get("router_cli_invoked", classified.router_cli_invoked)
135
- ),
136
- "plan_block_in_prompt": bool(raw.get("plan_block_in_prompt")),
137
- "read_verification_done": bool(
138
- raw.get("read_verification_done", classified.read_count > 0)
139
- ),
140
- "semantic_attempted": bool(
141
- raw.get("semantic_attempted", classified.semantic_attempted)
142
- ),
143
- "semantic_executed": bool(
144
- raw.get("semantic_executed", classified.semantic_executed)
145
- ),
146
- "semantic_outcome": str(raw.get("semantic_outcome", "unknown")),
147
- "semantic_success": bool(raw.get("semantic_success")),
148
- "semantic_skip_reason": raw.get("semantic_skip_reason"),
149
- "rg_candidate_count": raw.get("rg_candidate_count")
150
- if isinstance(raw.get("rg_candidate_count"), int)
151
- else None,
152
- "rg_corrob_status": str(raw.get("rg_corrob_status", "unknown")),
153
- "trap_only": bool(raw.get("trap_only")),
154
- "corroborated_files": list(raw.get("corroborated_files") or []),
155
- "adapter_errors": list(raw.get("adapter_errors") or []),
156
- "candidate_pool_recall": raw.get("candidate_pool_recall")
157
- if isinstance(raw.get("candidate_pool_recall"), (int, float))
158
- else None,
159
- "final_top_k_recall": raw.get("final_top_k_recall")
160
- if isinstance(raw.get("final_top_k_recall"), (int, float))
161
- else None,
162
- "answer_score": raw.get("answer_score")
163
- if isinstance(raw.get("answer_score"), (int, float))
164
- else None,
165
- "compliance_score": raw.get("compliance_score")
166
- if isinstance(raw.get("compliance_score"), (int, float))
167
- else None,
168
- "platform_semantic_executed": bool(
169
- raw.get("platform_semantic_executed", classified.platform_semantic_executed)
170
- ),
171
- "fast_context_count": int(raw.get("fast_context_count", classified.fast_context_count)),
172
- "cursor_fast_context_misuse": bool(
173
- raw.get("cursor_fast_context_misuse", classified.cursor_fast_context_misuse)
174
- ),
175
- }
176
- if record["compliance_score"] is None:
177
- record["compliance_score"] = compute_compliance_score(record)
178
- return record
179
-
180
-
181
- def derive_metrics(records: list[dict[str, Any]]) -> dict[str, Any]:
182
- total = len(records)
183
- outcome_counts = {k: 0 for k in SEMANTIC_OUTCOMES}
184
- skip_counts = {k: 0 for k in SEMANTIC_SKIP_REASONS}
185
-
186
- semantic_plan = semantic_exec = semantic_attempt = semantic_success = 0
187
- platform_sem_exec = fast_context_total = fast_context_misuse = 0
188
- codegraph_plan = codegraph_exec = router_cli = plan_block = read_verify = 0
189
- compliance_scores: list[float] = []
190
- answer_scores: list[float] = []
191
- pool_recalls: list[float] = []
192
- topk_recalls: list[float] = []
193
-
194
- for rec in records:
195
- if rec.get("semantic_in_plan"):
196
- semantic_plan += 1
197
- if rec.get("semantic_executed"):
198
- semantic_exec += 1
199
- if rec.get("platform_semantic_executed"):
200
- platform_sem_exec += 1
201
- fc = rec.get("fast_context_count")
202
- if isinstance(fc, int) and fc > 0:
203
- fast_context_total += fc
204
- if rec.get("cursor_fast_context_misuse"):
205
- fast_context_misuse += 1
206
- if rec.get("semantic_attempted"):
207
- semantic_attempt += 1
208
- if rec.get("semantic_success") and rec.get("semantic_outcome") == "success":
209
- semantic_success += 1
210
- if rec.get("codegraph_in_plan"):
211
- codegraph_plan += 1
212
- if rec.get("codegraph_executed"):
213
- codegraph_exec += 1
214
- if rec.get("router_cli_invoked"):
215
- router_cli += 1
216
- if rec.get("plan_block_in_prompt"):
217
- plan_block += 1
218
- if rec.get("read_verification_done"):
219
- read_verify += 1
220
-
221
- outcome = str(rec.get("semantic_outcome", "unknown"))
222
- if outcome in outcome_counts:
223
- outcome_counts[outcome] += 1
224
- skip = rec.get("semantic_skip_reason")
225
- if skip and skip in skip_counts:
226
- skip_counts[skip] += 1
227
-
228
- cs = rec.get("compliance_score")
229
- if isinstance(cs, (int, float)):
230
- compliance_scores.append(float(cs))
231
- ans = rec.get("answer_score")
232
- if isinstance(ans, (int, float)):
233
- answer_scores.append(float(ans))
234
- cpr = rec.get("candidate_pool_recall")
235
- if isinstance(cpr, (int, float)):
236
- pool_recalls.append(float(cpr))
237
- ftr = rec.get("final_top_k_recall")
238
- if isinstance(ftr, (int, float)):
239
- topk_recalls.append(float(ftr))
240
-
241
- avg_pool = _avg(pool_recalls)
242
- avg_topk = _avg(topk_recalls)
243
- recall_drop = 0.0 if avg_pool <= 0 else 1.0 - avg_topk / avg_pool
244
-
245
- return {
246
- "schema_version": SCHEMA_VERSION,
247
- "total_queries": total,
248
- "semantic_plan_count": semantic_plan,
249
- "semantic_exec_count": semantic_exec,
250
- "semantic_attempt_count": semantic_attempt,
251
- "semantic_exec_success_count": semantic_success,
252
- "semantic_plan_rate": _rate(semantic_plan, total),
253
- "semantic_exec_rate": _rate(semantic_exec, total),
254
- "platform_semantic_exec_count": platform_sem_exec,
255
- "platform_semantic_exec_rate": _rate(platform_sem_exec, total),
256
- "fast_context_invocation_total": fast_context_total,
257
- "cursor_fast_context_misuse_count": fast_context_misuse,
258
- "cursor_fast_context_misuse_rate": _rate(fast_context_misuse, total),
259
- "semantic_attempt_rate": _rate(semantic_attempt, total),
260
- "semantic_exec_success_rate": _rate(semantic_success, total),
261
- "codegraph_plan_count": codegraph_plan,
262
- "codegraph_exec_count": codegraph_exec,
263
- "codegraph_plan_rate": _rate(codegraph_plan, total),
264
- "codegraph_exec_rate": _rate(codegraph_exec, total),
265
- "router_cli_count": router_cli,
266
- "router_cli_rate": _rate(router_cli, total),
267
- "plan_block_count": plan_block,
268
- "plan_block_rate": _rate(plan_block, total),
269
- "read_verification_count": read_verify,
270
- "read_verification_rate": _rate(read_verify, total),
271
- "avg_compliance_score": _avg(compliance_scores),
272
- "avg_answer_score": _avg(answer_scores),
273
- "semantic_outcome_counts": outcome_counts,
274
- "semantic_skip_reason_counts": skip_counts,
275
- "avg_candidate_pool_recall": avg_pool,
276
- "avg_final_top_k_recall": avg_topk,
277
- "recall_drop_rate": recall_drop,
278
- }
279
-
280
-
281
- def load_jsonl(path: Path) -> list[dict[str, Any]]:
282
- records: list[dict[str, Any]] = []
283
- for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
284
- stripped = line.strip()
285
- if not stripped or stripped.startswith("#"):
286
- continue
287
- try:
288
- raw = json.loads(stripped)
289
- except json.JSONDecodeError as exc:
290
- raise ValueError(f"{path}:{line_no}: invalid JSON: {exc}") from exc
291
- if not isinstance(raw, dict):
292
- raise ValueError(f"{path}:{line_no}: expected JSON object per line")
293
- records.append(migrate_record(raw))
294
- return records
295
-
296
-
297
- def main() -> int:
298
- parser = argparse.ArgumentParser(description=__doc__)
299
- parser.add_argument(
300
- "jsonl",
301
- type=Path,
302
- help="Per-query telemetry JSONL (one object per line)",
303
- )
304
- parser.add_argument(
305
- "--markdown",
306
- action="store_true",
307
- help="Print a short markdown summary for eval reports",
308
- )
309
- args = parser.parse_args()
310
-
311
- if not args.jsonl.is_file():
312
- print(f"error: file not found: {args.jsonl}", file=sys.stderr)
313
- return 1
314
-
315
- records = load_jsonl(args.jsonl)
316
- metrics = derive_metrics(records)
317
- payload = {"metrics": metrics, "record_count": len(records)}
318
-
319
- if args.markdown:
320
- m = metrics
321
- print("## Retrieval telemetry (derived from JSONL)\n")
322
- print("| Metric | Value |")
323
- print("| --- | ---: |")
324
- print(f"| total_queries | {m['total_queries']} |")
325
- print(f"| semantic_plan_rate | {m['semantic_plan_rate']:.1%} |")
326
- print(f"| semantic_exec_rate | {m['semantic_exec_rate']:.1%} |")
327
- print(f"| platform_semantic_exec_rate | {m.get('platform_semantic_exec_rate', 0):.1%} |")
328
- print(f"| cursor_fast_context_misuse_rate | {m.get('cursor_fast_context_misuse_rate', 0):.1%} |")
329
- print(f"| codegraph_plan_rate | {m['codegraph_plan_rate']:.1%} |")
330
- print(f"| codegraph_exec_rate | {m['codegraph_exec_rate']:.1%} |")
331
- print(f"| router_cli_rate | {m['router_cli_rate']:.1%} |")
332
- print(f"| avg_compliance_score | {m['avg_compliance_score']:.3f} |")
333
- print(f"| avg_answer_score | {m['avg_answer_score']:.3f} |")
334
- print("\nDo not hand-copy plan/exec rates; regenerate from JSONL with this script.")
335
- else:
336
- print(json.dumps(payload, indent=2, ensure_ascii=False))
337
-
338
- return 0
339
-
340
-
341
- if __name__ == "__main__":
342
- raise SystemExit(main())
@@ -1,105 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Batch route_codebase_retrieval for plan metrics (REC-06: platform-semantic on Cursor)."""
3
-
4
- from __future__ import annotations
5
-
6
- import argparse
7
- import json
8
- import sys
9
- from pathlib import Path
10
-
11
- _SCRIPT_DIR = Path(__file__).resolve().parent
12
- if str(_SCRIPT_DIR) not in sys.path:
13
- sys.path.insert(0, str(_SCRIPT_DIR))
14
-
15
- from common.codebase_retrieval_router import route_codebase_retrieval # noqa: E402
16
- from common.retrieval_tool_classification import ( # noqa: E402
17
- platform_semantic_route_order,
18
- semantic_routes_in_plan,
19
- )
20
-
21
-
22
- def load_queries(path: Path) -> list[dict]:
23
- text = path.read_text(encoding="utf-8-sig")
24
- rows: list[dict] = []
25
- for line in text.splitlines():
26
- stripped = line.strip()
27
- if not stripped:
28
- continue
29
- rows.append(json.loads(stripped))
30
- return rows
31
-
32
-
33
- def main(argv: list[str] | None = None) -> int:
34
- parser = argparse.ArgumentParser(description=__doc__)
35
- parser.add_argument(
36
- "--queries",
37
- type=Path,
38
- default=None,
39
- help="JSONL query file (default: ./queries.jsonl under cwd)",
40
- )
41
- parser.add_argument("--pretty", action="store_true")
42
- args = parser.parse_args(argv)
43
-
44
- root = Path.cwd()
45
- queries_path = args.queries or (root / "queries.jsonl")
46
- if not queries_path.is_file():
47
- print(f"error: queries file not found: {queries_path}", file=sys.stderr)
48
- return 1
49
-
50
- rows = load_queries(queries_path)
51
- out: list[dict] = []
52
- plan_sem = 0
53
- platform_sem = 0
54
- fast_context_sem = 0
55
-
56
- for row in rows:
57
- query = str(row.get("query", ""))
58
- plan = route_codebase_retrieval(query)
59
- routes = plan.get("routes") or []
60
- route_ids = [str(r.get("id", "")) for r in routes if isinstance(r, dict)]
61
- sem_in_plan = semantic_routes_in_plan(route_ids)
62
- if sem_in_plan:
63
- plan_sem += 1
64
- if "platform-semantic" in route_ids:
65
- platform_sem += 1
66
- if "semantic-fast-context" in route_ids:
67
- fast_context_sem += 1
68
-
69
- out.append(
70
- {
71
- "id": row.get("id"),
72
- "intents": [i["id"] for i in plan.get("intents", []) if isinstance(i, dict)],
73
- "route_ids": route_ids,
74
- "semantic_in_plan": sem_in_plan,
75
- "platform_semantic_order": platform_semantic_route_order(
76
- [r for r in routes if isinstance(r, dict)]
77
- ),
78
- "fallback_rg_empty": any(
79
- "corroborated" in str(f.get("when", ""))
80
- for f in (plan.get("fallback") or [])
81
- if isinstance(f, dict)
82
- ),
83
- }
84
- )
85
-
86
- n = len(rows)
87
- summary = {
88
- "n": n,
89
- "semantic_plan_rate": plan_sem / n if n else 0.0,
90
- "platform_semantic_plan_rate": platform_sem / n if n else 0.0,
91
- "semantic_fast_context_plan_rate": fast_context_sem / n if n else 0.0,
92
- "fallback_hint_rate": sum(1 for x in out if x["fallback_rg_empty"]) / n if n else 0.0,
93
- "note": (
94
- "Plan rates only. semantic_exec_rate must come from session tool logs "
95
- "and classify_tool_calls(platform=cursor)."
96
- ),
97
- }
98
- payload = {"summary": summary, "queries": out}
99
- indent = 2 if args.pretty else None
100
- print(json.dumps(payload, ensure_ascii=False, indent=indent))
101
- return 0
102
-
103
-
104
- if __name__ == "__main__":
105
- raise SystemExit(main())
@@ -1,21 +0,0 @@
1
- MIT License
2
-
3
- Copyright (c) 2025 GuDaStudio
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy
6
- of this software and associated documentation files (the "Software"), to deal
7
- in the Software without restriction, including without limitation the rights
8
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
- copies of the Software, and to permit persons to whom the Software is
10
- furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in all
13
- copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
- SOFTWARE.