@blxzer/cursor-trellis 0.1.3 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/README.md +45 -23
- package/dist/configurators/workflow.d.ts +1 -1
- package/dist/configurators/workflow.d.ts.map +1 -1
- package/dist/configurators/workflow.js +20 -9
- package/dist/configurators/workflow.js.map +1 -1
- package/dist/migrations/manifests/0.1.3.json +9 -0
- package/dist/migrations/manifests/0.1.4.json +9 -0
- package/dist/templates/markdown/agents.md +1 -1
- package/dist/templates/markdown/index.d.ts +1 -0
- package/dist/templates/markdown/index.d.ts.map +1 -1
- package/dist/templates/markdown/index.js +1 -0
- package/dist/templates/markdown/index.js.map +1 -1
- package/dist/templates/markdown/prompts/run-semantic-slice-12.md.txt +13 -29
- package/dist/templates/markdown/spec/guides/code-reuse-thinking-guide.md.txt +0 -49
- package/dist/templates/markdown/spec/guides/cross-layer-thinking-guide.md.txt +0 -85
- package/dist/templates/markdown/spec/guides/cursor-semantic-compliance.md.txt +7 -21
- package/dist/templates/markdown/spec/guides/cursor-subagent-policy.md.txt +6 -10
- package/dist/templates/markdown/spec/guides/retrieval-daily-guide.md.txt +7 -13
- package/dist/templates/shared-hooks/inject-retrieval-plan.py +42 -23
- package/dist/templates/shared-hooks/inject-subagent-context.py +6 -2
- package/dist/templates/shared-hooks/session-start.py +17 -9
- package/dist/templates/trellis/index.d.ts +7 -4
- package/dist/templates/trellis/index.d.ts.map +1 -1
- package/dist/templates/trellis/index.js +11 -7
- package/dist/templates/trellis/index.js.map +1 -1
- package/dist/templates/trellis/scripts/common/codebase_retrieval_router.py +2 -2
- package/dist/templates/trellis/scripts/common/config.py +1 -1
- package/dist/templates/trellis/scripts/common/parent_orchestration.py +27 -15
- package/dist/templates/trellis/scripts/common/retrieval_agent_instructions.py +1 -1
- package/dist/templates/trellis/scripts/common/smart_search_resolve.py +28 -3
- package/dist/templates/trellis/scripts/common/task_gates.py +372 -73
- package/dist/templates/trellis/scripts/common/task_map.py +29 -1
- package/dist/templates/trellis/workflow.md +9 -53
- package/package.json +5 -44
- package/bin/smart-search.js +0 -79
- package/dist/templates/trellis/scripts/aggregate_retrieval_telemetry.py +0 -342
- package/dist/templates/trellis/scripts/batch_plan_envelope.py +0 -105
- package/vendor/smart-search/LICENSE +0 -21
- package/vendor/smart-search/README.md +0 -412
- package/vendor/smart-search/README.zh-CN.md +0 -430
- package/vendor/smart-search/npm/bin/smart-search.js +0 -63
- package/vendor/smart-search/npm/scripts/postinstall.js +0 -87
- package/vendor/smart-search/npm/scripts/resolve-prerelease-version.js +0 -108
- package/vendor/smart-search/npm/scripts/set-package-version.js +0 -35
- package/vendor/smart-search/npm/scripts/sync-python-version.js +0 -22
- package/vendor/smart-search/npm/scripts/test-wrapper-repair.js +0 -99
- package/vendor/smart-search/npm/scripts/test.js +0 -85
- package/vendor/smart-search/package.json +0 -42
- package/vendor/smart-search/pyproject.toml +0 -37
- package/vendor/smart-search/skills/smart-search-cli/SKILL.md +0 -387
- package/vendor/smart-search/skills/smart-search-cli/agents/openai.yaml +0 -3
- package/vendor/smart-search/skills/smart-search-cli/examples/batch-search.md +0 -98
- package/vendor/smart-search/skills/smart-search-cli/examples/evidence-gathering.md +0 -89
- package/vendor/smart-search/skills/smart-search-cli/references/cli-contract.md +0 -337
- package/vendor/smart-search/src/smart_search/__init__.py +0 -1
- package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/SKILL.md +0 -387
- package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/agents/openai.yaml +0 -3
- package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/batch-search.md +0 -98
- package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/evidence-gathering.md +0 -89
- package/vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/references/cli-contract.md +0 -337
- package/vendor/smart-search/src/smart_search/cli.py +0 -2003
- package/vendor/smart-search/src/smart_search/config.py +0 -580
- package/vendor/smart-search/src/smart_search/logger.py +0 -43
- package/vendor/smart-search/src/smart_search/providers/__init__.py +0 -16
- package/vendor/smart-search/src/smart_search/providers/base.py +0 -41
- package/vendor/smart-search/src/smart_search/providers/context7.py +0 -141
- package/vendor/smart-search/src/smart_search/providers/exa.py +0 -206
- package/vendor/smart-search/src/smart_search/providers/jina.py +0 -136
- package/vendor/smart-search/src/smart_search/providers/openai_compatible.py +0 -370
- package/vendor/smart-search/src/smart_search/providers/zhipu.py +0 -143
- package/vendor/smart-search/src/smart_search/service.py +0 -2865
- package/vendor/smart-search/src/smart_search/sources.py +0 -429
- package/vendor/smart-search/src/smart_search/utils.py +0 -220
|
@@ -41,8 +41,6 @@ python3 ./.trellis/scripts/get_context.py --mode packages # list packages / la
|
|
|
41
41
|
|
|
42
42
|
Every task has its own directory under `.trellis/tasks/{MM-DD-name}/` holding `task.json`, `prd.md`, optional `design.md`, optional `implement.md`, optional `research/`, and context manifests (`implement.jsonl`, `check.jsonl`) for Cursor (sub-agent dispatch).
|
|
43
43
|
|
|
44
|
-
Optional copied templates live under `.trellis/tasks/templates/` (for example `release-readiness/` and `release-execution/`). `task.py create` does not auto-apply them; copy files into a new task when starting that workflow.
|
|
45
|
-
|
|
46
44
|
```bash
|
|
47
45
|
# Task lifecycle
|
|
48
46
|
python3 ./.trellis/scripts/task.py create "<title>" [--slug <name>] [--parent <dir>]
|
|
@@ -265,7 +263,11 @@ python3 ./.trellis/scripts/task.py review-child <parent-task> <child-task> --dec
|
|
|
265
263
|
|
|
266
264
|
- `generate-child-prompt` reads parent `task-map.md` for `depends_on` and `touches`, child artifacts, and optional parent `child-prompts.md`. Use `--mode subagent` only as a delivery hint when the platform can spawn subagents; inline mode remains the portable default.
|
|
267
265
|
- `review-child` summarizes child `verify.md` / `handoff.md`, appends notes to parent `verify.md`, and can advance `accepted` / `integrating` / `integrated` in one flow (`--decision integrate-through`) while still using the same Stage 0 integration guards as `integrate-child`.
|
|
268
|
-
- Reviewer quality gates
|
|
266
|
+
- Reviewer quality gates are **not** auto-recorded. CLI enforces them at transition boundaries:
|
|
267
|
+
- **Full Child accept / integrate-through**: requires substantive `verify.md` evidence and `child-review/code-review` (plus configured architecture gates) before Parent marks the Child `accepted`.
|
|
268
|
+
- **Parent archive**: requires every structural Child `integrated` or `cancelled`, substantive Parent integration evidence, and `parent-integrated/integration-review`.
|
|
269
|
+
- **Lite closeout**: explicit no-gate chain; archive still requires validation, acceptance, and durable-learning evidence in `verify.md`.
|
|
270
|
+
- **`record-gate`**: rejects PASS/SKIPPED when transition evidence is missing or placeholder-only.
|
|
269
271
|
|
|
270
272
|
<!-- Per-turn breadcrumb: shown when no task is selected (before Phase 1) -->
|
|
271
273
|
|
|
@@ -724,10 +726,10 @@ Archive readiness by mode:
|
|
|
724
726
|
|---|---|
|
|
725
727
|
| No Task | No archive; upgrade to a durable task mode before archive is possible. |
|
|
726
728
|
| Micro-Grill | No archive unless upgraded into Lite, Full, or Parent/Child. |
|
|
727
|
-
| Lite | `verify.md` has validation
|
|
728
|
-
| Full | Lite evidence plus required completion gates, fresh fingerprints, and no unresolved required `FAIL` gates. |
|
|
729
|
-
| Child | Lite
|
|
730
|
-
| Parent |
|
|
729
|
+
| Lite | Explicit no-gate chain. `verify.md` has substantive validation, final acceptance, and durable-learning decision evidence. |
|
|
730
|
+
| Full | Lite evidence plus required completion gates (`full-task-complete/*`), substantive check/change-set evidence, fresh fingerprints, and no unresolved required `FAIL` gates. |
|
|
731
|
+
| Child | Lite or Full evidence (by child profile). Full Children require `child-review` gates before Parent `accepted`. Parent task-map marks the Child `integrated` or `cancelled`; integrated Children include `handoff.md`. |
|
|
732
|
+
| Parent | Archive evidence plus every Child `integrated` or `cancelled`, substantive final integration evidence, and `parent-integrated/integration-review`. |
|
|
731
733
|
|
|
732
734
|
Archive / Learning is terminal. After archive, do not silently mutate archived task artifacts; follow-up work requires a new task unless the user explicitly approves an archive amendment.
|
|
733
735
|
|
|
@@ -735,52 +737,6 @@ If archive is not being run in this session, report the passing or failing archi
|
|
|
735
737
|
|
|
736
738
|
---
|
|
737
739
|
|
|
738
|
-
## Release readiness and release execution
|
|
739
|
-
|
|
740
|
-
Package and CLI releases use **two tasks** (or two explicit phases with separate evidence). Readiness is non-mutating; execution requires explicit user approval before any remote mutation.
|
|
741
|
-
|
|
742
|
-
### When to split
|
|
743
|
-
|
|
744
|
-
| Task type | Purpose | Remote mutations |
|
|
745
|
-
| --- | --- | --- |
|
|
746
|
-
| **Release readiness** | Version recommendation, changelog, manifest notes, build/test/typecheck, pack dry-run, prepublish blockers | **None** — no publish, tag, push, or version bump |
|
|
747
|
-
| **Release execution** | Preflight immediately before publish; run approved `release.js` / `npm publish` / tag / push; post-publish smoke | **Only after explicit user approval** in the execution task |
|
|
748
|
-
|
|
749
|
-
Templates: `.trellis/tasks/templates/release-readiness/` and `.trellis/tasks/templates/release-execution/`. Copy `prd.md`, `design.md`, and `implement.md` into a new task directory; use `handoff-template.md` as the outline for `handoff.md`.
|
|
750
|
-
|
|
751
|
-
### Release status vocabulary (handoff / verify)
|
|
752
|
-
|
|
753
|
-
These labels describe **release posture** in `handoff.md` and `verify.md`. They are not `task.json.status` values (contract epoch 1).
|
|
754
|
-
|
|
755
|
-
| Label | Typical task |
|
|
756
|
-
| --- | --- |
|
|
757
|
-
| **ready to publish** | Readiness — evidence complete; operator may proceed after approval |
|
|
758
|
-
| **not published** | Readiness or aborted execution — no remote release in this task |
|
|
759
|
-
| **published** | Execution — registry and/or git remote actions completed |
|
|
760
|
-
| **blocked** | Required gate failed; do not publish until fixed or waived |
|
|
761
|
-
| **waived** | User explicitly accepted a known blocker (name approver + rationale) |
|
|
762
|
-
| **deferred** | Publish postponed by user (e.g. wait for dogfood / another child) |
|
|
763
|
-
|
|
764
|
-
Readiness `handoff.md` must include **Ready to publish**, **Not published**, and **Blockers**. Execution `handoff.md` must include **Published** (or **Not published** if aborted) and **Publish approval evidence** in `verify.md`.
|
|
765
|
-
|
|
766
|
-
### Standard evidence (readiness `verify.md`)
|
|
767
|
-
|
|
768
|
-
- Version recommendation (current → proposed; channel; npm dist-tag expectation)
|
|
769
|
-
- Changelog / release notes (draft)
|
|
770
|
-
- Migration manifest (id or none)
|
|
771
|
-
- Build evidence, test evidence, pack dry-run evidence
|
|
772
|
-
- Prepublish blockers with disposition: `blocking` / `fixed` / `waived` / `deferred`
|
|
773
|
-
|
|
774
|
-
Dry-run examples for `@blxzer/cursor-trellis` (from `packages/cli`): `check-manifest-continuity.js`, `release-preflight.js check-versions`, `publish-plan`, `verify-packed-cli`, `npm pack --dry-run`. Do not run `release.js`, `npm publish`, or `git push` during readiness.
|
|
775
|
-
|
|
776
|
-
### Execution approval gate
|
|
777
|
-
|
|
778
|
-
Before `release.js`, `npm publish`, `git tag`, `git push`, or GitHub release creation, stop and obtain **explicit user approval**. Record in execution `verify.md` under **Publish approval evidence** (approved version, dist-tag, and which remote actions are allowed). Passing preflight is not approval.
|
|
779
|
-
|
|
780
|
-
See also `.trellis/spec/Trellis/cli/release-and-vendor.md`.
|
|
781
|
-
|
|
782
|
-
---
|
|
783
|
-
|
|
784
740
|
## Customizing Trellis (for forks)
|
|
785
741
|
|
|
786
742
|
This section is for developers who want to modify the Trellis workflow itself. All customization is done by editing this file; the scripts are parsers only.
|
package/package.json
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@blxzer/cursor-trellis",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "AI capabilities grow like ivy — Trellis provides the structure to guide them along a disciplined path",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
7
7
|
"types": "./dist/index.d.ts",
|
|
8
8
|
"bin": {
|
|
9
9
|
"trellis": "./bin/trellis.js",
|
|
10
|
-
"tl": "./bin/trellis.js"
|
|
11
|
-
"smart-search": "./bin/smart-search.js"
|
|
10
|
+
"tl": "./bin/trellis.js"
|
|
12
11
|
},
|
|
13
12
|
"publishConfig": {
|
|
14
13
|
"access": "public"
|
|
@@ -28,6 +27,7 @@
|
|
|
28
27
|
"author": "Mindfold LLC",
|
|
29
28
|
"license": "AGPL-3.0-only",
|
|
30
29
|
"dependencies": {
|
|
30
|
+
"@blxzer/smart-search": "^0.1.0",
|
|
31
31
|
"chalk": "^5.3.0",
|
|
32
32
|
"commander": "^12.1.0",
|
|
33
33
|
"figlet": "^1.9.4",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"inquirer": "^9.3.7",
|
|
36
36
|
"undici": "^6.21.0",
|
|
37
37
|
"zod": "^4.4.2",
|
|
38
|
-
"@blxzer/cursor-trellis-core": "0.1
|
|
38
|
+
"@blxzer/cursor-trellis-core": "0.2.1"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@eslint/js": "^9.18.0",
|
|
@@ -59,43 +59,7 @@
|
|
|
59
59
|
"scripts/postinstall.js",
|
|
60
60
|
"README.md",
|
|
61
61
|
"CHANGELOG.md",
|
|
62
|
-
"LICENSE"
|
|
63
|
-
"vendor/smart-search/LICENSE",
|
|
64
|
-
"vendor/smart-search/README.md",
|
|
65
|
-
"vendor/smart-search/README.zh-CN.md",
|
|
66
|
-
"vendor/smart-search/npm/bin/smart-search.js",
|
|
67
|
-
"vendor/smart-search/npm/scripts/postinstall.js",
|
|
68
|
-
"vendor/smart-search/npm/scripts/resolve-prerelease-version.js",
|
|
69
|
-
"vendor/smart-search/npm/scripts/set-package-version.js",
|
|
70
|
-
"vendor/smart-search/npm/scripts/sync-python-version.js",
|
|
71
|
-
"vendor/smart-search/npm/scripts/test-wrapper-repair.js",
|
|
72
|
-
"vendor/smart-search/npm/scripts/test.js",
|
|
73
|
-
"vendor/smart-search/package.json",
|
|
74
|
-
"vendor/smart-search/pyproject.toml",
|
|
75
|
-
"vendor/smart-search/skills/smart-search-cli/SKILL.md",
|
|
76
|
-
"vendor/smart-search/skills/smart-search-cli/agents/openai.yaml",
|
|
77
|
-
"vendor/smart-search/skills/smart-search-cli/examples/batch-search.md",
|
|
78
|
-
"vendor/smart-search/skills/smart-search-cli/examples/evidence-gathering.md",
|
|
79
|
-
"vendor/smart-search/skills/smart-search-cli/references/cli-contract.md",
|
|
80
|
-
"vendor/smart-search/src/smart_search/__init__.py",
|
|
81
|
-
"vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/SKILL.md",
|
|
82
|
-
"vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/agents/openai.yaml",
|
|
83
|
-
"vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/batch-search.md",
|
|
84
|
-
"vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/examples/evidence-gathering.md",
|
|
85
|
-
"vendor/smart-search/src/smart_search/assets/skills/smart-search-cli/references/cli-contract.md",
|
|
86
|
-
"vendor/smart-search/src/smart_search/cli.py",
|
|
87
|
-
"vendor/smart-search/src/smart_search/config.py",
|
|
88
|
-
"vendor/smart-search/src/smart_search/logger.py",
|
|
89
|
-
"vendor/smart-search/src/smart_search/providers/__init__.py",
|
|
90
|
-
"vendor/smart-search/src/smart_search/providers/base.py",
|
|
91
|
-
"vendor/smart-search/src/smart_search/providers/context7.py",
|
|
92
|
-
"vendor/smart-search/src/smart_search/providers/exa.py",
|
|
93
|
-
"vendor/smart-search/src/smart_search/providers/jina.py",
|
|
94
|
-
"vendor/smart-search/src/smart_search/providers/openai_compatible.py",
|
|
95
|
-
"vendor/smart-search/src/smart_search/providers/zhipu.py",
|
|
96
|
-
"vendor/smart-search/src/smart_search/service.py",
|
|
97
|
-
"vendor/smart-search/src/smart_search/sources.py",
|
|
98
|
-
"vendor/smart-search/src/smart_search/utils.py"
|
|
62
|
+
"LICENSE"
|
|
99
63
|
],
|
|
100
64
|
"repository": {
|
|
101
65
|
"type": "git",
|
|
@@ -106,8 +70,6 @@
|
|
|
106
70
|
"clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"",
|
|
107
71
|
"copy-templates": "node scripts/copy-templates.js",
|
|
108
72
|
"postinstall": "node scripts/postinstall.js",
|
|
109
|
-
"sync:smart-search": "node scripts/sync-smart-search-vendor.js",
|
|
110
|
-
"check:smart-search": "node scripts/check-smart-search-vendor.js",
|
|
111
73
|
"check:pack-files": "node scripts/check-cli-pack-files.js",
|
|
112
74
|
"check:release-pack": "node scripts/check-release-pack-contents.js",
|
|
113
75
|
"check:router-copy-sync": "node scripts/check-router-copy-sync.js",
|
|
@@ -126,7 +88,6 @@
|
|
|
126
88
|
"typecheck": "tsc --noEmit",
|
|
127
89
|
"lint:py": "basedpyright",
|
|
128
90
|
"lint:all": "pnpm lint && pnpm lint:py",
|
|
129
|
-
"prepublishOnly_disabled": "pnpm test && pnpm run build && pnpm run copy:release-assets",
|
|
130
91
|
"release": "node scripts/release.js patch",
|
|
131
92
|
"release:minor": "node scripts/release.js minor",
|
|
132
93
|
"release:major": "node scripts/release.js major",
|
package/bin/smart-search.js
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
import { spawn, spawnSync } from "node:child_process";
|
|
4
|
-
import fs from "node:fs";
|
|
5
|
-
import path from "node:path";
|
|
6
|
-
import { fileURLToPath } from "node:url";
|
|
7
|
-
|
|
8
|
-
const packageRoot = path.resolve(
|
|
9
|
-
path.dirname(fileURLToPath(import.meta.url)),
|
|
10
|
-
"..",
|
|
11
|
-
);
|
|
12
|
-
const vendorRoot = path.join(packageRoot, "vendor", "smart-search");
|
|
13
|
-
const callerCwd = process.env.INIT_CWD || process.cwd();
|
|
14
|
-
const venvDir = path.join(vendorRoot, ".smart-search-python");
|
|
15
|
-
const pythonPath =
|
|
16
|
-
process.platform === "win32"
|
|
17
|
-
? path.join(venvDir, "Scripts", "python.exe")
|
|
18
|
-
: path.join(venvDir, "bin", "python");
|
|
19
|
-
const postinstall = path.join(vendorRoot, "npm", "scripts", "postinstall.js");
|
|
20
|
-
|
|
21
|
-
function fail(message, code = 5) {
|
|
22
|
-
console.error(message);
|
|
23
|
-
process.exit(code);
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
if (!fs.existsSync(path.join(vendorRoot, "pyproject.toml"))) {
|
|
27
|
-
fail(
|
|
28
|
-
"Trellis smart-search runtime is missing. Reinstall @blxzer/cursor-trellis or run the package build again.",
|
|
29
|
-
);
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
if (!fs.existsSync(pythonPath)) {
|
|
33
|
-
if (!fs.existsSync(postinstall)) {
|
|
34
|
-
fail(
|
|
35
|
-
`smart-search Python runtime is missing and repair script was not found: ${postinstall}`,
|
|
36
|
-
);
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
console.error("smart-search Python runtime is missing; attempting repair...");
|
|
40
|
-
const repaired = spawnSync(process.execPath, [postinstall], {
|
|
41
|
-
cwd: vendorRoot,
|
|
42
|
-
stdio: "inherit",
|
|
43
|
-
windowsHide: true,
|
|
44
|
-
});
|
|
45
|
-
if (repaired.error) {
|
|
46
|
-
fail(`smart-search runtime repair failed: ${repaired.error.message}`);
|
|
47
|
-
}
|
|
48
|
-
if (repaired.status !== 0 || !fs.existsSync(pythonPath)) {
|
|
49
|
-
console.error("Trellis could not find the smart-search Python runtime.");
|
|
50
|
-
console.error(`Expected: ${pythonPath}`);
|
|
51
|
-
console.error("Repair it by reinstalling Trellis or running:");
|
|
52
|
-
console.error(" node scripts/postinstall.js");
|
|
53
|
-
process.exit(repaired.status || 5);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
const child = spawn(pythonPath, ["-m", "smart_search.cli", ...process.argv.slice(2)], {
|
|
58
|
-
cwd: callerCwd,
|
|
59
|
-
stdio: "inherit",
|
|
60
|
-
env: {
|
|
61
|
-
...process.env,
|
|
62
|
-
SMART_SEARCH_PACKAGE_ROOT: vendorRoot,
|
|
63
|
-
PYTHONIOENCODING: process.env.PYTHONIOENCODING || "utf-8",
|
|
64
|
-
PYTHONUTF8: process.env.PYTHONUTF8 || "1",
|
|
65
|
-
},
|
|
66
|
-
windowsHide: true,
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
child.on("error", (error) => {
|
|
70
|
-
fail(`Failed to start smart-search: ${error.message}`);
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
child.on("close", (code, signal) => {
|
|
74
|
-
if (signal) {
|
|
75
|
-
process.kill(process.pid, signal);
|
|
76
|
-
return;
|
|
77
|
-
}
|
|
78
|
-
process.exit(code ?? 5);
|
|
79
|
-
});
|
|
@@ -1,342 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Aggregate per-query retrieval telemetry JSONL into summary metrics (schema v2)."""
|
|
3
|
-
|
|
4
|
-
from __future__ import annotations
|
|
5
|
-
|
|
6
|
-
import argparse
|
|
7
|
-
import json
|
|
8
|
-
import sys
|
|
9
|
-
from pathlib import Path
|
|
10
|
-
from typing import Any
|
|
11
|
-
|
|
12
|
-
_SCRIPT_DIR = Path(__file__).resolve().parent
|
|
13
|
-
if str(_SCRIPT_DIR) not in sys.path:
|
|
14
|
-
sys.path.insert(0, str(_SCRIPT_DIR))
|
|
15
|
-
|
|
16
|
-
from common.retrieval_tool_classification import ( # noqa: E402
|
|
17
|
-
classify_tool_calls,
|
|
18
|
-
semantic_routes_in_plan,
|
|
19
|
-
structural_routes_in_plan,
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
SCHEMA_VERSION = 2
|
|
23
|
-
|
|
24
|
-
SEMANTIC_OUTCOMES = (
|
|
25
|
-
"success",
|
|
26
|
-
"partial",
|
|
27
|
-
"resource_exhausted",
|
|
28
|
-
"timeout",
|
|
29
|
-
"not_configured",
|
|
30
|
-
"unavailable",
|
|
31
|
-
"not_run",
|
|
32
|
-
"unknown",
|
|
33
|
-
)
|
|
34
|
-
|
|
35
|
-
SEMANTIC_SKIP_REASONS = (
|
|
36
|
-
"rg_corrob_sufficient",
|
|
37
|
-
"rg_empty_semantic_required",
|
|
38
|
-
"trap_only_semantic_required",
|
|
39
|
-
"not_in_plan",
|
|
40
|
-
"not_applicable",
|
|
41
|
-
"adapter_unavailable",
|
|
42
|
-
"agent_stopped_early",
|
|
43
|
-
"unknown",
|
|
44
|
-
"manual_not_recorded",
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def _rate(count: int, total: int) -> float:
|
|
49
|
-
return 0.0 if total == 0 else count / total
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def _avg(values: list[float]) -> float:
|
|
53
|
-
return 0.0 if not values else sum(values) / len(values)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
def compute_compliance_score(record: dict[str, Any]) -> float:
|
|
57
|
-
earned = 0.0
|
|
58
|
-
possible = 0.0
|
|
59
|
-
|
|
60
|
-
if record.get("structural_in_plan"):
|
|
61
|
-
possible += 1.0
|
|
62
|
-
if record.get("codegraph_executed"):
|
|
63
|
-
earned += 1.0
|
|
64
|
-
|
|
65
|
-
if record.get("semantic_in_plan"):
|
|
66
|
-
possible += 1.0
|
|
67
|
-
if record.get("semantic_executed") or record.get("semantic_skip_reason") == "rg_corrob_sufficient":
|
|
68
|
-
earned += 1.0
|
|
69
|
-
|
|
70
|
-
possible += 1.0
|
|
71
|
-
if record.get("read_verification_done"):
|
|
72
|
-
earned += 1.0
|
|
73
|
-
|
|
74
|
-
if record.get("plan_block_in_prompt"):
|
|
75
|
-
possible += 0.25
|
|
76
|
-
if record.get("router_cli_invoked"):
|
|
77
|
-
earned += 0.25
|
|
78
|
-
|
|
79
|
-
return 1.0 if possible == 0 else earned / possible
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def migrate_record(raw: dict[str, Any]) -> dict[str, Any]:
|
|
83
|
-
routes = raw.get("routes") or raw.get("routes_in_plan") or []
|
|
84
|
-
if not isinstance(routes, list):
|
|
85
|
-
routes = []
|
|
86
|
-
tools = raw.get("tools_called") or []
|
|
87
|
-
if not isinstance(tools, list):
|
|
88
|
-
tools = []
|
|
89
|
-
platform = str(raw.get("platform", "cursor"))
|
|
90
|
-
classified = classify_tool_calls([str(t) for t in tools], platform=platform)
|
|
91
|
-
|
|
92
|
-
structural_in = raw.get("structural_in_plan")
|
|
93
|
-
if not isinstance(structural_in, bool):
|
|
94
|
-
structural_in = structural_routes_in_plan([str(r) for r in routes])
|
|
95
|
-
|
|
96
|
-
semantic_in = raw.get("semantic_in_plan")
|
|
97
|
-
if not isinstance(semantic_in, bool):
|
|
98
|
-
semantic_in = semantic_routes_in_plan([str(r) for r in routes])
|
|
99
|
-
|
|
100
|
-
record: dict[str, Any] = {
|
|
101
|
-
"schema_version": SCHEMA_VERSION,
|
|
102
|
-
"query_id": str(raw.get("query_id", "")),
|
|
103
|
-
"dataset": str(raw.get("dataset", "unknown")),
|
|
104
|
-
"query_text": str(raw.get("query_text", "")),
|
|
105
|
-
"run_id": str(raw.get("run_id", "")),
|
|
106
|
-
"platform": platform,
|
|
107
|
-
"semantic_in_plan": bool(semantic_in),
|
|
108
|
-
"semantic_order": raw.get("semantic_order")
|
|
109
|
-
if isinstance(raw.get("semantic_order"), int)
|
|
110
|
-
else None,
|
|
111
|
-
"structural_in_plan": bool(structural_in),
|
|
112
|
-
"codegraph_in_plan": bool(raw.get("codegraph_in_plan", structural_in)),
|
|
113
|
-
"fallback_hint_present": bool(raw.get("fallback_hint_present")),
|
|
114
|
-
"intents": list(raw.get("intents") or []),
|
|
115
|
-
"routes": [str(r) for r in routes],
|
|
116
|
-
"routes_in_plan": [str(r) for r in routes],
|
|
117
|
-
"project_file_count": raw.get("project_file_count")
|
|
118
|
-
if isinstance(raw.get("project_file_count"), int)
|
|
119
|
-
else None,
|
|
120
|
-
"tools_called": classified.tools_called,
|
|
121
|
-
"grep_count": raw.get("grep_count")
|
|
122
|
-
if isinstance(raw.get("grep_count"), int)
|
|
123
|
-
else classified.grep_count,
|
|
124
|
-
"read_count": raw.get("read_count")
|
|
125
|
-
if isinstance(raw.get("read_count"), int)
|
|
126
|
-
else classified.read_count,
|
|
127
|
-
"codegraph_attempted": bool(
|
|
128
|
-
raw.get("codegraph_attempted", classified.codegraph_attempted)
|
|
129
|
-
),
|
|
130
|
-
"codegraph_executed": bool(
|
|
131
|
-
raw.get("codegraph_executed", classified.codegraph_executed)
|
|
132
|
-
),
|
|
133
|
-
"router_cli_invoked": bool(
|
|
134
|
-
raw.get("router_cli_invoked", classified.router_cli_invoked)
|
|
135
|
-
),
|
|
136
|
-
"plan_block_in_prompt": bool(raw.get("plan_block_in_prompt")),
|
|
137
|
-
"read_verification_done": bool(
|
|
138
|
-
raw.get("read_verification_done", classified.read_count > 0)
|
|
139
|
-
),
|
|
140
|
-
"semantic_attempted": bool(
|
|
141
|
-
raw.get("semantic_attempted", classified.semantic_attempted)
|
|
142
|
-
),
|
|
143
|
-
"semantic_executed": bool(
|
|
144
|
-
raw.get("semantic_executed", classified.semantic_executed)
|
|
145
|
-
),
|
|
146
|
-
"semantic_outcome": str(raw.get("semantic_outcome", "unknown")),
|
|
147
|
-
"semantic_success": bool(raw.get("semantic_success")),
|
|
148
|
-
"semantic_skip_reason": raw.get("semantic_skip_reason"),
|
|
149
|
-
"rg_candidate_count": raw.get("rg_candidate_count")
|
|
150
|
-
if isinstance(raw.get("rg_candidate_count"), int)
|
|
151
|
-
else None,
|
|
152
|
-
"rg_corrob_status": str(raw.get("rg_corrob_status", "unknown")),
|
|
153
|
-
"trap_only": bool(raw.get("trap_only")),
|
|
154
|
-
"corroborated_files": list(raw.get("corroborated_files") or []),
|
|
155
|
-
"adapter_errors": list(raw.get("adapter_errors") or []),
|
|
156
|
-
"candidate_pool_recall": raw.get("candidate_pool_recall")
|
|
157
|
-
if isinstance(raw.get("candidate_pool_recall"), (int, float))
|
|
158
|
-
else None,
|
|
159
|
-
"final_top_k_recall": raw.get("final_top_k_recall")
|
|
160
|
-
if isinstance(raw.get("final_top_k_recall"), (int, float))
|
|
161
|
-
else None,
|
|
162
|
-
"answer_score": raw.get("answer_score")
|
|
163
|
-
if isinstance(raw.get("answer_score"), (int, float))
|
|
164
|
-
else None,
|
|
165
|
-
"compliance_score": raw.get("compliance_score")
|
|
166
|
-
if isinstance(raw.get("compliance_score"), (int, float))
|
|
167
|
-
else None,
|
|
168
|
-
"platform_semantic_executed": bool(
|
|
169
|
-
raw.get("platform_semantic_executed", classified.platform_semantic_executed)
|
|
170
|
-
),
|
|
171
|
-
"fast_context_count": int(raw.get("fast_context_count", classified.fast_context_count)),
|
|
172
|
-
"cursor_fast_context_misuse": bool(
|
|
173
|
-
raw.get("cursor_fast_context_misuse", classified.cursor_fast_context_misuse)
|
|
174
|
-
),
|
|
175
|
-
}
|
|
176
|
-
if record["compliance_score"] is None:
|
|
177
|
-
record["compliance_score"] = compute_compliance_score(record)
|
|
178
|
-
return record
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
def derive_metrics(records: list[dict[str, Any]]) -> dict[str, Any]:
|
|
182
|
-
total = len(records)
|
|
183
|
-
outcome_counts = {k: 0 for k in SEMANTIC_OUTCOMES}
|
|
184
|
-
skip_counts = {k: 0 for k in SEMANTIC_SKIP_REASONS}
|
|
185
|
-
|
|
186
|
-
semantic_plan = semantic_exec = semantic_attempt = semantic_success = 0
|
|
187
|
-
platform_sem_exec = fast_context_total = fast_context_misuse = 0
|
|
188
|
-
codegraph_plan = codegraph_exec = router_cli = plan_block = read_verify = 0
|
|
189
|
-
compliance_scores: list[float] = []
|
|
190
|
-
answer_scores: list[float] = []
|
|
191
|
-
pool_recalls: list[float] = []
|
|
192
|
-
topk_recalls: list[float] = []
|
|
193
|
-
|
|
194
|
-
for rec in records:
|
|
195
|
-
if rec.get("semantic_in_plan"):
|
|
196
|
-
semantic_plan += 1
|
|
197
|
-
if rec.get("semantic_executed"):
|
|
198
|
-
semantic_exec += 1
|
|
199
|
-
if rec.get("platform_semantic_executed"):
|
|
200
|
-
platform_sem_exec += 1
|
|
201
|
-
fc = rec.get("fast_context_count")
|
|
202
|
-
if isinstance(fc, int) and fc > 0:
|
|
203
|
-
fast_context_total += fc
|
|
204
|
-
if rec.get("cursor_fast_context_misuse"):
|
|
205
|
-
fast_context_misuse += 1
|
|
206
|
-
if rec.get("semantic_attempted"):
|
|
207
|
-
semantic_attempt += 1
|
|
208
|
-
if rec.get("semantic_success") and rec.get("semantic_outcome") == "success":
|
|
209
|
-
semantic_success += 1
|
|
210
|
-
if rec.get("codegraph_in_plan"):
|
|
211
|
-
codegraph_plan += 1
|
|
212
|
-
if rec.get("codegraph_executed"):
|
|
213
|
-
codegraph_exec += 1
|
|
214
|
-
if rec.get("router_cli_invoked"):
|
|
215
|
-
router_cli += 1
|
|
216
|
-
if rec.get("plan_block_in_prompt"):
|
|
217
|
-
plan_block += 1
|
|
218
|
-
if rec.get("read_verification_done"):
|
|
219
|
-
read_verify += 1
|
|
220
|
-
|
|
221
|
-
outcome = str(rec.get("semantic_outcome", "unknown"))
|
|
222
|
-
if outcome in outcome_counts:
|
|
223
|
-
outcome_counts[outcome] += 1
|
|
224
|
-
skip = rec.get("semantic_skip_reason")
|
|
225
|
-
if skip and skip in skip_counts:
|
|
226
|
-
skip_counts[skip] += 1
|
|
227
|
-
|
|
228
|
-
cs = rec.get("compliance_score")
|
|
229
|
-
if isinstance(cs, (int, float)):
|
|
230
|
-
compliance_scores.append(float(cs))
|
|
231
|
-
ans = rec.get("answer_score")
|
|
232
|
-
if isinstance(ans, (int, float)):
|
|
233
|
-
answer_scores.append(float(ans))
|
|
234
|
-
cpr = rec.get("candidate_pool_recall")
|
|
235
|
-
if isinstance(cpr, (int, float)):
|
|
236
|
-
pool_recalls.append(float(cpr))
|
|
237
|
-
ftr = rec.get("final_top_k_recall")
|
|
238
|
-
if isinstance(ftr, (int, float)):
|
|
239
|
-
topk_recalls.append(float(ftr))
|
|
240
|
-
|
|
241
|
-
avg_pool = _avg(pool_recalls)
|
|
242
|
-
avg_topk = _avg(topk_recalls)
|
|
243
|
-
recall_drop = 0.0 if avg_pool <= 0 else 1.0 - avg_topk / avg_pool
|
|
244
|
-
|
|
245
|
-
return {
|
|
246
|
-
"schema_version": SCHEMA_VERSION,
|
|
247
|
-
"total_queries": total,
|
|
248
|
-
"semantic_plan_count": semantic_plan,
|
|
249
|
-
"semantic_exec_count": semantic_exec,
|
|
250
|
-
"semantic_attempt_count": semantic_attempt,
|
|
251
|
-
"semantic_exec_success_count": semantic_success,
|
|
252
|
-
"semantic_plan_rate": _rate(semantic_plan, total),
|
|
253
|
-
"semantic_exec_rate": _rate(semantic_exec, total),
|
|
254
|
-
"platform_semantic_exec_count": platform_sem_exec,
|
|
255
|
-
"platform_semantic_exec_rate": _rate(platform_sem_exec, total),
|
|
256
|
-
"fast_context_invocation_total": fast_context_total,
|
|
257
|
-
"cursor_fast_context_misuse_count": fast_context_misuse,
|
|
258
|
-
"cursor_fast_context_misuse_rate": _rate(fast_context_misuse, total),
|
|
259
|
-
"semantic_attempt_rate": _rate(semantic_attempt, total),
|
|
260
|
-
"semantic_exec_success_rate": _rate(semantic_success, total),
|
|
261
|
-
"codegraph_plan_count": codegraph_plan,
|
|
262
|
-
"codegraph_exec_count": codegraph_exec,
|
|
263
|
-
"codegraph_plan_rate": _rate(codegraph_plan, total),
|
|
264
|
-
"codegraph_exec_rate": _rate(codegraph_exec, total),
|
|
265
|
-
"router_cli_count": router_cli,
|
|
266
|
-
"router_cli_rate": _rate(router_cli, total),
|
|
267
|
-
"plan_block_count": plan_block,
|
|
268
|
-
"plan_block_rate": _rate(plan_block, total),
|
|
269
|
-
"read_verification_count": read_verify,
|
|
270
|
-
"read_verification_rate": _rate(read_verify, total),
|
|
271
|
-
"avg_compliance_score": _avg(compliance_scores),
|
|
272
|
-
"avg_answer_score": _avg(answer_scores),
|
|
273
|
-
"semantic_outcome_counts": outcome_counts,
|
|
274
|
-
"semantic_skip_reason_counts": skip_counts,
|
|
275
|
-
"avg_candidate_pool_recall": avg_pool,
|
|
276
|
-
"avg_final_top_k_recall": avg_topk,
|
|
277
|
-
"recall_drop_rate": recall_drop,
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
def load_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
282
|
-
records: list[dict[str, Any]] = []
|
|
283
|
-
for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
|
284
|
-
stripped = line.strip()
|
|
285
|
-
if not stripped or stripped.startswith("#"):
|
|
286
|
-
continue
|
|
287
|
-
try:
|
|
288
|
-
raw = json.loads(stripped)
|
|
289
|
-
except json.JSONDecodeError as exc:
|
|
290
|
-
raise ValueError(f"{path}:{line_no}: invalid JSON: {exc}") from exc
|
|
291
|
-
if not isinstance(raw, dict):
|
|
292
|
-
raise ValueError(f"{path}:{line_no}: expected JSON object per line")
|
|
293
|
-
records.append(migrate_record(raw))
|
|
294
|
-
return records
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
def main() -> int:
|
|
298
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
299
|
-
parser.add_argument(
|
|
300
|
-
"jsonl",
|
|
301
|
-
type=Path,
|
|
302
|
-
help="Per-query telemetry JSONL (one object per line)",
|
|
303
|
-
)
|
|
304
|
-
parser.add_argument(
|
|
305
|
-
"--markdown",
|
|
306
|
-
action="store_true",
|
|
307
|
-
help="Print a short markdown summary for eval reports",
|
|
308
|
-
)
|
|
309
|
-
args = parser.parse_args()
|
|
310
|
-
|
|
311
|
-
if not args.jsonl.is_file():
|
|
312
|
-
print(f"error: file not found: {args.jsonl}", file=sys.stderr)
|
|
313
|
-
return 1
|
|
314
|
-
|
|
315
|
-
records = load_jsonl(args.jsonl)
|
|
316
|
-
metrics = derive_metrics(records)
|
|
317
|
-
payload = {"metrics": metrics, "record_count": len(records)}
|
|
318
|
-
|
|
319
|
-
if args.markdown:
|
|
320
|
-
m = metrics
|
|
321
|
-
print("## Retrieval telemetry (derived from JSONL)\n")
|
|
322
|
-
print("| Metric | Value |")
|
|
323
|
-
print("| --- | ---: |")
|
|
324
|
-
print(f"| total_queries | {m['total_queries']} |")
|
|
325
|
-
print(f"| semantic_plan_rate | {m['semantic_plan_rate']:.1%} |")
|
|
326
|
-
print(f"| semantic_exec_rate | {m['semantic_exec_rate']:.1%} |")
|
|
327
|
-
print(f"| platform_semantic_exec_rate | {m.get('platform_semantic_exec_rate', 0):.1%} |")
|
|
328
|
-
print(f"| cursor_fast_context_misuse_rate | {m.get('cursor_fast_context_misuse_rate', 0):.1%} |")
|
|
329
|
-
print(f"| codegraph_plan_rate | {m['codegraph_plan_rate']:.1%} |")
|
|
330
|
-
print(f"| codegraph_exec_rate | {m['codegraph_exec_rate']:.1%} |")
|
|
331
|
-
print(f"| router_cli_rate | {m['router_cli_rate']:.1%} |")
|
|
332
|
-
print(f"| avg_compliance_score | {m['avg_compliance_score']:.3f} |")
|
|
333
|
-
print(f"| avg_answer_score | {m['avg_answer_score']:.3f} |")
|
|
334
|
-
print("\nDo not hand-copy plan/exec rates; regenerate from JSONL with this script.")
|
|
335
|
-
else:
|
|
336
|
-
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
337
|
-
|
|
338
|
-
return 0
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
if __name__ == "__main__":
|
|
342
|
-
raise SystemExit(main())
|