@bendyline/gilde 0.1.53 → 0.1.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/authoring/chat-models/qwen3.5-27b-q4.json +147 -0
- package/authoring/gstack/evals/cso.json +21 -18
- package/authoring/gstack/evals/investigate.json +25 -22
- package/authoring/gstack/evals/plan-ceo-review.json +19 -16
- package/authoring/gstack/overlays/retro.json +2 -2
- package/authoring/gstack/wave.json +2 -2
- package/data/chat-models/index.json +1 -1
- package/data/chat-models/qw/qwen3.5-27b-q4/manifest.json +217 -0
- package/data/chat-models/qw/qwen3.5-27b-q4/versions/1.0.0/manifest.json +90 -0
- package/data/craftbook-templates/br/browser-qa-audit/versions/2.0.7/craftbook.json +620 -0
- package/data/craftbook-templates/br/browser-qa-audit/versions/2.0.7/test.json +376 -0
- package/data/craftbook-templates/de/design-system-consultation/versions/2.0.7/craftbook.json +616 -0
- package/data/craftbook-templates/de/design-system-consultation/versions/2.0.7/test.json +201 -0
- package/data/craftbook-templates/en/engineering-retrospective/versions/2.0.7/craftbook.json +566 -0
- package/data/craftbook-templates/en/engineering-retrospective/versions/2.0.7/test.json +191 -0
- package/data/craftbook-templates/ex/executive-level-review/versions/2.0.7/craftbook.json +595 -0
- package/data/craftbook-templates/ex/executive-level-review/versions/2.0.7/test.json +139 -0
- package/data/craftbook-templates/id/idea-office-hours/versions/2.0.7/craftbook.json +566 -0
- package/data/craftbook-templates/id/idea-office-hours/versions/2.0.7/test.json +141 -0
- package/data/craftbook-templates/index.json +1 -1
- package/data/craftbook-templates/pl/plan/versions/1.0.2/craftbook.json +222 -0
- package/data/craftbook-templates/pl/plan/versions/1.0.2/test.json +91 -0
- package/data/craftbook-templates/ro/root-cause-investigation/versions/2.0.7/craftbook.json +595 -0
- package/data/craftbook-templates/ro/root-cause-investigation/versions/2.0.7/test.json +157 -0
- package/data/craftbook-templates/se/security-architecture-review/versions/2.0.7/craftbook.json +597 -0
- package/data/craftbook-templates/se/security-architecture-review/versions/2.0.7/test.json +157 -0
- package/data/craftbook-templates/sh/ship/versions/1.1.2/craftbook.json +346 -0
- package/data/craftbook-templates/sh/ship/versions/1.1.2/test.json +228 -0
- package/data/craftbook-templates/sp/spec-authoring/versions/2.0.7/craftbook.json +599 -0
- package/data/craftbook-templates/sp/spec-authoring/versions/2.0.7/test.json +162 -0
- package/data/craftbook-templates/te/technical-documentation/versions/2.0.7/craftbook.json +577 -0
- package/data/craftbook-templates/te/technical-documentation/versions/2.0.7/test.json +174 -0
- package/package.json +1 -1
- package/schemas/craftbook-test.schema.json +7 -1
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "qwen3.5-27b-q4",
|
|
3
|
+
"name": "Qwen 3.5 (27B, Q4)",
|
|
4
|
+
"description": "Alibaba's dense Qwen 3.5 model at 27B parameters. Multimodal (text + image + video), native tool calling, and 262K native context. A capable on-device option for machines with a 24GB+ inference budget.",
|
|
5
|
+
"tags": [
|
|
6
|
+
"alibaba",
|
|
7
|
+
"multimodal",
|
|
8
|
+
"vision",
|
|
9
|
+
"video",
|
|
10
|
+
"tools",
|
|
11
|
+
"long-context"
|
|
12
|
+
],
|
|
13
|
+
"category": "general",
|
|
14
|
+
"maintainer": {
|
|
15
|
+
"name": "Alibaba",
|
|
16
|
+
"url": "https://huggingface.co/Qwen/Qwen3.5-27B"
|
|
17
|
+
},
|
|
18
|
+
"version": "1.0.0",
|
|
19
|
+
"updatedAt": "2026-09-04T00:00:00Z",
|
|
20
|
+
"license": "Apache-2.0",
|
|
21
|
+
"licenseClass": "open",
|
|
22
|
+
"licenseShortName": "Apache 2.0",
|
|
23
|
+
"parameterSize": "27B",
|
|
24
|
+
"approxSizeBytes": 17106773120,
|
|
25
|
+
"supportsTools": true,
|
|
26
|
+
"contextWindow": 262144,
|
|
27
|
+
"upstream": "https://huggingface.co/Qwen/Qwen3.5-27B",
|
|
28
|
+
"style": {
|
|
29
|
+
"family": "qwen",
|
|
30
|
+
"reasoningFormat": "think",
|
|
31
|
+
"toolCallFormat": "function-call"
|
|
32
|
+
},
|
|
33
|
+
"behaviors": [
|
|
34
|
+
"reasoning.strip-think-tags",
|
|
35
|
+
"prompt.private-reasoning-guidance",
|
|
36
|
+
"prompt.tool-cookbook-condensed",
|
|
37
|
+
"prompt.meester-build-prelude",
|
|
38
|
+
"fabrication.detect-past-tense-no-tools",
|
|
39
|
+
"fabrication.detect-claim-without-tool",
|
|
40
|
+
"mcp.compact-tool-schemas",
|
|
41
|
+
"mcp.relax-required-fields",
|
|
42
|
+
"mcp.default-missing-fields",
|
|
43
|
+
"turn.ollama-num-predict-bumped",
|
|
44
|
+
"turn.preamble-folding",
|
|
45
|
+
"turn.ramble-detection",
|
|
46
|
+
"tools.mlx-grammar",
|
|
47
|
+
"provider.merge-system-messages",
|
|
48
|
+
"validate.inline-js-parses",
|
|
49
|
+
"prompt.prefer-writefile-edits",
|
|
50
|
+
"prompt.derive-by-execution"
|
|
51
|
+
],
|
|
52
|
+
"tuning": {
|
|
53
|
+
"sampling": {
|
|
54
|
+
"temperature": 0.7,
|
|
55
|
+
"topP": 0.8,
|
|
56
|
+
"topK": 20,
|
|
57
|
+
"minP": 0,
|
|
58
|
+
"maxTokens": 16384,
|
|
59
|
+
"repetitionPenalty": 1.1,
|
|
60
|
+
"repetitionContext": 20
|
|
61
|
+
},
|
|
62
|
+
"samplingWhenThinking": {
|
|
63
|
+
"temperature": 0.6,
|
|
64
|
+
"topP": 0.95
|
|
65
|
+
},
|
|
66
|
+
"reasoning": {
|
|
67
|
+
"enableThinking": true,
|
|
68
|
+
"thinkingBudget": 2048
|
|
69
|
+
},
|
|
70
|
+
"promptTags": {
|
|
71
|
+
"enableThinkingTag": "/think",
|
|
72
|
+
"disableThinkingTag": "/no_think"
|
|
73
|
+
},
|
|
74
|
+
"profiles": {
|
|
75
|
+
"thinking-general": {
|
|
76
|
+
"sampling": {
|
|
77
|
+
"temperature": 0.6,
|
|
78
|
+
"topP": 0.95,
|
|
79
|
+
"topK": 20,
|
|
80
|
+
"repetitionPenalty": 1.05
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
"thinking-coding": {
|
|
84
|
+
"sampling": {
|
|
85
|
+
"temperature": 0.5,
|
|
86
|
+
"topP": 0.9,
|
|
87
|
+
"topK": 20,
|
|
88
|
+
"repetitionPenalty": 1.05,
|
|
89
|
+
"repetitionContext": 32,
|
|
90
|
+
"dry": {
|
|
91
|
+
"multiplier": 0.8,
|
|
92
|
+
"base": 1.75,
|
|
93
|
+
"allowedLength": 2
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
},
|
|
97
|
+
"thinking-precise": {
|
|
98
|
+
"sampling": {
|
|
99
|
+
"temperature": 0.3,
|
|
100
|
+
"topP": 0.9,
|
|
101
|
+
"topK": 10,
|
|
102
|
+
"repetitionPenalty": 1.05
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
"instruct": {
|
|
106
|
+
"sampling": {
|
|
107
|
+
"temperature": 0.7,
|
|
108
|
+
"topP": 0.8,
|
|
109
|
+
"topK": 20,
|
|
110
|
+
"repetitionPenalty": 1.1
|
|
111
|
+
}
|
|
112
|
+
},
|
|
113
|
+
"creative": {
|
|
114
|
+
"sampling": {
|
|
115
|
+
"temperature": 0.9,
|
|
116
|
+
"topP": 0.95,
|
|
117
|
+
"topK": 40,
|
|
118
|
+
"repetitionPenalty": 1.1
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
},
|
|
123
|
+
"ollama": {
|
|
124
|
+
"tag": "qwen3.5:27b"
|
|
125
|
+
},
|
|
126
|
+
"llamaCpp": {
|
|
127
|
+
"huggingfaceRepo": "unsloth/Qwen3.5-27B-MTP-GGUF",
|
|
128
|
+
"filename": "Qwen3.5-27B-Q4_K_M.gguf",
|
|
129
|
+
"quantization": "Q4_K_M"
|
|
130
|
+
},
|
|
131
|
+
"releasePatch": {
|
|
132
|
+
"tuning": {
|
|
133
|
+
"engine": {
|
|
134
|
+
"llamaCpp": {
|
|
135
|
+
"spec": {
|
|
136
|
+
"mtp": true,
|
|
137
|
+
"nMax": 4
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
},
|
|
143
|
+
"mlx": {
|
|
144
|
+
"huggingfaceRepo": "mlx-community/Qwen3.5-27B-4bit",
|
|
145
|
+
"quantization": "4bit"
|
|
146
|
+
}
|
|
147
|
+
}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"architecture-review",
|
|
9
9
|
"typescript"
|
|
10
10
|
],
|
|
11
|
-
"prompt": "Use the Security Architecture Review craftbook to review the seeded service slice. This is report-only: do not modify source files. Model assets, actors, trust boundaries, and data flows; audit authorization, command/SQL construction, secrets handling, and relevant common vulnerability classes. Produce the scope and findings register, then write `security/security-architecture-review.md`. Every confirmed finding must name severity, confidence, concrete workspace evidence, exploit preconditions, impact, and a specific remediation plus verification. Include working controls and rejected hypotheses. Do not invent CVEs, dependency versions, scans, or tests you did not run, and do not reproduce the example secret value in the report.",
|
|
11
|
+
"prompt": "Use the Security Architecture Review craftbook to review the seeded service slice. This is report-only: do not modify source files. Model assets, actors, trust boundaries, and data flows; audit authorization, command/SQL construction, secrets handling, and relevant common vulnerability classes. Produce the scope and findings register, then write `tasks/eval/security/security-architecture-review.md`. Every confirmed finding must name severity, confidence, concrete workspace evidence, exploit preconditions, impact, and a specific remediation plus verification. Include working controls and rejected hypotheses. Do not invent CVEs, dependency versions, scans, or tests you did not run, and do not reproduce the example secret value in the report.",
|
|
12
12
|
"setup": {
|
|
13
13
|
"projectName": "Admin export security review",
|
|
14
14
|
"about": "A hermetic source-review fixture. No network or dependency lookup is needed or authorized; conclusions must come from seeded files.",
|
|
@@ -39,64 +39,63 @@
|
|
|
39
39
|
"path": "config/example.env",
|
|
40
40
|
"content": "# Example only; deployment replaces this value\nSESSION_SECRET=local-dev-only-not-a-real-secret\n"
|
|
41
41
|
}
|
|
42
|
-
]
|
|
42
|
+
],
|
|
43
|
+
"craftbookParams": {
|
|
44
|
+
"workPath": "tasks/eval"
|
|
45
|
+
}
|
|
43
46
|
},
|
|
44
47
|
"mocks": [],
|
|
45
48
|
"success": {
|
|
46
|
-
"taskGraph": {
|
|
47
|
-
"requireCraftbookTask": true,
|
|
48
|
-
"requireTerminalStep": true
|
|
49
|
-
},
|
|
50
49
|
"summary": "The named craftbook produces a source-grounded risk posture with prioritized findings, positive controls, remediation, verification, and terminal task evidence.",
|
|
51
50
|
"deliverables": [
|
|
52
51
|
{
|
|
53
|
-
"path": "security/security-architecture-review.md",
|
|
52
|
+
"path": "tasks/eval/security/security-architecture-review.md",
|
|
54
53
|
"kind": "security-report",
|
|
55
54
|
"artifact": true,
|
|
56
55
|
"minBytes": 1700,
|
|
57
56
|
"checks": [
|
|
58
57
|
{
|
|
59
58
|
"kind": "contains",
|
|
60
|
-
"file": "security/security-architecture-review.md",
|
|
59
|
+
"file": "tasks/eval/security/security-architecture-review.md",
|
|
61
60
|
"pattern": "^#{1,3}\\s+Executive risk posture\\b[\\s\\S]*^#{1,3}\\s+What is working\\b[\\s\\S]*^#{1,3}\\s+Prioritized findings\\b[\\s\\S]*^#{1,3}\\s+Remediation plan\\b[\\s\\S]*^#{1,3}\\s+Verification\\b[\\s\\S]*^#{1,3}\\s+Unverified areas\\b",
|
|
62
61
|
"flags": "im",
|
|
63
62
|
"label": "complete security-review structure"
|
|
64
63
|
},
|
|
65
64
|
{
|
|
66
65
|
"kind": "contains",
|
|
67
|
-
"file": "security/security-architecture-review.md",
|
|
66
|
+
"file": "tasks/eval/security/security-architecture-review.md",
|
|
68
67
|
"pattern": "(command injection|shell injection)[\\s\\S]{0,500}src/admin-export\\.ts[\\s\\S]*(severity)[\\s\\S]*(confidence)",
|
|
69
68
|
"flags": "is",
|
|
70
69
|
"label": "command-injection finding with evidence metadata"
|
|
71
70
|
},
|
|
72
71
|
{
|
|
73
72
|
"kind": "contains",
|
|
74
|
-
"file": "security/security-architecture-review.md",
|
|
73
|
+
"file": "tasks/eval/security/security-architecture-review.md",
|
|
75
74
|
"pattern": "(SQL injection|parameteri[sz]ed quer)[\\s\\S]{0,500}src/customer-search\\.ts",
|
|
76
75
|
"flags": "is",
|
|
77
76
|
"label": "SQL-injection finding and remediation"
|
|
78
77
|
},
|
|
79
78
|
{
|
|
80
79
|
"kind": "contains",
|
|
81
|
-
"file": "security/security-architecture-review.md",
|
|
80
|
+
"file": "tasks/eval/security/security-architecture-review.md",
|
|
82
81
|
"pattern": "(requireUser|requireAdmin)[\\s\\S]{0,300}(working|control|authorization|authentication)[\\s\\S]*(safe-audit-log|audit)",
|
|
83
82
|
"flags": "is",
|
|
84
83
|
"label": "positive auth and audit controls"
|
|
85
84
|
},
|
|
86
85
|
{
|
|
87
86
|
"kind": "citationsResolve",
|
|
88
|
-
"file": "security/security-architecture-review.md",
|
|
87
|
+
"file": "tasks/eval/security/security-architecture-review.md",
|
|
89
88
|
"minCitations": 4
|
|
90
89
|
},
|
|
91
90
|
{
|
|
92
91
|
"kind": "notContains",
|
|
93
|
-
"file": "security/security-architecture-review.md",
|
|
92
|
+
"file": "tasks/eval/security/security-architecture-review.md",
|
|
94
93
|
"pattern": "CVE-\\d{4}-\\d+",
|
|
95
94
|
"label": "no invented CVEs"
|
|
96
95
|
},
|
|
97
96
|
{
|
|
98
97
|
"kind": "notContains",
|
|
99
|
-
"file": "security/security-architecture-review.md",
|
|
98
|
+
"file": "tasks/eval/security/security-architecture-review.md",
|
|
100
99
|
"pattern": "local-dev-only-not-a-real-secret",
|
|
101
100
|
"label": "example secret value is not reproduced"
|
|
102
101
|
}
|
|
@@ -105,16 +104,20 @@
|
|
|
105
104
|
],
|
|
106
105
|
"taskNotes": {
|
|
107
106
|
"minBytes": 180,
|
|
108
|
-
"requireCraftbookTask": true,
|
|
109
107
|
"checks": [
|
|
110
108
|
{
|
|
111
109
|
"kind": "contains",
|
|
112
110
|
"file": "task-notes.md",
|
|
113
|
-
"pattern": "\\bDONE\\b[\\s\\S]*security/security-architecture-review\\.md[\\s\\S]*(command|SQL)",
|
|
111
|
+
"pattern": "\\bDONE\\b[\\s\\S]*tasks/eval/security/security-architecture-review\\.md[\\s\\S]*(command|SQL)",
|
|
114
112
|
"flags": "i",
|
|
115
113
|
"label": "terminal note records report and principal risks"
|
|
116
114
|
}
|
|
117
|
-
]
|
|
115
|
+
],
|
|
116
|
+
"requireCraftbookTask": true
|
|
117
|
+
},
|
|
118
|
+
"taskGraph": {
|
|
119
|
+
"requireCraftbookTask": true,
|
|
120
|
+
"requireTerminalStep": true
|
|
118
121
|
},
|
|
119
122
|
"unchangedFixtures": [
|
|
120
123
|
"docs/architecture.md",
|
|
@@ -127,7 +130,7 @@
|
|
|
127
130
|
},
|
|
128
131
|
"rubric": {
|
|
129
132
|
"artifact": {
|
|
130
|
-
"path": "security/security-architecture-review.md",
|
|
133
|
+
"path": "tasks/eval/security/security-architecture-review.md",
|
|
131
134
|
"kind": "markdown"
|
|
132
135
|
},
|
|
133
136
|
"axes": [
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"code-with-tests",
|
|
9
9
|
"node"
|
|
10
10
|
],
|
|
11
|
-
"prompt": "Use the Root-Cause Investigation craftbook to diagnose the failing decimal cart-total behavior in this workspace. First reproduce it with the existing dependency-free test and preserve concrete evidence; do not edit until a causal hypothesis has been tested. Then make the smallest correct source change, keep the regression coverage, and write the craftbook's artifacts, including `reports/root-cause-investigation.md`. The final report must cite the files inspected, distinguish root cause from symptom, include the exact verification command and result, and describe a safe rollback.",
|
|
11
|
+
"prompt": "Use the Root-Cause Investigation craftbook to diagnose the failing decimal cart-total behavior in this workspace. First reproduce it with the existing dependency-free test and preserve concrete evidence; do not edit until a causal hypothesis has been tested. Then make the smallest correct source change, keep the regression coverage, and write the craftbook's artifacts, including `tasks/eval/reports/root-cause-investigation.md`. The final report must cite the files inspected, distinguish root cause from symptom, include the exact verification command and result, and describe a safe rollback.",
|
|
12
12
|
"setup": {
|
|
13
13
|
"projectName": "Decimal cart total regression",
|
|
14
14
|
"about": "A tiny dependency-free Node project with one reproducible arithmetic defect. The initial test is expected to fail.",
|
|
@@ -35,46 +35,45 @@
|
|
|
35
35
|
"content": "import assert from 'node:assert/strict';\nimport { cartTotal } from '../src/cart-total.js';\n\nconst cases = [\n [['19.99', '5.50'], '25.49'],\n [['0.10', '0.20'], '0.30'],\n [['12.34', '0.66'], '13.00'],\n [['7.00'], '7.00'],\n [['1.01', '2.02', '3.03'], '6.06'],\n];\nfor (const [prices, expected] of cases) assert.equal(cartTotal(prices), expected, prices.join(' + '));\nconsole.log('CART_TOTAL_ORACLE_OK 5/5');\n",
|
|
36
36
|
"surface": "harness"
|
|
37
37
|
}
|
|
38
|
-
]
|
|
38
|
+
],
|
|
39
|
+
"craftbookParams": {
|
|
40
|
+
"workPath": "tasks/eval"
|
|
41
|
+
}
|
|
39
42
|
},
|
|
40
43
|
"mocks": [],
|
|
41
44
|
"success": {
|
|
42
|
-
"taskGraph": {
|
|
43
|
-
"requireCraftbookTask": true,
|
|
44
|
-
"requireTerminalStep": true
|
|
45
|
-
},
|
|
46
45
|
"summary": "The decimal regression is fixed by a diagnosed causal change, the existing test passes, and the craftbook report preserves reproducible evidence.",
|
|
47
46
|
"deliverables": [
|
|
48
47
|
{
|
|
49
|
-
"path": "reports/root-cause-investigation.md",
|
|
50
|
-
"artifact": true,
|
|
48
|
+
"path": "tasks/eval/reports/root-cause-investigation.md",
|
|
51
49
|
"kind": "markdown-report",
|
|
50
|
+
"artifact": true,
|
|
52
51
|
"minBytes": 1200,
|
|
53
52
|
"checks": [
|
|
54
53
|
{
|
|
55
54
|
"kind": "contains",
|
|
56
|
-
"file": "reports/root-cause-investigation.md",
|
|
55
|
+
"file": "tasks/eval/reports/root-cause-investigation.md",
|
|
57
56
|
"pattern": "^#{1,3}\\s+Root cause\\b[\\s\\S]*^#{1,3}\\s+(Fix|Changed files)\\b[\\s\\S]*^#{1,3}\\s+(Regression coverage|Verification)\\b[\\s\\S]*^#{1,3}\\s+Rollback\\b",
|
|
58
57
|
"flags": "im",
|
|
59
58
|
"label": "root cause through rollback sections"
|
|
60
59
|
},
|
|
61
60
|
{
|
|
62
61
|
"kind": "contains",
|
|
63
|
-
"file": "reports/root-cause-investigation.md",
|
|
62
|
+
"file": "tasks/eval/reports/root-cause-investigation.md",
|
|
64
63
|
"pattern": "parseInt|truncat(?:e|ed|ion)|discard(?:s|ed)?\\s+(?:the\\s+)?(?:decimal|fractional|cent)",
|
|
65
64
|
"flags": "i",
|
|
66
65
|
"label": "causal mechanism"
|
|
67
66
|
},
|
|
68
67
|
{
|
|
69
68
|
"kind": "contains",
|
|
70
|
-
"file": "reports/root-cause-investigation.md",
|
|
69
|
+
"file": "tasks/eval/reports/root-cause-investigation.md",
|
|
71
70
|
"pattern": "node\\s+--test\\s+tests/cart-total\\.test\\.mjs[\\s\\S]*(pass|2\\s+tests?|exit(?:ed)?\\s+0)",
|
|
72
71
|
"flags": "i",
|
|
73
72
|
"label": "verification command and result"
|
|
74
73
|
},
|
|
75
74
|
{
|
|
76
75
|
"kind": "citationsResolve",
|
|
77
|
-
"file": "reports/root-cause-investigation.md",
|
|
76
|
+
"file": "tasks/eval/reports/root-cause-investigation.md",
|
|
78
77
|
"minCitations": 2
|
|
79
78
|
}
|
|
80
79
|
]
|
|
@@ -107,28 +106,32 @@
|
|
|
107
106
|
]
|
|
108
107
|
}
|
|
109
108
|
],
|
|
110
|
-
"unchangedFixtures": [
|
|
111
|
-
"package.json",
|
|
112
|
-
"tests/cart-total.test.mjs",
|
|
113
|
-
"source/incident.md"
|
|
114
|
-
],
|
|
115
109
|
"taskNotes": {
|
|
116
110
|
"minBytes": 180,
|
|
117
|
-
"requireCraftbookTask": true,
|
|
118
111
|
"checks": [
|
|
119
112
|
{
|
|
120
113
|
"kind": "contains",
|
|
121
114
|
"file": "task-notes.md",
|
|
122
|
-
"pattern": "\\bDONE\\b[\\s\\S]*(node\\s+--test|tests/cart-total\\.test\\.mjs)[\\s\\S]*reports/root-cause-investigation\\.md",
|
|
115
|
+
"pattern": "\\bDONE\\b[\\s\\S]*(node\\s+--test|tests/cart-total\\.test\\.mjs)[\\s\\S]*tasks/eval/reports/root-cause-investigation\\.md",
|
|
123
116
|
"flags": "i",
|
|
124
117
|
"label": "terminal note records proof and report"
|
|
125
118
|
}
|
|
126
|
-
]
|
|
127
|
-
|
|
119
|
+
],
|
|
120
|
+
"requireCraftbookTask": true
|
|
121
|
+
},
|
|
122
|
+
"taskGraph": {
|
|
123
|
+
"requireCraftbookTask": true,
|
|
124
|
+
"requireTerminalStep": true
|
|
125
|
+
},
|
|
126
|
+
"unchangedFixtures": [
|
|
127
|
+
"package.json",
|
|
128
|
+
"tests/cart-total.test.mjs",
|
|
129
|
+
"source/incident.md"
|
|
130
|
+
]
|
|
128
131
|
},
|
|
129
132
|
"rubric": {
|
|
130
133
|
"artifact": {
|
|
131
|
-
"path": "reports/root-cause-investigation.md",
|
|
134
|
+
"path": "tasks/eval/reports/root-cause-investigation.md",
|
|
132
135
|
"kind": "markdown"
|
|
133
136
|
},
|
|
134
137
|
"axes": [
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"executive-review",
|
|
9
9
|
"decision"
|
|
10
10
|
],
|
|
11
|
-
"prompt": "Use the Executive-Level Review craftbook to review `source/analytics-proposal.md`. Challenge whether the proposed scope is the best way to reach the outcome; compare at least three alternatives including the two-day manual export and waiting. Produce all craftbook artifacts and a decision-ready `reviews/executive-level-review.md`. Use exactly one allowed mode verdict, score every required dimension from 0–10 with a factual basis, separate unknowns from evidence, and state explicit approve, pause, and stop conditions. Do not invent customer or financial evidence.",
|
|
11
|
+
"prompt": "Use the Executive-Level Review craftbook to review `source/analytics-proposal.md`. Challenge whether the proposed scope is the best way to reach the outcome; compare at least three alternatives including the two-day manual export and waiting. Produce all craftbook artifacts and a decision-ready `tasks/eval/reviews/executive-level-review.md`. Use exactly one allowed mode verdict, score every required dimension from 0–10 with a factual basis, separate unknowns from evidence, and state explicit approve, pause, and stop conditions. Do not invent customer or financial evidence.",
|
|
12
12
|
"setup": {
|
|
13
13
|
"projectName": "Analytics expansion review",
|
|
14
14
|
"about": "A hermetic executive decision review. The seeded proposal is the complete evidence set.",
|
|
@@ -18,39 +18,38 @@
|
|
|
18
18
|
"path": "source/analytics-proposal.md",
|
|
19
19
|
"content": "# Embedded analytics proposal\n\n## Requested decision\nApprove an eight-week build of customer-facing dashboards with scheduled PDF delivery and CSV export.\n\n## Evidence\nTwelve of 86 active customers requested better reporting in support conversations. Three customers agreed to a design pilot, but none signed a paid commitment. Support currently creates weekly exports for five customers.\n\n## Cost and constraints\n- Estimate: two engineers for 8 weeks.\n- New charting service: $4,000 per month at current volume.\n- Security review is complete; accessibility and data-retention reviews are not.\n- The enterprise renewal decision is in six weeks.\n- A manual scheduled-export workflow can be built in two engineer-days and would test delivery cadence, but not dashboard engagement.\n\n## Unknowns\nWillingness to pay, which metrics matter, PDF accessibility, and whether scheduled delivery or interactive dashboards drives retention.\n"
|
|
20
20
|
}
|
|
21
|
-
]
|
|
21
|
+
],
|
|
22
|
+
"craftbookParams": {
|
|
23
|
+
"workPath": "tasks/eval"
|
|
24
|
+
}
|
|
22
25
|
},
|
|
23
26
|
"mocks": [],
|
|
24
27
|
"success": {
|
|
25
|
-
"taskGraph": {
|
|
26
|
-
"requireCraftbookTask": true,
|
|
27
|
-
"requireTerminalStep": true
|
|
28
|
-
},
|
|
29
28
|
"summary": "The named craftbook produces a grounded executive verdict, alternatives, scorecard, and terminal task record.",
|
|
30
29
|
"deliverables": [
|
|
31
30
|
{
|
|
32
|
-
"path": "reviews/executive-level-review.md",
|
|
33
|
-
"artifact": true,
|
|
31
|
+
"path": "tasks/eval/reviews/executive-level-review.md",
|
|
34
32
|
"kind": "markdown-report",
|
|
33
|
+
"artifact": true,
|
|
35
34
|
"minBytes": 1400,
|
|
36
35
|
"checks": [
|
|
37
36
|
{
|
|
38
37
|
"kind": "contains",
|
|
39
|
-
"file": "reviews/executive-level-review.md",
|
|
38
|
+
"file": "tasks/eval/reviews/executive-level-review.md",
|
|
40
39
|
"pattern": "^#{1,3}\\s+Executive verdict\\b[\\s\\S]*(SCOPE EXPANSION|SELECTIVE|HOLD|SCOPE REDUCTION)[\\s\\S]*^#{1,3}\\s+Scorecard\\b[\\s\\S]*^#{1,3}\\s+Alternatives\\b[\\s\\S]*^#{1,3}\\s+Recommendation\\b[\\s\\S]*^#{1,3}\\s+Decision conditions\\b",
|
|
41
40
|
"flags": "im",
|
|
42
41
|
"label": "verdict, scorecard, alternatives, recommendation, and conditions"
|
|
43
42
|
},
|
|
44
43
|
{
|
|
45
44
|
"kind": "contains",
|
|
46
|
-
"file": "reviews/executive-level-review.md",
|
|
45
|
+
"file": "tasks/eval/reviews/executive-level-review.md",
|
|
47
46
|
"pattern": "(user value).*?(strategic fit).*?(evidence strength).*?(feasibility).*?(sequencing).*?(reversibility).*?(operating cost).*?(downside)",
|
|
48
47
|
"flags": "is",
|
|
49
48
|
"label": "complete executive scorecard"
|
|
50
49
|
},
|
|
51
50
|
{
|
|
52
51
|
"kind": "valueGrounding",
|
|
53
|
-
"file": "reviews/executive-level-review.md",
|
|
52
|
+
"file": "tasks/eval/reviews/executive-level-review.md",
|
|
54
53
|
"facts": [
|
|
55
54
|
{
|
|
56
55
|
"id": "customer-requests",
|
|
@@ -86,7 +85,7 @@
|
|
|
86
85
|
},
|
|
87
86
|
{
|
|
88
87
|
"kind": "citationsResolve",
|
|
89
|
-
"file": "reviews/executive-level-review.md",
|
|
88
|
+
"file": "tasks/eval/reviews/executive-level-review.md",
|
|
90
89
|
"minCitations": 1
|
|
91
90
|
}
|
|
92
91
|
]
|
|
@@ -94,15 +93,19 @@
|
|
|
94
93
|
],
|
|
95
94
|
"taskNotes": {
|
|
96
95
|
"minBytes": 160,
|
|
97
|
-
"requireCraftbookTask": true,
|
|
98
96
|
"checks": [
|
|
99
97
|
{
|
|
100
98
|
"kind": "contains",
|
|
101
99
|
"file": "task-notes.md",
|
|
102
|
-
"pattern": "\\bDONE\\b[\\s\\S]*reviews/executive-level-review\\.md",
|
|
100
|
+
"pattern": "\\bDONE\\b[\\s\\S]*tasks/eval/reviews/executive-level-review\\.md",
|
|
103
101
|
"label": "terminal craftbook note names the review"
|
|
104
102
|
}
|
|
105
|
-
]
|
|
103
|
+
],
|
|
104
|
+
"requireCraftbookTask": true
|
|
105
|
+
},
|
|
106
|
+
"taskGraph": {
|
|
107
|
+
"requireCraftbookTask": true,
|
|
108
|
+
"requireTerminalStep": true
|
|
106
109
|
},
|
|
107
110
|
"unchangedFixtures": [
|
|
108
111
|
"source/analytics-proposal.md"
|
|
@@ -110,7 +113,7 @@
|
|
|
110
113
|
},
|
|
111
114
|
"rubric": {
|
|
112
115
|
"artifact": {
|
|
113
|
-
"path": "reviews/executive-level-review.md",
|
|
116
|
+
"path": "tasks/eval/reviews/executive-level-review.md",
|
|
114
117
|
"kind": "markdown"
|
|
115
118
|
},
|
|
116
119
|
"axes": [
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"workflow": {
|
|
3
|
-
"plan": "Build an engineering retrospective from reproducible
|
|
3
|
+
"plan": "Build an engineering retrospective from reproducible evidence, not memory or a leaderboard. Prefer repository history when it is available, and support authoritative workspace exports when it is not. State the time window, evidence sources, and identity mapping; preserve both counts and qualitative examples; and explain data limits. Interpret activity in context: commit volume is not productivity, review visibility is incomplete, and individual notes must be constructive growth observations rather than performance ratings.",
|
|
4
4
|
"phases": [
|
|
5
5
|
{
|
|
6
6
|
"id": "scope-retro",
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"name": "Collect and reconcile the engineering evidence",
|
|
41
41
|
"description": "Gather reproducible activity, churn, hotspot, collaboration, and quality signals with caveats.",
|
|
42
42
|
"suggestedRole": "engineering data analyst",
|
|
43
|
-
"prompt": "
|
|
43
|
+
"prompt": "Collect the evidence contract defined in `retro/scope.md`. When `run_git` is available in your function schema, use its read-only log, show, and diff subcommands; do not use shell pipelines. Otherwise, inventory and read the authoritative repository exports or workspace records named in the scope, and explicitly record that live history is unavailable. Preserve the exact query arguments or source paths and result counts. Reconcile totals across authors and date boundaries, sample the work behind the numbers, and identify hotspots, change concentration, review or pairing signals visible in the evidence, rework, test activity, documentation, maintenance, and operational work. Compare with the chosen baseline where possible. Treat missing data as missing, not zero, and never invent unavailable repository or review metrics.",
|
|
44
44
|
"output": {
|
|
45
45
|
"path": "retro/evidence.md",
|
|
46
46
|
"minBytes": 1400,
|