@bendyline/gilde 0.1.65 → 0.1.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/craftbook-templates/index.json +1 -1
- package/data/craftbook-templates/pu/pull-request-review/versions/1.9.17/craftbook.json +602 -0
- package/data/craftbook-templates/pu/pull-request-review/versions/1.9.17/test.json +156 -0
- package/package.json +1 -1
- package/schemas/craftbook-doc.schema.json +24 -0
- package/schemas/craftbook-template-version.schema.json +24 -0
- package/schemas/craftbook-test.schema.json +12 -0
- package/schemas/knowledge-catalog-version.schema.json +1 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"title": "Complete-corpus pull request review",
|
|
4
|
+
"objective": "Verify that Pull Request Review fans a chunk-readable PR corpus out to per-batch reviewers, deterministically merges their exact coverage shards into a provenance-checked run-wide ledger, preserves an API-use question through the structured cross-file verification channel and resolves it against a late definition held by a different batch, finds a late-file defect, reports CI evidence separately, and writes a cited local verdict without modifying source or posting to GitHub.",
|
|
5
|
+
"tags": [
|
|
6
|
+
"external",
|
|
7
|
+
"pull-request",
|
|
8
|
+
"large-corpus",
|
|
9
|
+
"coverage",
|
|
10
|
+
"fanout"
|
|
11
|
+
],
|
|
12
|
+
"prompt": "Review the deterministic large pull-request corpus completely. Publish the review batches, let the crew review them in parallel, let the runtime merge and provenance-check their coverage shards, verify the early API call against its late definition, find the late-file authorization defect, and write tasks/eval/pr-review.md. Do not modify source or post a GitHub comment.",
|
|
13
|
+
"setup": {
|
|
14
|
+
"projectName": "Large PR Review Eval",
|
|
15
|
+
"about": "A deterministic PR-review fixture. The registered hand-authored workflow scenario materializes 120 changed-file records and more than 80 KB of patch text, then dispatches the real craftbook over that artifacts corpus.",
|
|
16
|
+
"missionObjectives": "Account for every changed path, avoid a false missing-API finding, catch the late authorization defect, and deliver a local request-changes report with exact path:line evidence.",
|
|
17
|
+
"craftbookParams": {
|
|
18
|
+
"workPath": "tasks/eval",
|
|
19
|
+
"number": "52"
|
|
20
|
+
},
|
|
21
|
+
"files": [
|
|
22
|
+
{
|
|
23
|
+
"path": "src/api/use-assured-api.ts",
|
|
24
|
+
"content": "import { assuredApi } from './assured-api.js';\n\nexport function loadWidget(id: string): string {\n return assuredApi(id);\n}\n"
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"path": "src/api/assured-api.ts",
|
|
28
|
+
"content": "export function assuredApi(id: string): string {\n return `widget:${id}`;\n}\n"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"path": "src/security/late-authorization.ts",
|
|
32
|
+
"content": "export function canDeleteProject(userId: string, ownerId: string): boolean {\n if (!userId) return false;\n // Regression: any authenticated user can delete another owner's project.\n return true;\n}\n"
|
|
33
|
+
}
|
|
34
|
+
],
|
|
35
|
+
"worker": {
|
|
36
|
+
"name": "Rina",
|
|
37
|
+
"role": "Reviewer",
|
|
38
|
+
"description": "A careful code reviewer who works through large change sets in bounded batches and verifies cross-file claims."
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
"mocks": [],
|
|
42
|
+
"success": {
|
|
43
|
+
"summary": "The batches are published, every exact batch shard is deterministically merged into a provenance-stamped 120-file ledger, the assuredApi cross-file candidate survives deterministic synthesis and the report recognizes that its definition exists, cites the late authorization defect, requests changes, and leaves the seeded source unchanged.",
|
|
44
|
+
"taskGraph": {
|
|
45
|
+
"requireCraftbookTask": true,
|
|
46
|
+
"requireTerminalStep": true
|
|
47
|
+
},
|
|
48
|
+
"deliverables": [
|
|
49
|
+
{
|
|
50
|
+
"path": "tasks/eval/pr-review.md",
|
|
51
|
+
"kind": "markdown-report",
|
|
52
|
+
"minBytes": 500,
|
|
53
|
+
"checks": [
|
|
54
|
+
{
|
|
55
|
+
"kind": "contains",
|
|
56
|
+
"file": "tasks/eval/pr-review.md",
|
|
57
|
+
"pattern": "Coverage:\\s*120\\s*/\\s*120\\s+changed files",
|
|
58
|
+
"flags": "i"
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"kind": "contains",
|
|
62
|
+
"file": "tasks/eval/pr-review.md",
|
|
63
|
+
"pattern": "src/security/late-authorization\\.ts(?::|\\s*\\|\\s*)4",
|
|
64
|
+
"flags": "i"
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"kind": "contains",
|
|
68
|
+
"file": "tasks/eval/pr-review.md",
|
|
69
|
+
"pattern": "Verdict:\\s*request-changes",
|
|
70
|
+
"flags": "i"
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"kind": "notContains",
|
|
74
|
+
"file": "tasks/eval/pr-review.md",
|
|
75
|
+
"pattern": "assuredApi.{0,80}(missing|does not exist|undefined)|(?:missing|does not exist|undefined).{0,80}assuredApi",
|
|
76
|
+
"flags": "is"
|
|
77
|
+
}
|
|
78
|
+
]
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"path": "tasks/eval/pr-review-coverage.json",
|
|
82
|
+
"kind": "json",
|
|
83
|
+
"minBytes": 500
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"path": "tasks/eval/pr-review/batches.json",
|
|
87
|
+
"kind": "json",
|
|
88
|
+
"minBytes": 500,
|
|
89
|
+
"checks": [
|
|
90
|
+
{
|
|
91
|
+
"kind": "contains",
|
|
92
|
+
"file": "tasks/eval/pr-review/batches.json",
|
|
93
|
+
"pattern": "\"batchNumber\"[\\s\\S]*\"paths\"[\\s\\S]*\"records\""
|
|
94
|
+
}
|
|
95
|
+
]
|
|
96
|
+
}
|
|
97
|
+
],
|
|
98
|
+
"unchangedFixtures": [
|
|
99
|
+
"src/api/use-assured-api.ts",
|
|
100
|
+
"src/api/assured-api.ts",
|
|
101
|
+
"src/security/late-authorization.ts"
|
|
102
|
+
]
|
|
103
|
+
},
|
|
104
|
+
"rubric": {
|
|
105
|
+
"artifact": {
|
|
106
|
+
"path": "tasks/eval/pr-review.md",
|
|
107
|
+
"kind": "markdown"
|
|
108
|
+
},
|
|
109
|
+
"axes": [
|
|
110
|
+
{
|
|
111
|
+
"name": "coverage",
|
|
112
|
+
"description": "The review demonstrates complete 120-file coverage across every batch rather than reasoning from an early prefix or from whichever shards happened to land first."
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
"name": "cross-file-verification",
|
|
116
|
+
"description": "The use-side shard emits a structured verificationCandidates[] entry for assuredApi, deterministic synthesis preserves it across batches, and the final report resolves it against the checkout without alleging that assuredApi is missing."
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"name": "late-defect",
|
|
120
|
+
"description": "The report finds and cites the authorization defect deliberately placed in the final changed file — the last batch is reviewed as carefully as the first."
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"name": "synthesis",
|
|
124
|
+
"description": "The final report draws on every per-batch observations shard, merging duplicate findings raised by different reviewers instead of dropping or double-listing them."
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
"name": "side-effects",
|
|
128
|
+
"description": "The review remains local and source fixtures stay unchanged."
|
|
129
|
+
}
|
|
130
|
+
]
|
|
131
|
+
},
|
|
132
|
+
"qualityFocus": [
|
|
133
|
+
"complete large-PR coverage",
|
|
134
|
+
"per-batch fanout and deterministic provenance-checked shard merging",
|
|
135
|
+
"late-file defect detection",
|
|
136
|
+
"safe local-only delivery",
|
|
137
|
+
"structured cross-file candidate preservation and checkout verification",
|
|
138
|
+
"bounded phase latency and restart-safe fanout completion"
|
|
139
|
+
],
|
|
140
|
+
"extensions": {
|
|
141
|
+
"legacySimulators": [
|
|
142
|
+
{
|
|
143
|
+
"id": "large-pr-artifact-corpus",
|
|
144
|
+
"kind": "data-source",
|
|
145
|
+
"status": "implemented",
|
|
146
|
+
"description": "The linked hand-authored workflow scenario seeds 120 per-file PR records totaling more than 80 KB, with an API use early, its valid definition late, and a real authorization defect in the final record. It dispatches the real craftbook and grades fanout, read/write receipts, exact coverage shards, deterministic merge provenance, terminal host synthesis, and source immutability."
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
"id": "github-pr-fixture-api",
|
|
150
|
+
"kind": "http",
|
|
151
|
+
"status": "planned",
|
|
152
|
+
"description": "A trial-local fake GitHub PR API must supply realistic pull metadata, paginated changed files, comments, and a full diff so the workflow eval can launch real connector prep without network credentials. Include success and partial/error modes."
|
|
153
|
+
}
|
|
154
|
+
]
|
|
155
|
+
}
|
|
156
|
+
}
|
package/package.json
CHANGED
|
@@ -509,6 +509,12 @@
|
|
|
509
509
|
"mode"
|
|
510
510
|
]
|
|
511
511
|
},
|
|
512
|
+
"promptProfile": {
|
|
513
|
+
"type": "string",
|
|
514
|
+
"enum": [
|
|
515
|
+
"focused"
|
|
516
|
+
]
|
|
517
|
+
},
|
|
512
518
|
"toolPolicy": {
|
|
513
519
|
"type": "object",
|
|
514
520
|
"properties": {
|
|
@@ -2038,6 +2044,9 @@
|
|
|
2038
2044
|
"type": "string",
|
|
2039
2045
|
"minLength": 1
|
|
2040
2046
|
},
|
|
2047
|
+
"requireVerificationCandidates": {
|
|
2048
|
+
"type": "boolean"
|
|
2049
|
+
},
|
|
2041
2050
|
"artifact": {
|
|
2042
2051
|
"type": "boolean"
|
|
2043
2052
|
}
|
|
@@ -3130,6 +3139,9 @@
|
|
|
3130
3139
|
"type": "string",
|
|
3131
3140
|
"minLength": 1
|
|
3132
3141
|
},
|
|
3142
|
+
"requireVerificationCandidates": {
|
|
3143
|
+
"type": "boolean"
|
|
3144
|
+
},
|
|
3133
3145
|
"artifact": {
|
|
3134
3146
|
"type": "boolean"
|
|
3135
3147
|
}
|
|
@@ -3719,6 +3731,12 @@
|
|
|
3719
3731
|
"mode"
|
|
3720
3732
|
]
|
|
3721
3733
|
},
|
|
3734
|
+
"promptProfile": {
|
|
3735
|
+
"type": "string",
|
|
3736
|
+
"enum": [
|
|
3737
|
+
"focused"
|
|
3738
|
+
]
|
|
3739
|
+
},
|
|
3722
3740
|
"toolPolicy": {
|
|
3723
3741
|
"type": "object",
|
|
3724
3742
|
"properties": {
|
|
@@ -5248,6 +5266,9 @@
|
|
|
5248
5266
|
"type": "string",
|
|
5249
5267
|
"minLength": 1
|
|
5250
5268
|
},
|
|
5269
|
+
"requireVerificationCandidates": {
|
|
5270
|
+
"type": "boolean"
|
|
5271
|
+
},
|
|
5251
5272
|
"artifact": {
|
|
5252
5273
|
"type": "boolean"
|
|
5253
5274
|
}
|
|
@@ -6340,6 +6361,9 @@
|
|
|
6340
6361
|
"type": "string",
|
|
6341
6362
|
"minLength": 1
|
|
6342
6363
|
},
|
|
6364
|
+
"requireVerificationCandidates": {
|
|
6365
|
+
"type": "boolean"
|
|
6366
|
+
},
|
|
6343
6367
|
"artifact": {
|
|
6344
6368
|
"type": "boolean"
|
|
6345
6369
|
}
|
|
@@ -93,6 +93,12 @@
|
|
|
93
93
|
"mode"
|
|
94
94
|
]
|
|
95
95
|
},
|
|
96
|
+
"promptProfile": {
|
|
97
|
+
"type": "string",
|
|
98
|
+
"enum": [
|
|
99
|
+
"focused"
|
|
100
|
+
]
|
|
101
|
+
},
|
|
96
102
|
"toolPolicy": {
|
|
97
103
|
"type": "object",
|
|
98
104
|
"properties": {
|
|
@@ -1622,6 +1628,9 @@
|
|
|
1622
1628
|
"type": "string",
|
|
1623
1629
|
"minLength": 1
|
|
1624
1630
|
},
|
|
1631
|
+
"requireVerificationCandidates": {
|
|
1632
|
+
"type": "boolean"
|
|
1633
|
+
},
|
|
1625
1634
|
"artifact": {
|
|
1626
1635
|
"type": "boolean"
|
|
1627
1636
|
}
|
|
@@ -2714,6 +2723,9 @@
|
|
|
2714
2723
|
"type": "string",
|
|
2715
2724
|
"minLength": 1
|
|
2716
2725
|
},
|
|
2726
|
+
"requireVerificationCandidates": {
|
|
2727
|
+
"type": "boolean"
|
|
2728
|
+
},
|
|
2717
2729
|
"artifact": {
|
|
2718
2730
|
"type": "boolean"
|
|
2719
2731
|
}
|
|
@@ -3297,6 +3309,12 @@
|
|
|
3297
3309
|
"mode"
|
|
3298
3310
|
]
|
|
3299
3311
|
},
|
|
3312
|
+
"promptProfile": {
|
|
3313
|
+
"type": "string",
|
|
3314
|
+
"enum": [
|
|
3315
|
+
"focused"
|
|
3316
|
+
]
|
|
3317
|
+
},
|
|
3300
3318
|
"toolPolicy": {
|
|
3301
3319
|
"type": "object",
|
|
3302
3320
|
"properties": {
|
|
@@ -4826,6 +4844,9 @@
|
|
|
4826
4844
|
"type": "string",
|
|
4827
4845
|
"minLength": 1
|
|
4828
4846
|
},
|
|
4847
|
+
"requireVerificationCandidates": {
|
|
4848
|
+
"type": "boolean"
|
|
4849
|
+
},
|
|
4829
4850
|
"artifact": {
|
|
4830
4851
|
"type": "boolean"
|
|
4831
4852
|
}
|
|
@@ -5918,6 +5939,9 @@
|
|
|
5918
5939
|
"type": "string",
|
|
5919
5940
|
"minLength": 1
|
|
5920
5941
|
},
|
|
5942
|
+
"requireVerificationCandidates": {
|
|
5943
|
+
"type": "boolean"
|
|
5944
|
+
},
|
|
5921
5945
|
"artifact": {
|
|
5922
5946
|
"type": "boolean"
|
|
5923
5947
|
}
|
|
@@ -1200,6 +1200,9 @@
|
|
|
1200
1200
|
"type": "string",
|
|
1201
1201
|
"minLength": 1
|
|
1202
1202
|
},
|
|
1203
|
+
"requireVerificationCandidates": {
|
|
1204
|
+
"type": "boolean"
|
|
1205
|
+
},
|
|
1203
1206
|
"artifact": {
|
|
1204
1207
|
"type": "boolean"
|
|
1205
1208
|
}
|
|
@@ -2370,6 +2373,9 @@
|
|
|
2370
2373
|
"type": "string",
|
|
2371
2374
|
"minLength": 1
|
|
2372
2375
|
},
|
|
2376
|
+
"requireVerificationCandidates": {
|
|
2377
|
+
"type": "boolean"
|
|
2378
|
+
},
|
|
2373
2379
|
"artifact": {
|
|
2374
2380
|
"type": "boolean"
|
|
2375
2381
|
}
|
|
@@ -3540,6 +3546,9 @@
|
|
|
3540
3546
|
"type": "string",
|
|
3541
3547
|
"minLength": 1
|
|
3542
3548
|
},
|
|
3549
|
+
"requireVerificationCandidates": {
|
|
3550
|
+
"type": "boolean"
|
|
3551
|
+
},
|
|
3543
3552
|
"artifact": {
|
|
3544
3553
|
"type": "boolean"
|
|
3545
3554
|
}
|
|
@@ -4711,6 +4720,9 @@
|
|
|
4711
4720
|
"type": "string",
|
|
4712
4721
|
"minLength": 1
|
|
4713
4722
|
},
|
|
4723
|
+
"requireVerificationCandidates": {
|
|
4724
|
+
"type": "boolean"
|
|
4725
|
+
},
|
|
4714
4726
|
"artifact": {
|
|
4715
4727
|
"type": "boolean"
|
|
4716
4728
|
}
|