@dailephd/my-dev-kit-lab 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +23 -113
  2. package/benchmarks/contracts/todo-behavior.md +1 -1
  3. package/benchmarks/projects/task-analytics-large-mixed/README.md +2 -0
  4. package/benchmarks/projects/task-workflow-medium-ts/README.md +3 -1
  5. package/dist/src/evaluation/stageContextDeterminism/calculateStageContextDeterminism.js +52 -0
  6. package/dist/src/evaluation/stageContextDeterminism/canonicalizeStageContextRun.js +53 -0
  7. package/dist/src/evaluation/stageContextDeterminism/index.js +2 -0
  8. package/dist/src/evaluation/stageContextDeterminism/types.js +1 -0
  9. package/dist/src/evaluation/stageContextExpectations/index.js +2 -0
  10. package/dist/src/evaluation/stageContextExpectations/readStageContextExpectationFixtureV1.js +50 -0
  11. package/dist/src/evaluation/stageContextExpectations/readTypes.js +1 -0
  12. package/dist/src/evaluation/stageContextExpectations/types.js +1 -0
  13. package/dist/src/evaluation/stageContextExpectations/validation.js +568 -0
  14. package/dist/src/evaluation/stageContextMetrics/calculateArtifactStateMetrics.js +130 -0
  15. package/dist/src/evaluation/stageContextMetrics/calculateContextSizeMetrics.js +85 -0
  16. package/dist/src/evaluation/stageContextMetrics/calculateExpectationMetrics.js +103 -0
  17. package/dist/src/evaluation/stageContextMetrics/calculateResponsibilityMappingMetrics.js +36 -0
  18. package/dist/src/evaluation/stageContextMetrics/collectContextCapsuleEvidence.js +312 -0
  19. package/dist/src/evaluation/stageContextMetrics/collectRetrievalAuditEvidence.js +35 -0
  20. package/dist/src/evaluation/stageContextMetrics/collectWorkflowInstructionEvidence.js +137 -0
  21. package/dist/src/evaluation/stageContextMetrics/evaluateV043StageContextExecution.js +247 -0
  22. package/dist/src/evaluation/stageContextMetrics/index.js +10 -0
  23. package/dist/src/evaluation/stageContextMetrics/matchStageContextExpectations.js +38 -0
  24. package/dist/src/evaluation/stageContextMetrics/targetKeys.js +35 -0
  25. package/dist/src/evaluation/stageContextMetrics/types.js +1 -0
  26. package/dist/src/evaluation/stageContextSelectors/contextArtifactConsistency.js +79 -0
  27. package/dist/src/evaluation/stageContextSelectors/contextCapsuleSelectors.js +126 -0
  28. package/dist/src/evaluation/stageContextSelectors/index.js +4 -0
  29. package/dist/src/evaluation/stageContextSelectors/retrievalAuditRecordSelectors.js +54 -0
  30. package/dist/src/evaluation/stageContextSelectors/workflowInstructionPacketSelectors.js +51 -0
  31. package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +208 -0
  32. package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +56 -0
  33. package/dist/src/evaluation/targetImmutability/index.js +3 -0
  34. package/dist/src/evaluation/targetImmutability/types.js +1 -0
  35. package/dist/src/evaluation/targetImmutability/validateTargetImmutabilityConfig.js +130 -0
  36. package/dist/src/evaluation/upstreamArtifacts/artifactReadTypes.js +1 -0
  37. package/dist/src/evaluation/upstreamArtifacts/index.js +3 -0
  38. package/dist/src/evaluation/upstreamArtifacts/jsonTypes.js +1 -0
  39. package/dist/src/evaluation/upstreamArtifacts/myDevKitClassificationTypesV1.js +1 -0
  40. package/dist/src/evaluation/upstreamArtifacts/myDevKitContextArtifactsV1.js +1 -0
  41. package/dist/src/evaluation/upstreamArtifacts/myDevKitSemanticTypesV1.js +1 -0
  42. package/dist/src/evaluation/upstreamArtifacts/orchestratorWorkflowInstructionPacketV1.js +1 -0
  43. package/dist/src/evaluation/upstreamArtifacts/readJsonArtifact.js +53 -0
  44. package/dist/src/evaluation/upstreamArtifacts/readMyDevKitContextCapsuleV1.js +9 -0
  45. package/dist/src/evaluation/upstreamArtifacts/readMyDevKitRetrievalAuditRecordV1.js +9 -0
  46. package/dist/src/evaluation/upstreamArtifacts/readOrchestratorWorkflowInstructionPacketV1.js +9 -0
  47. package/dist/src/evaluation/upstreamArtifacts/runtimeValidation.js +131 -0
  48. package/dist/src/evaluation/upstreamArtifacts/schemaVersion.js +32 -0
  49. package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitContextCapsuleV1.js +1208 -0
  50. package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitRetrievalAuditRecordV1.js +105 -0
  51. package/dist/src/evaluation/upstreamArtifacts/validateOrchestratorWorkflowInstructionPacketV1.js +235 -0
  52. package/dist/src/experiments/plugins/contextStrategyComparison/executeV043StageContextStrategy.js +46 -0
  53. package/dist/src/experiments/plugins/contextStrategyComparison/index.js +9 -0
  54. package/dist/src/experiments/plugins/contextStrategyComparison/loadV043StrategyArtifacts.js +286 -0
  55. package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +39 -1
  56. package/dist/src/experiments/plugins/contextStrategyComparison/readV043FullWorkflowLibraryFixture.js +162 -0
  57. package/dist/src/experiments/plugins/contextStrategyComparison/resolveV043StrategyInputs.js +43 -0
  58. package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +3 -0
  59. package/dist/src/experiments/plugins/contextStrategyComparison/runV043StageContextStrategyWithAssurance.js +190 -0
  60. package/dist/src/experiments/plugins/contextStrategyComparison/v043FullWorkflowLibraryFixture.js +1 -0
  61. package/dist/src/experiments/plugins/contextStrategyComparison/v043RunAssuranceTypes.js +1 -0
  62. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyExecutionTypes.js +1 -0
  63. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyIds.js +13 -0
  64. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyInputContracts.js +1 -0
  65. package/dist/src/experiments/plugins/contextStrategyComparison/validateV043RunAssuranceConfig.js +59 -0
  66. package/dist/src/experiments/plugins/contextStrategyComparison/validateV043StrategyInput.js +252 -0
  67. package/dist/src/report/experiments/buildContextStrategyComparisonV043Report.js +398 -0
  68. package/dist/src/report/experiments/buildPluginExperimentReport.js +21 -6
  69. package/dist/src/report/experiments/contextStrategyComparisonV043ReportModel.js +1 -0
  70. package/dist/src/report/experiments/index.js +4 -0
  71. package/dist/src/report/experiments/renderContextStrategyComparisonV043Html.js +234 -0
  72. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
  73. package/dist/src/report/experiments/renderPluginExperimentReportText.js +225 -0
  74. package/dist/src/report/experiments/writePluginExperimentReports.js +3 -0
  75. package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +215 -9
  76. package/docs/METRICS.md +56 -10
  77. package/package.json +1 -1
  78. package/tests/fixtures/full-workflow-library/complete-v1.0.0.json +24 -0
  79. package/tests/fixtures/full-workflow-library/future-minor-v1.1.0-additive.json +25 -0
  80. package/tests/fixtures/stage-context-expectations/complete-v1.0.0.json +230 -0
  81. package/tests/fixtures/stage-context-expectations/future-minor-v1.1.0-additive.json +232 -0
  82. package/tests/fixtures/stage-context-expectations/invalid/malformed-json.txt +1 -0
  83. package/tests/fixtures/stage-context-expectations/invalid/non-object-root.json +1 -0
  84. package/tests/fixtures/stage-context-expectations/minimal-v1.0.0.json +18 -0
  85. package/tests/fixtures/upstream-artifacts/invalid/malformed-json.txt +1 -0
  86. package/tests/fixtures/upstream-artifacts/invalid/non-object-root.json +1 -0
  87. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/complete-v1.0.0.json +504 -0
  88. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/future-minor-v1.1.0-additive.json +955 -0
  89. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/complete-v1.0.0.json +190 -0
  90. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/future-minor-v1.1.0-additive.json +327 -0
  91. package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/complete-v1.0.0.json +160 -0
  92. package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/future-minor-v1.1.0-additive.json +186 -0
@@ -12,11 +12,215 @@ const SECRET_PATTERNS = [
12
12
  { kind: "github-token", regex: /gh[pousr]_[A-Za-z0-9]{16,}/g },
13
13
  { kind: "openai-token", regex: /sk-[A-Za-z0-9]{16,}/g },
14
14
  { kind: "aws-key", regex: /AKIA[0-9A-Z]{12,}/g },
15
- {
16
- kind: "generic-secret-assignment",
17
- regex: /(?:password|passwd|secret|api[_-]?key|token)\s*[:=]\s*["']?([A-Za-z0-9_\-]{8,})["']?/gi,
18
- },
19
15
  ];
16
+ // Only recognized configuration file forms may carry an unquoted literal
17
+ // value; every other file (including all TypeScript/JavaScript/JVM/Gradle
18
+ // source) requires a genuine quoted string literal before a match is even
19
+ // considered.
20
+ function genericSecretFileMode(relativePath) {
21
+ const normalized = relativePath.replace(/\\/g, "/");
22
+ const basename = normalized.slice(normalized.lastIndexOf("/") + 1).toLowerCase();
23
+ if (basename === ".env" || basename.startsWith(".env."))
24
+ return "allow-unquoted";
25
+ if (/\.(properties|yaml|yml|toml)$/.test(basename))
26
+ return "allow-unquoted";
27
+ return "quoted-only";
28
+ }
29
+ // Deterministic line-by-line comment filter. Preserves the original string
30
+ // length and every newline so line numbers computed from the filtered
31
+ // content remain identical to the original file; comment content (line or
32
+ // block) is replaced with spaces so it can never contribute to a match.
33
+ // Quote-aware: a comment marker inside a single- or double-quoted literal
34
+ // does not start a comment, and a backslash-escaped quote does not end one.
35
+ function stripCodeCommentsPreservingLength(content) {
36
+ const result = new Array(content.length);
37
+ let index = 0;
38
+ let inLineComment = false;
39
+ let inBlockComment = false;
40
+ let inSingleQuote = false;
41
+ let inDoubleQuote = false;
42
+ while (index < content.length) {
43
+ const ch = content[index];
44
+ const next = index + 1 < content.length ? content[index + 1] : "";
45
+ if (inLineComment) {
46
+ if (ch === "\n") {
47
+ inLineComment = false;
48
+ result[index] = "\n";
49
+ }
50
+ else {
51
+ result[index] = " ";
52
+ }
53
+ index++;
54
+ continue;
55
+ }
56
+ if (inBlockComment) {
57
+ if (ch === "*" && next === "/") {
58
+ result[index] = " ";
59
+ result[index + 1] = " ";
60
+ inBlockComment = false;
61
+ index += 2;
62
+ continue;
63
+ }
64
+ result[index] = ch === "\n" ? "\n" : " ";
65
+ index++;
66
+ continue;
67
+ }
68
+ if (inSingleQuote || inDoubleQuote) {
69
+ result[index] = ch;
70
+ if (ch === "\\" && next !== "") {
71
+ result[index + 1] = next;
72
+ index += 2;
73
+ continue;
74
+ }
75
+ if ((inSingleQuote && ch === "'") || (inDoubleQuote && ch === '"')) {
76
+ inSingleQuote = false;
77
+ inDoubleQuote = false;
78
+ }
79
+ index++;
80
+ continue;
81
+ }
82
+ if (ch === "/" && next === "/") {
83
+ inLineComment = true;
84
+ result[index] = " ";
85
+ index++;
86
+ continue;
87
+ }
88
+ if (ch === "/" && next === "*") {
89
+ inBlockComment = true;
90
+ result[index] = " ";
91
+ index++;
92
+ continue;
93
+ }
94
+ if (ch === "'") {
95
+ inSingleQuote = true;
96
+ result[index] = ch;
97
+ index++;
98
+ continue;
99
+ }
100
+ if (ch === '"') {
101
+ inDoubleQuote = true;
102
+ result[index] = ch;
103
+ index++;
104
+ continue;
105
+ }
106
+ result[index] = ch;
107
+ index++;
108
+ }
109
+ return result.join("");
110
+ }
111
+ const GENERIC_SENSITIVE_SINGLE_WORDS = new Set(["password", "passwd", "secret", "token"]);
112
+ function splitGenericIdentifierWords(identifier) {
113
+ const unquoted = identifier.replace(/^['"]|['"]$/g, "");
114
+ return unquoted
115
+ .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
116
+ .split(/[_\-.\s]+/)
117
+ .map((word) => word.toLowerCase())
118
+ .filter((word) => word.length > 0);
119
+ }
120
+ // Classifies only the complete left-side identifier already isolated by the
121
+ // caller. Never inspects right-side text, so a function name, type name, or
122
+ // argument on the right of an assignment can never itself be treated as a
123
+ // sensitive identifier.
124
+ //
125
+ // The sensitive word must be the identifier's final component (e.g.
126
+ // storePassword, DB_PASSWORD, accessToken) rather than appearing anywhere in
127
+ // a compound name — this keeps unrelated metadata fields that merely mention
128
+ // "token" as a leading/middle word (tokenUsageSource, tokenCountMethod) from
129
+ // being treated as credential-shaped identifiers, while every required
130
+ // sensitive example (password, storePassword, keyPassword, DB_PASSWORD,
131
+ // SECRET_TOKEN, clientSecret, accessToken, apiKey, api_key) still ends in
132
+ // its sensitive word.
133
+ function isGenericSecretIdentifier(identifier) {
134
+ const words = splitGenericIdentifierWords(identifier);
135
+ if (words.length === 0)
136
+ return false;
137
+ const lastWord = words[words.length - 1];
138
+ if (GENERIC_SENSITIVE_SINGLE_WORDS.has(lastWord))
139
+ return true;
140
+ if (words.length >= 2 && words[words.length - 2] === "api" && lastWord === "key")
141
+ return true;
142
+ return false;
143
+ }
144
+ function stripSurroundingQuotes(value) {
145
+ if (value.length >= 2 && ((value[0] === '"' && value.endsWith('"')) || (value[0] === "'" && value.endsWith("'")))) {
146
+ return value.slice(1, -1);
147
+ }
148
+ return value;
149
+ }
150
+ const DYNAMIC_VALUE_PREFIXES = ["$", "process.env", "system.getenv", "providers.gradleproperty"];
151
+ function isDynamicUnquotedValue(rawValue) {
152
+ const lower = rawValue.toLowerCase();
153
+ return DYNAMIC_VALUE_PREFIXES.some((prefix) => lower.startsWith(prefix));
154
+ }
155
+ // Requires an identifier, a separator, and a value immediately following the
156
+ // separator whose first non-whitespace character is a quote — this alone
157
+ // structurally excludes function calls, identifier references, environment
158
+ // lookups, and Gradle-property lookups, none of which begin with a quote.
159
+ const QUOTED_ASSIGNMENT_PATTERN = /(?<![A-Za-z0-9_$])(["']?)([A-Za-z_$][A-Za-z0-9_$]*)\1\s*[:=]\s*(["'])((?:\\.|(?!\3)[^\\\r\n])*)\3/g;
160
+ const UNQUOTED_CONFIG_PATTERN = /^([A-Za-z_][A-Za-z0-9_.-]*)\s*[:=]\s*(\S.*?)\s*$/;
161
+ const MIN_GENERIC_SECRET_LENGTH = 8;
162
+ function scanGenericSecretAssignments(relativePath, content) {
163
+ const findings = [];
164
+ const mode = genericSecretFileMode(relativePath);
165
+ const scannedContent = mode === "quoted-only" ? stripCodeCommentsPreservingLength(content) : content;
166
+ const lines = scannedContent.split("\n");
167
+ for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
168
+ const line = lines[lineIndex];
169
+ if (mode === "allow-unquoted") {
170
+ const trimmed = line.trim();
171
+ if (trimmed.length === 0 || trimmed.startsWith("#") || trimmed.startsWith(";"))
172
+ continue;
173
+ const unquotedMatch = UNQUOTED_CONFIG_PATTERN.exec(trimmed);
174
+ if (!unquotedMatch)
175
+ continue;
176
+ const identifier = unquotedMatch[1];
177
+ if (!isGenericSecretIdentifier(identifier))
178
+ continue;
179
+ const rawCandidate = unquotedMatch[2];
180
+ if (isDynamicUnquotedValue(rawCandidate))
181
+ continue;
182
+ const rawValue = stripSurroundingQuotes(rawCandidate);
183
+ if (rawValue.length < MIN_GENERIC_SECRET_LENGTH)
184
+ continue;
185
+ if (isPlaceholder(rawValue))
186
+ continue;
187
+ findings.push({
188
+ kind: "generic-secret-assignment",
189
+ file: relativePath,
190
+ line: lineIndex + 1,
191
+ matchedText: `${identifier} ${line.includes(":") ? ":" : "="} [REDACTED]`,
192
+ matchLength: rawValue.length,
193
+ severity: SEVERITY_FOR_KIND["generic-secret-assignment"],
194
+ });
195
+ continue;
196
+ }
197
+ QUOTED_ASSIGNMENT_PATTERN.lastIndex = 0;
198
+ let match;
199
+ while ((match = QUOTED_ASSIGNMENT_PATTERN.exec(line)) !== null) {
200
+ const identifier = match[2];
201
+ const rawValue = match[4];
202
+ if (match[0].length === 0) {
203
+ QUOTED_ASSIGNMENT_PATTERN.lastIndex += 1;
204
+ continue;
205
+ }
206
+ if (!isGenericSecretIdentifier(identifier))
207
+ continue;
208
+ if (rawValue.length < MIN_GENERIC_SECRET_LENGTH)
209
+ continue;
210
+ if (isPlaceholder(rawValue))
211
+ continue;
212
+ findings.push({
213
+ kind: "generic-secret-assignment",
214
+ file: relativePath,
215
+ line: lineIndex + 1,
216
+ matchedText: `${identifier} [REDACTED]`,
217
+ matchLength: rawValue.length,
218
+ severity: SEVERITY_FOR_KIND["generic-secret-assignment"],
219
+ });
220
+ }
221
+ }
222
+ return findings;
223
+ }
20
224
  const PLACEHOLDER_MARKERS = [
21
225
  /\bexample\b/i,
22
226
  /change[-_]?me/i,
@@ -49,10 +253,7 @@ function scanFileForSecrets(relativePath, content) {
49
253
  let match;
50
254
  while ((match = re.exec(content)) !== null) {
51
255
  const matchedText = match[0];
52
- // For the generic assignment pattern, judge placeholder-ness on the
53
- // captured value; for shaped tokens, judge on the whole match.
54
- const judgedText = kind === "generic-secret-assignment" ? (match[1] ?? matchedText) : matchedText;
55
- if (isPlaceholder(judgedText) || isPlaceholder(matchedText)) {
256
+ if (isPlaceholder(matchedText)) {
56
257
  // Downgrade: not a high-confidence finding, skip entirely rather
57
258
  // than add noise — placeholders are expected in source/config.
58
259
  continue;
@@ -72,12 +273,17 @@ function scanFileForSecrets(relativePath, content) {
72
273
  });
73
274
  }
74
275
  }
276
+ // Generic sensitive-identifier assignment scanning is syntax-aware (see
277
+ // scanGenericSecretAssignments above) and is appended after the shaped-
278
+ // token findings for this file; the caller's global sort by file/line
279
+ // keeps overall ordering deterministic.
280
+ findings.push(...scanGenericSecretAssignments(relativePath, content));
75
281
  return findings;
76
282
  }
77
283
  export const SECRET_LEAKAGE_SCENARIO = {
78
284
  id: "secret-leakage-bounded-scan",
79
285
  title: "Secret leakage: bounded static scan finds no high-confidence secret-shaped literals",
80
- description: "Scans a bounded, deterministic set of project files (manifests, tsconfig, src/, scripts/, .env*) for high-confidence secret-shaped literal values (private keys, GitHub/OpenAI/AWS-style tokens, generic password/token/secret assignments). Placeholder values and bare env-var-name references are excluded from findings.",
286
+ description: "Scans a bounded, deterministic set of project files (manifests, tsconfig, src/, scripts/, .env*) for high-confidence secret-shaped literal values (private keys, GitHub/OpenAI/AWS-style tokens, and generic password/token/secret assignments). Generic assignments in code files require a genuine quoted string literal value; recognized configuration files (.env*, .properties, .yaml/.yml, .toml) may also carry an unquoted literal. Comments, type declarations, function calls, identifier references, and environment/Gradle-property lookups are never treated as literal values. Placeholder values are excluded from findings.",
81
287
  checkId: "secrets",
82
288
  applicableProfiles: [],
83
289
  severityBaseline: "informational",
package/docs/METRICS.md CHANGED
@@ -3,17 +3,25 @@
3
3
  This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports.
4
4
 
5
5
  Related documentation:
6
- - [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) — how metrics flow through the pipeline
7
- - [docs/TUTORIAL.md](docs/TUTORIAL.md) — how to read token savings and correctness scores in the report
8
- - [docs/CURRENT_STATE.md](docs/CURRENT_STATE.md) — current baseline and limitations
6
+ - [ARCHITECTURE.md](ARCHITECTURE.md) — how metrics flow through the pipeline
7
+ - [TUTORIAL.md](TUTORIAL.md) — how to read token savings and correctness scores in the report
8
+ - [CURRENT_STATE.md](CURRENT_STATE.md) — current capabilities, plans, and limitations
9
9
 
10
10
  ## Metric interpretation quick reference
11
11
 
12
- | Metric | Positive means | Negative means | N/A means |
12
+ | Metric | Higher or positive value | Lower or negative value | Unavailable or N/A |
13
13
  |---|---|---|---|
14
- | `tokenSavings` | my-dev-kit used fewer tokens | my-dev-kit used more tokens | Token totals unavailable for one or both runs |
15
- | `correctnessScore` | More answer-key facts matched | Fewer answer-key facts matched | Run did not complete |
16
- | `complexityScore` | Higher project complexity | Lower project complexity | — |
14
+ | `tokenSavingsPercent` | my-dev-kit used fewer tokens | my-dev-kit used more tokens | One or both runs lacked total-token data |
15
+ | `correctnessScore` | More weighted answer-key evidence matched | Less weighted answer-key evidence matched | The run did not produce a scorable answer |
16
+ | `complexityScore` | The heuristic indicates greater project complexity | The heuristic indicates lower project complexity | The project was not profiled |
17
+
18
+ Each entry below uses the same fields: **Meaning**, **Appears in** (the artifact or report source), **Formula**, **Interpretation**, and **Caveat**. Availability follows these repository-wide rules:
19
+
20
+ - Provider-reported tokens and character-based estimates are different sources and must remain labeled.
21
+ - Unavailable means the required input was not supplied or exposed; it is not zero.
22
+ - Zero is a measured or derived numeric result with a valid denominator.
23
+ - Not applicable means the metric does not describe that run or strategy.
24
+ - Invalid zero-denominator cases are reported explicitly rather than divided or coerced to zero.
17
25
 
18
26
  **Token savings notes:**
19
27
  - A positive token savings value means the my-dev-kit-guided strategy used fewer tokens than raw-full-file for that run pair.
@@ -30,7 +38,7 @@ Related documentation:
30
38
  - Small projects may show negative token savings because raw-full-file is cheaper when the entire project fits easily in context.
31
39
  - Larger, more localized tasks are where my-dev-kit is expected to become more useful.
32
40
 
33
- ## Project Complexity Metrics
41
+ ## Project complexity metrics
34
42
 
35
43
  - `fileCount`
36
44
  Meaning: total non-generated files captured in the benchmark file tree.
@@ -129,7 +137,7 @@ Related documentation:
129
137
  Interpretation: human-readable size category.
130
138
  Caveat: coarse label; use the score and metrics for detail.
131
139
 
132
- ## Prompt Complexity Metrics
140
+ ## Prompt complexity metrics
133
141
 
134
142
  - `promptChars`
135
143
  Meaning: prompt length in characters.
@@ -198,7 +206,7 @@ Related documentation:
198
206
  Interpretation: `true` means retrieval commands are part of the task.
199
207
  Caveat: prompt intent only.
200
208
 
201
- ## Experiment And Run Metrics
209
+ ## Experiment and run metrics
202
210
 
203
211
  - `durationMs`
204
212
  Meaning: measured wall-clock duration of a normalized run.
@@ -284,3 +292,41 @@ Related documentation:
284
292
  Formula: derived from paired run outcomes and metric availability.
285
293
  Interpretation: stronger labels mean safer aggregate interpretation.
286
294
  Caveat: comparison reliability is not the same as correctness.
295
+
296
+ ## Planned v0.4.3 metrics
297
+
298
+ Implemented in `src/evaluation/stageContextMetrics` and published in `v0.4.3`. This section is retained under its original "Planned v0.4.3 metrics" heading for documentation-consistency tooling; the metrics below are implemented and published, not planned. See [ROADMAP.md](ROADMAP.md) and [CHANGELOG.md](../CHANGELOG.md) for its release state.
299
+
300
+ Every ratio metric below reports `availability` (`available`, `unavailable`, or `not-applicable`), `numerator`, `denominator`, and `rate` explicitly. Missing metric input is reported as `unavailable`, never coerced to zero; `not-applicable` is distinct from both. Every count metric reports `availability`, `count`, and bounded `evidenceKeys`.
301
+
302
+ - `requiredEvidenceRecall`
303
+ Meaning: fraction of fixture-required evidence (files, symbols, source ranges, contracts, validators, constants, errors, schemas/serializers, production responsibilities, package scripts, test commands, and workflow/stage/command/rule/report-contract IDs) found in observed evidence.
304
+ Formula: matched required evidence / total required evidence.
305
+ Caveat: requires an explicit `StageContextExpectationFixtureV1`; not computable without it.
306
+ - `allowedEvidenceCoverage`
307
+ Meaning: fraction of fixture-allowed (optional) evidence found in observed evidence.
308
+ Formula: matched allowed evidence / total allowed evidence.
309
+ - `forbiddenEvidenceInclusion`
310
+ Meaning: fraction of fixture-forbidden evidence that was nonetheless observed.
311
+ Formula: observed forbidden evidence / total forbidden evidence.
312
+ Caveat: a `not-applicable` availability when no forbidden evidence is defined is not the same as zero forbidden inclusion.
313
+ - `irrelevantFileInclusion` / `irrelevantInstructionInclusion`
314
+ Meaning: count of observed files/instructions that are not required or allowed by the fixture.
315
+ Caveat: zero irrelevant items is a real, available zero, not the same as `unavailable`.
316
+ - `requiredProvenanceRecall`
317
+ Meaning: fraction of fixture-required provenance evidence IDs found in observed evidence.
318
+ - `responsibilityMappingCompleteness`
319
+ Meaning: per source instance, whether context-capsule/retrieval-audit-record responsibility mappings are mapped, partially mapped, unmapped, or not applicable, with a mapped rate; partial mappings do not count as complete.
320
+ - `stateComparisons`
321
+ Meaning: explicit expected-versus-actual comparisons of artifact state fields (for example context/role adequacy status, freshness state, truncation, unresolved-item counts, warning counts) declared in the fixture's expected states, each with its own `available`/`unavailable`/`not-applicable` availability.
322
+ - `contextSize`
323
+ Meaning: per-source character count and estimated token count, plus totals, across all context sources supplied to the strategy.
324
+ Formula: estimated tokens use `ceil(characterCount / 4)` per source — an estimate, not provider-reported tokens.
325
+ - `consideredButUnselectedReads` / `unnecessaryReads`
326
+ Meaning: counts of considered-but-unselected and unnecessary reads.
327
+ Caveat: the published upstream `RetrievalAuditRecord` does not expose this evidence; these metrics report `unavailable` with an explicit reason, never zero.
328
+ - `targetImmutability`
329
+ Meaning: count of target mutations detected between the before and after target snapshots for a run.
330
+ Caveat: reports `unavailable` when no target-immutability configuration was supplied for the run; a configured, unchanged target reports an available count of zero.
331
+
332
+ Repeated-run determinism (`StageContextDeterminismResultV1` in `src/evaluation/stageContextDeterminism`) reports whether repeated canonical runs (`repeatCount` 1 through 10, run 1 as baseline) produce identical canonicalized run values via SHA-256 digest comparison. A single run reports `not-applicable`; a canonicalization failure (for example a circular reference) reports `unavailable`.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dailephd/my-dev-kit-lab",
3
- "version": "0.4.2",
3
+ "version": "0.4.3",
4
4
  "type": "module",
5
5
  "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
6
6
  "bin": {
@@ -0,0 +1,24 @@
1
+ {
2
+ "schemaVersion": "1.0.0",
3
+ "fixtureId": "FULL-WORKFLOW-LIBRARY-V043-001",
4
+ "title": "Complete workflow instruction library",
5
+ "description": "Deterministic full-workflow-library fixture for v0.4.3 strategy execution.",
6
+ "workflowIds": ["workflow.feature", "workflow.release"],
7
+ "stageIds": [
8
+ "stage.feature.architecture",
9
+ "stage.feature.implementation",
10
+ "stage.feature.test-implementation",
11
+ "stage.release.readiness"
12
+ ],
13
+ "commandIds": [
14
+ "command.context.inspect",
15
+ "command.test.focused",
16
+ "command.test.full",
17
+ "command.release.publish"
18
+ ],
19
+ "ruleIds": ["rule.scope.bounded", "rule.testing.required", "rule.release.publish-last"],
20
+ "reportContractIds": ["report.implementation", "report.release"],
21
+ "provenanceEvidenceIds": ["provenance.workflow.feature", "provenance.workflow.release"],
22
+ "rawText": "A deterministic complete workflow instruction library fixture containing architecture, implementation, test, validation, release, and reporting instructions.",
23
+ "warnings": []
24
+ }
@@ -0,0 +1,25 @@
1
+ {
2
+ "schemaVersion": "1.1.0",
3
+ "fixtureId": "FULL-WORKFLOW-LIBRARY-V043-001",
4
+ "title": "Complete workflow instruction library",
5
+ "description": "Deterministic full-workflow-library fixture for v0.4.3 strategy execution.",
6
+ "futureMinorRootField": "preserved",
7
+ "workflowIds": ["workflow.feature", "workflow.release"],
8
+ "stageIds": [
9
+ "stage.feature.architecture",
10
+ "stage.feature.implementation",
11
+ "stage.feature.test-implementation",
12
+ "stage.release.readiness"
13
+ ],
14
+ "commandIds": [
15
+ "command.context.inspect",
16
+ "command.test.focused",
17
+ "command.test.full",
18
+ "command.release.publish"
19
+ ],
20
+ "ruleIds": ["rule.scope.bounded", "rule.testing.required", "rule.release.publish-last"],
21
+ "reportContractIds": ["report.implementation", "report.release"],
22
+ "provenanceEvidenceIds": ["provenance.workflow.feature", "provenance.workflow.release"],
23
+ "rawText": "A deterministic complete workflow instruction library fixture containing architecture, implementation, test, validation, release, and reporting instructions.",
24
+ "warnings": []
25
+ }
@@ -0,0 +1,230 @@
1
+ {
2
+ "schemaVersion": "1.0.0",
3
+ "caseId": "CASE-STAGE-CONTEXT-002",
4
+ "title": "Complete stage-context expectation",
5
+ "description": "Complete valid expectation fixture covering every expectation category.",
6
+ "expectedEvidence": [
7
+ {
8
+ "expectationId": "REQ-FILE-001",
9
+ "inclusion": "required",
10
+ "sourceArtifact": "context-capsule",
11
+ "category": "file",
12
+ "match": { "path": "src/example.ts" },
13
+ "notes": []
14
+ },
15
+ {
16
+ "expectationId": "REQ-SYMBOL-001",
17
+ "inclusion": "required",
18
+ "sourceArtifact": "context-capsule",
19
+ "category": "symbol",
20
+ "match": { "symbolId": "symbol:src/example.ts#Example" },
21
+ "notes": []
22
+ },
23
+ {
24
+ "expectationId": "REQ-SOURCE-RANGE-001",
25
+ "inclusion": "required",
26
+ "sourceArtifact": "context-capsule",
27
+ "category": "source-range",
28
+ "match": { "filePath": "src/example.ts", "startLine": 1, "endLine": 10 },
29
+ "notes": []
30
+ },
31
+ {
32
+ "expectationId": "ALLOW-CONTRACT-001",
33
+ "inclusion": "allowed",
34
+ "sourceArtifact": "context-capsule",
35
+ "category": "contract",
36
+ "match": { "symbolId": "contract:src/example.ts#ExampleContract" },
37
+ "notes": []
38
+ },
39
+ {
40
+ "expectationId": "ALLOW-VALIDATOR-001",
41
+ "inclusion": "allowed",
42
+ "sourceArtifact": "context-capsule",
43
+ "category": "validator",
44
+ "match": { "symbolId": "validator:src/example.ts#validateExample" },
45
+ "notes": []
46
+ },
47
+ {
48
+ "expectationId": "ALLOW-CONSTANT-001",
49
+ "inclusion": "allowed",
50
+ "sourceArtifact": "context-capsule",
51
+ "category": "constant",
52
+ "match": { "symbolId": "constant:src/example.ts#EXAMPLE_CONST" },
53
+ "notes": []
54
+ },
55
+ {
56
+ "expectationId": "FORBID-ERROR-001",
57
+ "inclusion": "forbidden",
58
+ "sourceArtifact": "context-capsule",
59
+ "category": "error",
60
+ "match": { "symbolId": "error:src/example.ts#ExampleError" },
61
+ "notes": []
62
+ },
63
+ {
64
+ "expectationId": "ALLOW-SCHEMA-001",
65
+ "inclusion": "allowed",
66
+ "sourceArtifact": "context-capsule",
67
+ "category": "schema-or-serializer",
68
+ "match": { "symbolId": "schema:src/example.ts#ExampleSchema" },
69
+ "notes": []
70
+ },
71
+ {
72
+ "expectationId": "REQ-RESPONSIBILITY-001",
73
+ "inclusion": "required",
74
+ "sourceArtifact": "context-capsule",
75
+ "category": "production-responsibility",
76
+ "match": { "responsibilityId": "fixture.responsibility.001" },
77
+ "notes": []
78
+ },
79
+ {
80
+ "expectationId": "ALLOW-RESPONSIBILITY-001",
81
+ "inclusion": "allowed",
82
+ "sourceArtifact": "retrieval-audit-record",
83
+ "category": "production-responsibility",
84
+ "match": { "responsibilityId": "fixture.responsibility.002" },
85
+ "notes": []
86
+ },
87
+ {
88
+ "expectationId": "REQ-TEST-FILE-001",
89
+ "inclusion": "required",
90
+ "sourceArtifact": "context-capsule",
91
+ "category": "test-file",
92
+ "match": { "path": "tests/example.spec.ts" },
93
+ "notes": []
94
+ },
95
+ {
96
+ "expectationId": "ALLOW-FIXTURE-001",
97
+ "inclusion": "allowed",
98
+ "sourceArtifact": "context-capsule",
99
+ "category": "fixture",
100
+ "match": { "path": "tests/fixtures/example.json" },
101
+ "notes": []
102
+ },
103
+ {
104
+ "expectationId": "ALLOW-FACTORY-001",
105
+ "inclusion": "allowed",
106
+ "sourceArtifact": "context-capsule",
107
+ "category": "factory",
108
+ "match": { "path": "tests/factories/exampleFactory.ts" },
109
+ "notes": []
110
+ },
111
+ {
112
+ "expectationId": "FORBID-MOCK-001",
113
+ "inclusion": "forbidden",
114
+ "sourceArtifact": "context-capsule",
115
+ "category": "mock",
116
+ "match": { "path": "tests/mocks/exampleMock.ts" },
117
+ "notes": []
118
+ },
119
+ {
120
+ "expectationId": "ALLOW-SETUP-FILE-001",
121
+ "inclusion": "allowed",
122
+ "sourceArtifact": "context-capsule",
123
+ "category": "setup-file",
124
+ "match": { "path": "tests/setup.ts" },
125
+ "notes": []
126
+ },
127
+ {
128
+ "expectationId": "REQ-TEST-CONFIGURATION-001",
129
+ "inclusion": "required",
130
+ "sourceArtifact": "context-capsule",
131
+ "category": "test-configuration",
132
+ "match": { "path": "vitest.config.ts" },
133
+ "notes": []
134
+ },
135
+ {
136
+ "expectationId": "REQ-PACKAGE-SCRIPT-001",
137
+ "inclusion": "required",
138
+ "sourceArtifact": "context-capsule",
139
+ "category": "package-script",
140
+ "match": { "name": "test" },
141
+ "notes": []
142
+ },
143
+ {
144
+ "expectationId": "ALLOW-TEST-COMMAND-001",
145
+ "inclusion": "allowed",
146
+ "sourceArtifact": "context-capsule",
147
+ "category": "test-command",
148
+ "match": { "commandText": "vitest run tests/example.spec.ts" },
149
+ "notes": []
150
+ },
151
+ {
152
+ "expectationId": "REQ-WORKFLOW-001",
153
+ "inclusion": "required",
154
+ "sourceArtifact": "workflow-instruction-packet",
155
+ "category": "workflow",
156
+ "match": { "id": "workflow.feature" },
157
+ "notes": []
158
+ },
159
+ {
160
+ "expectationId": "REQ-STAGE-001",
161
+ "inclusion": "required",
162
+ "sourceArtifact": "workflow-instruction-packet",
163
+ "category": "stage",
164
+ "match": { "id": "stage.feature.implementation" },
165
+ "notes": []
166
+ },
167
+ {
168
+ "expectationId": "ALLOW-COMMAND-001",
169
+ "inclusion": "allowed",
170
+ "sourceArtifact": "workflow-instruction-packet",
171
+ "category": "command",
172
+ "match": { "id": "command.repo.lint" },
173
+ "notes": []
174
+ },
175
+ {
176
+ "expectationId": "ALLOW-RULE-001",
177
+ "inclusion": "allowed",
178
+ "sourceArtifact": "full-workflow-library",
179
+ "category": "rule",
180
+ "match": { "id": "rule.quality.prefer-tests" },
181
+ "notes": []
182
+ },
183
+ {
184
+ "expectationId": "FORBID-REPORT-CONTRACT-001",
185
+ "inclusion": "forbidden",
186
+ "sourceArtifact": "workflow-instruction-packet",
187
+ "category": "report-contract",
188
+ "match": { "id": "report.other-summary" },
189
+ "notes": []
190
+ },
191
+ {
192
+ "expectationId": "REQ-PROVENANCE-001",
193
+ "inclusion": "required",
194
+ "sourceArtifact": "workflow-instruction-packet",
195
+ "category": "provenance",
196
+ "match": { "evidenceId": "symbol:src/example.ts#Example" },
197
+ "notes": []
198
+ }
199
+ ],
200
+ "expectedStates": {
201
+ "contextCapsule": {
202
+ "contextAdequacyStatus": "context sufficient with listed assumptions",
203
+ "roleAdequacyStatus": "context sufficient with listed assumptions",
204
+ "freshnessState": "fresh",
205
+ "truncated": false,
206
+ "fullFileFallbackUsed": 0,
207
+ "unresolvedItemIds": ["fixture.unresolved.001"],
208
+ "warningCount": 1
209
+ },
210
+ "retrievalAuditRecord": {
211
+ "contextAdequacyStatus": "context sufficient with listed assumptions",
212
+ "roleAdequacyStatus": "context sufficient with listed assumptions",
213
+ "freshnessState": "fresh",
214
+ "truncated": false,
215
+ "fullFileFallbackUsed": 1,
216
+ "unresolvedItemIds": [],
217
+ "warningCount": 1
218
+ },
219
+ "workflowInstructionPacket": {
220
+ "adequacyStatus": "adequate",
221
+ "truncated": false,
222
+ "unresolvedReferences": ["command.repo.unresolved-example"],
223
+ "warningCount": 0
224
+ },
225
+ "targetImmutability": {
226
+ "newMutationCount": 0
227
+ }
228
+ },
229
+ "warnings": []
230
+ }