@dailephd/my-dev-kit-lab 0.7.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/README.md +6 -4
  3. package/benchmarks/projects/README.md +3 -1
  4. package/dist/src/cli/help.js +20 -3
  5. package/dist/src/commands/runExperimentDescribeCommand.js +10 -1
  6. package/dist/src/commands/runExperimentRunCommand.js +169 -19
  7. package/dist/src/core/fileGlobs.js +44 -0
  8. package/dist/src/evaluation/localRepositorySubject/config.js +296 -0
  9. package/dist/src/evaluation/localRepositorySubject/evaluationCase.js +36 -0
  10. package/dist/src/evaluation/localRepositorySubject/expectedFiles.js +51 -0
  11. package/dist/src/evaluation/localRepositorySubject/gitRepository.js +128 -0
  12. package/dist/src/evaluation/localRepositorySubject/index.js +8 -0
  13. package/dist/src/evaluation/localRepositorySubject/inventory.js +127 -0
  14. package/dist/src/evaluation/localRepositorySubject/loadLocalRepositorySubject.js +48 -0
  15. package/dist/src/evaluation/localRepositorySubject/manifest.js +52 -0
  16. package/dist/src/evaluation/localRepositorySubject/types.js +55 -0
  17. package/dist/src/evaluation/projectFileTree.js +45 -1
  18. package/dist/src/evaluation/runMyDevKitRetrieval.js +3 -1
  19. package/dist/src/evaluation/runRawFullFileBaseline.js +18 -4
  20. package/dist/src/evaluation/syntheticRepository/config.js +304 -0
  21. package/dist/src/evaluation/syntheticRepository/evaluationCase.js +43 -0
  22. package/dist/src/evaluation/syntheticRepository/identity.js +41 -0
  23. package/dist/src/evaluation/syntheticRepository/index.js +16 -0
  24. package/dist/src/evaluation/syntheticRepository/inspection.js +92 -0
  25. package/dist/src/evaluation/syntheticRepository/layout.js +50 -0
  26. package/dist/src/evaluation/syntheticRepository/manifest.js +281 -0
  27. package/dist/src/evaluation/syntheticRepository/manifestVerification.js +112 -0
  28. package/dist/src/evaluation/syntheticRepository/materialize.js +170 -0
  29. package/dist/src/evaluation/syntheticRepository/planning.js +479 -0
  30. package/dist/src/evaluation/syntheticRepository/prng.js +47 -0
  31. package/dist/src/evaluation/syntheticRepository/renderPython.js +68 -0
  32. package/dist/src/evaluation/syntheticRepository/renderRepository.js +39 -0
  33. package/dist/src/evaluation/syntheticRepository/renderShared.js +93 -0
  34. package/dist/src/evaluation/syntheticRepository/renderTypeScript.js +82 -0
  35. package/dist/src/evaluation/syntheticRepository/topology.js +96 -0
  36. package/dist/src/evaluation/syntheticRepository/types.js +26 -0
  37. package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +29 -7
  38. package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +4 -0
  39. package/dist/src/experiments/plugins/contextWindowScaling/index.js +1 -0
  40. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectErrors.js +35 -0
  41. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExclusions.js +75 -0
  42. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExecution.js +136 -0
  43. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js +128 -0
  44. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectScratch.js +85 -0
  45. package/dist/src/experiments/plugins/contextWindowScaling/metadata.js +2 -2
  46. package/dist/src/experiments/plugins/contextWindowScaling/plugin.js +93 -0
  47. package/dist/src/experiments/plugins/contextWindowScaling/projectProfile.js +4 -9
  48. package/dist/src/experiments/plugins/contextWindowScaling/syntheticInputs.js +56 -0
  49. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +4 -3
  50. package/dist/src/report/experiments/writePluginExperimentReports.js +8 -2
  51. package/docs/METRICS.md +5 -1
  52. package/package.json +1 -1
@@ -3,6 +3,7 @@ import { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
3
3
  import { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeStalenessHtml.js";
4
4
  import { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
5
5
  export function renderPluginExperimentReportHtml(report) {
6
+ const redactedTarget = report.target.privacyProjection === "external-local-redacted";
6
7
  return `<!DOCTYPE html>
7
8
  <html lang="en">
8
9
  <head>
@@ -49,10 +50,10 @@ export function renderPluginExperimentReportHtml(report) {
49
50
  ["Mode", report.target.mode],
50
51
  ["Tool root", report.target.toolRoot],
51
52
  ["Target root", report.target.targetRoot],
52
- ["Target package", report.target.packageName ? `${report.target.packageName}@${report.target.packageVersion ?? "unknown"}` : "unavailable"],
53
+ ["Target package", redactedTarget ? "redacted" : report.target.packageName ? `${report.target.packageName}@${report.target.packageVersion ?? "unknown"}` : "unavailable"],
53
54
  ["Target branch/commit", report.target.hasGit ? `${report.target.branch ?? "unknown"} / ${report.target.commit ?? "unknown"}` : "unavailable"],
54
- ["Has package.json", String(report.target.hasPackageJson)],
55
- ["Has lockfile", String(report.target.hasLockfile)],
55
+ ["Has package.json", redactedTarget ? "redacted" : String(report.target.hasPackageJson)],
56
+ ["Has lockfile", redactedTarget ? "redacted" : String(report.target.hasLockfile)],
56
57
  ["Output root", report.metadata.outputRoot ?? "unavailable"],
57
58
  ])}
58
59
  </section>
@@ -17,13 +17,19 @@ export async function writePluginExperimentReports(args) {
17
17
  textPath: resolveWithinRoot(outDir, "report.txt"),
18
18
  };
19
19
  await mkdir(outputPaths.outDir, { recursive: true });
20
- const report = buildPluginExperimentReport({
20
+ const builtReport = buildPluginExperimentReport({
21
21
  run: args.run,
22
22
  plugin: args.plugin,
23
23
  outputRoot: outputPaths.outDir,
24
24
  generatedAt: args.generatedAt,
25
25
  });
26
- await writeFile(outputPaths.jsonPath, `${JSON.stringify({ report, outputPaths }, null, 2)}\n`, "utf8");
26
+ const report = args.redactOutputRoot
27
+ ? { ...builtReport, metadata: { ...builtReport.metadata, outputRoot: "[redacted]" } }
28
+ : builtReport;
29
+ const serializedPaths = args.redactOutputRoot
30
+ ? { outDir: "[redacted]", jsonPath: "report.json", htmlPath: "report.html", textPath: "report.txt" }
31
+ : outputPaths;
32
+ await writeFile(outputPaths.jsonPath, `${JSON.stringify({ report, outputPaths: serializedPaths }, null, 2)}\n`, "utf8");
27
33
  await writeFile(outputPaths.htmlPath, renderPluginExperimentReportHtml(report), "utf8");
28
34
  await writeFile(outputPaths.textPath, renderPluginExperimentReportText(report), "utf8");
29
35
  return { report, outputPaths };
package/docs/METRICS.md CHANGED
@@ -598,7 +598,7 @@ All six metrics are warm-side only (they are not on the raw-full-file side, beca
598
598
 
599
599
  ## Context-window scaling evidence (v0.7.0)
600
600
 
601
- Status: **released in v0.7.0**. The `context-window-scaling` plugin compares `raw-full-file` and `my-dev-kit-guided` treatment contexts against selected estimated-token budgets. It runs the bundled case catalog against the fixed self target.
601
+ Status: **released in v0.7.0**. The `context-window-scaling` plugin compares `raw-full-file` and `my-dev-kit-guided` treatment contexts against selected estimated-token budgets. By default it runs the bundled case catalog against the fixed self target; the v0.7.1 `--synthetic-config` input runs the same metrics over deterministic generated repositories without changing any formula below.
602
602
 
603
603
  ### Measurement and budget-cell semantics
604
604
 
@@ -620,6 +620,10 @@ The generic `ExperimentMetric` projection emits run-level case and budget counts
620
620
 
621
621
  The report artifact schema is `my-dev-kit-lab-context-window-scaling-report-v1`. It includes run summaries, budget-treatment summaries, per-treatment context measurements, budget cells, relevant-file summary, and fixed interpretation limits. The execution artifact is `context-window-scaling-execution.json` (`my-dev-kit-lab-context-window-scaling-execution-v1`) and does not embed context text.
622
622
 
623
+ **Synthetic-repository runs (v0.7.1).** The synthetic input introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact schema, the report schema, and the plot set are identical to the bundled-corpus run. Each generated repository's `synthetic-repository-manifest.json` records generation evidence (per-file byte length, character count, SHA-256, generated-content estimated tokens, aggregate totals, and `repositoryContentIdentity`); these are reproducibility measurements, not context-window outcome metrics. The raw-full-file treatment context is measured by the existing baseline, which adds file headers and separators, so its estimated tokens differ from the manifest's generated-content estimate. Retrieval precision/recall remains v0.8.0 scope.
624
+
625
+ **Local-repository runs (v0.7.2).** Running over an explicitly selected local Git repository introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact and report schemas, and the plot set are the same. In this mode file identity lists are replaced by numbered placeholders (`<redacted file N>`) with the same length as the real list, and each treatment carries `fileIdentityRedaction: "redacted"`; a placeholder is a withheld identity, not an empty result, and an empty list or `null` keeps its usual meaning. Operational compatibility with a repository, the 1 MiB file-size limit, and the Git-ignore exclusions are safety policy, not quality measures: they do not imply retrieval precision, recall, ranking quality, or a retrieval winner.
626
+
623
627
  There is no composite score, winner, or ranking. Budgets are experiment thresholds, not assertions about actual provider context-window capacities. See [COMMANDS.md](COMMANDS.md), [WORKFLOWS.md](WORKFLOWS.md), and [ARCHITECTURE.md](ARCHITECTURE.md).
624
628
 
625
629
  ## Incremental-change and staleness evidence (v0.6.2)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dailephd/my-dev-kit-lab",
3
- "version": "0.7.0",
3
+ "version": "0.7.2",
4
4
  "type": "module",
5
5
  "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
6
6
  "bin": {