@dailephd/my-dev-kit-lab 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +6 -4
- package/benchmarks/projects/README.md +3 -1
- package/dist/src/cli/help.js +20 -3
- package/dist/src/commands/runExperimentDescribeCommand.js +10 -1
- package/dist/src/commands/runExperimentRunCommand.js +169 -19
- package/dist/src/core/fileGlobs.js +44 -0
- package/dist/src/evaluation/localRepositorySubject/config.js +296 -0
- package/dist/src/evaluation/localRepositorySubject/evaluationCase.js +36 -0
- package/dist/src/evaluation/localRepositorySubject/expectedFiles.js +51 -0
- package/dist/src/evaluation/localRepositorySubject/gitRepository.js +128 -0
- package/dist/src/evaluation/localRepositorySubject/index.js +8 -0
- package/dist/src/evaluation/localRepositorySubject/inventory.js +127 -0
- package/dist/src/evaluation/localRepositorySubject/loadLocalRepositorySubject.js +48 -0
- package/dist/src/evaluation/localRepositorySubject/manifest.js +52 -0
- package/dist/src/evaluation/localRepositorySubject/types.js +55 -0
- package/dist/src/evaluation/projectFileTree.js +45 -1
- package/dist/src/evaluation/runMyDevKitRetrieval.js +3 -1
- package/dist/src/evaluation/runRawFullFileBaseline.js +18 -4
- package/dist/src/evaluation/syntheticRepository/config.js +304 -0
- package/dist/src/evaluation/syntheticRepository/evaluationCase.js +43 -0
- package/dist/src/evaluation/syntheticRepository/identity.js +41 -0
- package/dist/src/evaluation/syntheticRepository/index.js +16 -0
- package/dist/src/evaluation/syntheticRepository/inspection.js +92 -0
- package/dist/src/evaluation/syntheticRepository/layout.js +50 -0
- package/dist/src/evaluation/syntheticRepository/manifest.js +281 -0
- package/dist/src/evaluation/syntheticRepository/manifestVerification.js +112 -0
- package/dist/src/evaluation/syntheticRepository/materialize.js +170 -0
- package/dist/src/evaluation/syntheticRepository/planning.js +479 -0
- package/dist/src/evaluation/syntheticRepository/prng.js +47 -0
- package/dist/src/evaluation/syntheticRepository/renderPython.js +68 -0
- package/dist/src/evaluation/syntheticRepository/renderRepository.js +39 -0
- package/dist/src/evaluation/syntheticRepository/renderShared.js +93 -0
- package/dist/src/evaluation/syntheticRepository/renderTypeScript.js +82 -0
- package/dist/src/evaluation/syntheticRepository/topology.js +96 -0
- package/dist/src/evaluation/syntheticRepository/types.js +26 -0
- package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +29 -7
- package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +4 -0
- package/dist/src/experiments/plugins/contextWindowScaling/index.js +1 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectErrors.js +35 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExclusions.js +75 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExecution.js +136 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js +128 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectScratch.js +85 -0
- package/dist/src/experiments/plugins/contextWindowScaling/metadata.js +2 -2
- package/dist/src/experiments/plugins/contextWindowScaling/plugin.js +93 -0
- package/dist/src/experiments/plugins/contextWindowScaling/projectProfile.js +4 -9
- package/dist/src/experiments/plugins/contextWindowScaling/syntheticInputs.js +56 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +4 -3
- package/dist/src/report/experiments/writePluginExperimentReports.js +8 -2
- package/docs/METRICS.md +5 -1
- package/package.json +1 -1
|
@@ -3,6 +3,7 @@ import { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
|
|
|
3
3
|
import { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeStalenessHtml.js";
|
|
4
4
|
import { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
|
|
5
5
|
export function renderPluginExperimentReportHtml(report) {
|
|
6
|
+
const redactedTarget = report.target.privacyProjection === "external-local-redacted";
|
|
6
7
|
return `<!DOCTYPE html>
|
|
7
8
|
<html lang="en">
|
|
8
9
|
<head>
|
|
@@ -49,10 +50,10 @@ export function renderPluginExperimentReportHtml(report) {
|
|
|
49
50
|
["Mode", report.target.mode],
|
|
50
51
|
["Tool root", report.target.toolRoot],
|
|
51
52
|
["Target root", report.target.targetRoot],
|
|
52
|
-
["Target package", report.target.packageName ? `${report.target.packageName}@${report.target.packageVersion ?? "unknown"}` : "unavailable"],
|
|
53
|
+
["Target package", redactedTarget ? "redacted" : report.target.packageName ? `${report.target.packageName}@${report.target.packageVersion ?? "unknown"}` : "unavailable"],
|
|
53
54
|
["Target branch/commit", report.target.hasGit ? `${report.target.branch ?? "unknown"} / ${report.target.commit ?? "unknown"}` : "unavailable"],
|
|
54
|
-
["Has package.json", String(report.target.hasPackageJson)],
|
|
55
|
-
["Has lockfile", String(report.target.hasLockfile)],
|
|
55
|
+
["Has package.json", redactedTarget ? "redacted" : String(report.target.hasPackageJson)],
|
|
56
|
+
["Has lockfile", redactedTarget ? "redacted" : String(report.target.hasLockfile)],
|
|
56
57
|
["Output root", report.metadata.outputRoot ?? "unavailable"],
|
|
57
58
|
])}
|
|
58
59
|
</section>
|
|
@@ -17,13 +17,19 @@ export async function writePluginExperimentReports(args) {
|
|
|
17
17
|
textPath: resolveWithinRoot(outDir, "report.txt"),
|
|
18
18
|
};
|
|
19
19
|
await mkdir(outputPaths.outDir, { recursive: true });
|
|
20
|
-
const
|
|
20
|
+
const builtReport = buildPluginExperimentReport({
|
|
21
21
|
run: args.run,
|
|
22
22
|
plugin: args.plugin,
|
|
23
23
|
outputRoot: outputPaths.outDir,
|
|
24
24
|
generatedAt: args.generatedAt,
|
|
25
25
|
});
|
|
26
|
-
|
|
26
|
+
const report = args.redactOutputRoot
|
|
27
|
+
? { ...builtReport, metadata: { ...builtReport.metadata, outputRoot: "[redacted]" } }
|
|
28
|
+
: builtReport;
|
|
29
|
+
const serializedPaths = args.redactOutputRoot
|
|
30
|
+
? { outDir: "[redacted]", jsonPath: "report.json", htmlPath: "report.html", textPath: "report.txt" }
|
|
31
|
+
: outputPaths;
|
|
32
|
+
await writeFile(outputPaths.jsonPath, `${JSON.stringify({ report, outputPaths: serializedPaths }, null, 2)}\n`, "utf8");
|
|
27
33
|
await writeFile(outputPaths.htmlPath, renderPluginExperimentReportHtml(report), "utf8");
|
|
28
34
|
await writeFile(outputPaths.textPath, renderPluginExperimentReportText(report), "utf8");
|
|
29
35
|
return { report, outputPaths };
|
package/docs/METRICS.md
CHANGED
|
@@ -598,7 +598,7 @@ All six metrics are warm-side only (they are not on the raw-full-file side, beca
|
|
|
598
598
|
|
|
599
599
|
## Context-window scaling evidence (v0.7.0)
|
|
600
600
|
|
|
601
|
-
Status: **released in v0.7.0**. The `context-window-scaling` plugin compares `raw-full-file` and `my-dev-kit-guided` treatment contexts against selected estimated-token budgets.
|
|
601
|
+
Status: **released in v0.7.0**. The `context-window-scaling` plugin compares `raw-full-file` and `my-dev-kit-guided` treatment contexts against selected estimated-token budgets. By default it runs the bundled case catalog against the fixed self target; the v0.7.1 `--synthetic-config` input runs the same metrics over deterministic generated repositories without changing any formula below.
|
|
602
602
|
|
|
603
603
|
### Measurement and budget-cell semantics
|
|
604
604
|
|
|
@@ -620,6 +620,10 @@ The generic `ExperimentMetric` projection emits run-level case and budget counts
|
|
|
620
620
|
|
|
621
621
|
The report artifact schema is `my-dev-kit-lab-context-window-scaling-report-v1`. It includes run summaries, budget-treatment summaries, per-treatment context measurements, budget cells, relevant-file summary, and fixed interpretation limits. The execution artifact is `context-window-scaling-execution.json` (`my-dev-kit-lab-context-window-scaling-execution-v1`) and does not embed context text.
|
|
622
622
|
|
|
623
|
+
**Synthetic-repository runs (v0.7.1).** The synthetic input introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact schema, the report schema, and the plot set are identical to the bundled-corpus run. Each generated repository's `synthetic-repository-manifest.json` records generation evidence (per-file byte length, character count, SHA-256, generated-content estimated tokens, aggregate totals, and `repositoryContentIdentity`); these are reproducibility measurements, not context-window outcome metrics. The raw-full-file treatment context is measured by the existing baseline, which adds file headers and separators, so its estimated tokens differ from the manifest's generated-content estimate. Retrieval precision/recall remains v0.8.0 scope.
|
|
624
|
+
|
|
625
|
+
**Local-repository runs (v0.7.2).** Running over an explicitly selected local Git repository introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact and report schemas, and the plot set are the same. In this mode file identity lists are replaced by numbered placeholders (`<redacted file N>`) with the same length as the real list, and each treatment carries `fileIdentityRedaction: "redacted"`; a placeholder is a withheld identity, not an empty result, and an empty list or `null` keeps its usual meaning. Operational compatibility with a repository, the 1 MiB file-size limit, and the Git-ignore exclusions are safety policy, not quality measures: they do not imply retrieval precision, recall, ranking quality, or a retrieval winner.
|
|
626
|
+
|
|
623
627
|
There is no composite score, winner, or ranking. Budgets are experiment thresholds, not assertions about actual provider context-window capacities. See [COMMANDS.md](COMMANDS.md), [WORKFLOWS.md](WORKFLOWS.md), and [ARCHITECTURE.md](ARCHITECTURE.md).
|
|
624
628
|
|
|
625
629
|
## Incremental-change and staleness evidence (v0.6.2)
|