@dailephd/my-dev-kit-lab 0.4.2 → 0.4.3

package/dist/src/report/experiments/buildContextStrategyComparisonV043Report.js

@@ -0,0 +1,398 @@
+import { V043_REPORT_DETAIL_LIMIT, } from "./contextStrategyComparisonV043ReportModel.js";
+const INTERPRETATION_SUMMARY = "Stage-context strategies are reported independently using direct evidence, explicit availability states, target-immutability evidence, and repeated-run determinism. No composite ranking or winning strategy is calculated.";
+const INTERPRETATION_LIMITATIONS = [
+    "The published upstream artifacts do not expose considered-but-unselected reads.",
+    "The published upstream artifacts do not expose unnecessary-read evidence.",
+    "Estimated token counts use ceil(characterCount / 4) per source and are not provider telemetry.",
+    "Unavailable and not-applicable values are distinct from zero.",
+];
+function boundItems(source) {
+    const items = source.slice(0, V043_REPORT_DETAIL_LIMIT);
+    return {
+        totalCount: source.length,
+        displayedCount: items.length,
+        omittedCount: source.length - items.length,
+        items,
+    };
+}
+export function buildContextStrategyComparisonV043Report(run) {
+    if (run.pluginId !== "context-strategy-comparison")
+        return null;
+    const candidate = run;
+    const executions = candidate.v043StageContextExecutions;
+    const evaluations = candidate.v043StageContextEvaluations;
+    const assurances = candidate.v043StageContextRunAssurance;
+    if (executions === undefined && evaluations === undefined && assurances === undefined) {
+        return null;
+    }
+    if (!Array.isArray(executions) || !Array.isArray(evaluations) || !Array.isArray(assurances)) {
+        throw new Error("Invalid v0.4.3 report source: execution, evaluation, and assurance arrays are inconsistent.");
+    }
+    if (executions.length !== evaluations.length || executions.length !== assurances.length) {
+        throw new Error("Invalid v0.4.3 report source: execution, evaluation, and assurance arrays are inconsistent.");
+    }
+    for (let index = 0; index < executions.length; index += 1) {
+        const ids = [
+            executions[index].strategyId,
+            evaluations[index].strategyId,
+            assurances[index].strategyId,
+        ].filter((id) => id !== null && id !== undefined);
+        const distinct = new Set(ids);
+        if (distinct.size > 1) {
+            throw new Error("Invalid v0.4.3 report source: execution, evaluation, and assurance arrays are inconsistent.");
+        }
+    }
+    const strategies = executions.map((execution, index) => buildStrategyReport(execution, evaluations[index], assurances[index]));
+    return {
+        schemaVersion: "1.0.0",
+        detailLimit: V043_REPORT_DETAIL_LIMIT,
+        summary: buildSummary(executions, evaluations, assurances),
+        strategies,
+        interpretation: {
+            summary: INTERPRETATION_SUMMARY,
+            limitations: [...INTERPRETATION_LIMITATIONS],
+        },
+    };
+}
+function buildSummary(executions, evaluations, assurances) {
+    return {
+        strategyCount: executions.length,
+        completedExecutionCount: executions.filter((e) => e.status === "completed").length,
+        invalidInputExecutionCount: executions.filter((e) => e.status === "invalid-input").length,
+        failedExecutionCount: executions.filter((e) => e.status === "failed").length,
+        completedEvaluationCount: evaluations.filter((e) => e.status === "completed").length,
+        notApplicableEvaluationCount: evaluations.filter((e) => e.status === "not-applicable").length,
+        failedEvaluationCount: evaluations.filter((e) => e.status === "failed").length,
+        passedAssuranceCount: assurances.filter((a) => a.status === "passed").length,
+        failedAssuranceCount: assurances.filter((a) => a.status === "failed").length,
+        notApplicableAssuranceCount: assurances.filter((a) => a.status === "not-applicable").length,
+    };
+}
+function buildStrategyReport(execution, evaluation, assurance) {
+    return {
+        strategyId: assurance.strategyId,
+        artifacts: buildArtifactMetadata(execution),
+        execution: buildExecutionSummary(execution),
+        evaluation: buildEvaluationSummary(evaluation),
+        assurance: buildAssuranceSummary(assurance),
+    };
+}
+function buildExecutionSummary(execution) {
+    if (execution.status === "completed") {
+        return { status: "completed", issues: boundItems([]) };
+    }
+    const issues = execution.issues.map(buildIssueFromExecutionIssue);
+    return { status: execution.status, issues: boundItems(issues) };
+}
+function buildIssueFromExecutionIssue(issue) {
+    return {
+        code: issue.code,
+        fieldPath: issue.fieldPath ?? null,
+        message: issue.message,
+        runNumber: null,
+    };
+}
+function buildArtifactMetadata(execution) {
+    if (execution.status !== "completed") {
+        return boundItems([]);
+    }
+    const entries = [];
+    entries.push({
+        sourceInstance: "expectations",
+        artifactKind: "expectation-fixture",
+        sourcePath: execution.expectationsSourcePath,
+        schemaVersion: execution.expectations.schemaVersion,
+        generatedAt: null,
+        producerName: null,
+        producerVersion: null,
+        role: null,
+        workflowId: null,
+        stageId: null,
+        catalogSchemaVersion: null,
+        catalogVersion: null,
+        fixtureId: null,
+        caseId: execution.expectations.caseId,
+        title: execution.expectations.title,
+    });
+    const payload = execution.payload;
+    if ("contextArtifacts" in payload) {
+        const contextArtifacts = payload.contextArtifacts;
+        contextArtifacts.forEach((pair, index) => {
+            entries.push(...buildContextArtifactPairMetadata(pair, `contextArtifacts[${index}]`));
+        });
+        entries.push(buildWorkflowInstructionPacketMetadata(payload.workflowInstructionPacketSourcePath, payload.workflowInstructionPacket));
+    }
+    else if ("fullWorkflowLibrary" in payload) {
+        entries.push(buildFullWorkflowLibraryMetadata(payload.fullWorkflowLibrarySourcePath, payload.fullWorkflowLibrary));
+    }
+    else if ("workflowInstructionPacket" in payload) {
+        entries.push(buildWorkflowInstructionPacketMetadata(payload.workflowInstructionPacketSourcePath, payload.workflowInstructionPacket));
+    }
+    else if ("testImplementation" in payload) {
+        entries.push(...buildContextArtifactPairMetadata(payload.architecture, "architecture"));
+        entries.push(...buildContextArtifactPairMetadata(payload.implementation, "implementation"));
+        entries.push(...buildContextArtifactPairMetadata(payload.testImplementation, "testImplementation"));
+    }
+    else if ("implementation" in payload) {
+        entries.push(...buildContextArtifactPairMetadata(payload.architecture, "architecture"));
+        entries.push(...buildContextArtifactPairMetadata(payload.implementation, "implementation"));
+    }
+    else if ("architecture" in payload) {
+        entries.push(...buildContextArtifactPairMetadata(payload.architecture, "architecture"));
+    }
+    return boundItems(entries);
+}
+function buildContextArtifactPairMetadata(pair, sourceInstancePrefix) {
+    const entries = [
+        {
+            sourceInstance: `${sourceInstancePrefix}.contextCapsule`,
+            artifactKind: "context-capsule",
+            sourcePath: pair.contextCapsuleSourcePath,
+            schemaVersion: pair.contextCapsule.schemaVersion,
+            generatedAt: pair.contextCapsule.generatedAt,
+            producerName: pair.contextCapsule.tool.name,
+            producerVersion: pair.contextCapsule.tool.version,
+            role: pair.contextCapsule.request.role,
+            workflowId: null,
+            stageId: null,
+            catalogSchemaVersion: null,
+            catalogVersion: null,
+            fixtureId: null,
+            caseId: null,
+            title: null,
+        },
+    ];
+    if (pair.retrievalAuditRecord) {
+        entries.push({
+            sourceInstance: `${sourceInstancePrefix}.retrievalAuditRecord`,
+            artifactKind: "retrieval-audit-record",
+            sourcePath: pair.retrievalAuditRecordSourcePath ?? "",
+            schemaVersion: pair.retrievalAuditRecord.schemaVersion,
+            generatedAt: pair.retrievalAuditRecord.generatedAt,
+            producerName: pair.retrievalAuditRecord.tool.name,
+            producerVersion: pair.retrievalAuditRecord.tool.version,
+            role: pair.retrievalAuditRecord.request.role,
+            workflowId: null,
+            stageId: null,
+            catalogSchemaVersion: null,
+            catalogVersion: null,
+            fixtureId: null,
+            caseId: null,
+            title: null,
+        });
+    }
+    return entries;
+}
+function buildWorkflowInstructionPacketMetadata(sourcePath, packet) {
+    return {
+        sourceInstance: "workflowInstructionPacket",
+        artifactKind: "workflow-instruction-packet",
+        sourcePath,
+        schemaVersion: packet.schemaVersion,
+        generatedAt: null,
+        producerName: null,
+        producerVersion: null,
+        role: null,
+        workflowId: packet.workflowId,
+        stageId: packet.stageId,
+        catalogSchemaVersion: packet.catalogSchemaVersion,
+        catalogVersion: packet.catalogVersion,
+        fixtureId: null,
+        caseId: null,
+        title: null,
+    };
+}
+function buildFullWorkflowLibraryMetadata(sourcePath, library) {
+    return {
+        sourceInstance: "fullWorkflowLibrary",
+        artifactKind: "full-workflow-library",
+        sourcePath,
+        schemaVersion: library.schemaVersion,
+        generatedAt: null,
+        producerName: null,
+        producerVersion: null,
+        role: null,
+        workflowId: null,
+        stageId: null,
+        catalogSchemaVersion: null,
+        catalogVersion: null,
+        fixtureId: library.fixtureId,
+        caseId: null,
+        title: library.title,
+    };
+}
+function buildEvaluationSummary(evaluation) {
+    if (evaluation.status !== "completed") {
+        return {
+            status: evaluation.status,
+            reason: evaluation.reason,
+            warnings: boundItems([]),
+            expectationMatches: boundItems([]),
+            observedEvidence: boundItems([]),
+            metrics: null,
+        };
+    }
+    return {
+        status: "completed",
+        reason: null,
+        warnings: boundItems(evaluation.warnings),
+        expectationMatches: boundItems(evaluation.expectationMatches.map(buildExpectationMatch)),
+        observedEvidence: boundItems(evaluation.observedEvidence.map(buildObservedEvidence)),
+        metrics: buildMetrics(evaluation.metrics),
+    };
+}
+function buildExpectationMatch(match) {
+    return {
+        expectationId: match.expectationId,
+        inclusion: match.inclusion,
+        sourceArtifact: match.sourceArtifact,
+        category: match.category,
+        targetKey: match.targetKey,
+        outcome: match.outcome,
+        matchedSourceInstances: match.matchedSourceInstances,
+        matchedSourceFieldPaths: match.matchedSourceFieldPaths,
+    };
+}
+function buildObservedEvidence(evidence) {
+    return {
+        sourceArtifact: evidence.sourceArtifact,
+        sourceInstance: evidence.sourceInstance,
+        category: evidence.category,
+        targetKey: evidence.targetKey,
+        sourceFieldPath: evidence.sourceFieldPath,
+    };
+}
+function buildMetrics(metrics) {
+    return {
+        requiredEvidenceRecall: buildRatioMetric(metrics.requiredEvidenceRecall),
+        allowedEvidenceCoverage: buildRatioMetric(metrics.allowedEvidenceCoverage),
+        forbiddenEvidenceInclusion: buildRatioMetric(metrics.forbiddenEvidenceInclusion),
+        irrelevantFileInclusion: buildCountMetric(metrics.irrelevantFileInclusion),
+        irrelevantInstructionInclusion: buildCountMetric(metrics.irrelevantInstructionInclusion),
+        requiredProvenanceRecall: buildRatioMetric(metrics.requiredProvenanceRecall),
+        responsibilityMappingCompleteness: boundItems(metrics.responsibilityMappingCompleteness.map(buildResponsibilityMapping)),
+        stateComparisons: boundItems(metrics.stateComparisons.map(buildStateComparison)),
+        contextSize: buildContextSize(metrics.contextSize),
+        consideredButUnselectedReads: buildCountMetric(metrics.consideredButUnselectedReads),
+        unnecessaryReads: buildCountMetric(metrics.unnecessaryReads),
+        targetImmutability: buildCountMetric(metrics.targetImmutability),
+    };
+}
+function buildRatioMetric(metric) {
+    return {
+        availability: metric.availability,
+        numerator: metric.numerator,
+        denominator: metric.denominator,
+        rate: metric.rate,
+        matchedExpectationIds: boundItems(metric.matchedExpectationIds),
+        missingExpectationIds: boundItems(metric.missingExpectationIds),
+        reason: metric.reason,
+    };
+}
+function buildCountMetric(metric) {
+    return {
+        availability: metric.availability,
+        count: metric.count,
+        evidenceKeys: boundItems(metric.evidenceKeys),
+        reason: metric.reason,
+    };
+}
+function buildResponsibilityMapping(mapping) {
+    return {
+        sourceArtifact: mapping.sourceArtifact,
+        sourceInstance: mapping.sourceInstance,
+        requested: mapping.requested,
+        operational: mapping.operational,
+        mappedCount: mapping.mappedCount,
+        partiallyMappedCount: mapping.partiallyMappedCount,
+        unmappedCount: mapping.unmappedCount,
+        notApplicableCount: mapping.notApplicableCount,
+        denominator: mapping.denominator,
+        mappedRate: mapping.mappedRate,
+    };
+}
+function buildStateComparison(comparison) {
+    return {
+        sourceArtifact: comparison.sourceArtifact,
+        sourceInstance: comparison.sourceInstance,
+        expectationFieldPath: comparison.expectationFieldPath,
+        artifactFieldPath: comparison.artifactFieldPath,
+        availability: comparison.availability,
+        expected: comparison.expected,
+        actual: comparison.actual,
+        matched: comparison.matched,
+        reason: comparison.reason,
+    };
+}
+function buildContextSize(size) {
+    return {
+        sources: boundItems(size.sources.map((source) => ({
+            sourceInstance: source.sourceInstance,
+            sourceKind: source.sourceKind,
+            characterCount: source.characterCount,
+            estimatedTokenCount: source.estimatedTokenCount,
+        }))),
+        totalCharacterCount: size.totalCharacterCount,
+        totalEstimatedTokenCount: size.totalEstimatedTokenCount,
+        tokenEstimateFormula: "ceil(characterCount / 4) per source",
+    };
+}
+function buildAssuranceSummary(assurance) {
+    return {
+        status: assurance.status,
+        repeatCount: assurance.repeatCount,
+        runRecords: boundItems(assurance.runRecords.map(buildRunRecord)),
+        determinism: buildDeterminism(assurance.determinism),
+        issues: boundItems(assurance.issues.map((issue) => ({
+            code: issue.code,
+            fieldPath: issue.fieldPath,
+            message: issue.message,
+            runNumber: issue.runNumber,
+        }))),
+    };
+}
+function buildRunRecord(record) {
+    const targetImmutability = record.targetImmutability;
+    if (targetImmutability.availability === "available") {
+        return {
+            runNumber: record.runNumber,
+            executionStatus: record.executionStatus,
+            evaluationStatus: record.evaluationStatus,
+            targetImmutabilityAvailability: "available",
+            targetImmutabilityStatus: targetImmutability.comparison.status,
+            newMutationCount: targetImmutability.comparison.newMutationCount,
+            targetImmutabilityReason: null,
+            mutations: boundItems(targetImmutability.comparison.mutations.map(buildTargetMutation)),
+        };
+    }
+    return {
+        runNumber: record.runNumber,
+        executionStatus: record.executionStatus,
+        evaluationStatus: record.evaluationStatus,
+        targetImmutabilityAvailability: "unavailable",
+        targetImmutabilityStatus: null,
+        newMutationCount: null,
+        targetImmutabilityReason: targetImmutability.reason,
+        mutations: boundItems([]),
+    };
+}
+function buildTargetMutation(mutation) {
+    return {
+        id: mutation.id,
+        kind: mutation.kind,
+        fieldPath: mutation.fieldPath,
+        before: mutation.before,
+        after: mutation.after,
+    };
+}
+function buildDeterminism(determinism) {
+    return {
+        availability: determinism.availability,
+        repeatCount: determinism.repeatCount,
+        deterministic: determinism.deterministic,
+        baselineSha256: determinism.baselineSha256,
+        runDigests: boundItems(determinism.runDigests.map((digest) => ({ runNumber: digest.runNumber, sha256: digest.sha256 }))),
+        mismatchRunNumbers: [...determinism.mismatchRunNumbers],
+        reason: determinism.reason,
+    };
+}

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -1,7 +1,18 @@
 import path from "node:path";
+import { buildContextStrategyComparisonV043Report } from "./buildContextStrategyComparisonV043Report.js";
+const V043_BULK_ARRAY_KEYS = [
+    "v043StageContextExecutions",
+    "v043StageContextEvaluations",
+    "v043StageContextRunAssurance",
+];
 export function buildPluginExperimentReport(args) {
     const outputRoot = args.outputRoot ?? readString(args.run.metadata?.outputRoot) ?? null;
     const allOutcomes = args.run.cases.flatMap((experimentCase) => experimentCase.outcomes);
+    const contextStrategyComparisonV043 = buildContextStrategyComparisonV043Report(args.run);
+    const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
+    for (const key of V043_BULK_ARRAY_KEYS) {
+        delete rawRun[key];
+    }
     return {
         metadata: {
             generatedAt: args.generatedAt ?? new Date().toISOString(),
@@ -25,11 +36,9 @@ export function buildPluginExperimentReport(args) {
         failures: args.run.failures,
         skippedOutcomes: allOutcomes.filter((outcome) => outcome.status === "skipped"),
         findings: buildFindings(args.run),
-        interpretation: buildInterpretation(args.run),
-        rawRun: {
-            ...args.run,
-            artifacts: relativizeArtifacts(args.run.artifacts, outputRoot),
-        },
+        contextStrategyComparisonV043,
+        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043),
+        rawRun,
     };
 }
 function buildVariantSummary(variant, outcomes) {
@@ -85,7 +94,13 @@ function buildFindings(run) {
         ...skipped,
     ];
 }
-function buildInterpretation(run) {
+function buildInterpretation(run, contextStrategyComparisonV043) {
+    if (run.pluginId === "context-strategy-comparison" && (contextStrategyComparisonV043?.summary.strategyCount ?? 0) > 0) {
+        return {
+            summary: `Stage-context evidence was recorded for ${contextStrategyComparisonV043.summary.strategyCount} strategy executions. Review each strategy independently; this report does not calculate a composite ranking or winning strategy.`,
+            recommendedNextStep: "Review required evidence, irrelevant inclusion, state comparisons, target immutability, determinism, and explicit unavailable or not-applicable metrics for each strategy.",
+        };
+    }
     if (run.pluginId === "context-strategy-comparison") {
         const tokenSavings = metricNumber(run.metrics, "average-token-savings-percent");
         const correctnessDelta = metricNumber(run.metrics, "average-correctness-delta");

package/dist/src/report/experiments/contextStrategyComparisonV043ReportModel.js

@@ -0,0 +1 @@
+export const V043_REPORT_DETAIL_LIMIT = 100;

package/dist/src/report/experiments/index.js

@@ -2,3 +2,7 @@ export * from "./buildPluginExperimentReport.js";
 export * from "./experimentReportModel.js";
 export * from "./renderPluginExperimentReportHtml.js";
 export * from "./writePluginExperimentReports.js";
+export * from "./contextStrategyComparisonV043ReportModel.js";
+export { buildContextStrategyComparisonV043Report } from "./buildContextStrategyComparisonV043Report.js";
+export { renderContextStrategyComparisonV043Html } from "./renderContextStrategyComparisonV043Html.js";
+export { renderPluginExperimentReportText } from "./renderPluginExperimentReportText.js";