@compaction/cli 0.1.0

package/dist/core/billing-delta/billing-delta-record.d.ts

@@ -0,0 +1,290 @@
+/**
+ * Billing-delta measurement record — operator-run A/B protocol (local-first).
+ *
+ * Implements Alternative 1 / §3 of docs/design/billing-delta-loop.md (the accepted
+ * methodology direction): a LOCAL, OPERATOR-RUN measurement record. The OPERATOR
+ * performs the real workflow runs themselves (control arm = original context;
+ * treatment arm = apply-context-approved compacted context), reads the billed costs
+ * from THEIR OWN provider console / billing export, and ENTERS them. This module
+ * records runs with provenance, computes per-arm statistics and the delta of means,
+ * applies the accepted methodology's honesty rubric, and emits a caveated verdict.
+ *
+ * HARD HONESTY RAILS (see the accepted doc §2.4, §1(a)-(e), Risks):
+ * - This module makes NO provider call, reads NO credential, has NO network access.
+ * - Operator-entered figures are labeled `operator_entered: true`. The tool CANNOT
+ *   verify them against the provider.
+ * - The tool NEVER emits the phrase "billing-confirmed savings" as a claim and NEVER
+ *   sets `billing_confirmed: true`. This is NOT a Tier-3 claim. `ValueProof` is
+ *   untouched (its `billing_confirmed_savings.claimed` stays false).
+ * - The strongest verdict is a "measured, caveated delta on N runs of ONE workflow
+ *   under recorded conditions" — explicitly NOT a general savings claim.
+ * - Aggregate cost figures only (LOW sensitivity). NO prompt/completion content is
+ *   stored anywhere in the record.
+ */
+export declare const BILLING_DELTA_RECORD_ROOT = ".compaction/billing-delta";
+export declare const BILLING_DELTA_RECORD_SCHEMA: "billing-delta.measurement-record.v1";
+export declare const BILLING_DELTA_REPORT_SCHEMA: "billing-delta.report.v1";
+/** The two A/B arms. Control = original (un-compacted) context. Treatment = apply-context-approved compacted context. */
+export type BillingDeltaArm = "control" | "treatment";
+/**
+ * Machine-readable privacy posture, self-documenting in the artifact the LOW /
+ * aggregate-only classification from docs/design/billing-delta-loop.md (previously
+ * stated only in the command banner and the design doc). Literal types: these facts
+ * are structural properties of the substrate, not per-record choices.
+ */
+export interface BillingDeltaPrivacyPosture {
+    /** Maps to the design doc's LOW sensitivity classification. */
+    sensitivity: "low";
+    /** Aggregate billed-cost figures only — never prompt/completion content. */
+    aggregate_cost_only: true;
+    /** All billing figures are operator-entered; the tool reads nothing from a provider. */
+    operator_entered_figures: true;
+    /** The record is a local file; the tool makes no provider call and has no network access. */
+    local_only: true;
+}
+export declare const BILLING_DELTA_PRIVACY_POSTURE: BillingDeltaPrivacyPosture;
+/** The documented decision criterion. Named in every report. */
+export declare const BILLING_DELTA_CRITERION: string;
+/** The standard-error multiplier used to form the conservative interval. Documented and fixed (no over-engineered statistics). */
+export declare const BILLING_DELTA_INTERVAL_K = 2;
+/** Verbatim-ish anecdote note required by the accepted doc (§2.4: "a single A/B run is an anecdote, not proof"). */
+export declare const BILLING_DELTA_ANECDOTE_NOTE = "A single A/B pair is an anecdote, not proof: N=1 per arm cannot distinguish a compaction effect from run-to-run noise.";
+/** Provenance link to an apply-context approval record (the treatment context provenance), recorded by path + hash. */
+export interface TreatmentApprovalLink {
+    /** Path to the apply-approval-record JSON emitted by `compaction apply-context --approve-in-workflow-use`. */
+    apply_approval_record_path: string;
+    /** sha256 of the apply-approval-record file contents, captured at add-run time. */
+    apply_approval_record_sha256: string;
+}
+/** One operator-entered run on one arm. Aggregate cost only — NO prompt/completion content. */
+export interface BillingDeltaRun {
+    run_id: string;
+    arm: BillingDeltaArm;
+    /** Operator-entered billed cost read from THEIR provider console/billing export. */
+    billed_cost: number;
+    currency: string;
+    /**
+     * The tool cannot verify this figure against the provider. ALWAYS true on every run:
+     * these are operator-entered, not provider-verified by this tool.
+     */
+    operator_entered: true;
+    /** Optional provider-reported usage the operator has on hand (aggregate token counts only — no content). */
+    provider_reported_usage?: {
+        input_tokens?: number;
+        output_tokens?: number;
+        total_tokens?: number;
+    };
+    /** ISO 8601 timestamp the operator recorded for the run. */
+    run_timestamp: string;
+    /** Free-text operator notes / confounders (e.g. cache state, retries, model version drift). Aggregate, non-content. */
+    notes?: string;
+    /** For treatment runs: optional provenance link to the apply-approval record (the compacted-context provenance). */
+    treatment_approval_link?: TreatmentApprovalLink;
+}
+/** Fixed conditions pinned at init time — requirement (a): the workflow/run identity contract. */
+export interface BillingDeltaFixedConditions {
+    /** Model identifier (e.g. provider model name). */
+    model: string;
+    /** Pinned model version where the provider exposes it; "unknown" if not pinned. */
+    model_version: string;
+    /** Plain description of the fixed input/task (NOT the content itself — a description). */
+    input_description: string;
+    /** Time/pricing window the arms were run in (free text, e.g. "2026-06-11 within one billing window"). */
+    window: string;
+}
+/** The measurement record persisted under .compaction/billing-delta/. */
+export interface BillingDeltaMeasurementRecord {
+    schema: typeof BILLING_DELTA_RECORD_SCHEMA;
+    measurement_id: string;
+    created_at: string;
+    /** Operator-chosen workflow id. */
+    workflow_id: string;
+    /** Plain-language workflow description. */
+    workflow_description: string;
+    fixed_conditions: BillingDeltaFixedConditions;
+    /**
+     * Optional link to the apply-approval record from `apply-context` (the treatment context
+     * provenance). Recorded by path + hash. Mirrors the per-run treatment_approval_link but at
+     * the record level for the whole measurement.
+     */
+    apply_approval_link?: TreatmentApprovalLink;
+    runs: BillingDeltaRun[];
+    /**
+     * Machine-readable privacy posture (additive; same schema version). Optional because
+     * records written before the field existed honestly lack it — absence means "written
+     * pre-field", not a different posture.
+     */
+    privacy_posture?: BillingDeltaPrivacyPosture;
+    /** All figures are operator-entered; the tool performs no provider interaction. */
+    operator_entered: true;
+    /** This record is NOT a Tier-3 billing-confirmed claim. Stays false. */
+    billing_confirmed: false;
+    /** The tool makes no provider call and holds no credential. */
+    tool_makes_provider_call: false;
+}
+export type BillingDeltaVerdict = "insufficient_data" | "inconclusive" | "measured_caveated_delta";
+export interface ArmStatistics {
+    arm: BillingDeltaArm;
+    n: number;
+    /** null when n === 0. */
+    mean: number | null;
+    min: number | null;
+    max: number | null;
+    /** Sample standard deviation (n-1 denominator); null when n < 2. */
+    sample_sd: number | null;
+}
+/**
+ * Claim-level labels for the token-delta section. Token-level, provider-reported — never a
+ * billing claim. The label is selected by the ACTUAL sign/status of the token delta so it
+ * never calls a non-reduction a "reduction" (an overclaim by mislabel): a negative input
+ * delta means the treatment context GREW (an INCREASE), zero means NO CHANGE, and the
+ * insufficient/suppressed statuses have no computed delta at all. Every variant preserves the
+ * "NOT billing-confirmed savings" disclaimer and makes no savings claim.
+ */
+export declare const BILLING_DELTA_TOKEN_DELTA_LABELS: {
+    /** Genuine reduction: treatment used fewer input tokens than control (positive input delta). */
+    readonly reduction: "provider-reported token reduction (workflow-scoped) — NOT billing-confirmed savings";
+    /** Increase: treatment used MORE input tokens than control (negative input delta). Not a saving. */
+    readonly increase: "provider-reported token INCREASE (workflow-scoped) — NOT a saving; NOT billing-confirmed savings";
+    /** No change: the input token delta is exactly zero. */
+    readonly no_change: "provider-reported token delta: no change (workflow-scoped) — NOT billing-confirmed savings";
+    /** Not computed: insufficient token data or deltas suppressed by a failed task. */
+    readonly not_computed: "provider-reported token delta not computed (workflow-scoped) — NOT billing-confirmed savings";
+};
+export type BillingDeltaTokenDeltaLabel = (typeof BILLING_DELTA_TOKEN_DELTA_LABELS)[keyof typeof BILLING_DELTA_TOKEN_DELTA_LABELS];
+/** Per-arm token statistics from the runs' provider_reported_usage fields (operator-entered from API responses). */
+export interface TokenArmStatistics {
+    arm: BillingDeltaArm;
+    /** Runs in this arm that carried provider-reported input_tokens. */
+    n_with_tokens: number;
+    /** Distinct input_tokens values observed (input is deterministic for a fixed context+prompt; >1 distinct value is surfaced, not averaged away). */
+    input_tokens_distinct: number[];
+    /** True when every recorded input_tokens in the arm is identical. */
+    input_deterministic: boolean;
+    mean_input_tokens: number | null;
+    mean_output_tokens: number | null;
+    output_min: number | null;
+    output_max: number | null;
+}
+export type TokenDeltaStatus = "computed" | "insufficient_token_data" | "suppressed_task_failure";
+/**
+ * Select the sign/status-accurate claim label. Reduction is used ONLY when the input delta is
+ * genuinely positive (treatment used fewer tokens); a negative delta is an increase, zero is no
+ * change, and a null delta (insufficient/suppressed status) is "not computed". This never
+ * strengthens a claim — it only makes the existing label honest for the actual sign/status.
+ */
+export declare function selectTokenDeltaLabel(status: TokenDeltaStatus, inputTokenDelta: number | null): BillingDeltaTokenDeltaLabel;
+/** Task-quality flags parsed from run notes ("task: ok" / "task: failed"). */
+export interface TokenDeltaTaskQuality {
+    control: string[];
+    treatment: string[];
+    any_failed: boolean;
+}
+/**
+ * Provider-reported token-delta section. Token counts come from the provider's own API
+ * response usage metadata (operator-entered per run); the cost delta is a price-table
+ * ESTIMATE applied to those token deltas — never a billed figure. This section makes
+ * NO billing-confirmed claim and does not affect the billed-cost verdict/criterion.
+ */
+export interface BillingDeltaTokenDelta {
+    /** Sign/status-accurate label (see selectTokenDeltaLabel): reduction ONLY for a genuine reduction. */
+    claim_label: BillingDeltaTokenDeltaLabel;
+    status: TokenDeltaStatus;
+    control: TokenArmStatistics;
+    treatment: TokenArmStatistics;
+    /** control mean input - treatment mean input. Positive = treatment used fewer input tokens. */
+    input_token_delta: number | null;
+    /** input_token_delta / control mean input. */
+    input_token_delta_pct: number | null;
+    /** control mean output - treatment mean output (noisy — read with the per-arm spread). */
+    output_token_delta: number | null;
+    /** Price-table ESTIMATE applied to the token deltas. null when the model is not in the price table. */
+    estimated_cost_delta: {
+        amount_usd: number;
+        cost_source: "price_table_estimate";
+        note: string;
+    } | null;
+    task_quality: TokenDeltaTaskQuality;
+    notes: string[];
+    /** Never a Tier-3 claim. */
+    billing_confirmed: false;
+}
+export interface BillingDeltaReport {
+    schema: typeof BILLING_DELTA_REPORT_SCHEMA;
+    measurement_id: string;
+    workflow_id: string;
+    generated_at: string;
+    currency: string | null;
+    fixed_conditions: BillingDeltaFixedConditions;
+    control: ArmStatistics;
+    treatment: ArmStatistics;
+    /** control_mean - treatment_mean (positive = treatment cheaper = a saving). null when an arm is empty. */
+    mean_delta: number | null;
+    /** Standard error of the delta of means. null when either arm has n < 2. */
+    delta_standard_error: number | null;
+    /** Conservative interval [mean_delta - k*SE, mean_delta + k*SE]. null when SE is null. */
+    delta_interval: {
+        lower: number;
+        upper: number;
+    } | null;
+    /** True only when the interval is computable AND excludes zero. */
+    delta_interval_excludes_zero: boolean;
+    /** The named, documented decision criterion. */
+    criterion: string;
+    verdict: BillingDeltaVerdict;
+    /** Notes attached to the verdict (e.g. the anecdote note for insufficient_data). */
+    verdict_notes: string[];
+    /** The measured workflow/conditions ONLY — never a general claim. */
+    claim_scope: string;
+    /** This is NOT a general savings claim. */
+    not_a_general_claim: true;
+    /** This record is NOT a Tier-3 billing-confirmed claim. Stays false. */
+    billing_confirmed: false;
+    /** All input figures are operator-entered; the tool did no provider interaction. */
+    operator_entered: true;
+    /** Confounders the operator recorded across the runs (notes), surfaced for honesty. */
+    confounders: string[];
+    /** Plain-language caveat block — always present. */
+    caveats: string[];
+    /** Treatment-run provenance links recorded (path + hash), surfaced for provenance. */
+    treatment_provenance_links: TreatmentApprovalLink[];
+    /** Provider-reported token-delta section (token-level; never a billing claim; does not affect the verdict). */
+    token_delta: BillingDeltaTokenDelta;
+}
+export interface InitMeasurementInput {
+    workflowId: string;
+    workflowDescription: string;
+    model: string;
+    modelVersion?: string;
+    inputDescription: string;
+    window: string;
+    measurementId?: string;
+    generatedAt: string;
+    applyApprovalRecordPath?: string;
+    applyApprovalRecordContent?: string;
+}
+export declare function buildTreatmentApprovalLink(path: string, fileContent: string): TreatmentApprovalLink;
+export declare function initMeasurementRecord(input: InitMeasurementInput): BillingDeltaMeasurementRecord;
+export interface AddRunInput {
+    arm: BillingDeltaArm;
+    billedCost: number;
+    currency: string;
+    runTimestamp: string;
+    notes?: string;
+    providerReportedUsage?: {
+        input_tokens?: number;
+        output_tokens?: number;
+        total_tokens?: number;
+    };
+    treatmentApprovalRecordPath?: string;
+    treatmentApprovalRecordContent?: string;
+    runId?: string;
+}
+export declare function addRun(record: BillingDeltaMeasurementRecord, input: AddRunInput): BillingDeltaMeasurementRecord;
+/**
+ * Compute the provider-reported token-delta section. Input tokens are deterministic for a
+ * fixed context+prompt, so one tokened run per arm pins the input delta exactly; the
+ * billed-cost verdict and its N>=3 criterion are unaffected by this section.
+ */
+export declare function computeTokenDelta(record: BillingDeltaMeasurementRecord): BillingDeltaTokenDelta;
+export declare function computeReport(record: BillingDeltaMeasurementRecord, generatedAt: string): BillingDeltaReport;