cloving 1.2.0 → 1.4.0

package/dist/src/report-quality.d.ts

@@ -0,0 +1,221 @@
+import type { Fields } from "./protocol.ts";
+import { type ReportDatum, type ReportWindow } from "./report-statistics.ts";
+export interface QualityJob {
+    readonly id: number;
+    readonly runId: number;
+    readonly commit: string;
+    readonly name: string;
+    readonly attempt: number;
+    readonly at: number;
+    readonly conclusion: string | null;
+}
+export declare function reportFailureData(input: {
+    readonly jobs: readonly QualityJob[];
+    readonly names: Readonly<Record<number, readonly string[]>>;
+}): {
+    flaky: ReportDatum[];
+    real: ReportDatum[];
+};
+export interface ReportQualityData {
+    readonly effects?: readonly Fields[] | null;
+    readonly daemonObservedSince?: number | null;
+    readonly reopened?: readonly ReportDatum[] | null;
+    readonly reverts?: readonly ReportDatum[] | null;
+    readonly jobLogs?: {
+        readonly read: number;
+        readonly notRead: number;
+        readonly limit: number;
+        readonly maximumBytes: number;
+    } | null;
+    readonly mainFirstPass?: readonly ReportDatum[] | null;
+    readonly failures?: {
+        readonly flaky: readonly ReportDatum[];
+        readonly real: readonly ReportDatum[];
+    } | null;
+}
+export declare function reportQualityMetrics(input: ReportQualityData, window: ReportWindow): {
+    reviewScore: {
+        [k: string]: {
+            evidence: Readonly<Record<string, unknown>>[];
+            mark?: string;
+            status: string;
+            value: number | null;
+            comparisonValue: number | null;
+            baseline: {
+                evidence: Readonly<Record<string, unknown>>[];
+                history: string;
+                days: number;
+                median: null;
+                band: null;
+            } | {
+                evidence: Readonly<Record<string, unknown>>[];
+                history: string;
+                days: number;
+                median: number;
+                band: null;
+            } | {
+                evidence: Readonly<Record<string, unknown>>[];
+                history: string;
+                days: number;
+                median: number;
+                band: {
+                    low: number;
+                    high: number;
+                };
+            };
+        };
+    };
+    reopenedTickets: {
+        evidence: Readonly<Record<string, unknown>>[];
+        mark?: string;
+        status: string;
+        value: number | null;
+        comparisonValue: number | null;
+        baseline: {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: null;
+            band: null;
+        } | {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: number;
+            band: null;
+        } | {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: number;
+            band: {
+                low: number;
+                high: number;
+            };
+        };
+    };
+    revertedPullRequests: {
+        evidence: Readonly<Record<string, unknown>>[];
+        mark?: string;
+        status: string;
+        value: number | null;
+        comparisonValue: number | null;
+        baseline: {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: null;
+            band: null;
+        } | {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: number;
+            band: null;
+        } | {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: number;
+            band: {
+                low: number;
+                high: number;
+            };
+        };
+    };
+    jobLogs: {
+        readonly read: number;
+        readonly notRead: number;
+        readonly limit: number;
+        readonly maximumBytes: number;
+    } | {
+        status: string;
+    };
+    mainFirstPassShare: {
+        evidence: Readonly<Record<string, unknown>>[];
+        mark?: string;
+        status: string;
+        value: number | null;
+        comparisonValue: number | null;
+        baseline: {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: null;
+            band: null;
+        } | {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: number;
+            band: null;
+        } | {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: number;
+            band: {
+                low: number;
+                high: number;
+            };
+        };
+    };
+    flakyFailures: {
+        evidence: Readonly<Record<string, unknown>>[];
+        mark?: string;
+        status: string;
+        value: number | null;
+        comparisonValue: number | null;
+        baseline: {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: null;
+            band: null;
+        } | {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: number;
+            band: null;
+        } | {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: number;
+            band: {
+                low: number;
+                high: number;
+            };
+        };
+    };
+    realFailures: {
+        evidence: Readonly<Record<string, unknown>>[];
+        mark?: string;
+        status: string;
+        value: number | null;
+        comparisonValue: number | null;
+        baseline: {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: null;
+            band: null;
+        } | {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: number;
+            band: null;
+        } | {
+            evidence: Readonly<Record<string, unknown>>[];
+            history: string;
+            days: number;
+            median: number;
+            band: {
+                low: number;
+                high: number;
+            };
+        };
+    };
+};

package/dist/src/report-quality.js

@@ -0,0 +1,94 @@
+import { reportReviewScores } from "./report-effects.js";
+import { reportMetric, } from "./report-statistics.js";
+function later(left, right) {
+    if (left.runId === right.runId)
+        return left.attempt > right.attempt;
+    return left.at > right.at;
+}
+export function reportFailureData(input) {
+    const groups = new Map();
+    for (const job of input.jobs) {
+        const key = JSON.stringify([job.commit, job.name]);
+        const group = groups.get(key) ?? [];
+        group.push(job);
+        groups.set(key, group);
+    }
+    const flaky = [];
+    const real = [];
+    for (const jobs of groups.values()) {
+        const failed = jobs.filter((job) => job.conclusion === "failure");
+        const passed = jobs.filter((job) => job.conclusion === "success");
+        const flakyJobs = failed.filter((job) => passed.some((success) => later(success, job)));
+        const realJobs = failed.filter((job) => !passed.some((success) => later(success, job)));
+        const successes = passed.filter((job) => flakyJobs.some((failure) => later(job, failure)));
+        flaky.push(...failureDatum({
+            failed: flakyJobs,
+            passed: successes,
+            names: input.names,
+        }));
+        real.push(...failureDatum({ failed: realJobs, passed: [], names: input.names }));
+    }
+    return { flaky, real };
+}
+export function reportQualityMetrics(input, window) {
+    const reviewScore = reviewMetrics(input, window);
+    return {
+        reviewScore,
+        reopenedTickets: reportMetric({
+            data: input.reopened ?? null,
+            window,
+            aggregation: "count",
+        }),
+        revertedPullRequests: reportMetric({
+            data: input.reverts ?? null,
+            window,
+            aggregation: "count",
+        }),
+        jobLogs: input.jobLogs ?? { status: "unavailable" },
+        mainFirstPassShare: reportMetric({
+            data: input.mainFirstPass ?? null,
+            window,
+            aggregation: "mean",
+        }),
+        flakyFailures: reportMetric({
+            data: input.failures?.flaky ?? null,
+            window,
+            aggregation: "distinct",
+        }),
+        realFailures: reportMetric({
+            data: input.failures?.real ?? null,
+            window,
+            aggregation: "distinct",
+        }),
+    };
+}
+function reviewMetrics(input, window) {
+    const rounds = reportReviewScores(input.effects ?? null);
+    const reviewScore = Object.fromEntries(Object.entries(rounds ?? { round1: null }).map(([key, data]) => [
+        key,
+        reportMetric({
+            data,
+            window: { ...window, observedSince: input.daemonObservedSince ?? null },
+            aggregation: "median",
+        }),
+    ]));
+    return reviewScore;
+}
+function failureDatum(input) {
+    return input.failed.map((failed) => {
+        const passed = input.passed.filter((job) => later(job, failed));
+        const jobs = [failed, ...passed];
+        return {
+            at: failed.at,
+            value: 1,
+            key: JSON.stringify([failed.commit, failed.name]),
+            evidence: {
+                commit: failed.commit,
+                jobName: failed.name,
+                testNames: input.names[failed.id] ?? ["unknown"],
+                jobIds: jobs.map((job) => job.id),
+                runIds: [...new Set(jobs.map((job) => job.runId))],
+            },
+        };
+    });
+}

package/dist/src/report-ready.d.ts

@@ -0,0 +1,23 @@
+import type { ReportFlow, ReportReadyTicket } from "./report-flow-types.ts";
+import type { ReportDatum, ReportWindow } from "./report-statistics.ts";
+interface ReadyInput {
+    readonly tickets: readonly ReportReadyTicket[] | null;
+    readonly flow: ReportFlow | undefined;
+}
+export declare function reportReadyData(input: ReadyInput, window: ReportWindow): {
+    dependency: null;
+    noLane: null;
+    idle: null;
+} | {
+    dependency: ReportDatum[];
+    noLane: ReportDatum[] | null;
+    idle: {
+        at: number;
+        value: number;
+        evidence: {
+            issues: number[];
+            runIds: string[];
+        };
+    }[] | null;
+};
+export {};

package/dist/src/report-ready.js

@@ -0,0 +1,117 @@
+function boundaries(input, window) {
+    const times = [
+        window.start,
+        window.end,
+        ...(input.tickets ?? []).flatMap((ticket) => [
+            ...ticket.intervals.flatMap((interval) => [
+                interval.start,
+                interval.end ?? window.end,
+            ]),
+            ...ticket.dependencies.flatMap((dependency) => dependency.closedAt === null ? [] : [dependency.closedAt]),
+        ]),
+        ...(input.flow?.runs ?? []).flatMap((run) => [
+            run.start,
+            run.end ?? window.end,
+        ]),
+    ];
+    return [...new Set(times)]
+        .filter((at) => at >= window.start && at <= window.end)
+        .toSorted((a, b) => a - b);
+}
+function readyAt(ticket, at) {
+    return ticket.intervals.some((interval) => interval.start <= at && (interval.end === null || interval.end > at));
+}
+function blockedAt(ticket, at) {
+    return ticket.dependencies.some((dependency) => dependency.closedAt === null || dependency.closedAt > at);
+}
+function knownQueue(flow, window) {
+    return (flow?.queue != null &&
+        flow.runs !== null &&
+        flow.runs.every((run) => run.workflowId !== null ||
+            run.start >= window.end ||
+            (run.end !== null && run.end <= window.start)));
+}
+function addHours(waits, input) {
+    waits.set(input.ticket.issue, (waits.get(input.ticket.issue) ?? 0) + input.hours);
+}
+function activeQueueRuns(flow, at) {
+    return (flow?.runs ?? []).filter((run) => run.workflowId === flow?.queue?.workflowId &&
+        run.start <= at &&
+        (run.end === null || run.end > at));
+}
+function addSlice(summary, input) {
+    const ready = input.tickets.filter((ticket) => readyAt(ticket, input.at));
+    const eligible = ready.filter((ticket) => !blockedAt(ticket, input.at));
+    for (const ticket of ready.filter((ticket) => blockedAt(ticket, input.at)))
+        addHours(summary.dependency, { ticket, hours: input.hours });
+    const queue = input.flow?.queue;
+    if (queue == null || eligible.length === 0)
+        return;
+    const active = activeQueueRuns(input.flow, input.at);
+    if (active.length < queue.lanes) {
+        summary.idle += input.hours;
+        for (const ticket of eligible)
+            summary.idleIssues.add(ticket.issue);
+        for (const run of active)
+            summary.idleRuns.add(run.runId);
+        return;
+    }
+    for (const ticket of eligible)
+        addHours(summary.noLane, { ticket, hours: input.hours });
+    for (const run of active)
+        summary.fullRuns.add(run.runId);
+}
+export function reportReadyData(input, window) {
+    const tickets = input.tickets;
+    if (tickets === null)
+        return { dependency: null, noLane: null, idle: null };
+    const times = boundaries(input, window);
+    const summary = {
+        dependency: new Map(),
+        noLane: new Map(),
+        idleIssues: new Set(),
+        idleRuns: new Set(),
+        fullRuns: new Set(),
+        idle: 0,
+    };
+    for (let index = 0; index < times.length - 1; index += 1) {
+        const at = times[index] ?? window.end;
+        const hours = ((times[index + 1] ?? at) - at) / 3_600_000;
+        addSlice(summary, { tickets, flow: input.flow, at, hours });
+    }
+    const known = knownQueue(input.flow, window);
+    return {
+        dependency: waitData({ tickets, waits: summary.dependency }, window),
+        noLane: known
+            ? waitData({ tickets, waits: summary.noLane, runIds: [...summary.fullRuns] }, window)
+            : null,
+        idle: known
+            ? [
+                {
+                    at: window.start,
+                    value: summary.idle,
+                    evidence: {
+                        issues: [...summary.idleIssues],
+                        runIds: [...summary.idleRuns],
+                    },
+                },
+            ]
+            : null,
+    };
+}
+function waitData(input, window) {
+    const data = input.tickets
+        .filter((ticket) => input.waits.has(ticket.issue))
+        .map((ticket) => ({
+        at: window.start,
+        value: input.waits.get(ticket.issue) ?? 0,
+        evidence: {
+            issue: ticket.issue,
+            dependencies: ticket.dependencies.map((dependency) => dependency.issue),
+            ...(input.runIds === undefined ? {} : { runIds: input.runIds }),
+        },
+    }));
+    return data.length === 0
+        ? [{ at: window.start, value: 0, evidence: { issues: [] } }]
+        : data;
+}

package/dist/src/report-statistics.d.ts

@@ -0,0 +1,51 @@
+export interface ReportDatum {
+    readonly key?: string;
+    readonly at: number;
+    readonly value: number | null;
+    readonly evidence: Readonly<Record<string, unknown>>;
+}
+export interface ReportWindow {
+    readonly start: number;
+    readonly end: number;
+    readonly observedSince: number | null;
+}
+interface MetricInput {
+    readonly data: readonly ReportDatum[] | null;
+    readonly window: ReportWindow;
+    readonly aggregation: "count" | "median" | "mean" | "distinct";
+}
+export declare function reportMedian(values: readonly number[]): number | null;
+export declare function reportMetric(input: MetricInput): {
+    evidence: Readonly<Record<string, unknown>>[];
+    mark?: string;
+    status: string;
+    value: number | null;
+    comparisonValue: number | null;
+    baseline: {
+        evidence: Readonly<Record<string, unknown>>[];
+        history: string;
+        days: number;
+        median: null;
+        band: null;
+    } | {
+        evidence: Readonly<Record<string, unknown>>[];
+        history: string;
+        days: number;
+        median: number;
+        band: null;
+    } | {
+        evidence: Readonly<Record<string, unknown>>[];
+        history: string;
+        days: number;
+        median: number;
+        band: {
+            low: number;
+            high: number;
+        };
+    };
+};
+export declare function outsideReportBand(value: number | null, band: {
+    readonly low: number;
+    readonly high: number;
+} | null): boolean;
+export {};

package/dist/src/report-statistics.js

@@ -0,0 +1,106 @@
+export function reportMedian(values) {
+    if (values.length === 0)
+        return null;
+    const sorted = values.toSorted((left, right) => left - right);
+    const middle = Math.floor(sorted.length / 2);
+    return ((sorted[middle] ?? 0) / 2 +
+        (sorted[Math.ceil(sorted.length / 2) - 1] ?? 0) / 2);
+}
+function selected(data, window) {
+    return data.filter((datum) => datum.at >= window.start && datum.at < window.end);
+}
+function aggregate(data, aggregation) {
+    if (data.some((datum) => datum.value === null))
+        return null;
+    const values = data.flatMap((datum) => datum.value === null ? [] : [datum.value]);
+    if (aggregation === "mean")
+        return values.length === 0
+            ? null
+            : values.reduce((sum, value) => sum + value, 0) / values.length;
+    return aggregation === "count" || aggregation === "distinct"
+        ? values.reduce((sum, value) => sum + value, 0)
+        : reportMedian(values);
+}
+function baseline(input) {
+    const day = 86_400_000;
+    const origin = input.window.observedSince;
+    const samples = [];
+    const evidence = [];
+    for (let index = 1; index <= 7; index += 1) {
+        const end = input.window.start - (index - 1) * day;
+        const start = end - day;
+        if (origin === null || origin > start || input.data === null)
+            continue;
+        const data = metricData(input, { start, end });
+        const value = aggregate(data, input.aggregation);
+        if (value === null)
+            continue;
+        samples.push(value);
+        evidence.push(...data.map((datum) => datum.evidence));
+    }
+    return { ...historySummary(samples), evidence };
+}
+function historySummary(samples) {
+    const median = reportMedian(samples);
+    const deviation = median === null
+        ? null
+        : reportMedian(samples.map((value) => Math.abs(value - median)));
+    if (samples.length < 5 || median === null || deviation === null)
+        return { history: "short", days: samples.length, median: null, band: null };
+    const band = { low: median - 3 * deviation, high: median + 3 * deviation };
+    if (!Number.isFinite(band.low) || !Number.isFinite(band.high))
+        return { history: "unavailable", days: samples.length, median, band: null };
+    return { history: "available", days: samples.length, median, band };
+}
+export function reportMetric(input) {
+    const data = metricData(input, input.window);
+    const value = input.data === null ? null : aggregate(data, input.aggregation);
+    const comparisonValue = value !== null && ["count", "distinct"].includes(input.aggregation)
+        ? (value * 86_400_000) / (input.window.end - input.window.start)
+        : value;
+    const history = baseline(input);
+    const outside = outsideReportBand(comparisonValue, history.band);
+    return {
+        status: value === null ? "unavailable" : "available",
+        value,
+        comparisonValue,
+        baseline: history,
+        ...(outside ? { mark: "outside" } : {}),
+        evidence: data.map((datum) => datum.evidence),
+    };
+}
+export function outsideReportBand(value, band) {
+    return (value !== null && band !== null && (value < band.low || value > band.high));
+}
+function metricData(input, window) {
+    const data = selected(input.data ?? [], window);
+    if (input.aggregation !== "distinct")
+        return data;
+    const groups = new Map();
+    for (const datum of data) {
+        if (datum.key === undefined)
+            throw new Error("missing-report-datum-key");
+        const previous = groups.get(datum.key);
+        groups.set(datum.key, previous === undefined
+            ? datum
+            : {
+                ...datum,
+                evidence: combinedEvidence(previous.evidence, datum.evidence),
+            });
+    }
+    return [...groups.values()];
+}
+function combinedEvidence(left, right) {
+    return Object.fromEntries(Object.entries(right).map(([key, value]) => {
+        const earlier = left[key];
+        return [
+            key,
+            unknownList(earlier) && unknownList(value)
+                ? [...new Set([...earlier, ...value])]
+                : value,
+        ];
+    }));
+}
+function unknownList(value) {
+    return Array.isArray(value);
+}