@appeeky/spec 0.0.0-stage → 0.2.0

package/dist/loops.js

@@ -0,0 +1,197 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { readChanges } from "./competitors.js";
+import { readExperiments } from "./experiments.js";
+import { SPEC_DIR } from "./files.js";
+import { readReleases } from "./history.js";
+import { CHECK_DAYS } from "./measure.js";
+import { formatChange, METRICS } from "./metrics.js";
+/**
+ * Every measurement loop in the repo, in one place: each listing push (a release) and each
+ * experiment, what it measures, when it is checked next and what it shows so far. Written to
+ * appeeky/loops.md by the commands that open, measure or close a loop, and shown by `status`.
+ */
+const DAY = 86_400_000;
+/** A check this many days past its date with no result has stalled (CI not running, a key missing). */
+export const LATE_AFTER_DAYS = 2;
+/** Fewer installs than this in a window: the numbers are a hint, not a result. */
+export const FEW_INSTALLS = 100;
+const iso = (ms) => new Date(ms).toISOString().slice(0, 10);
+function stateFor(nextCheckMs, now) {
+    if (nextCheckMs === undefined)
+        return "closed";
+    if (now.getTime() < nextCheckMs)
+        return "measuring";
+    return now.getTime() >= nextCheckMs + LATE_AFTER_DAYS * DAY ? "late" : "due";
+}
+function releaseWhat(r) {
+    const fields = [...new Set(r.changes.map((c) => `${c.locale} ${c.field}`))];
+    const head = fields.slice(0, 3).join(", ") + (fields.length > 3 ? ` +${fields.length - 3}` : "");
+    return r.note ? `${head}: ${r.note}` : head || r.target;
+}
+/** Installs per window behind a release's latest check, when it has them. */
+function windowInstalls(check) {
+    const m = check.metrics;
+    if (!m)
+        return undefined;
+    const per = (w) => (w.values.organic_installs ?? w.values.downloads);
+    const b = per(m.before);
+    const a = per(m.after);
+    if (b === undefined || a === undefined)
+        return undefined;
+    return Math.round(Math.min(b * m.before.days, a * m.after.days));
+}
+export function releaseLoop(r, now = new Date()) {
+    const done = new Set((r.checks ?? []).map((c) => c.day));
+    const next = r.baseline ? CHECK_DAYS.find((d) => !done.has(d)) : undefined;
+    const nextMs = next === undefined ? undefined : Date.parse(r.pushedAt) + next * DAY;
+    const last = r.checks?.at(-1);
+    const warnings = [];
+    if (!r.baseline)
+        warnings.push("no baseline was saved at push: it cannot be measured");
+    const installs = last ? windowInstalls(last) : undefined;
+    if (installs !== undefined && installs < FEW_INSTALLS)
+        warnings.push(`${installs} installs in the window: treat the result as a hint`);
+    return {
+        kind: "release",
+        id: r.id,
+        store: r.store,
+        what: releaseWhat(r),
+        metric: last?.decidedBy ? last.decidedBy.map((m) => METRICS[m]?.label ?? m).join(", ") : "target ranks",
+        startedAt: r.pushedAt,
+        ...(nextMs !== undefined ? { nextCheck: iso(nextMs) } : {}),
+        state: r.baseline ? stateFor(nextMs, now) : "closed",
+        ...(last ? { result: `day ${last.day} ${last.verdict}${last.decidedBy ? " by outcomes" : ""}` } : {}),
+        ...(last
+            ? {
+                detail: [
+                    ...(last.improved.length || last.dropped.length ? [`${last.improved.length} ${last.improved.length === 1 ? "target" : "targets"} up, ${last.dropped.length} down`] : []),
+                    ...(last.control ? [`net ${last.control.net > 0 ? "+" : ""}${last.control.net} places vs unchanged storefronts`] : []),
+                    ...(last.note ? [last.note] : []),
+                    ...(last.metrics?.changes ?? []).filter((c) => c.outcome !== "flat").slice(0, 3).map((c) => formatChange(c, last.metrics?.after.currency)),
+                ],
+            }
+            : {}),
+        warnings,
+    };
+}
+export function experimentLoop(e, now = new Date()) {
+    const running = e.status === "running";
+    const nextMs = running ? Date.parse(e.startedAt) + e.checkAfterDays * DAY : undefined;
+    const value = e.result?.value;
+    const goal = e.target !== undefined ? `target ${e.target}` : e.baseline !== undefined ? `baseline ${round(e.baseline)}` : "";
+    return {
+        kind: "experiment",
+        id: e.id,
+        ...(e.store ? { store: e.store } : {}),
+        what: e.hypothesis,
+        metric: METRICS[e.metric]?.label ?? e.metric,
+        startedAt: e.startedAt,
+        ...(nextMs !== undefined ? { nextCheck: iso(nextMs) } : {}),
+        state: stateFor(nextMs, now),
+        ...(running ? {} : { result: `${e.status}${value !== undefined ? ` (${round(value)}${goal ? ` vs ${goal}` : ""})` : ""}` }),
+        ...(running && goal ? { detail: [goal] } : {}),
+        warnings: [],
+    };
+}
+const round = (n) => (Math.abs(n) >= 100 ? Math.round(n) : Math.round(n * 1000) / 1000);
+/**
+ * Releases on the same store pushed within the measuring span of each other are measured
+ * over the same days, so their results mix. Both get a warning while either is open.
+ */
+function markOverlaps(releases, loops) {
+    const span = Math.max(...CHECK_DAYS) * DAY;
+    for (const a of releases)
+        for (const b of releases) {
+            if (a.id >= b.id || a.store !== b.store)
+                continue;
+            if (Math.abs(Date.parse(a.pushedAt) - Date.parse(b.pushedAt)) >= span)
+                continue;
+            const la = loops.get(a.id);
+            const lb = loops.get(b.id);
+            if (la.state === "closed" && lb.state === "closed")
+                continue;
+            la.warnings.push(`measured over the same days as ${b.id}: the results mix`);
+            lb.warnings.push(`measured over the same days as ${a.id}: the results mix`);
+        }
+}
+const ORDER = { late: 0, due: 1, measuring: 2, closed: 3 };
+export async function collectLoops(root, now = new Date()) {
+    const releases = await readReleases(root);
+    const experiments = await readExperiments(root);
+    const byId = new Map();
+    for (const r of releases)
+        byId.set(r.id, releaseLoop(r, now));
+    markOverlaps(releases, byId);
+    // A competitor's change inside a release's window moves the same numbers.
+    const changes = await readChanges(root);
+    const span = Math.max(...CHECK_DAYS) * DAY;
+    for (const r of releases) {
+        const loop = byId.get(r.id);
+        if (loop.state === "closed")
+            continue;
+        const from = Date.parse(r.pushedAt) - span;
+        const to = Date.parse(r.pushedAt) + span;
+        for (const ch of changes) {
+            const t = Date.parse(ch.date);
+            if (t >= from && t <= to && (ch.change.startsWith(r.store === "app-store" ? "App Store" : "Google Play")))
+                loop.warnings.push(`${ch.name} changed on ${ch.date}, inside this window: ${ch.change}`);
+        }
+    }
+    const loops = [...byId.values(), ...experiments.map((e) => experimentLoop(e, now))];
+    return loops.sort((a, b) => ORDER[a.state] - ORDER[b.state] || (a.nextCheck ?? "9").localeCompare(b.nextCheck ?? "9") || b.startedAt.localeCompare(a.startedAt));
+}
+const STATE_LABEL = { late: "late", due: "due", measuring: "measuring", closed: "closed" };
+const cell = (s) => s.replace(/\|/g, "\\|").replace(/\n+/g, " ");
+/** appeeky/loops.md: what is being measured, what is due, what came out. Rewritten, not appended. */
+export function loopsMarkdown(loops, now = new Date()) {
+    const open = loops.filter((l) => l.state !== "closed");
+    const closed = loops.filter((l) => l.state === "closed").slice(0, 20);
+    const lines = [
+        "# Loops",
+        "",
+        `Every change being measured, generated by appeeky-spec on ${iso(now.getTime())}. Do not edit: run \`appeeky-spec status\`.`,
+        "Releases are checked on day 7 and day 14 (`verify`); experiments on their check date (`verify`, `experiment close`).",
+        "",
+        "## Open",
+        "",
+    ];
+    if (open.length === 0)
+        lines.push("Nothing is being measured.");
+    else {
+        lines.push("| State | Next check | Kind | What | Decided by | So far |", "| --- | --- | --- | --- | --- | --- |");
+        for (const l of open)
+            lines.push(`| ${STATE_LABEL[l.state]} | ${l.nextCheck ?? "—"} | ${l.kind}${l.store ? ` · ${l.store}` : ""} | ${cell(l.what)} | ${cell(l.metric)} | ${cell([l.result, ...(l.detail ?? [])].filter(Boolean).join("; ") || "—")} |`);
+    }
+    const warned = loops.filter((l) => l.warnings.length && l.state !== "closed");
+    if (warned.length) {
+        lines.push("", "## Watch", "");
+        for (const l of warned)
+            for (const w of l.warnings)
+                lines.push(`- \`${l.id}\`: ${w}`);
+    }
+    lines.push("", "## Closed", "");
+    if (closed.length === 0)
+        lines.push("Nothing closed yet.");
+    else {
+        lines.push("| Closed loop | What | Result |", "| --- | --- | --- |");
+        for (const l of closed)
+            lines.push(`| \`${l.id}\` | ${cell(l.what)} | ${cell([l.result, ...(l.detail ?? [])].filter(Boolean).join("; ") || "—")} |`);
+    }
+    return lines.join("\n") + "\n";
+}
+export async function writeLoops(root, now = new Date()) {
+    const loops = await collectLoops(root, now);
+    await mkdir(join(root, SPEC_DIR), { recursive: true });
+    await writeFile(join(root, SPEC_DIR, "loops.md"), loopsMarkdown(loops, now));
+    return loops;
+}
+/**
+ * Stores with no release being measured: a new listing change there will not mix with an
+ * open one. This is what a scheduled agent checks before it proposes anything.
+ */
+export function freeStores(loops, stores) {
+    const busy = new Set(loops.filter((l) => l.kind === "release" && l.state !== "closed").map((l) => l.store));
+    return stores.filter((s) => !busy.has(s));
+}
+//# sourceMappingURL=loops.js.map

package/dist/measure.d.ts

@@ -0,0 +1,40 @@
+import { type Experiment } from "./experiments.js";
+import { type ReleaseRecord } from "./history.js";
+import type { Project, StoreId } from "./schema.js";
+import { type MetricSource } from "./metrics.js";
+import { type ConversionSource, type RankSource } from "./verify.js";
+/** Days after a push when a release is measured. */
+export declare const CHECK_DAYS: readonly [7, 14];
+export interface VerifyResult {
+    release: ReleaseRecord;
+    check: NonNullable<ReleaseRecord["checks"]>[number];
+}
+/** The next milestone a release is due for, or null when it is not due (or already checked). */
+export declare function dueMilestone(record: ReleaseRecord, now?: Date): number | null;
+/**
+ * Measure releases against their baseline ranks and write the result into
+ * each release file. By default only releases that reached a milestone
+ * (day 7, day 14) are measured; `force` measures the given ones now.
+ */
+export declare function verifyReleases(project: Project, source: RankSource, options?: {
+    ids?: string[];
+    force?: boolean;
+    now?: Date;
+    conversion?: ConversionSource;
+    metrics?: MetricSource;
+}): Promise<VerifyResult[]>;
+/** One playbook line: what changed, the hypothesis, and what the ranks did. */
+export declare function playbookLine(result: VerifyResult): string;
+/** Append lines to appeeky/playbook.md, skipping ones already there. */
+export declare function appendPlaybook(root: string, lines: string[]): Promise<number>;
+/** The app an experiment is about: its store, or the first store in the spec. */
+export declare function experimentApp(project: Project, e: Pick<Experiment, "store">): {
+    store: StoreId;
+    appId: string;
+} | null;
+/**
+ * Close due experiments whose metric Appeeky can measure (conversion, revenue,
+ * trials, cpa, ...): the value is the metric over the experiment's run.
+ * Experiments on other metrics stay due for `experiment close --value`.
+ */
+export declare function measureExperiments(project: Project, source: MetricSource, now?: Date): Promise<Experiment[]>;

package/dist/measure.js

@@ -0,0 +1,159 @@
+import { appendFile, readFile, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { SPEC_DIR } from "./files.js";
+import { closeExperiment, dueExperiments, readExperiments, writeExperiment } from "./experiments.js";
+import { readReleases, releaseAge, updateRelease } from "./history.js";
+import { formatChange, isMetricName, measureAround, metricAfter, outcomeVerdict } from "./metrics.js";
+import { compareWithControl, MIN_NET_PLACES } from "./control.js";
+import { evaluateRanks, measureConversion, rankAppIds } from "./verify.js";
+/** Days after a push when a release is measured. */
+export const CHECK_DAYS = [7, 14];
+/** The next milestone a release is due for, or null when it is not due (or already checked). */
+export function dueMilestone(record, now = new Date()) {
+    if (!record.baseline)
+        return null;
+    const age = releaseAge(record, now);
+    const done = new Set((record.checks ?? []).map((c) => c.day));
+    const due = CHECK_DAYS.filter((d) => age >= d && !done.has(d));
+    return due.length > 0 ? due.at(-1) : null;
+}
+/**
+ * Measure releases against their baseline ranks and write the result into
+ * each release file. By default only releases that reached a milestone
+ * (day 7, day 14) are measured; `force` measures the given ones now.
+ */
+export async function verifyReleases(project, source, options = {}) {
+    const now = options.now ?? new Date();
+    const ids = rankAppIds(project);
+    const releases = (await readReleases(project.root)).filter((r) => !options.ids || options.ids.some((id) => r.id.startsWith(id)));
+    const out = [];
+    for (const release of releases) {
+        if (!release.baseline)
+            continue;
+        const day = options.force ? releaseAge(release, now) : dueMilestone(release, now);
+        if (day === null)
+            continue;
+        const appId = ids[release.store];
+        if (!appId)
+            continue;
+        const current = {};
+        for (const [country, ranks] of Object.entries(release.baseline.ranks)) {
+            try {
+                current[country] = await source.ranks(release.store, appId, country, Object.keys(ranks));
+            }
+            catch {
+                // leave the storefront out of this check
+            }
+        }
+        const span = Math.min(Math.max(day, 1), 14);
+        const metrics = options.metrics ? await measureAround(options.metrics, release.store, appId, release.pushedAt, span).catch(() => null) : null;
+        const conversion = options.conversion
+            ? await measureConversion(options.conversion, release.store, appId, release.pushedAt, span).catch(() => null)
+            : metrics
+                ? conversionFromMetrics(metrics)
+                : null;
+        const evaluation = evaluateRanks(release.baseline.ranks, current);
+        // Unchanged storefronts as a control: a store-wide lift is not the release's win.
+        const control = compareWithControl(project, release, release.baseline.ranks, current);
+        if (control) {
+            evaluation.control = control;
+            if (evaluation.verdict === "won" && control.net < MIN_NET_PLACES) {
+                evaluation.verdict = "mixed";
+                evaluation.note = `the unchanged storefronts (${control.control.join(", ")}) moved as much: the store moved, not the listing`;
+            }
+        }
+        // Ranks are a means; when none moved, organic installs and revenue (or conversion) decide.
+        const byOutcome = evaluation.verdict === "flat" && metrics ? outcomeVerdict(metrics.changes) : null;
+        if (byOutcome)
+            Object.assign(evaluation, { verdict: byOutcome.verdict, decidedBy: byOutcome.by });
+        const check = {
+            at: now.toISOString(),
+            day,
+            ranks: current,
+            ...evaluation,
+            ...(conversion ? { conversion } : {}),
+            ...(metrics ? { metrics } : {}),
+        };
+        release.checks = [...(release.checks ?? []).filter((c) => c.day !== day), check].sort((a, b) => a.day - b.day);
+        await updateRelease(project.root, release);
+        out.push({ release, check });
+    }
+    return out;
+}
+/** The conversion field of a check, from outcome metrics when no conversion source is given. */
+function conversionFromMetrics(m) {
+    const window = (w) => {
+        const rate = w.values.conversion;
+        const views = w.values.page_views;
+        if (rate === undefined || views === undefined)
+            return null;
+        const visitors = Math.round(views * w.days);
+        return { rate, visitors, installs: Math.round(rate * visitors), days: w.days };
+    };
+    const before = window(m.before);
+    const after = window(m.after);
+    return before && after ? { before, after, change: before.rate > 0 ? after.rate / before.rate - 1 : 0 } : null;
+}
+/** One playbook line: what changed, the hypothesis, and what the ranks did. */
+export function playbookLine(result) {
+    const { release, check } = result;
+    const fields = [...new Set(release.changes.map((c) => `${c.locale} ${c.field}`))].join(", ");
+    const moves = [
+        ...check.improved.slice(0, 3).map((m) => `"${m.keyword}" ${m.before ?? "–"}→${m.after ?? "–"}`),
+        ...check.regressions.slice(0, 3).map((m) => `lost "${m.keyword}" ${m.before}→${m.after ?? "out"}`),
+    ].join("; ");
+    // Conversion and the money metrics say whether the listing sold better, not just ranked better.
+    const outcomes = (check.metrics?.changes ?? []).filter((m) => m.outcome !== "flat" && ["conversion", "downloads", "revenue", "proceeds", "trials", "rc_trials", "organic_installs", "organic_revenue"].includes(m.metric));
+    // The metrics that decided the verdict go first.
+    if (check.decidedBy)
+        outcomes.sort((a, b) => Number(check.decidedBy.includes(b.metric)) - Number(check.decidedBy.includes(a.metric)));
+    const conv = outcomes.length
+        ? outcomes.slice(0, 3).map((m) => formatChange(m, check.metrics?.after.currency)).join("; ")
+        : check.conversion
+            ? `conversion ${(check.conversion.before.rate * 100).toFixed(1)}% → ${(check.conversion.after.rate * 100).toFixed(1)}%`
+            : "";
+    const control = check.control ? `net ${check.control.net > 0 ? "+" : ""}${check.control.net} places vs unchanged ${check.control.control.join(", ")}` : "";
+    const detail = [moves, control, conv].filter(Boolean).join("; ");
+    return `- ${release.pushedAt.slice(0, 10)}, ${release.store}, ${fields}${release.note ? `: ${release.note}` : ""} → day ${check.day} ${check.verdict}${check.decidedBy ? " by outcomes" : ""}${detail ? ` (${detail})` : ""}`;
+}
+/** Append lines to appeeky/playbook.md, skipping ones already there. */
+export async function appendPlaybook(root, lines) {
+    const file = join(root, SPEC_DIR, "playbook.md");
+    const existing = existsSync(file) ? await readFile(file, "utf8") : "# Playbook\n";
+    const fresh = lines.filter((l) => !existing.includes(l));
+    if (fresh.length === 0)
+        return 0;
+    if (!existsSync(file))
+        await writeFile(file, existing);
+    await appendFile(file, `${existing.endsWith("\n") ? "" : "\n"}${fresh.join("\n")}\n`);
+    return fresh.length;
+}
+/** The app an experiment is about: its store, or the first store in the spec. */
+export function experimentApp(project, e) {
+    const ids = rankAppIds(project);
+    const store = e.store ?? Object.keys(ids)[0];
+    const appId = store ? ids[store] : undefined;
+    return store && appId ? { store, appId } : null;
+}
+/**
+ * Close due experiments whose metric Appeeky can measure (conversion, revenue,
+ * trials, cpa, ...): the value is the metric over the experiment's run.
+ * Experiments on other metrics stay due for `experiment close --value`.
+ */
+export async function measureExperiments(project, source, now = new Date()) {
+    const closed = [];
+    for (const e of dueExperiments(await readExperiments(project.root), now)) {
+        if (!isMetricName(e.metric))
+            continue;
+        const app = experimentApp(project, e);
+        if (!app)
+            continue;
+        const value = await metricAfter(source, app.store, app.appId, e.metric, e.startedAt, e.checkAfterDays).catch(() => undefined);
+        if (value === undefined)
+            continue;
+        closed.push(await writeExperiment(project.root, closeExperiment(e, value, `Measured by appeeky-spec over ${e.checkAfterDays} days.`, now)));
+    }
+    return closed;
+}
+//# sourceMappingURL=measure.js.map