@viccydev/pi-fpa 0.4.1 → 0.6.0

package/extensions/fpa-dashboard/actuals.ts

@@ -1,11 +1,84 @@
 import { createHash } from "node:crypto";
+import { constants } from "node:fs";
+import { lstat, open, realpath, stat } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve } from "node:path";
 
 import type { Period } from "../fpa-artifacts/contracts.ts";
+import { sliceKey } from "../fpa-artifacts/contracts.ts";
+import { stableJson } from "../fpa-artifacts/store.ts";
 import { toNumber, type NumericLike } from "../fpa-data/calc.ts";
-import { runStructuredQuery } from "../fpa-data/runtime.ts";
-import type { SqlRow } from "../fpa-data/supabase.ts";
+import { runStructuredQuery, shapeQueryRows } from "../fpa-data/runtime.ts";
+import { buildQuery, type QuerySpec } from "../fpa-data/sql.ts";
+import { runQuery, type SqlRow } from "../fpa-data/supabase.ts";
 import type { DashboardActualsSnapshot, DashboardDailyPoint, DashboardScopeMode, QueryReceipt } from "./source.ts";
-import { ACTUALS_DATA_AS_OF_UNAVAILABLE } from "./source.ts";
+import { ACTUALS_DATA_AS_OF_UNAVAILABLE, validateActualsSnapshot } from "./source.ts";
+
+export interface ActualsSourceCloseSignal {
+	kind: "fpa.actuals.source-close";
+	schema_version: 1;
+	dataset: "ua_spend";
+	signal_id: string;
+	closed_through: string;
+	emitted_at: string;
+}
+
+function validateSourceCloseSignal(value: unknown): (ActualsSourceCloseSignal & { signal_sha256: string }) | null {
+	if (value === undefined || value === null) return null;
+	if (typeof value !== "object" || Array.isArray(value)) throw new Error("Actuals source close signal must be an object.");
+	const source = value as Record<string, unknown>;
+	if (source.kind !== "fpa.actuals.source-close" || source.schema_version !== 1) throw new Error("Actuals source close signal has an unsupported contract.");
+	if (source.dataset !== "ua_spend") throw new Error("Actuals source close signal must identify ua_spend.");
+	if (typeof source.signal_id !== "string" || !/^[A-Za-z0-9._:-]{1,256}$/.test(source.signal_id)) throw new Error("Actuals source close signal_id is invalid.");
+	if (typeof source.closed_through !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(source.closed_through)) throw new Error("Actuals source closed_through must use YYYY-MM-DD.");
+	const closedDate = new Date(`${source.closed_through}T00:00:00Z`);
+	if (Number.isNaN(closedDate.getTime()) || closedDate.toISOString().slice(0, 10) !== source.closed_through) {
+		throw new Error("Actuals source closed_through must be a real canonical calendar date.");
+	}
+	if (typeof source.emitted_at !== "string" || Number.isNaN(Date.parse(source.emitted_at))) throw new Error("Actuals source close emitted_at must be an ISO timestamp.");
+	const canonical = {
+		kind: "fpa.actuals.source-close" as const,
+		schema_version: 1 as const,
+		dataset: "ua_spend" as const,
+		signal_id: source.signal_id as string,
+		closed_through: source.closed_through as string,
+		emitted_at: new Date(source.emitted_at as string).toISOString(),
+	};
+	return { ...canonical, signal_sha256: createHash("sha256").update(stableJson(canonical)).digest("hex") };
+}
+
+async function configuredSourceCloseSignal(): Promise<ActualsSourceCloseSignal | null> {
+	const path = process.env.FPA_ACTUALS_CLOSE_SIGNAL_PATH;
+	if (!path) return null;
+	const rootConfig = process.env.FPA_ACTUALS_CLOSE_SIGNAL_ROOT;
+	if (!rootConfig) throw new Error("FPA_ACTUALS_CLOSE_SIGNAL_ROOT is required when a source close signal path is configured.");
+	const resolvedPath = resolve(path);
+	const configuredRoot = resolve(rootConfig);
+	if ((await lstat(configuredRoot)).isSymbolicLink()) throw new Error("FPA_ACTUALS_CLOSE_SIGNAL_ROOT must not be a symbolic link.");
+	const trustedRoot = await realpath(configuredRoot);
+	const parent = await realpath(dirname(resolvedPath));
+	if (parent !== resolve(dirname(resolvedPath))) throw new Error("Actuals source close signal parent path must not contain symbolic links.");
+	const relation = relative(trustedRoot, parent);
+	if (relation.startsWith("..") || isAbsolute(relation)) throw new Error("Actuals source close signal escapes FPA_ACTUALS_CLOSE_SIGNAL_ROOT.");
+	const processUid = typeof process.getuid === "function" ? process.getuid() : null;
+	if (processUid === null) throw new Error("Configured source close signals require POSIX ownership checks.");
+	for (let directory = parent; ; directory = dirname(directory)) {
+		const metadata = await stat(directory);
+		if (!metadata.isDirectory() || (metadata.mode & 0o022) !== 0 || metadata.uid === processUid) {
+			throw new Error("Actuals source close signal directory chain must be owned outside the Agent identity and not be group- or world-writable.");
+		}
+		if (dirname(directory) === directory) break;
+	}
+	const handle = await open(resolvedPath, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+	try {
+		const metadata = await handle.stat();
+		if (!metadata.isFile() || metadata.size > 16 * 1024) throw new Error("FPA_ACTUALS_CLOSE_SIGNAL_PATH must be a bounded regular file.");
+		if ((metadata.mode & 0o022) !== 0) throw new Error("Actuals source close signal must not be group- or world-writable.");
+		if (metadata.uid === processUid) throw new Error("Actuals source close signal must be owned outside the Agent identity.");
+		return validateSourceCloseSignal(JSON.parse(await handle.readFile("utf8")));
+	} finally {
+		await handle.close();
+	}
+}
 
 function dateInTimezone(timestamp: string, timezone: string, label: string): string {
 	const instant = new Date(timestamp);
@@ -51,6 +124,74 @@ function receipt(dataset: string, sql: string, rowCount: number): QueryReceipt {
 	return { dataset, query_fingerprint: queryFingerprint(sql), row_count: rowCount };
 }
 
+function approximatelyEqual(left: number, right: number): boolean {
+	return Math.abs(left - right) <= Math.max(0.000001, Math.max(Math.abs(left), Math.abs(right)) * 0.000001);
+}
+
+function jsonValue(value: unknown): unknown {
+	if (typeof value !== "string") return value;
+	try {
+		return JSON.parse(value);
+	} catch {
+		return value;
+	}
+}
+
+function jsonRows(value: unknown, label: string): SqlRow[] {
+	const parsed = jsonValue(value);
+	if (!Array.isArray(parsed)) throw new Error(`Unified Actuals snapshot ${label} must be a JSON array.`);
+	if (parsed.some((row) => row === null || typeof row !== "object" || Array.isArray(row))) throw new Error(`Unified Actuals snapshot ${label} contains an invalid row.`);
+	return parsed as SqlRow[];
+}
+
+function jsonRow(value: unknown, label: string): SqlRow | undefined {
+	const parsed = jsonValue(value);
+	if (parsed === null || parsed === undefined) return undefined;
+	if (typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(`Unified Actuals snapshot ${label} must be a JSON object or null.`);
+	return parsed as SqlRow;
+}
+
+function unifiedSnapshotSql(queries: Record<string, ReturnType<typeof buildQuery>>): string {
+	return [
+		"with",
+		`snapshot_current as (${queries.current.sql}),`,
+		`snapshot_comparison as (${queries.comparison.sql}),`,
+		`snapshot_daily as (${queries.daily.sql}),`,
+		`snapshot_approved as (${queries.approved.sql}),`,
+		`snapshot_discovered as (${queries.discovered.sql}),`,
+		`snapshot_unscoped as (${queries.unscoped.sql})`,
+		"select",
+		"txid_current_snapshot()::text as snapshot_id,",
+		"statement_timestamp()::text as available_at,",
+		"(select row_to_json(snapshot_current) from snapshot_current limit 1) as current,",
+		"(select row_to_json(snapshot_comparison) from snapshot_comparison limit 1) as comparison,",
+		"coalesce((select json_agg(row_to_json(snapshot_daily) order by period) from snapshot_daily), '[]'::json) as daily,",
+		"coalesce((select json_agg(row_to_json(snapshot_approved)) from snapshot_approved), '[]'::json) as approved,",
+		"coalesce((select json_agg(row_to_json(snapshot_discovered)) from snapshot_discovered), '[]'::json) as discovered,",
+		"(select row_to_json(snapshot_unscoped) from snapshot_unscoped limit 1) as unscoped",
+	].join(" ");
+}
+
+export function dashboardActualsWatermark(actuals: DashboardActualsSnapshot): string {
+	const normalized = validateActualsSnapshot(actuals);
+	return createHash("sha256").update(stableJson({
+		period: normalized.period,
+		data_as_of: normalized.data_as_of,
+		coverage: normalized.coverage,
+		current: normalized.current,
+		comparison: normalized.comparison,
+		daily: normalized.daily,
+		slices: normalized.slices,
+		period_has_unscoped_actuals: normalized.period_has_unscoped_actuals,
+		period_complete: normalized.snapshot_evidence.period_complete,
+		reconciled: normalized.snapshot_evidence.reconciled,
+	})).digest("hex");
+}
+
+export function dashboardActualsSnapshotFingerprint(actuals: DashboardActualsSnapshot): string {
+	return createHash("sha256").update(stableJson(validateActualsSnapshot(actuals))).digest("hex");
+}
+
 export function calendarQueryDates(period: Period): {
 	currentFrom: string;
 	currentTo: string;
@@ -97,6 +238,32 @@ export interface DashboardActualsScope {
 	slices: Array<{ app_id: string; store: string; channel_group: string }>;
 }
 
+/**
+ * Why every approved slice can be missing, and which of the two reasons it is.
+ *
+ * An early cycle and a forecast keyed on values ua_spend never uses produce
+ * byte-identical Actuals: nothing. The difference is only visible next to the
+ * unscoped probe — if the window holds paid rows that no approved key matched,
+ * the keys are wrong. Returns the operator-facing explanation, or null when the
+ * evidence does not support that conclusion.
+ */
+export function detectSliceKeyMismatch(
+	allocation: Array<{ app_id: string; store: string; channel_group: string }>,
+	actuals: Pick<DashboardActualsSnapshot, "slices" | "period_has_unscoped_actuals" | "period">,
+): string | null {
+	if (allocation.length === 0) return null;
+	if (!actuals.period_has_unscoped_actuals) return null;
+	const actualKeys = new Set(actuals.slices.map(sliceKey));
+	const missing = allocation.filter((slice) => !actualKeys.has(sliceKey(slice)));
+	if (missing.length < allocation.length) return null;
+	return (
+		`All ${allocation.length} approved slices have no Actuals, yet ua_spend holds paid rows in `
+		+ `${actuals.period.start_inclusive.slice(0, 10)}..${actuals.period.end_exclusive.slice(0, 10)}. `
+		+ "The frozen slice keys do not match the dimension values ua_spend uses — check store against "
+		+ "ua_spend.platform and channel_group against ua_spend.media_source, then re-freeze the forecast."
+	);
+}
+
 /**
  * Rows carry app_code only in "app" scope mode. In "portfolio" mode the App axis was
  * collapsed onto a placeholder before the query ran, so the caller supplies it back.
@@ -148,8 +315,12 @@ export async function loadDashboardActuals(
 	period: Period,
 	scope: DashboardActualsScope,
 	signal?: AbortSignal,
+	options: { sourceCloseSignal?: ActualsSourceCloseSignal | null } = {},
 ): Promise<DashboardActualsSnapshot> {
 	if (scope.slices.length === 0) throw new Error("Dashboard Actuals scope requires at least one approved slice.");
+	const closeSignal = options.sourceCloseSignal === undefined
+		? await configuredSourceCloseSignal()
+		: validateSourceCloseSignal(options.sourceCloseSignal);
 	const dates = calendarQueryDates(period);
 	const uniqueSlices = [...new Map(scope.slices.map((slice) => [`${slice.app_id}\u0000${slice.store}\u0000${slice.channel_group}`, slice])).values()];
 	const common = {
@@ -169,41 +340,40 @@ export async function loadDashboardActuals(
 		? { filters: { platform: [...new Set(uniqueSlices.map((slice) => slice.store))] } }
 		: { exactUaPortfolioScope };
 	const sliceDimensions = portfolio ? ["platform", "media_source"] : ["app_code", "platform", "media_source"];
-	const current = await runStructuredQuery({ ...common, ...sliceScope, dateFrom: dates.currentFrom, dateTo: dates.currentTo, limit: 1 }, signal);
-	const currentRow = current.rows[0];
+	const periodDays = calendarDayDistance(dates.currentFrom, dates.currentTo) + 1;
+	const priorTo = addCalendarDays(dates.currentFrom, -1);
+	const priorFrom = addCalendarDays(priorTo, -(periodDays - 1));
+	const specs: Record<string, QuerySpec> = {
+		current: { ...common, ...sliceScope, dateFrom: dates.currentFrom, dateTo: dates.currentTo, limit: 1 },
+		comparison: { ...common, ...sliceScope, dateFrom: priorFrom, dateTo: priorTo, limit: 1 },
+		daily: { ...common, ...sliceScope, dateFrom: dates.currentFrom, dateTo: dates.currentTo, timeGrain: "day", limit: 1000 },
+		approved: { ...common, dateFrom: dates.currentFrom, dateTo: dates.currentTo, ...sliceScope, dimensions: sliceDimensions, limit: 500 },
+		discovered: { ...common, dateFrom: dates.currentFrom, dateTo: dates.currentTo, ...discoveryScope, dimensions: sliceDimensions, limit: 1000 },
+		unscoped: { ...common, dateFrom: dates.currentFrom, dateTo: dates.currentTo, limit: 1 },
+	};
+	const built = Object.fromEntries(Object.entries(specs).map(([name, spec]) => [name, buildQuery(spec)])) as Record<string, ReturnType<typeof buildQuery>>;
+	const snapshotSql = unifiedSnapshotSql(built);
+	const snapshotRows = await runQuery(snapshotSql, { signal });
+	if (snapshotRows.length !== 1) throw new Error("Unified Actuals snapshot query must return exactly one envelope row.");
+	const envelope = snapshotRows[0];
+	const shape = (name: string, rows: SqlRow[]) => shapeQueryRows(rows, built[name].measures, built[name].derived);
+	const currentRow = shape("current", jsonRow(envelope.current, "current") ? [jsonRow(envelope.current, "current") as SqlRow] : [])[0];
+	const comparisonRow = shape("comparison", jsonRow(envelope.comparison, "comparison") ? [jsonRow(envelope.comparison, "comparison") as SqlRow] : [])[0];
+	const dailyRows = shape("daily", jsonRows(envelope.daily, "daily"));
+	const approvedRows = shape("approved", jsonRows(envelope.approved, "approved"));
+	const discoveredRows = shape("discovered", jsonRows(envelope.discovered, "discovered"));
+	const unscopedRow = shape("unscoped", jsonRow(envelope.unscoped, "unscoped") ? [jsonRow(envelope.unscoped, "unscoped") as SqlRow] : [])[0];
 	const coverageMin = currentRow?.date_min == null ? null : String(currentRow.date_min);
 	const coverageMax = currentRow?.date_max == null ? null : String(currentRow.date_max);
-	const comparisonDates = coverageMax ? comparisonDatesForCoverage(dates.currentFrom, coverageMax) : null;
-	const [comparison, daily, approvedSlices, discoveredSlices] = await Promise.all([
-		comparisonDates
-			? runStructuredQuery({ ...common, ...sliceScope, dateFrom: comparisonDates.comparisonFrom, dateTo: comparisonDates.comparisonTo, limit: 1 }, signal)
-			: Promise.resolve(null),
-		runStructuredQuery({ ...common, ...sliceScope, dateFrom: dates.currentFrom, dateTo: dates.currentTo, timeGrain: "day", limit: 1000 }, signal),
-		runStructuredQuery({
-			...common,
-			dateFrom: dates.currentFrom,
-			dateTo: dates.currentTo,
-			...sliceScope,
-			dimensions: sliceDimensions,
-			limit: 500,
-		}, signal),
-		runStructuredQuery({
-			...common,
-			dateFrom: dates.currentFrom,
-			dateTo: dates.currentTo,
-			...discoveryScope,
-			dimensions: sliceDimensions,
-			limit: 1000,
-		}, signal),
-	]);
-
-	const comparisonRow = comparison?.rows[0];
+	const periodHasUnscopedActuals = unscopedRow?.date_max != null;
 	const queryReceipts = [
-		receipt("ua_spend.current", current.built.sql, current.rows.length),
-		...(comparison ? [receipt("ua_spend.comparison", comparison.built.sql, comparison.rows.length)] : []),
-		receipt("ua_spend.daily", daily.built.sql, daily.rows.length),
-		receipt("ua_spend.slices.approved", approvedSlices.built.sql, approvedSlices.rows.length),
-		receipt("ua_spend.slices.discovery", discoveredSlices.built.sql, discoveredSlices.rows.length),
+		receipt("ua_spend.snapshot", snapshotSql, snapshotRows.length),
+		receipt("ua_spend.current", snapshotSql, currentRow ? 1 : 0),
+		receipt("ua_spend.comparison", snapshotSql, comparisonRow ? 1 : 0),
+		receipt("ua_spend.daily", snapshotSql, dailyRows.length),
+		receipt("ua_spend.slices.approved", snapshotSql, approvedRows.length),
+		receipt("ua_spend.slices.discovery", snapshotSql, discoveredRows.length),
+		receipt("ua_spend.period.unscoped", snapshotSql, periodHasUnscopedActuals ? 1 : 0),
 	];
 	const approvedKeys = new Set(
 		portfolio
@@ -213,25 +383,58 @@ export async function loadDashboardActuals(
 	const rowKey = (row: SqlRow): string =>
 		portfolio ? `${row.platform}\u0000${row.media_source}` : `${row.app_code}\u0000${row.platform}\u0000${row.media_source}`;
 	const sliceRows = [
-		...approvedSlices.rows,
-		...discoveredSlices.rows.filter((row) => !approvedKeys.has(rowKey(row))),
+		...approvedRows,
+		...discoveredRows.filter((row) => !approvedKeys.has(rowKey(row))),
 	];
+	const dailySeries = completeDailySeries(dailyRows, dates.currentFrom, coverageMax);
+	const currentSpend = value(currentRow, "spend");
+	const currentRevenue = value(currentRow, "revenue");
+	const dailyComplete = coverageMin === dates.currentFrom && coverageMax === dates.currentTo
+		&& dailySeries.length === periodDays
+		&& dailySeries.every((point) => point.spend !== null && point.revenue !== null);
+	const approvedSpend = approvedRows.reduce<number | null>((sum, row) => sum === null || value(row, "spend") === null ? null : sum + (value(row, "spend") as number), 0);
+	const approvedRevenue = approvedRows.reduce<number | null>((sum, row) => sum === null || value(row, "revenue") === null ? null : sum + (value(row, "revenue") as number), 0);
+	const dailySpend = dailySeries.reduce<number | null>((sum, point) => sum === null || point.spend === null ? null : sum + point.spend, 0);
+	const dailyRevenue = dailySeries.reduce<number | null>((sum, point) => sum === null || point.revenue === null ? null : sum + point.revenue, 0);
+	const reconciled = dailyComplete && currentSpend !== null && currentRevenue !== null
+		&& approvedSpend !== null && approvedRevenue !== null && dailySpend !== null && dailyRevenue !== null
+		&& approximatelyEqual(currentSpend, approvedSpend) && approximatelyEqual(currentRevenue, approvedRevenue)
+		&& approximatelyEqual(currentSpend, dailySpend) && approximatelyEqual(currentRevenue, dailyRevenue);
+	const snapshotId = envelope.snapshot_id == null ? null : String(envelope.snapshot_id);
+	const availableAt = envelope.available_at == null ? null : new Date(String(envelope.available_at)).toISOString();
+	// Date coverage and elapsed time are not close signals. Only an explicit ETL
+	// control-plane marker may close a cycle; absent evidence fails closed.
+	const periodComplete = dailyComplete && closeSignal !== null && availableAt !== null
+		&& closeSignal.closed_through >= dates.currentTo
+		&& Date.parse(availableAt) >= Date.parse(closeSignal.emitted_at);
 
 	return {
 		period,
 		data_as_of: coverageMax ?? ACTUALS_DATA_AS_OF_UNAVAILABLE,
+		snapshot_evidence: {
+			snapshot_id: snapshotId,
+			available_at: availableAt,
+			consistency: "single_statement",
+			source_close_signal_id: closeSignal?.signal_id ?? null,
+			source_closed_through: closeSignal?.closed_through ?? null,
+			source_close_emitted_at: closeSignal?.emitted_at ?? null,
+			source_close_signal_sha256: closeSignal?.signal_sha256 ?? null,
+			period_complete: periodComplete,
+			reconciled,
+		},
 		reporting_currency: "USD",
 		scope_mode: mode,
+		period_has_unscoped_actuals: periodHasUnscopedActuals,
 		coverage: {
 			date_min: coverageMin,
 			date_max: coverageMax,
 			source_rows: value(currentRow, "source_rows") ?? 0,
 		},
-		current: { spend: value(currentRow, "spend"), revenue: value(currentRow, "revenue") },
-		comparison: comparisonRow
+		current: { spend: currentSpend, revenue: currentRevenue },
+		comparison: periodComplete && comparisonRow
 			? { spend: value(comparisonRow, "spend"), revenue: value(comparisonRow, "revenue") }
 			: null,
-		daily: completeDailySeries(daily.rows, dates.currentFrom, coverageMax),
+		daily: dailySeries,
 		slices: mapDashboardSlices(sliceRows, portfolioAppId ?? undefined),
 		query_receipts: queryReceipts,
 	};