@viccydev/pi-fpa 0.3.1 → 0.3.3

package/README.md

@@ -39,6 +39,8 @@ Skill：
 | `fpa_calc` | 确定性计算器：命名公式求值（四则、abs/min/max/round），NULL 与除零安全传播 |
 | `fpa_compare` | 双期间对比：差值、百分比变化、逐行贡献度全部由代码计算 |
 
+`fpa_data_catalog` 默认只读取数据字典和各数据集日期覆盖，不执行随表规模增长的精确行数扫描。确实需要精确 `row_count`、App 数量和 cohort-size 覆盖时显式传入 `include_stats: true`；需要 App 列表时传入 `include_apps: true`。这些补充信息都有独立的 5 秒预算，单个数据集超时会返回对应的 `*_unavailable` 说明，不会丢失其它数据集的覆盖信息。认证、网络、服务端错误和调用方取消仍然抛出。
+
 设计契约：**模型不写 SQL、不做任何算术**。模型只从注册表中选择数据集、指标和维度；SQL 生成、数据库聚合和全部派生计算（比率、差异、LTV/ROAS/留存、临时公式）都在 Extension 代码内完成，缺数据或除零返回 `NULL`，绝不编造数值。
 
 Extension 内置的关键防护：
@@ -108,12 +110,12 @@ pi list
 团队分发建议使用固定 Git tag：
 
 ```bash
-pi install git:github.com/linyqh/pi-fpa@v0.3.0
+pi install git:github.com/linyqh/pi-fpa@v0.3.2
 ```
 
 ## 发布到 npm
 
-发布动作由 GitHub Release 触发。Release 标签必须严格使用 `v<package.json version>`，例如当前 `0.3.0` 对应 `v0.3.0`。工作流会检出该标签，执行 `npm ci`、`npm test` 和包内容预检，全部通过后发布公开包 `@viccydev/pi-fpa`。普通 Release 发布到 `latest`，Prerelease 发布到 `next`。
+发布动作由 GitHub Release 触发。Release 标签必须严格使用 `v<package.json version>`，例如版本 `0.3.3` 对应 `v0.3.3`。工作流会检出该标签，执行 `npm ci`、`npm test` 和包内容预检，全部通过后发布公开包 `@viccydev/pi-fpa`。普通 Release 发布到 `latest`，Prerelease 发布到 `next`。
 
 首次发布前需要完成一次仓库配置：
 

package/extensions/fpa-data/index.ts

@@ -23,16 +23,33 @@ import { CATALOG_CAVEATS, DATASETS, DATASET_IDS } from "./registry.ts";
 import {
 	buildCohortQuery,
 	buildCoverageSql,
+	buildCoverageStatsSql,
 	buildCohortSizeCoverageSql,
+	COVERAGE_TARGETS,
 	MAX_LIMIT,
 	type QuerySpec,
 } from "./sql.ts";
 import { runStructuredQuery } from "./runtime.ts";
-import { runQuery } from "./supabase.ts";
+import { runQuery, SupabaseQueryTimeoutError, type SqlRow } from "./supabase.ts";
 
 const MAX_TOOL_TEXT_CHARS = 100_000;
 const MAX_DISPLAY_ROWS = 200;
 
+/** Coverage is essential but one slow dataset must not consume the whole tool budget. */
+const CATALOG_COVERAGE_TIMEOUT_MS = 5_000;
+
+/** Explicitly requested supplementary catalog data has a bounded, non-fatal budget. */
+const CATALOG_OPTIONAL_TIMEOUT_MS = 5_000;
+
+async function fallbackOnTimeout<T>(work: Promise<T>, signal?: AbortSignal): Promise<T | null> {
+	try {
+		return await work;
+	} catch (error) {
+		if (signal?.aborted || !(error instanceof SupabaseQueryTimeoutError)) throw error;
+		return null;
+	}
+}
+
 const DatasetIdSchema = StringEnum(DATASET_IDS as [string, ...string[]], {
 	description: "Dataset id from fpa_data_catalog",
 });
@@ -126,6 +143,12 @@ export default function fpaDataExtension(pi: ExtensionAPI): void {
 		],
 		parameters: Type.Object(
 			{
+				include_stats: Type.Optional(
+					Type.Boolean({
+						description:
+							"Also request exact row/app counts and cohort-size availability. These are scan-bound and best-effort.",
+					}),
+				),
 				include_apps: Type.Optional(
 					Type.Boolean({ description: "Also list distinct app codes (up to 200)." }),
 				),
@@ -134,18 +157,82 @@ export default function fpaDataExtension(pi: ExtensionAPI): void {
 		),
 		executionMode: "parallel",
 		async execute(_toolCallId, params, signal) {
-			const [coverage, cohortSize, apps] = await Promise.all([
-				runQuery(buildCoverageSql(), { signal }),
-				runQuery(buildCohortSizeCoverageSql(), { signal }),
+			// Date coverage is the part callers must have. Keep each dataset in its own
+			// timeout domain so a missing leading-date index cannot sink the whole catalog;
+			// exact scan-bound statistics remain opt-in.
+			const coverageRows = await Promise.all(
+				COVERAGE_TARGETS.map(async (target) => {
+					const rows = await fallbackOnTimeout(
+						runQuery(buildCoverageSql(target), {
+							signal,
+							timeoutMs: CATALOG_COVERAGE_TIMEOUT_MS,
+						}),
+						signal,
+					);
+					return rows?.[0] ?? {
+						dataset: target.dataset,
+						date_min: null,
+						date_max: null,
+						...(rows === null
+							? {
+									coverage_unavailable:
+										"Date coverage timed out for this dataset; other datasets are unaffected.",
+								}
+							: {}),
+					};
+				}),
+			);
+			const statsBySource = new Map<string, Promise<SqlRow[] | null>>();
+			const statsFor = (target: (typeof COVERAGE_TARGETS)[number]) => {
+				const key = `${target.table}\u0000${target.appColumn}`;
+				let pending = statsBySource.get(key);
+				if (!pending) {
+					pending = fallbackOnTimeout(
+						runQuery(buildCoverageStatsSql(target), {
+							signal,
+							timeoutMs: CATALOG_OPTIONAL_TIMEOUT_MS,
+						}),
+						signal,
+					);
+					statsBySource.set(key, pending);
+				}
+				return pending;
+			};
+			const [statsRows, cohortSize, apps] = await Promise.all([
+				params.include_stats
+					? Promise.all(
+							COVERAGE_TARGETS.map((target) => statsFor(target)),
+						)
+					: Promise.resolve(COVERAGE_TARGETS.map(() => null)),
+				params.include_stats
+					? fallbackOnTimeout(runQuery(buildCohortSizeCoverageSql(), { signal, timeoutMs: CATALOG_OPTIONAL_TIMEOUT_MS }), signal)
+					: Promise.resolve(null),
 				params.include_apps
-					? runQuery(
-							"select app_code, string_agg(distinct platform, ',' order by platform) as platforms " +
-								"from appsflyer_ua_campaign_daily where app_code is not null " +
-								"group by 1 order by 1 limit 200",
-							{ signal },
+					? fallbackOnTimeout(
+							runQuery(
+								"select app_code, string_agg(distinct platform, ',' order by platform) as platforms " +
+									"from appsflyer_ua_campaign_daily where app_code is not null " +
+									"group by 1 order by 1 limit 200",
+								{ signal, timeoutMs: CATALOG_OPTIONAL_TIMEOUT_MS },
+							),
+							signal,
 						)
 					: Promise.resolve<SqlRow[]>([]),
 			]);
+			const coverage = coverageRows.map((row, index) => {
+				const stats = statsRows[index]?.[0];
+				return stats
+					? { ...row, ...stats }
+					: {
+							...row,
+							row_count: null,
+							apps: null,
+							updated_at: null,
+							stats_unavailable: params.include_stats
+								? "Row counts timed out; date coverage above is unaffected."
+								: "Not requested; set include_stats=true for exact row and app counts.",
+						};
+			});
 
 			return toolResult(
 				{
@@ -165,9 +252,26 @@ export default function fpaDataExtension(pi: ExtensionAPI): void {
 						notes: d.notes,
 					})),
 					coverage,
-					cohort_size_available: cohortSize[0] ?? null,
+					cohort_size_available: cohortSize?.[0] ?? null,
+					...(cohortSize?.[0]
+						? {}
+						: {
+								cohort_size_unavailable: params.include_stats
+									? "Cohort-size coverage timed out; dataset date coverage is unaffected."
+									: "Not requested; set include_stats=true for cohort-size availability.",
+							}),
 					caveats: CATALOG_CAVEATS,
-					...(params.include_apps ? { apps } : {}),
+					...(params.include_apps
+						? {
+								apps,
+								...(apps === null
+									? {
+											apps_unavailable:
+												"App list timed out; dataset definitions and date coverage are unaffected.",
+										}
+									: {}),
+							}
+						: {}),
 				},
 				"coverage",
 			);

package/extensions/fpa-data/sql.ts

@@ -442,17 +442,50 @@ export function buildCohortQuery(spec: CohortSpec): BuiltCohortQuery {
 	return { sql, spendSql, asOfSql, horizons, bucket, groupBy };
 }
 
-/** Coverage catalog query used by fpa_data_catalog. */
-export function buildCoverageSql(): string {
-	const parts = [
-		`select 'ua_spend' as dataset, count(*) as row_count, min(install_date)::text as date_min, max(install_date)::text as date_max, count(distinct app_code) as apps, max(updated_at)::text as updated_at from appsflyer_ua_campaign_daily`,
-		`select 'revenue_activity', count(*), min(event_date)::text, max(event_date)::text, count(distinct app_code), max(updated_at)::text from appsflyer_install_cohort_daily`,
-		`select 'cohort (fpa_cohort)', count(*), min(install_date)::text, max(install_date)::text, count(distinct app_code), max(updated_at)::text from appsflyer_install_cohort_daily`,
-		`select 'apple_store', count(*), min(metric_date)::text, max(metric_date)::text, count(distinct adam_id), max(updated_at)::text from apple_app_daily`,
-		`select 'product_events', count(*), min(event_date)::text, max(event_date)::text, count(distinct app_code), max(updated_at)::text from appsflyer_event_daily`,
-		`select 'mixpanel_product', count(*), min(business_date)::text, max(business_date)::text, count(distinct app_code), max(updated_at)::text from mixpanel_app_daily`,
-	];
-	return assertReadOnly(parts.join(" union all "));
+/** One row of the fpa_data_catalog coverage block. */
+export interface CoverageTarget {
+	dataset: string;
+	table: string;
+	dateColumn: string;
+	appColumn: string;
+}
+
+export const COVERAGE_TARGETS: CoverageTarget[] = [
+	{ dataset: "ua_spend", table: "appsflyer_ua_campaign_daily", dateColumn: "install_date", appColumn: "app_code" },
+	{ dataset: "revenue_activity", table: "appsflyer_install_cohort_daily", dateColumn: "event_date", appColumn: "app_code" },
+	{ dataset: "cohort (fpa_cohort)", table: "appsflyer_install_cohort_daily", dateColumn: "install_date", appColumn: "app_code" },
+	{ dataset: "apple_store", table: "apple_app_daily", dateColumn: "metric_date", appColumn: "adam_id" },
+	{ dataset: "product_events", table: "appsflyer_event_daily", dateColumn: "event_date", appColumn: "app_code" },
+	{ dataset: "mixpanel_product", table: "mixpanel_app_daily", dateColumn: "business_date", appColumn: "app_code" },
+];
+
+/**
+ * Answerable date range for one dataset.
+ *
+ * min()/max() alone let Postgres walk the date index instead of scanning the table.
+ * Mixing them with count(*) in a single statement — as the old six-way UNION did —
+ * forces a sequential scan per arm, which is what pushed fpa_data_catalog past its
+ * timeout as the mart grew.
+ */
+export function buildCoverageSql(target: CoverageTarget): string {
+	return assertReadOnly(
+		`select ${escapeLiteral(target.dataset)} as dataset, ` +
+			`min(${target.dateColumn})::text as date_min, max(${target.dateColumn})::text as date_max ` +
+			`from ${target.table}`,
+	);
+}
+
+/**
+ * Row counts and freshness for one dataset. These need a full scan (count(distinct)
+ * most of all), so callers treat them as best-effort and report null when they time out
+ * rather than failing the whole catalog.
+ */
+export function buildCoverageStatsSql(target: CoverageTarget): string {
+	return assertReadOnly(
+		`select count(*) as row_count, count(distinct ${target.appColumn}) as apps, ` +
+			`max(updated_at)::text as updated_at ` +
+			`from ${target.table}`,
+	);
 }
 
 /** Cohort-size availability window, reported by fpa_data_catalog. */

package/extensions/fpa-data/supabase.ts

@@ -35,6 +35,15 @@ export function resolveConfig(env: Record<string, string | undefined> = process.
 
 export type SqlRow = Record<string, unknown>;
 
+export class SupabaseQueryTimeoutError extends Error {
+	readonly code = "SUPABASE_QUERY_TIMEOUT";
+
+	constructor() {
+		super("Supabase query timed out. Narrow the date range or filters.");
+		this.name = "SupabaseQueryTimeoutError";
+	}
+}
+
 export async function runQuery(
 	sql: string,
 	options: { signal?: AbortSignal; timeoutMs?: number; config?: SupabaseConfig } = {},
@@ -45,6 +54,7 @@ export async function runQuery(
 	if (options.signal) signals.push(options.signal);
 
 	let response: Response;
+	let body: string;
 	try {
 		response = await fetch(
 			`${API_BASE}/v1/projects/${config.projectRef}/database/query`,
@@ -58,16 +68,19 @@ export async function runQuery(
 				signal: AbortSignal.any(signals),
 			},
 		);
+		body = await response.text();
 	} catch (error) {
+		if (options.signal?.aborted) {
+			throw options.signal.reason ?? error;
+		}
 		if (error instanceof Error && error.name === "TimeoutError") {
-			throw new Error("Supabase query timed out. Narrow the date range or filters.");
+			throw new SupabaseQueryTimeoutError();
 		}
 		throw new Error(
 			`Could not reach the Supabase Management API: ${error instanceof Error ? error.message : String(error)}`,
 		);
 	}
 
-	const body = await response.text();
 	if (new TextEncoder().encode(body).byteLength > MAX_RESPONSE_BYTES) {
 		throw new Error(
 			`Supabase response exceeds ${MAX_RESPONSE_BYTES} bytes. Narrow the query with filters, a shorter date range, or a lower limit.`,
@@ -86,6 +99,12 @@ export async function runQuery(
 			payload && typeof payload === "object" && "message" in payload
 				? String((payload as { message: unknown }).message)
 				: body.slice(0, 500);
+		if (
+			response.status === 504 ||
+			/(?:statement|query|gateway) (?:timed out|timeout)/i.test(detail)
+		) {
+			throw new SupabaseQueryTimeoutError();
+		}
 		throw new Error(`Supabase Management API error ${response.status}: ${detail}`);
 	}
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@viccydev/pi-fpa",
-  "version": "0.3.1",
+  "version": "0.3.3",
   "type": "module",
   "description": "Full-cycle FP&A planning, strategy, forecast, and review prompts, skills, and data tools for Pi",
   "license": "UNLICENSED",
@@ -23,9 +23,9 @@
     "extensions"
   ],
   "scripts": {
-    "test": "node tests/package-structure.test.mjs && node tests/extension-unit.test.mjs && node --test tests/artifact-store.test.mjs tests/dashboard-actuals.test.mjs tests/dashboard-projector.test.mjs tests/dashboard-publisher.test.mjs && node tests/pi-loader-smoke.mjs && node tests/publish-workflow.test.mjs",
+    "test": "node tests/package-structure.test.mjs && node tests/extension-unit.test.mjs && node --test tests/catalog-timeout.test.mjs tests/artifact-store.test.mjs tests/dashboard-actuals.test.mjs tests/dashboard-projector.test.mjs tests/dashboard-publisher.test.mjs && node tests/pi-loader-smoke.mjs && node tests/publish-workflow.test.mjs",
     "test:structure": "node tests/package-structure.test.mjs",
-    "test:unit": "node tests/extension-unit.test.mjs && node --test tests/artifact-store.test.mjs tests/dashboard-actuals.test.mjs tests/dashboard-projector.test.mjs tests/dashboard-publisher.test.mjs",
+    "test:unit": "node tests/extension-unit.test.mjs && node --test tests/catalog-timeout.test.mjs tests/artifact-store.test.mjs tests/dashboard-actuals.test.mjs tests/dashboard-projector.test.mjs tests/dashboard-publisher.test.mjs",
     "test:loader": "node tests/pi-loader-smoke.mjs",
     "test:live": "node tests/live-smoke.mjs",
     "pack:check": "npm pack --dry-run"