apphud-mcp 0.2.2 → 0.2.4

package/dist/src/services/etlService.js

@@ -1,22 +1,13 @@
+import { execFile } from "node:child_process";
 import { createHash } from "node:crypto";
 import { mkdirSync } from "node:fs";
-import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
+import { promisify } from "node:util";
 import { gunzipSync } from "node:zlib";
-import Database from "better-sqlite3";
 import { ApphudMcpError } from "../errors/toolError.js";
 const LOCAL_ROWS_GUARD_LIMIT = 1_000_000;
-function interpolatePath(template, vars) {
-    return template.replace(/\{([a-zA-Z0-9_]+)\}/g, (_, key) => vars[key] ?? "");
-}
-function normalizeToPath(baseUrl, rawPath) {
-    if (rawPath.startsWith("http://") || rawPath.startsWith("https://")) {
-        return rawPath;
-    }
-    const normalizedBase = baseUrl.replace(/\/$/, "");
-    const normalizedPath = rawPath.startsWith("/") ? rawPath : `/${rawPath}`;
-    return `${normalizedBase}${normalizedPath}`;
-}
+const execFileAsync = promisify(execFile);
 function parseCsvLine(line) {
     const result = [];
     let current = "";
@@ -50,16 +41,11 @@ function parseCsv(content) {
         .map((line) => line.trimEnd())
         .filter((line) => line.length > 0);
     if (lines.length === 0) {
-        return {
-            headers: [],
-            rows: [],
-        };
+        return { headers: [], rows: [] };
     }
-    const headers = parseCsvLine(lines[0] ?? "");
-    const rows = lines.slice(1).map((line) => parseCsvLine(line));
     return {
-        headers,
-        rows,
+        headers: parseCsvLine(lines[0] ?? ""),
+        rows: lines.slice(1).map((line) => parseCsvLine(line)),
     };
 }
 function rowToRecord(headers, values) {
@@ -71,45 +57,6 @@ function rowToRecord(headers, values) {
     }
     return result;
 }
-function extractExportCandidates(payload, appId) {
-    const candidates = [];
-    const visited = new WeakSet();
-    const walk = (node) => {
-        if (Array.isArray(node)) {
-            for (const item of node) {
-                walk(item);
-            }
-            return;
-        }
-        if (!node || typeof node !== "object") {
-            return;
-        }
-        const record = node;
-        if (visited.has(record)) {
-            return;
-        }
-        visited.add(record);
-        const downloadUrl = (typeof record.download_url === "string" && record.download_url) ||
-            (typeof record.downloadUrl === "string" && record.downloadUrl) ||
-            (typeof record.url === "string" && record.url) ||
-            (typeof record.file_url === "string" && record.file_url);
-        const fileName = (typeof record.file_name === "string" && record.file_name) ||
-            (typeof record.filename === "string" && record.filename) ||
-            (typeof record.name === "string" && record.name);
-        if (downloadUrl && fileName) {
-            candidates.push({
-                appId,
-                fileName,
-                downloadUrl,
-            });
-        }
-        for (const value of Object.values(record)) {
-            walk(value);
-        }
-    };
-    walk(payload);
-    return candidates;
-}
 async function listCsvGzFiles(root) {
     const entries = await readdir(root, { withFileTypes: true });
     const files = [];
@@ -128,25 +75,24 @@ async function listCsvGzFiles(root) {
 export class EtlService {
     config;
     apphudClient;
-    db;
     timer = null;
     running = false;
+    stateLoaded = false;
+    appKeys = [];
+    manifest = new Map();
+    alerts = [];
     constructor(config, apphudClient) {
         this.config = config;
         this.apphudClient = apphudClient;
-        const sqlitePath = path.resolve(config.sqlitePath ?? ".apphud-mcp/apphud.db");
-        mkdirSync(path.dirname(sqlitePath), { recursive: true });
-        this.db = new Database(sqlitePath);
-        this.db.pragma("journal_mode = WAL");
-        this.db.pragma("busy_timeout = 5000");
-        this.initializeSchema();
+        mkdirSync(path.resolve(this.config.etlStorageDir), { recursive: true });
+        mkdirSync(path.resolve(this.config.etlIncomingDir), { recursive: true });
+        mkdirSync(path.resolve(this.stateDir()), { recursive: true });
     }
     async start() {
         if (!this.config.etlEnabled) {
             return;
         }
-        await mkdir(path.resolve(this.config.etlStorageDir), { recursive: true });
-        await mkdir(path.resolve(this.config.etlIncomingDir), { recursive: true });
+        await this.ensureStateLoaded();
         await this.runOnce();
         const intervalMs = Math.max(this.config.etlPollIntervalMinutes, 1) * 60_000;
         this.timer = setInterval(() => {
@@ -158,63 +104,75 @@ export class EtlService {
             clearInterval(this.timer);
             this.timer = null;
         }
-        this.db.close();
+        await this.persistState();
     }
     async runOnce() {
         if (!this.config.etlEnabled || this.running) {
             return;
         }
+        await this.ensureStateLoaded();
         this.running = true;
         try {
-            const keySync = await this.syncAppApiKeys();
-            if (this.config.etlRemoteFetchEnabled) {
-                await this.fetchRemoteExports(keySync);
-            }
-            await this.ingestIncomingFiles();
+            await this.syncAppApiKeys();
+            await this.syncFromStorage();
+            await this.refreshManifestFromIncoming();
             this.applyAlertTodos();
+            await this.persistState();
         }
         catch (error) {
             this.insertAlertTodo("ETL_RUN_FAILED", error instanceof Error ? error.message : String(error));
+            await this.persistState();
         }
         finally {
             this.running = false;
         }
     }
     async localStatus() {
+        await this.ensureStateLoaded();
         const now = new Date().toISOString();
-        const manifest = this.db
-            .prepare(`SELECT COUNT(*) AS total, SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) AS done, MAX(processed_at) AS last_processed
-         FROM etl_manifest WHERE tenant_id = ?`)
-            .get(this.config.etlTenantId);
-        const rawRows = this.db
-            .prepare(`SELECT COUNT(*) AS total FROM etl_raw_rows WHERE tenant_id = ?`)
-            .get(this.config.etlTenantId);
-        const alerts = this.db
-            .prepare(`SELECT code, message, created_at FROM etl_alerts_todo ORDER BY id DESC LIMIT 20`)
-            .all();
+        const manifestEntries = Array.from(this.manifest.values()).filter((entry) => entry.tenant_id === this.config.etlTenantId);
+        const doneEntries = manifestEntries.filter((entry) => entry.status === "done");
+        const lastProcessed = doneEntries
+            .map((entry) => entry.processed_at)
+            .filter((value) => Boolean(value))
+            .sort()
+            .at(-1) ?? null;
+        const rawRowsTotal = doneEntries.reduce((sum, entry) => sum + entry.row_count, 0);
         return {
-            source: "apphud_local_etl",
+            source: "apphud_storage_etl",
             tenant_id: this.config.etlTenantId,
             etl_enabled: this.config.etlEnabled,
-            remote_fetch_enabled: this.config.etlRemoteFetchEnabled,
+            etl_source: this.config.etlSource,
+            source_config: {
+                gcs_bucket: this.config.etlGcsBucket,
+                gcs_prefix: this.config.etlGcsPrefix,
+                s3_bucket: this.config.etlS3Bucket,
+                s3_prefix: this.config.etlS3Prefix,
+            },
             poll_interval_minutes: this.config.etlPollIntervalMinutes,
-            sqlite_path: path.resolve(this.config.sqlitePath ?? ".apphud-mcp/apphud.db"),
             incoming_dir: path.resolve(this.config.etlIncomingDir),
-            manifest_total: manifest.total,
-            manifest_done: manifest.done ?? 0,
-            raw_rows_total: rawRows.total,
-            last_processed_at: manifest.last_processed,
-            alerts_todo: alerts,
+            manifest_total: manifestEntries.length,
+            manifest_done: doneEntries.length,
+            raw_rows_total: rawRowsTotal,
+            last_processed_at: lastProcessed,
+            alerts_todo: this.alerts.slice(-20).reverse(),
             retrieved_at: now,
         };
     }
     async localAppsList(input = {}) {
         await this.ensureLocalBootstrap(false);
-        const apps = this.db
-            .prepare(`SELECT app_id, app_name, platform, key_source AS source, updated_at FROM etl_app_api_keys WHERE tenant_id = ? ORDER BY app_name`)
-            .all(this.config.etlTenantId);
+        const apps = this.appKeys
+            .filter((app) => app.tenant_id === this.config.etlTenantId)
+            .map((app) => ({
+            app_id: app.app_id,
+            app_name: app.app_name,
+            platform: app.platform ?? null,
+            source: app.key_source,
+            updated_at: app.updated_at,
+        }))
+            .sort((a, b) => a.app_name.localeCompare(b.app_name));
         return {
-            source: "apphud_local_etl",
+            source: "apphud_storage_etl",
             count: apps.length,
             apps,
             retrieved_at: new Date().toISOString(),
@@ -227,7 +185,7 @@ export class EtlService {
         const from = input.from ?? new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString();
         const to = input.to ?? new Date().toISOString();
         const limit = Math.min(Math.max(input.limit ?? 100, 1), 500);
-        const rows = this.loadRows(appId, from, to, input.filters);
+        const rows = await this.loadRows(appId, from, to, input.filters);
         const events = rows.slice(0, limit).map((row) => ({
             event_id: this.readString(row.record, ["event_id", "id", "uuid"]),
             event_type: this.readString(row.record, ["event_type", "event", "type", "name"]) ?? "unknown",
@@ -253,8 +211,8 @@ export class EtlService {
             events_count: events.length,
             next_cursor: undefined,
             has_more: rows.length > limit,
-            source: "apphud_local_etl",
-            source_used: "etl_raw_rows",
+            source: "apphud_storage_etl",
+            source_used: "incoming_csv_gz",
             retrieved_at: new Date().toISOString(),
             raw_payload: input.include_raw ? rows.map((row) => row.record) : undefined,
         };
@@ -267,7 +225,7 @@ export class EtlService {
             app_id: appId,
             apphud_app_id: appId,
             analytics_available: true,
-            source: "apphud_local_etl",
+            source: "apphud_storage_etl",
             supported_tools: [
                 "apphud_analytics_events_list_local",
                 "apphud_analytics_metrics_list_local",
@@ -284,7 +242,7 @@ export class EtlService {
             supported_shapes: ["value", "timeseries", "breakdown", "raw"],
             metrics_count: metrics.length,
             metrics_sample: metrics.slice(0, 20),
-            source_used: "etl_raw_rows",
+            source_used: "incoming_csv_gz",
             retrieved_at: new Date().toISOString(),
             raw_payload: input.include_raw ? metrics : undefined,
         };
@@ -298,7 +256,7 @@ export class EtlService {
             apphud_app_id: appId,
             metrics,
             count: metrics.length,
-            source_used: "etl_raw_rows",
+            source_used: "incoming_csv_gz",
             retrieved_at: new Date().toISOString(),
             raw_payload: input.include_raw ? metrics : undefined,
         };
@@ -307,19 +265,17 @@ export class EtlService {
         await this.ensureLocalBootstrap(true);
         const appId = this.resolveLocalAppId(input.app_id, input.apphud_app_id);
         const metricKey = input.metric_key ?? "events_count";
-        const from = input.from;
-        const to = input.to;
-        const rows = this.loadRows(appId, from, to, input.filters);
+        const rows = await this.loadRows(appId, input.from, input.to, input.filters);
         const value = this.computeMetric(metricKey, rows);
         return {
             app_id: appId,
             apphud_app_id: appId,
             metric_key: metricKey,
-            from,
-            to,
+            from: input.from,
+            to: input.to,
             value,
-            source: "apphud_local_etl",
-            source_used: "etl_raw_rows",
+            source: "apphud_storage_etl",
+            source_used: "incoming_csv_gz",
             warnings: [],
             retrieved_at: new Date().toISOString(),
             raw_payload: input.include_raw ? rows.map((row) => row.record) : undefined,
@@ -330,7 +286,7 @@ export class EtlService {
         const appId = this.resolveLocalAppId(input.app_id, input.apphud_app_id);
         const metricKey = input.metric_key ?? "events_count";
         const granularity = input.granularity ?? "day";
-        const rows = this.loadRows(appId, input.from, input.to, input.filters);
+        const rows = await this.loadRows(appId, input.from, input.to, input.filters);
         const buckets = new Map();
         for (const row of rows) {
             const bucket = this.timeBucket(row.occurredAt, granularity);
@@ -356,8 +312,8 @@ export class EtlService {
             points_count: points.length,
             total: Number(total.toFixed(4)),
             average: points.length > 0 ? Number((total / points.length).toFixed(4)) : null,
-            source: "apphud_local_etl",
-            source_used: "etl_raw_rows",
+            source: "apphud_storage_etl",
+            source_used: "incoming_csv_gz",
             warnings: [],
             retrieved_at: new Date().toISOString(),
             raw_payload: input.include_raw ? rows.map((row) => row.record) : undefined,
@@ -369,7 +325,7 @@ export class EtlService {
         const metricKey = input.metric_key ?? "events_count";
         const dimension = input.dimension ?? "country";
         const limit = Math.min(Math.max(input.limit ?? 20, 1), 200);
-        const rows = this.loadRows(appId, input.from, input.to, input.filters);
+        const rows = await this.loadRows(appId, input.from, input.to, input.filters);
         const groups = new Map();
         for (const row of rows) {
             const key = this.dimensionValue(row.record, dimension);
@@ -399,8 +355,8 @@ export class EtlService {
             granularity: input.granularity ?? "day",
             rows: rowsWithShare,
             rows_count: rowsWithShare.length,
-            source: "apphud_local_etl",
-            source_used: "etl_raw_rows",
+            source: "apphud_storage_etl",
+            source_used: "incoming_csv_gz",
             warnings: [],
             retrieved_at: new Date().toISOString(),
             raw_payload: input.include_raw ? rows.map((row) => row.record) : undefined,
@@ -408,9 +364,8 @@ export class EtlService {
     }
     async localRevenueSummary(input) {
         await this.ensureLocalBootstrap(true);
-        const base = { ...input };
-        const gross = await this.localMetricValue({ ...base, metric_key: "revenue_gross", include_raw: false });
-        const refunds = await this.localMetricValue({ ...base, metric_key: "refunds", include_raw: false });
+        const gross = await this.localMetricValue({ ...input, metric_key: "revenue_gross", include_raw: false });
+        const refunds = await this.localMetricValue({ ...input, metric_key: "refunds", include_raw: false });
         const grossValue = typeof gross.value === "number" ? gross.value : 0;
         const refundValue = Math.abs(typeof refunds.value === "number" ? refunds.value : 0);
         return {
@@ -421,8 +376,8 @@ export class EtlService {
             revenue_gross: gross.value,
             refunds: refunds.value,
             net_revenue_estimate: Number((grossValue - refundValue).toFixed(4)),
-            source: "apphud_local_etl",
-            source_used: ["etl_raw_rows"],
+            source: "apphud_storage_etl",
+            source_used: ["incoming_csv_gz"],
             warnings: [],
             compare_prev_period: input.compare_prev_period ? { note: "TODO local compare_prev_period" } : undefined,
             retrieved_at: new Date().toISOString(),
@@ -449,8 +404,8 @@ export class EtlService {
             renewals: renewals.value,
             cancellations: cancellations.value,
             churn_rate_estimate: activePaidValue > 0 ? Number((cancellationsValue / activePaidValue).toFixed(6)) : null,
-            source: "apphud_local_etl",
-            source_used: ["etl_raw_rows"],
+            source: "apphud_storage_etl",
+            source_used: ["incoming_csv_gz"],
             warnings: [],
             retrieved_at: new Date().toISOString(),
             raw_payload: input.include_raw ? { activePaid, activeTrials, newSubs, renewals, cancellations } : undefined,
@@ -471,8 +426,8 @@ export class EtlService {
             trials_converted: converted.value,
             conversion_rate: startedValue > 0 ? Number((convertedValue / startedValue).toFixed(6)) : null,
             median_time_to_convert: null,
-            source: "apphud_local_etl",
-            source_used: ["etl_raw_rows"],
+            source: "apphud_storage_etl",
+            source_used: ["incoming_csv_gz"],
             warnings: [],
             retrieved_at: new Date().toISOString(),
             raw_payload: input.include_raw ? { started, converted } : undefined,
@@ -483,11 +438,6 @@ export class EtlService {
         const timeseries = await this.localMetricTimeseries({ ...input, metric_key: "active_subs", granularity: input.granularity ?? "week" });
         const points = timeseries.points ?? [];
         const base = points[0]?.value ?? 0;
-        const periods = points.map((point, index) => ({
-            period_index: index,
-            retention_rate: base > 0 ? Number((point.value / base).toFixed(6)) : 0,
-            users_count: Math.round(point.value),
-        }));
         return {
             app_id: timeseries.app_id,
             apphud_app_id: timeseries.apphud_app_id,
@@ -495,10 +445,20 @@ export class EtlService {
             to: input.to,
             granularity: input.granularity ?? "week",
             cohort_by: "subscription_started",
-            rows: [{ cohort_start_date: (input.from ?? new Date().toISOString()).slice(0, 10), users_count: base, periods }],
-            source: "apphud_local_etl",
-            source_used: "etl_raw_rows",
-            warnings: ["Retention is approximated from local event stream."],
+            rows: [
+                {
+                    cohort_start_date: (input.from ?? new Date().toISOString()).slice(0, 10),
+                    users_count: base,
+                    periods: points.map((point, index) => ({
+                        period_index: index,
+                        retention_rate: base > 0 ? Number((point.value / base).toFixed(6)) : 0,
+                        users_count: Math.round(point.value),
+                    })),
+                },
+            ],
+            source: "apphud_storage_etl",
+            source_used: "incoming_csv_gz",
+            warnings: ["Retention is approximated from storage event stream."],
             retrieved_at: new Date().toISOString(),
             raw_payload: input.include_raw ? points : undefined,
         };
@@ -507,21 +467,25 @@ export class EtlService {
         await this.ensureLocalBootstrap(true);
         const timeseries = await this.localMetricTimeseries({ ...input, metric_key: "cumulative_ltv", granularity: input.granularity ?? "week" });
         const points = timeseries.points ?? [];
-        const periods = points.map((point, index) => ({
-            period_index: index,
-            ltv_value: Number(point.value.toFixed(6)),
-            date: point.date,
-        }));
         return {
             app_id: timeseries.app_id,
             apphud_app_id: timeseries.apphud_app_id,
             from: input.from,
             to: input.to,
             granularity: input.granularity ?? "week",
-            rows: [{ cohort_start_date: (input.from ?? new Date().toISOString()).slice(0, 10), periods }],
-            source: "apphud_local_etl",
-            source_used: "etl_raw_rows",
-            warnings: ["LTV is approximated from local event stream."],
+            rows: [
+                {
+                    cohort_start_date: (input.from ?? new Date().toISOString()).slice(0, 10),
+                    periods: points.map((point, index) => ({
+                        period_index: index,
+                        ltv_value: Number(point.value.toFixed(6)),
+                        date: point.date,
+                    })),
+                },
+            ],
+            source: "apphud_storage_etl",
+            source_used: "incoming_csv_gz",
+            warnings: ["LTV is approximated from storage event stream."],
             retrieved_at: new Date().toISOString(),
             raw_payload: input.include_raw ? points : undefined,
         };
@@ -531,7 +495,13 @@ export class EtlService {
         const query = (input.query ?? {});
         const shape = String(query.shape ?? "raw");
         if (shape === "value") {
-            return this.localMetricValue({ ...input, metric_key: String(query.metric_key ?? "") });
+            return this.localMetricValue({
+                ...input,
+                metric_key: String(query.metric_key ?? ""),
+                from: typeof query.from === "string" ? query.from : input.from,
+                to: typeof query.to === "string" ? query.to : input.to,
+                filters: query.filters ?? input.filters,
+            });
         }
         if (shape === "timeseries") {
             return this.localMetricTimeseries({
@@ -540,6 +510,7 @@ export class EtlService {
                 from: typeof query.from === "string" ? query.from : input.from,
                 to: typeof query.to === "string" ? query.to : input.to,
                 granularity: query.granularity === "week" ? "week" : "day",
+                filters: query.filters ?? input.filters,
             });
         }
         if (shape === "breakdown") {
@@ -550,15 +521,16 @@ export class EtlService {
                 from: typeof query.from === "string" ? query.from : input.from,
                 to: typeof query.to === "string" ? query.to : input.to,
                 limit: typeof query.limit === "number" ? query.limit : input.limit,
+                filters: query.filters ?? input.filters,
             });
         }
         const appId = this.resolveLocalAppId(input.app_id, input.apphud_app_id, true);
-        const rows = this.loadRows(appId, input.from, input.to, input.filters);
+        const rows = await this.loadRows(appId, input.from, input.to, input.filters);
         return {
             app_id: appId,
             apphud_app_id: appId,
             shape,
-            source: "apphud_local_etl",
+            source: "apphud_storage_etl",
             status: 200,
             retrieved_at: new Date().toISOString(),
             response: input.include_raw ? rows.map((row) => row.record) : { rows_count: rows.length },
@@ -581,8 +553,8 @@ export class EtlService {
             this.localMetricValue({ app_id: appId, metric_key: "new_subscriptions", from: rangeFrom, to: rangeTo, include_raw: false }),
             this.localMetricValue({ app_id: appId, metric_key: "renewals", from: rangeFrom, to: rangeTo, include_raw: false }),
         ]);
-        const selectedRows = this.loadRows(appId, rangeFrom, rangeTo, input.filters);
-        const allRows = this.loadRows(appId, undefined, undefined, input.filters);
+        const selectedRows = await this.loadRows(appId, rangeFrom, rangeTo, input.filters);
+        const allRows = await this.loadRows(appId, undefined, undefined, input.filters);
         const selectedRevenueGross = typeof revenueGross.value === "number" ? revenueGross.value : 0;
         const selectedRefunds = typeof refunds.value === "number" ? refunds.value : 0;
         const selectedProceeds = Number((selectedRevenueGross - Math.abs(selectedRefunds)).toFixed(4));
@@ -591,7 +563,6 @@ export class EtlService {
             .filter((row) => ["subscription_started", "renewal", "trial_converted"].includes(this.eventType(row.record)))
             .map((row) => this.userId(row.record))
             .filter((value) => Boolean(value))).size;
-        const refundCount = selectedRows.filter((row) => this.eventType(row.record) === "refund").length;
         const failedCharges = selectedRows.filter((row) => {
             const type = this.eventType(row.record);
             return type === "billing_issue" || type === "failed_charge";
@@ -599,14 +570,14 @@ export class EtlService {
         const newUsers = new Set(selectedRows
             .map((row) => this.userId(row.record))
             .filter((value) => Boolean(value))).size;
+        const trialCancellations = selectedRows.filter((row) => this.eventType(row.record) === "cancellation").length;
         const mrr = Number(selectedProceeds.toFixed(2));
         const arr = Number((mrr * 12).toFixed(2));
         const arpu = newUsers > 0 ? Number((selectedProceeds / newUsers).toFixed(2)) : null;
         const arppu = paidUsersInRange > 0 ? Number((selectedProceeds / paidUsersInRange).toFixed(2)) : null;
         const refundRate = selectedSales > 0 ? Number((Math.abs(selectedRefunds) / selectedSales).toFixed(4)) : 0;
-        const trialCancellations = selectedRows.filter((row) => this.eventType(row.record) === "cancellation").length;
         return {
-            source: "apphud_local_etl",
+            source: "apphud_storage_etl",
             app_id: appId,
             apphud_app_id: appId,
             where: {
@@ -637,8 +608,8 @@ export class EtlService {
                         refunds: Math.abs(selectedRefunds),
                         refund_rate: refundRate,
                         failed_charges: failedCharges,
-                        arpu: arpu,
-                        arppu: arppu,
+                        arpu,
+                        arppu,
                         prevented_refund_requests: "N/A",
                     },
                     events: {
@@ -659,55 +630,50 @@ export class EtlService {
                 rows_in_selected_range: selectedRows.length,
                 rows_total_local: allRows.length,
             },
-            warnings: [
-                "Local dashboard metrics are computed from ingested ETL rows and may differ from Apphud dashboard exact formulas.",
-            ],
+            warnings: ["Dashboard metrics are computed from storage exports and may differ from Apphud dashboard exact formulas."],
             retrieved_at: nowIso,
             raw_payload: input.include_raw ? selectedRows.slice(0, 1000).map((row) => row.record) : undefined,
         };
     }
-    initializeSchema() {
-        this.db.exec(`
-      CREATE TABLE IF NOT EXISTS etl_app_api_keys (
-        tenant_id TEXT NOT NULL,
-        app_id TEXT NOT NULL,
-        app_name TEXT NOT NULL,
-        platform TEXT,
-        api_key TEXT NOT NULL,
-        key_source TEXT NOT NULL,
-        updated_at TEXT NOT NULL,
-        PRIMARY KEY (tenant_id, app_id)
-      );
-
-      CREATE TABLE IF NOT EXISTS etl_manifest (
-        tenant_id TEXT NOT NULL,
-        app_id TEXT,
-        source_file TEXT NOT NULL,
-        checksum TEXT NOT NULL,
-        status TEXT NOT NULL,
-        row_count INTEGER NOT NULL DEFAULT 0,
-        processed_at TEXT,
-        error_message TEXT,
-        PRIMARY KEY (tenant_id, source_file)
-      );
-
-      CREATE TABLE IF NOT EXISTS etl_raw_rows (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        tenant_id TEXT NOT NULL,
-        app_id TEXT,
-        source_file TEXT NOT NULL,
-        row_index INTEGER NOT NULL,
-        raw_json TEXT NOT NULL,
-        ingested_at TEXT NOT NULL
-      );
-
-      CREATE TABLE IF NOT EXISTS etl_alerts_todo (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        code TEXT NOT NULL,
-        message TEXT NOT NULL,
-        created_at TEXT NOT NULL
-      );
-    `);
+    async ensureStateLoaded() {
+        if (this.stateLoaded) {
+            return;
+        }
+        this.appKeys = await this.readJsonFile(this.appKeysPath(), []);
+        const manifestRows = await this.readJsonFile(this.manifestPath(), []);
+        this.manifest = new Map(manifestRows.map((row) => [row.source_file, row]));
+        this.alerts = await this.readJsonFile(this.alertsPath(), []);
+        this.stateLoaded = true;
+    }
+    async persistState() {
+        await this.writeJsonFile(this.appKeysPath(), this.appKeys);
+        await this.writeJsonFile(this.manifestPath(), Array.from(this.manifest.values()));
+        await this.writeJsonFile(this.alertsPath(), this.alerts.slice(-1000));
+    }
+    stateDir() {
+        return path.join(path.resolve(this.config.etlStorageDir), "state");
+    }
+    appKeysPath() {
+        return path.join(this.stateDir(), "app_api_keys.json");
+    }
+    manifestPath() {
+        return path.join(this.stateDir(), "manifest.json");
+    }
+    alertsPath() {
+        return path.join(this.stateDir(), "alerts_todo.json");
+    }
+    async readJsonFile(filePath, fallback) {
+        try {
+            const raw = await readFile(filePath, "utf8");
+            return JSON.parse(raw);
+        }
+        catch {
+            return fallback;
+        }
+    }
+    async writeJsonFile(filePath, payload) {
+        await mkdir(path.dirname(filePath), { recursive: true });
+        await writeFile(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
     }
     metricCatalog() {
         return [
@@ -731,68 +697,57 @@ export class EtlService {
         if (!this.config.etlEnabled) {
             return;
         }
-        const keysCount = this.db
-            .prepare(`SELECT COUNT(*) AS total FROM etl_app_api_keys WHERE tenant_id = ?`)
-            .get(this.config.etlTenantId);
-        const doneCount = this.db
-            .prepare(`SELECT COUNT(*) AS total FROM etl_manifest WHERE tenant_id = ? AND status = 'done'`)
-            .get(this.config.etlTenantId);
-        const needsBootstrap = keysCount.total === 0 || (requireRows && doneCount.total === 0);
-        if (!needsBootstrap) {
-            return;
+        await this.ensureStateLoaded();
+        const keysCount = this.appKeys.filter((key) => key.tenant_id === this.config.etlTenantId).length;
+        const doneCount = Array.from(this.manifest.values()).filter((entry) => entry.tenant_id === this.config.etlTenantId && entry.status === "done").length;
+        if (keysCount === 0 || (requireRows && doneCount === 0)) {
+            await this.runOnce();
         }
-        await this.runOnce();
     }
     resolveLocalAppId(appId, apphudAppId, allowEmpty = false) {
         const explicit = appId?.trim() || apphudAppId?.trim();
         if (explicit) {
             return explicit;
         }
-        const fromKeys = this.db
-            .prepare(`SELECT app_id FROM etl_app_api_keys WHERE tenant_id = ? ORDER BY updated_at DESC LIMIT 1`)
-            .get(this.config.etlTenantId);
-        if (fromKeys?.app_id) {
-            return fromKeys.app_id;
+        const latestKey = this.appKeys
+            .filter((key) => key.tenant_id === this.config.etlTenantId)
+            .sort((a, b) => b.updated_at.localeCompare(a.updated_at))[0];
+        if (latestKey?.app_id) {
+            return latestKey.app_id;
         }
-        const fromRows = this.db
-            .prepare(`SELECT app_id FROM etl_raw_rows WHERE tenant_id = ? AND app_id IS NOT NULL ORDER BY id DESC LIMIT 1`)
-            .get(this.config.etlTenantId);
-        if (fromRows?.app_id) {
-            return fromRows.app_id;
+        const latestManifest = Array.from(this.manifest.values())
+            .filter((entry) => entry.tenant_id === this.config.etlTenantId && entry.app_id)
+            .sort((a, b) => (b.processed_at ?? "").localeCompare(a.processed_at ?? ""))[0];
+        if (latestManifest?.app_id) {
+            return latestManifest.app_id;
         }
         if (allowEmpty) {
             return "unknown_app";
         }
-        throw new ApphudMcpError("APP_NOT_FOUND", "No local app data found", {
+        throw new ApphudMcpError("APP_NOT_FOUND", "No storage data found", {
             statusCode: 404,
-            actionHint: "Enable etl.enabled=true and etl.remote_fetch_enabled=true for automatic historical import, or place .csv.gz files into etl.incoming_dir.",
+            actionHint: "Set etl.enabled=true and etl.source=gcs|s3 with bucket settings, or place .csv.gz files into etl.incoming_dir.",
         });
     }
-    loadRows(appId, from, to, filters) {
-        const where = ["tenant_id = ?"];
-        const params = [this.config.etlTenantId];
-        if (appId) {
-            where.push("app_id = ?");
-            params.push(appId);
-        }
-        if (from) {
-            where.push("ingested_at >= ?");
-            params.push(from);
-        }
-        if (to) {
-            where.push("ingested_at <= ?");
-            params.push(to);
-        }
-        const whereSql = where.join(" AND ");
-        const count = this.db
-            .prepare(`SELECT COUNT(*) AS total FROM etl_raw_rows WHERE ${whereSql}`)
-            .get(...params);
-        if (count.total > LOCAL_ROWS_GUARD_LIMIT) {
-            throw new ApphudMcpError("INVALID_PAYLOAD", "Local dataset for selected range is too large", {
+    async loadRows(appId, from, to, filters) {
+        const fromMs = from ? new Date(from).getTime() : Number.NEGATIVE_INFINITY;
+        const toMs = to ? new Date(to).getTime() : Number.POSITIVE_INFINITY;
+        const candidates = Array.from(this.manifest.values()).filter((entry) => {
+            if (entry.tenant_id !== this.config.etlTenantId || entry.status !== "done") {
+                return false;
+            }
+            if (appId && entry.app_id !== appId) {
+                return false;
+            }
+            return true;
+        });
+        const totalRowsHint = candidates.reduce((sum, entry) => sum + entry.row_count, 0);
+        if (totalRowsHint > LOCAL_ROWS_GUARD_LIMIT) {
+            throw new ApphudMcpError("INVALID_PAYLOAD", "Storage dataset for selected range is too large", {
                 statusCode: 400,
-                actionHint: "Narrow date range (from/to) or add stricter filters. Large local queries should be requested in smaller windows.",
+                actionHint: "Narrow date range (from/to) or add stricter filters.",
                 details: {
-                    rows: count.total,
+                    rows_hint: totalRowsHint,
                     limit: LOCAL_ROWS_GUARD_LIMIT,
                     from,
                     to,
@@ -800,31 +755,31 @@ export class EtlService {
                 },
             });
         }
-        const records = this.db
-            .prepare(`SELECT raw_json, ingested_at, app_id FROM etl_raw_rows WHERE ${whereSql} ORDER BY id DESC`)
-            .all(...params);
-        return records
-            .map((row) => {
-            const record = this.safeParseJson(row.raw_json);
-            const occurredAt = this.detectOccurredAt(record) ?? row.ingested_at;
-            return { occurredAt, record };
-        })
-            .filter((row) => this.passesFilters(row.record, filters));
-    }
-    safeParseJson(raw) {
-        try {
-            const parsed = JSON.parse(raw);
-            if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
-                return parsed;
+        const rows = [];
+        for (const entry of candidates) {
+            try {
+                const content = await readFile(entry.source_file);
+                const unzipped = gunzipSync(content).toString("utf8");
+                const parsed = parseCsv(unzipped);
+                for (const row of parsed.rows) {
+                    const record = rowToRecord(parsed.headers, row);
+                    const normalized = record;
+                    const occurredAt = this.detectOccurredAt(normalized) ?? entry.processed_at ?? new Date().toISOString();
+                    const ts = new Date(occurredAt).getTime();
+                    if (Number.isFinite(ts) && (ts < fromMs || ts > toMs)) {
+                        continue;
+                    }
+                    if (!this.passesFilters(normalized, filters)) {
+                        continue;
+                    }
+                    rows.push({ occurredAt, record: normalized });
+                }
+            }
+            catch {
+                continue;
             }
-            return {};
-        }
-        catch {
-            return {};
         }
-    }
-    detectOccurredAt(record) {
-        return this.readString(record, ["occurred_at", "timestamp", "time", "date", "event_date", "created_at", "purchased_at"]);
+        return rows.sort((a, b) => b.occurredAt.localeCompare(a.occurredAt));
     }
     readString(record, keys) {
         for (const key of keys) {
@@ -853,6 +808,9 @@ export class EtlService {
         }
         return null;
     }
+    detectOccurredAt(record) {
+        return this.readString(record, ["occurred_at", "timestamp", "time", "date", "event_date", "created_at", "purchased_at"]);
+    }
     eventType(record) {
         return (this.readString(record, ["event_type", "event", "type", "name"]) ?? "unknown").toLowerCase();
     }
@@ -870,8 +828,7 @@ export class EtlService {
         const distinctUsers = (types) => {
             const set = new Set();
             for (const row of rows) {
-                const type = this.eventType(row.record);
-                if (!types.includes(type)) {
+                if (!types.includes(this.eventType(row.record))) {
                     continue;
                 }
                 const uid = this.userId(row.record);
@@ -881,27 +838,20 @@ export class EtlService {
             }
             return set.size;
         };
-        if (lower === "events_count") {
+        if (lower === "events_count")
             return rows.length;
-        }
-        if (lower === "trials_started") {
+        if (lower === "trials_started")
             return countByType(["trial_started"]);
-        }
-        if (lower === "trials_converted") {
+        if (lower === "trials_converted")
             return countByType(["trial_converted"]);
-        }
-        if (lower === "new_subscriptions") {
+        if (lower === "new_subscriptions")
             return countByType(["subscription_started"]);
-        }
-        if (lower === "renewals") {
+        if (lower === "renewals")
             return countByType(["renewal"]);
-        }
-        if (lower === "cancellations") {
+        if (lower === "cancellations")
             return countByType(["cancellation", "expiration"]);
-        }
-        if (lower === "active_trials") {
+        if (lower === "active_trials")
             return distinctUsers(["trial_started"]);
-        }
         if (lower === "active_subs" || lower === "subscribers_retention") {
             return distinctUsers(["subscription_started", "renewal", "trial_converted"]);
         }
@@ -924,12 +874,10 @@ export class EtlService {
     }
     dimensionValue(record, dimension) {
         const normalized = dimension.toLowerCase();
-        if (normalized === "country") {
+        if (normalized === "country")
             return this.readString(record, ["country", "country_code"]) ?? "unknown";
-        }
-        if (normalized === "platform") {
+        if (normalized === "platform")
             return this.readString(record, ["platform", "store", "os"]) ?? "unknown";
-        }
         if (normalized === "product" || normalized === "product_id") {
             return this.readString(record, ["product_id", "sku", "subscription_product_id"]) ?? "unknown";
         }
@@ -948,9 +896,8 @@ export class EtlService {
                 if (String(actual ?? "").toLowerCase() !== rawValue.toLowerCase()) {
                     return false;
                 }
-                continue;
             }
-            if (typeof rawValue === "number" || typeof rawValue === "boolean") {
+            else if (typeof rawValue === "number" || typeof rawValue === "boolean") {
                 if (actual !== rawValue) {
                     return false;
                 }
@@ -973,162 +920,124 @@ export class EtlService {
     async syncAppApiKeys() {
         const listed = await this.apphudClient.listDashboardAppApiKeys();
         const now = new Date().toISOString();
-        const upsert = this.db.prepare(`
-      INSERT INTO etl_app_api_keys (tenant_id, app_id, app_name, platform, api_key, key_source, updated_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?)
-      ON CONFLICT(tenant_id, app_id) DO UPDATE SET
-        app_name = excluded.app_name,
-        platform = excluded.platform,
-        api_key = excluded.api_key,
-        key_source = excluded.key_source,
-        updated_at = excluded.updated_at
-    `);
-        const tx = this.db.transaction((apps) => {
-            for (const app of apps) {
-                upsert.run(this.config.etlTenantId, app.app_id, app.app_name, app.platform ?? null, app.api_key, app.source, now);
-            }
-        });
-        tx(listed.apps);
-        return listed.apps;
-    }
-    async fetchRemoteExports(apiKeys) {
-        for (const app of apiKeys) {
-            const listPath = interpolatePath(this.config.apphudEtlExportsListPath, {
+        const map = new Map(this.appKeys.map((item) => [`${item.tenant_id}:${item.app_id}`, item]));
+        for (const app of listed.apps) {
+            map.set(`${this.config.etlTenantId}:${app.app_id}`, {
+                tenant_id: this.config.etlTenantId,
                 app_id: app.app_id,
+                app_name: app.app_name,
+                platform: app.platform,
+                api_key: app.api_key,
+                key_source: app.source,
+                updated_at: now,
             });
-            const listUrl = normalizeToPath(this.config.apphudEtlExportsApiBaseUrl, listPath);
-            try {
-                const response = await fetch(listUrl, {
-                    method: "GET",
-                    headers: {
-                        Authorization: `Bearer ${app.api_key}`,
-                        Accept: "application/json",
-                    },
-                    signal: AbortSignal.timeout(20_000),
-                });
-                if (!response.ok) {
-                    this.insertAlertTodo("ETL_EXPORT_LIST_FAILED", `app=${app.app_id} status=${response.status}`);
-                    continue;
-                }
-                const payload = (await response.json());
-                const exportsList = extractExportCandidates(payload, app.app_id);
-                for (const item of exportsList) {
-                    await this.downloadExport(item, app.api_key);
-                }
-            }
-            catch (error) {
-                this.insertAlertTodo("ETL_EXPORT_LIST_FAILED", `app=${app.app_id} ${error instanceof Error ? error.message : String(error)}`);
-            }
         }
+        this.appKeys = Array.from(map.values());
     }
-    async downloadExport(item, apiKey) {
-        const now = new Date();
-        const destinationDir = path.resolve(this.config.etlIncomingDir, item.appId, String(now.getUTCFullYear()), String(now.getUTCMonth() + 1).padStart(2, "0"), String(now.getUTCDate()).padStart(2, "0"));
-        await mkdir(destinationDir, { recursive: true });
-        const targetPath = path.join(destinationDir, item.fileName);
-        try {
-            const existing = await stat(targetPath).catch(() => null);
-            if (existing && existing.isFile()) {
-                return;
-            }
-            const response = await fetch(normalizeToPath(this.config.apphudEtlExportsApiBaseUrl, item.downloadUrl), {
-                method: "GET",
-                headers: {
-                    Authorization: `Bearer ${apiKey}`,
-                    Accept: "application/octet-stream,application/gzip,application/json",
-                },
-                signal: AbortSignal.timeout(60_000),
-            });
-            if (!response.ok) {
-                this.insertAlertTodo("ETL_EXPORT_DOWNLOAD_FAILED", `file=${item.fileName} status=${response.status}`);
-                return;
-            }
-            const contentType = response.headers.get("content-type") ?? "";
-            if (contentType.includes("application/json")) {
-                const payload = (await response.json());
-                const nestedUrl = (typeof payload.download_url === "string" && payload.download_url) ||
-                    (typeof payload.url === "string" && payload.url);
-                if (!nestedUrl) {
-                    this.insertAlertTodo("ETL_EXPORT_DOWNLOAD_FAILED", `file=${item.fileName} missing nested download url`);
-                    return;
-                }
-                await this.downloadExport({
-                    ...item,
-                    downloadUrl: nestedUrl,
-                }, apiKey);
-                return;
-            }
-            const bytes = Buffer.from(await response.arrayBuffer());
-            await writeFile(targetPath, bytes);
+    async syncFromStorage() {
+        if (this.config.etlSource === "none") {
+            return;
         }
-        catch (error) {
-            this.insertAlertTodo("ETL_EXPORT_DOWNLOAD_FAILED", `file=${item.fileName} ${error instanceof Error ? error.message : String(error)}`);
+        if (this.config.etlSource === "gcs") {
+            await this.syncFromGcs();
+            return;
+        }
+        if (this.config.etlSource === "s3") {
+            await this.syncFromS3();
+            return;
         }
     }
-    async ingestIncomingFiles() {
-        const incomingRoot = path.resolve(this.config.etlIncomingDir);
-        await mkdir(incomingRoot, { recursive: true });
-        const files = await listCsvGzFiles(incomingRoot);
-        for (const filePath of files) {
-            await this.ingestCsvGzFile(filePath);
+    async syncFromGcs() {
+        const bucket = this.config.etlGcsBucket;
+        if (!bucket) {
+            this.insertAlertTodo("ETL_GCS_CONFIG_MISSING", "Missing ETL_GCS_BUCKET for etl.source=gcs");
+            return;
+        }
+        const sourceUri = `gs://${bucket}${this.normalizeStoragePrefix(this.config.etlGcsPrefix)}`;
+        const destination = path.resolve(this.config.etlIncomingDir);
+        try {
+            await execFileAsync("gsutil", ["-m", "rsync", "-r", sourceUri, destination], { timeout: 10 * 60 * 1000 });
+        }
+        catch (error) {
+            this.insertAlertTodo("ETL_GCS_SYNC_FAILED", this.stringifyExecError(error, "gsutil"));
         }
     }
-    async ingestCsvGzFile(filePath) {
-        const absolutePath = path.resolve(filePath);
-        const tenantId = this.config.etlTenantId;
-        const appId = this.extractAppIdFromPath(absolutePath);
-        const content = await readFile(absolutePath);
-        const checksum = createHash("sha256").update(content).digest("hex");
-        const now = new Date().toISOString();
-        const existing = this.db
-            .prepare(`SELECT checksum, status FROM etl_manifest WHERE tenant_id = ? AND source_file = ?`)
-            .get(tenantId, absolutePath);
-        if (existing?.checksum === checksum && existing.status === "done") {
+    async syncFromS3() {
+        const bucket = this.config.etlS3Bucket;
+        if (!bucket) {
+            this.insertAlertTodo("ETL_S3_CONFIG_MISSING", "Missing ETL_S3_BUCKET for etl.source=s3");
             return;
         }
-        const markFailed = this.db.prepare(`
-      INSERT INTO etl_manifest (tenant_id, app_id, source_file, checksum, status, row_count, processed_at, error_message)
-      VALUES (?, ?, ?, ?, 'failed', 0, ?, ?)
-      ON CONFLICT(tenant_id, source_file) DO UPDATE SET
-        checksum = excluded.checksum,
-        app_id = excluded.app_id,
-        status = 'failed',
-        row_count = 0,
-        processed_at = excluded.processed_at,
-        error_message = excluded.error_message
-    `);
+        const sourceUri = `s3://${bucket}${this.normalizeStoragePrefix(this.config.etlS3Prefix)}`;
+        const destination = path.resolve(this.config.etlIncomingDir);
         try {
-            const unzipped = gunzipSync(content).toString("utf8");
-            const parsed = parseCsv(unzipped);
-            const deleteExistingRows = this.db.prepare(`DELETE FROM etl_raw_rows WHERE tenant_id = ? AND source_file = ?`);
-            const insertRaw = this.db.prepare(`
-        INSERT INTO etl_raw_rows (tenant_id, app_id, source_file, row_index, raw_json, ingested_at)
-        VALUES (?, ?, ?, ?, ?, ?)
-      `);
-            const upsertManifest = this.db.prepare(`
-        INSERT INTO etl_manifest (tenant_id, app_id, source_file, checksum, status, row_count, processed_at, error_message)
-        VALUES (?, ?, ?, ?, 'done', ?, ?, NULL)
-        ON CONFLICT(tenant_id, source_file) DO UPDATE SET
-          checksum = excluded.checksum,
-          app_id = excluded.app_id,
-          status = 'done',
-          row_count = excluded.row_count,
-          processed_at = excluded.processed_at,
-          error_message = NULL
-      `);
-            const tx = this.db.transaction(() => {
-                deleteExistingRows.run(tenantId, absolutePath);
-                parsed.rows.forEach((row, index) => {
-                    const mapped = rowToRecord(parsed.headers, row);
-                    insertRaw.run(tenantId, appId, absolutePath, index + 1, JSON.stringify(mapped), now);
-                });
-                upsertManifest.run(tenantId, appId, absolutePath, checksum, parsed.rows.length, now);
+            await execFileAsync("aws", ["s3", "sync", sourceUri, destination, "--no-progress"], {
+                timeout: 10 * 60 * 1000,
             });
-            tx();
         }
         catch (error) {
-            markFailed.run(tenantId, appId, absolutePath, checksum, now, error instanceof Error ? error.message : String(error));
-            this.insertAlertTodo("ETL_INGEST_FAILED", `file=${path.basename(absolutePath)} ${error instanceof Error ? error.message : String(error)}`);
+            this.insertAlertTodo("ETL_S3_SYNC_FAILED", this.stringifyExecError(error, "aws"));
+        }
+    }
+    normalizeStoragePrefix(prefix) {
+        if (!prefix || prefix.trim().length === 0) {
+            return "";
+        }
+        const trimmed = prefix.trim().replace(/^\/+/, "").replace(/\/+$/, "");
+        return trimmed.length > 0 ? `/${trimmed}` : "";
+    }
+    stringifyExecError(error, command) {
+        const err = error;
+        if (err?.code === "ENOENT") {
+            return `${command} command not found. Install ${command} CLI and configure credentials.`;
+        }
+        const stderr = typeof err?.stderr === "string" ? err.stderr.trim() : "";
+        const message = typeof err?.message === "string" ? err.message : "unknown error";
+        return `${command} sync failed: ${stderr || message}`.slice(0, 800);
+    }
+    async refreshManifestFromIncoming() {
+        const incomingRoot = path.resolve(this.config.etlIncomingDir);
+        await mkdir(incomingRoot, { recursive: true });
+        const files = await listCsvGzFiles(incomingRoot);
+        for (const filePath of files) {
+            const absolutePath = path.resolve(filePath);
+            const tenantId = this.config.etlTenantId;
+            const appId = this.extractAppIdFromPath(absolutePath);
+            const now = new Date().toISOString();
+            try {
+                const content = await readFile(absolutePath);
+                const checksum = createHash("sha256").update(content).digest("hex");
+                const existing = this.manifest.get(absolutePath);
+                if (existing?.checksum === checksum && existing.status === "done") {
+                    continue;
+                }
+                const unzipped = gunzipSync(content).toString("utf8");
+                const parsed = parseCsv(unzipped);
+                this.manifest.set(absolutePath, {
+                    tenant_id: tenantId,
+                    app_id: appId,
+                    source_file: absolutePath,
+                    checksum,
+                    status: "done",
+                    row_count: parsed.rows.length,
+                    processed_at: now,
+                    error_message: null,
+                });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                this.manifest.set(absolutePath, {
+                    tenant_id: tenantId,
+                    app_id: appId,
+                    source_file: absolutePath,
+                    checksum: "",
+                    status: "failed",
+                    row_count: 0,
+                    processed_at: now,
+                    error_message: message,
+                });
+                this.insertAlertTodo("ETL_INGEST_FAILED", `file=${path.basename(absolutePath)} ${message}`);
+            }
         }
     }
     extractAppIdFromPath(filePath) {
@@ -1137,10 +1046,10 @@ export class EtlService {
         return parts.length > 0 ? parts[0] ?? null : null;
     }
     applyAlertTodos() {
-        const latestDone = this.db
-            .prepare(`SELECT MAX(processed_at) AS processed_at FROM etl_manifest WHERE tenant_id = ? AND status = 'done'`)
-            .get(this.config.etlTenantId);
-        if (!latestDone.processed_at) {
+        const latestDone = Array.from(this.manifest.values())
+            .filter((entry) => entry.tenant_id === this.config.etlTenantId && entry.status === "done" && entry.processed_at)
+            .sort((a, b) => (b.processed_at ?? "").localeCompare(a.processed_at ?? ""))[0];
+        if (!latestDone?.processed_at) {
             return;
         }
         const ageMs = Date.now() - new Date(latestDone.processed_at).getTime();
@@ -1150,8 +1059,14 @@ export class EtlService {
         }
     }
     insertAlertTodo(code, message) {
-        this.db
-            .prepare(`INSERT INTO etl_alerts_todo (code, message, created_at) VALUES (?, ?, ?)`)
-            .run(code, message, new Date().toISOString());
+        const now = new Date().toISOString();
+        const lastSame = this.alerts[this.alerts.length - 1];
+        if (lastSame && lastSame.code === code && lastSame.message === message) {
+            return;
+        }
+        this.alerts.push({ code, message, created_at: now });
+        if (this.alerts.length > 1000) {
+            this.alerts = this.alerts.slice(this.alerts.length - 1000);
+        }
     }
 }