@agent-native/core 0.122.1 → 0.122.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -13
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +12 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/production-agent.ts +65 -1
- package/corpus/core/src/agent/run-loop-with-resume.ts +12 -0
- package/corpus/core/src/agent/run-store.ts +61 -1
- package/corpus/core/src/integrations/a2a-continuation-processor.ts +182 -0
- package/corpus/core/src/integrations/a2a-continuations-store.ts +122 -12
- package/corpus/core/src/integrations/integration-campaign-recovery.ts +127 -0
- package/corpus/core/src/integrations/integration-campaigns-store.ts +1039 -0
- package/corpus/core/src/integrations/integration-durable-dispatch.ts +28 -3
- package/corpus/core/src/integrations/pending-tasks-retry-job.ts +8 -0
- package/corpus/core/src/integrations/plugin.ts +453 -35
- package/corpus/core/src/integrations/webhook-handler.ts +469 -11
- package/corpus/core/src/server/builder-design-systems.ts +1 -1
- package/corpus/templates/analytics/.agents/skills/dashboard-management/SKILL.md +15 -0
- package/corpus/templates/analytics/app/pages/adhoc/sql-dashboard/report-panel-window.ts +3 -3
- package/corpus/templates/analytics/changelog/2026-07-25-analytics-dashboards-and-scheduled-reports-are-faster-and-mo.md +6 -0
- package/corpus/templates/analytics/server/db/schema.ts +12 -0
- package/corpus/templates/analytics/server/lib/dashboard-panel-query.ts +8 -4
- package/corpus/templates/analytics/server/lib/dashboard-report.ts +136 -5
- package/corpus/templates/analytics/server/lib/dashboard-time-scope.ts +250 -1
- package/corpus/templates/analytics/server/lib/first-party-analytics-cache.ts +167 -0
- package/corpus/templates/analytics/server/lib/first-party-analytics.ts +34 -9
- package/corpus/templates/analytics/server/lib/first-party-dashboard-repair.ts +105 -23
- package/corpus/templates/analytics/server/lib/first-party-unbounded-panel-repair.ts +169 -0
- package/corpus/templates/analytics/server/plugins/db.ts +41 -1
- package/corpus/templates/analytics/shared/dashboard-report-timeouts.ts +6 -3
- package/corpus/templates/content/app/components/editor/DocumentEditor.tsx +7 -3
- package/corpus/templates/content/app/components/editor/body-hydration.ts +10 -0
- package/corpus/templates/content/app/components/editor/database/DatabaseView.tsx +1 -0
- package/corpus/templates/content/app/root.tsx +12 -1
- package/corpus/templates/content/changelog/2026-07-23-new-pages-open-immediately-while-slow-network-work-continues.md +6 -0
- package/corpus/templates/content/changelog/2026-07-25-database-preview-actions-open-reliably.md +6 -0
- package/corpus/templates/content/e2e/playwright.config.ts +1 -1
- package/corpus/templates/design/actions/edit-design.ts +66 -27
- package/corpus/templates/design/changelog/2026-07-25-design-edits-retry-concurrent-changes.md +6 -0
- package/corpus/templates/design/server/source-workspace.ts +5 -1
- package/corpus/templates/forms/.agents/skills/form-responses/SKILL.md +6 -4
- package/corpus/templates/forms/actions/create-form.ts +5 -1
- package/corpus/templates/forms/actions/export-responses.ts +22 -11
- package/corpus/templates/forms/actions/update-form.ts +5 -1
- package/corpus/templates/forms/changelog/2026-07-25-fields-can-be-created-without-ids.md +6 -0
- package/corpus/templates/forms/changelog/2026-07-25-response-exports-use-file-storage.md +6 -0
- package/corpus/templates/forms/server/lib/validate-fields.ts +43 -0
- package/corpus/templates/slides/actions/extract-pdf.ts +4 -4
- package/corpus/templates/slides/actions/import-file.ts +3 -4
- package/corpus/templates/slides/changelog/2026-07-25-gemini-image-models-updated.md +6 -0
- package/corpus/templates/slides/changelog/2026-07-25-pdf-imports-tolerate-missing-canvas.md +6 -0
- package/corpus/templates/slides/server/handlers/image-providers/gemini.ts +5 -2
- package/corpus/templates/slides/server/lib/pdf-parse-setup.ts +150 -0
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +63 -1
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/run-loop-with-resume.d.ts +7 -1
- package/dist/agent/run-loop-with-resume.d.ts.map +1 -1
- package/dist/agent/run-loop-with-resume.js +7 -0
- package/dist/agent/run-loop-with-resume.js.map +1 -1
- package/dist/agent/run-store.d.ts.map +1 -1
- package/dist/agent/run-store.js +49 -0
- package/dist/agent/run-store.js.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/file-upload/actions/upload-image.d.ts +2 -2
- package/dist/integrations/a2a-continuation-processor.d.ts +14 -0
- package/dist/integrations/a2a-continuation-processor.d.ts.map +1 -1
- package/dist/integrations/a2a-continuation-processor.js +134 -1
- package/dist/integrations/a2a-continuation-processor.js.map +1 -1
- package/dist/integrations/a2a-continuations-store.d.ts +23 -0
- package/dist/integrations/a2a-continuations-store.d.ts.map +1 -1
- package/dist/integrations/a2a-continuations-store.js +99 -13
- package/dist/integrations/a2a-continuations-store.js.map +1 -1
- package/dist/integrations/integration-campaign-recovery.d.ts +16 -0
- package/dist/integrations/integration-campaign-recovery.d.ts.map +1 -0
- package/dist/integrations/integration-campaign-recovery.js +94 -0
- package/dist/integrations/integration-campaign-recovery.js.map +1 -0
- package/dist/integrations/integration-campaigns-store.d.ts +121 -0
- package/dist/integrations/integration-campaigns-store.d.ts.map +1 -0
- package/dist/integrations/integration-campaigns-store.js +784 -0
- package/dist/integrations/integration-campaigns-store.js.map +1 -0
- package/dist/integrations/integration-durable-dispatch.d.ts +3 -0
- package/dist/integrations/integration-durable-dispatch.d.ts.map +1 -1
- package/dist/integrations/integration-durable-dispatch.js +22 -3
- package/dist/integrations/integration-durable-dispatch.js.map +1 -1
- package/dist/integrations/pending-tasks-retry-job.d.ts.map +1 -1
- package/dist/integrations/pending-tasks-retry-job.js +6 -0
- package/dist/integrations/pending-tasks-retry-job.js.map +1 -1
- package/dist/integrations/plugin.d.ts.map +1 -1
- package/dist/integrations/plugin.js +325 -18
- package/dist/integrations/plugin.js.map +1 -1
- package/dist/integrations/webhook-handler.d.ts +16 -1
- package/dist/integrations/webhook-handler.d.ts.map +1 -1
- package/dist/integrations/webhook-handler.js +368 -13
- package/dist/integrations/webhook-handler.js.map +1 -1
- package/dist/observability/routes.d.ts +1 -1
- package/dist/provider-api/actions/custom-provider-registration.d.ts +14 -14
- package/dist/provider-api/actions/provider-api.d.ts +8 -8
- package/dist/secrets/routes.d.ts +9 -9
- package/dist/server/builder-design-systems.js +1 -1
- package/dist/server/builder-design-systems.js.map +1 -1
- package/package.json +3 -3
- package/src/agent/production-agent.ts +65 -1
- package/src/agent/run-loop-with-resume.ts +12 -0
- package/src/agent/run-store.ts +61 -1
- package/src/integrations/a2a-continuation-processor.ts +182 -0
- package/src/integrations/a2a-continuations-store.ts +122 -12
- package/src/integrations/integration-campaign-recovery.ts +127 -0
- package/src/integrations/integration-campaigns-store.ts +1039 -0
- package/src/integrations/integration-durable-dispatch.ts +28 -3
- package/src/integrations/pending-tasks-retry-job.ts +8 -0
- package/src/integrations/plugin.ts +453 -35
- package/src/integrations/webhook-handler.ts +469 -11
- package/src/server/builder-design-systems.ts +1 -1
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { MAX_CONCURRENT_SQL_QUERIES } from "@shared/sql-query-limits";
|
|
2
|
-
|
|
3
1
|
import type { SqlPanel } from "./types";
|
|
4
2
|
|
|
5
|
-
|
|
3
|
+
// Keep each report window within the browser-side SQL query concurrency limit
|
|
4
|
+
// so a cold window does not serialize into multiple waves before readiness.
|
|
5
|
+
export const REPORT_PANEL_CHUNK_SIZE = 4;
|
|
6
6
|
|
|
7
7
|
export function listReportablePanelIds(panels: SqlPanel[]): string[] {
|
|
8
8
|
return panels
|
|
@@ -193,6 +193,18 @@ export const bigqueryCache = table("bigquery_cache", {
|
|
|
193
193
|
expiresAt: text("expires_at").notNull(),
|
|
194
194
|
});
|
|
195
195
|
|
|
196
|
+
/**
|
|
197
|
+
* First-party dashboard panel result cache — see
|
|
198
|
+
* server/lib/first-party-analytics-cache.ts.
|
|
199
|
+
*/
|
|
200
|
+
export const firstPartyAnalyticsCache = table("first_party_analytics_cache", {
|
|
201
|
+
key: text("key").primaryKey(),
|
|
202
|
+
sql: text("sql").notNull(),
|
|
203
|
+
result: text("result").notNull(),
|
|
204
|
+
createdAt: text("created_at").notNull(),
|
|
205
|
+
expiresAt: text("expires_at").notNull(),
|
|
206
|
+
});
|
|
207
|
+
|
|
196
208
|
/**
|
|
197
209
|
* Public write keys for the first-party analytics ingestion endpoint.
|
|
198
210
|
* The key is intentionally public/write-only: it can create events for the
|
|
@@ -446,10 +446,14 @@ export async function runDashboardPanelQuery(args: {
|
|
|
446
446
|
}
|
|
447
447
|
|
|
448
448
|
if (source === "first-party") {
|
|
449
|
-
return await queryFirstPartyAnalytics(
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
449
|
+
return await queryFirstPartyAnalytics(
|
|
450
|
+
query,
|
|
451
|
+
{
|
|
452
|
+
userEmail: ctx.userEmail,
|
|
453
|
+
orgId: ctx.orgId ?? null,
|
|
454
|
+
},
|
|
455
|
+
{ cache: true },
|
|
456
|
+
);
|
|
453
457
|
}
|
|
454
458
|
|
|
455
459
|
if (source === "demo") {
|
|
@@ -15,6 +15,8 @@ import {
|
|
|
15
15
|
EMBED_TOKEN_QUERY_PARAM,
|
|
16
16
|
} from "@agent-native/core/shared";
|
|
17
17
|
|
|
18
|
+
import { interpolate } from "../../app/pages/adhoc/sql-dashboard/interpolate";
|
|
19
|
+
import { serializePanelSql } from "../../app/pages/adhoc/sql-dashboard/panel-sql";
|
|
18
20
|
import {
|
|
19
21
|
listReportablePanelIds,
|
|
20
22
|
REPORT_PANEL_CHUNK_SIZE,
|
|
@@ -32,6 +34,7 @@ import {
|
|
|
32
34
|
type DashboardReportCaptureOutcome,
|
|
33
35
|
type DashboardReportSubscription,
|
|
34
36
|
} from "./dashboard-report-subscriptions";
|
|
37
|
+
import { queryFirstPartyAnalytics } from "./first-party-analytics.js";
|
|
35
38
|
|
|
36
39
|
type ReportSnapshot = {
|
|
37
40
|
dashboardId: string;
|
|
@@ -42,6 +45,8 @@ type ReportSnapshot = {
|
|
|
42
45
|
reportSettingsUrl: string;
|
|
43
46
|
generatedAt: string;
|
|
44
47
|
panelIds: string[];
|
|
48
|
+
panels: SqlDashboardConfig["panels"];
|
|
49
|
+
variables?: Record<string, string>;
|
|
45
50
|
};
|
|
46
51
|
|
|
47
52
|
const DATE_FILTER_TYPES: ReadonlySet<FilterType> = new Set([
|
|
@@ -58,7 +63,9 @@ const LOCAL_SCREENSHOT_TIMEOUT_MS = 90_000;
|
|
|
58
63
|
const SERVERLESS_SCREENSHOT_TIMEOUT_MS = 90_000;
|
|
59
64
|
const SERVERLESS_SECOND_READY_TIMEOUT_MS = 45_000;
|
|
60
65
|
// Cap browser work separately from the sweep-level delivery deadline.
|
|
61
|
-
|
|
66
|
+
// Reserve time under Netlify's 300s background-function limit for cleanup and
|
|
67
|
+
// email delivery (see MAX_CAPTURE_CLEANUP_RESERVE_MS / delivery reserves below).
|
|
68
|
+
const SERVERLESS_CHUNKED_ATTEMPT_TIMEOUT_MS = 210_000;
|
|
62
69
|
const BROWSER_CLEANUP_TIMEOUT_MS = 10_000;
|
|
63
70
|
const DASHBOARD_REPORT_EMAIL_TIMEOUT_MS = 10_000;
|
|
64
71
|
const DASHBOARD_REPORT_EMAIL_OVERHEAD_RESERVE_MS = 5_000;
|
|
@@ -226,6 +233,8 @@ async function collectReportSnapshot(
|
|
|
226
233
|
}),
|
|
227
234
|
generatedAt: new Date().toISOString(),
|
|
228
235
|
panelIds: listReportablePanelIds(config.panels),
|
|
236
|
+
panels: config.panels,
|
|
237
|
+
variables: config.variables,
|
|
229
238
|
};
|
|
230
239
|
}
|
|
231
240
|
|
|
@@ -375,6 +384,14 @@ function boundedStageTimeout(capMs: number, deadlineAt?: number): number {
|
|
|
375
384
|
return Math.max(1, Math.min(capMs, remainingMs));
|
|
376
385
|
}
|
|
377
386
|
|
|
387
|
+
function reportDeliveryReserveMs(recipientCount: number): number {
|
|
388
|
+
return (
|
|
389
|
+
MAX_CAPTURE_CLEANUP_RESERVE_MS +
|
|
390
|
+
recipientCount * DASHBOARD_REPORT_EMAIL_TIMEOUT_MS +
|
|
391
|
+
DASHBOARD_REPORT_EMAIL_OVERHEAD_RESERVE_MS
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
|
|
378
395
|
async function runWithinCaptureDeadline<T>(
|
|
379
396
|
operation: () => Promise<T>,
|
|
380
397
|
deadlineAt?: number,
|
|
@@ -448,7 +465,15 @@ async function waitForDashboardReportReady(
|
|
|
448
465
|
if (root.getAttribute("data-dashboard-report-ready") !== "true") {
|
|
449
466
|
return false;
|
|
450
467
|
}
|
|
451
|
-
|
|
468
|
+
if (root.querySelector("[data-dashboard-report-loading='true']")) {
|
|
469
|
+
return false;
|
|
470
|
+
}
|
|
471
|
+
// A panel that errored (including its own query-call timeout) clears
|
|
472
|
+
// its loading marker just like a successful panel, so without this
|
|
473
|
+
// check the capture would treat a broken panel as "ready" and bake
|
|
474
|
+
// its raw error text into the delivered screenshot instead of
|
|
475
|
+
// failing the attempt so the retry sweep can pick it up.
|
|
476
|
+
return !root.querySelector("[role='alert']");
|
|
452
477
|
})()`,
|
|
453
478
|
undefined,
|
|
454
479
|
{ timeout },
|
|
@@ -505,6 +530,101 @@ async function waitForDashboardReportReady(
|
|
|
505
530
|
}
|
|
506
531
|
}
|
|
507
532
|
|
|
533
|
+
const PREWARM_CONCURRENCY = 3;
|
|
534
|
+
const PREWARM_PANEL_TIMEOUT_MS = 20_000;
|
|
535
|
+
const PREWARM_MAX_BUDGET_MS = 45_000;
|
|
536
|
+
|
|
537
|
+
/**
|
|
538
|
+
* Runs first-party panel queries once, directly, before the browser capture
|
|
539
|
+
* starts, so the capture's own query-dashboard-panel calls hit a warm cache
|
|
540
|
+
* instead of several panels racing each other cold for database time inside
|
|
541
|
+
* a single capture window — that contention, not any one query alone, is
|
|
542
|
+
* what was pushing panels past their capture timeout. Best-effort: a panel
|
|
543
|
+
* that fails or times out here is simply computed cold by the capture later,
|
|
544
|
+
* exactly as it always was, so this can never make a report worse than
|
|
545
|
+
* before it existed. Bounded to time left after reserving the full chunked
|
|
546
|
+
* capture attempt and delivery tail, so prewarming cannot starve capture.
|
|
547
|
+
* The per-panel timeout only bounds how long a worker waits before logging;
|
|
548
|
+
* it is also passed to the database/cache layers, and the original query is
|
|
549
|
+
* still awaited before this function returns so it cannot overlap capture.
|
|
550
|
+
*/
|
|
551
|
+
async function prewarmFirstPartyPanelCache(
|
|
552
|
+
sub: DashboardReportSubscription,
|
|
553
|
+
snapshot: ReportSnapshot,
|
|
554
|
+
deadlineAt?: number,
|
|
555
|
+
recipientCount = 0,
|
|
556
|
+
): Promise<void> {
|
|
557
|
+
const panels = (snapshot.panels ?? []).filter(
|
|
558
|
+
(panel) => panel.source === "first-party" && panel.sql != null,
|
|
559
|
+
);
|
|
560
|
+
if (!panels.length) return;
|
|
561
|
+
|
|
562
|
+
const remaining = deadlineAt ? deadlineAt - Date.now() : Infinity;
|
|
563
|
+
const budgetMs = Math.max(
|
|
564
|
+
0,
|
|
565
|
+
Math.min(
|
|
566
|
+
PREWARM_MAX_BUDGET_MS,
|
|
567
|
+
deadlineAt
|
|
568
|
+
? remaining -
|
|
569
|
+
SERVERLESS_CHUNKED_ATTEMPT_TIMEOUT_MS -
|
|
570
|
+
reportDeliveryReserveMs(recipientCount)
|
|
571
|
+
: PREWARM_MAX_BUDGET_MS,
|
|
572
|
+
),
|
|
573
|
+
);
|
|
574
|
+
if (budgetMs <= 0) return;
|
|
575
|
+
const prewarmDeadline = Date.now() + budgetMs;
|
|
576
|
+
|
|
577
|
+
const vars: Record<string, string> = { ...snapshot.variables };
|
|
578
|
+
for (const [key, value] of Object.entries(snapshot.filters)) {
|
|
579
|
+
if (key.startsWith("f_")) vars[key.slice(2)] = value;
|
|
580
|
+
}
|
|
581
|
+
const scope = { userEmail: sub.ownerEmail, orgId: sub.orgId ?? null };
|
|
582
|
+
|
|
583
|
+
let index = 0;
|
|
584
|
+
async function worker(): Promise<void> {
|
|
585
|
+
while (index < panels.length && Date.now() < prewarmDeadline) {
|
|
586
|
+
const panel = panels[index++];
|
|
587
|
+
const sql = interpolate(serializePanelSql(panel.sql), vars, {
|
|
588
|
+
failClosedTimeVariables: true,
|
|
589
|
+
});
|
|
590
|
+
if (!sql.trim()) continue;
|
|
591
|
+
const panelTimeoutMs = Math.max(
|
|
592
|
+
1,
|
|
593
|
+
Math.min(PREWARM_PANEL_TIMEOUT_MS, prewarmDeadline - Date.now()),
|
|
594
|
+
);
|
|
595
|
+
const query = queryFirstPartyAnalytics(sql, scope, {
|
|
596
|
+
cache: true,
|
|
597
|
+
timeoutMs: panelTimeoutMs,
|
|
598
|
+
});
|
|
599
|
+
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
600
|
+
try {
|
|
601
|
+
await Promise.race([
|
|
602
|
+
query,
|
|
603
|
+
new Promise<never>((_, reject) => {
|
|
604
|
+
timeout = setTimeout(
|
|
605
|
+
() => reject(new Error("prewarm panel timeout")),
|
|
606
|
+
panelTimeoutMs,
|
|
607
|
+
);
|
|
608
|
+
timeout.unref?.();
|
|
609
|
+
}),
|
|
610
|
+
]);
|
|
611
|
+
} catch (err) {
|
|
612
|
+
console.warn(
|
|
613
|
+
`[dashboard-report] prewarm failed for panel ${panel.id}:`,
|
|
614
|
+
err instanceof Error ? err.message : err,
|
|
615
|
+
);
|
|
616
|
+
} finally {
|
|
617
|
+
if (timeout) clearTimeout(timeout);
|
|
618
|
+
await query.catch(() => undefined);
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
await Promise.all(
|
|
624
|
+
Array.from({ length: PREWARM_CONCURRENCY }, () => worker()),
|
|
625
|
+
);
|
|
626
|
+
}
|
|
627
|
+
|
|
508
628
|
async function assertDashboardReportPanelWindow(
|
|
509
629
|
page: any,
|
|
510
630
|
expectedPanelIds: string[],
|
|
@@ -1358,14 +1478,25 @@ export async function sendDashboardReportSubscription(
|
|
|
1358
1478
|
() => collectReportSnapshot(sub),
|
|
1359
1479
|
options.deadlineAt,
|
|
1360
1480
|
);
|
|
1481
|
+
try {
|
|
1482
|
+
await prewarmFirstPartyPanelCache(
|
|
1483
|
+
sub,
|
|
1484
|
+
snapshot,
|
|
1485
|
+
options.deadlineAt,
|
|
1486
|
+
recipients.length,
|
|
1487
|
+
);
|
|
1488
|
+
} catch (err) {
|
|
1489
|
+
console.warn(
|
|
1490
|
+
"[dashboard-report] panel cache prewarm failed (non-fatal):",
|
|
1491
|
+
err instanceof Error ? err.message : err,
|
|
1492
|
+
);
|
|
1493
|
+
}
|
|
1361
1494
|
const captureTimeoutMs = options.deadlineAt
|
|
1362
1495
|
? Math.min(
|
|
1363
1496
|
SERVERLESS_CHUNKED_ATTEMPT_TIMEOUT_MS,
|
|
1364
1497
|
options.deadlineAt -
|
|
1365
1498
|
Date.now() -
|
|
1366
|
-
|
|
1367
|
-
recipients.length * DASHBOARD_REPORT_EMAIL_TIMEOUT_MS -
|
|
1368
|
-
DASHBOARD_REPORT_EMAIL_OVERHEAD_RESERVE_MS,
|
|
1499
|
+
reportDeliveryReserveMs(recipients.length),
|
|
1369
1500
|
)
|
|
1370
1501
|
: undefined;
|
|
1371
1502
|
const capture = await captureDashboardPngWithFallback(
|
|
@@ -14,7 +14,7 @@ type DashboardFilterLike = {
|
|
|
14
14
|
default?: unknown;
|
|
15
15
|
};
|
|
16
16
|
|
|
17
|
-
type DashboardPanelLike = {
|
|
17
|
+
export type DashboardPanelLike = {
|
|
18
18
|
id?: unknown;
|
|
19
19
|
title?: unknown;
|
|
20
20
|
chartType?: unknown;
|
|
@@ -64,6 +64,241 @@ function hasExplicitLowerBound(sql: string): boolean {
|
|
|
64
64
|
);
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
+
const ANALYTICS_SCAN_RE = /\b(?:FROM|JOIN)\s+analytics_events\b/gi;
|
|
68
|
+
const TOP_LEVEL_CTE_HEAD_RE = /^\s*WITH\b/i;
|
|
69
|
+
const CTE_NAME_RE =
|
|
70
|
+
/^\s*((?:[A-Za-z_]\w*|"(?:[^"]|"")*"|`(?:[^`]|``)*`|\[(?:[^\]]|\]\])*\]))\s*(?:\([^)]*\)\s*)?AS\s*(?:(?:NOT\s+)?MATERIALIZED\s*)?\(/i;
|
|
71
|
+
|
|
72
|
+
type TopLevelCte = { name: string; bodyStart: number; bodyEnd: number };
|
|
73
|
+
|
|
74
|
+
type TopLevelCteParse = {
|
|
75
|
+
ctes: TopLevelCte[];
|
|
76
|
+
outerQueryStart: number;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
function findClosingParenthesis(sql: string, bodyStart: number): number {
|
|
80
|
+
let depth = 1;
|
|
81
|
+
let quote: "'" | '"' | "`" | null = null;
|
|
82
|
+
let lineComment = false;
|
|
83
|
+
let blockComment = false;
|
|
84
|
+
|
|
85
|
+
for (let i = bodyStart; i < sql.length; i += 1) {
|
|
86
|
+
const char = sql[i];
|
|
87
|
+
const next = sql[i + 1];
|
|
88
|
+
|
|
89
|
+
if (lineComment) {
|
|
90
|
+
if (char === "\n") lineComment = false;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (blockComment) {
|
|
94
|
+
if (char === "*" && next === "/") {
|
|
95
|
+
blockComment = false;
|
|
96
|
+
i += 1;
|
|
97
|
+
}
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (quote) {
|
|
101
|
+
if (char === quote) {
|
|
102
|
+
if (next === quote) {
|
|
103
|
+
i += 1;
|
|
104
|
+
} else {
|
|
105
|
+
quote = null;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (char === "-" && next === "-") {
|
|
111
|
+
lineComment = true;
|
|
112
|
+
i += 1;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
if (char === "/" && next === "*") {
|
|
116
|
+
blockComment = true;
|
|
117
|
+
i += 1;
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
if (char === "'" || char === '"' || char === "`") {
|
|
121
|
+
quote = char;
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (char === "(") depth += 1;
|
|
125
|
+
else if (char === ")") {
|
|
126
|
+
depth -= 1;
|
|
127
|
+
if (depth === 0) return i;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return -1;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function skipSqlTrivia(sql: string, start: number): number {
|
|
135
|
+
let i = start;
|
|
136
|
+
while (i < sql.length) {
|
|
137
|
+
if (/\s/.test(sql[i])) {
|
|
138
|
+
i += 1;
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (sql[i] === "-" && sql[i + 1] === "-") {
|
|
142
|
+
i += 2;
|
|
143
|
+
while (i < sql.length && sql[i] !== "\n") i += 1;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (sql[i] === "/" && sql[i + 1] === "*") {
|
|
147
|
+
const end = sql.indexOf("*/", i + 2);
|
|
148
|
+
if (end === -1) return sql.length;
|
|
149
|
+
i = end + 2;
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
return i;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function splitTopLevelUnionBranches(sql: string): string[] {
|
|
158
|
+
const branches: string[] = [];
|
|
159
|
+
let start = 0;
|
|
160
|
+
let depth = 0;
|
|
161
|
+
let quote: "'" | '"' | "`" | null = null;
|
|
162
|
+
let lineComment = false;
|
|
163
|
+
let blockComment = false;
|
|
164
|
+
|
|
165
|
+
for (let i = 0; i < sql.length; i += 1) {
|
|
166
|
+
const char = sql[i];
|
|
167
|
+
const next = sql[i + 1];
|
|
168
|
+
|
|
169
|
+
if (lineComment) {
|
|
170
|
+
if (char === "\n") lineComment = false;
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
if (blockComment) {
|
|
174
|
+
if (char === "*" && next === "/") {
|
|
175
|
+
blockComment = false;
|
|
176
|
+
i += 1;
|
|
177
|
+
}
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (quote) {
|
|
181
|
+
if (char === quote) {
|
|
182
|
+
if (next === quote) i += 1;
|
|
183
|
+
else quote = null;
|
|
184
|
+
}
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
if (char === "-" && next === "-") {
|
|
188
|
+
lineComment = true;
|
|
189
|
+
i += 1;
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
if (char === "/" && next === "*") {
|
|
193
|
+
blockComment = true;
|
|
194
|
+
i += 1;
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
if (char === "'" || char === '"' || char === "`") {
|
|
198
|
+
quote = char;
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
if (char === "(") {
|
|
202
|
+
depth += 1;
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
if (char === ")") {
|
|
206
|
+
depth = Math.max(0, depth - 1);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
if (depth !== 0) continue;
|
|
210
|
+
|
|
211
|
+
const union = /^UNION\b/i.exec(sql.slice(i));
|
|
212
|
+
if (!union) continue;
|
|
213
|
+
branches.push(sql.slice(start, i));
|
|
214
|
+
start = skipSqlTrivia(sql, i + union[0].length);
|
|
215
|
+
const modifier = /^(?:ALL|DISTINCT)\b/i.exec(sql.slice(start));
|
|
216
|
+
if (modifier) start = skipSqlTrivia(sql, start + modifier[0].length);
|
|
217
|
+
i = start - 1;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
branches.push(sql.slice(start));
|
|
221
|
+
return branches;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** One top-level `name AS (...)` common-table-expression's body span. */
|
|
225
|
+
function topLevelCtes(sql: string): TopLevelCteParse | null {
|
|
226
|
+
const sqlStart = skipSqlTrivia(sql, 0);
|
|
227
|
+
const head = TOP_LEVEL_CTE_HEAD_RE.exec(sql.slice(sqlStart));
|
|
228
|
+
if (!head) return null;
|
|
229
|
+
const ctes: TopLevelCte[] = [];
|
|
230
|
+
let i = skipSqlTrivia(sql, sqlStart + head[0].length);
|
|
231
|
+
const recursive = /^RECURSIVE\b/i.exec(sql.slice(i));
|
|
232
|
+
if (recursive) i = skipSqlTrivia(sql, i + recursive[0].length);
|
|
233
|
+
while (i < sql.length) {
|
|
234
|
+
const nameMatch = CTE_NAME_RE.exec(sql.slice(i));
|
|
235
|
+
if (!nameMatch) return null;
|
|
236
|
+
const name = nameMatch[1];
|
|
237
|
+
const bodyStart = i + nameMatch[0].length;
|
|
238
|
+
const bodyEnd = findClosingParenthesis(sql, bodyStart);
|
|
239
|
+
if (bodyEnd === -1) return null;
|
|
240
|
+
ctes.push({ name, bodyStart, bodyEnd });
|
|
241
|
+
const k = skipSqlTrivia(sql, bodyEnd + 1);
|
|
242
|
+
if (sql[k] === ",") {
|
|
243
|
+
i = k + 1;
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
return { ctes, outerQueryStart: k };
|
|
247
|
+
}
|
|
248
|
+
return null;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function hasAnyTimeBound(text: string): boolean {
|
|
252
|
+
// .search() ignores the shared global-flagged regex's lastIndex state,
|
|
253
|
+
// unlike .test(), which would otherwise give wrong results across calls.
|
|
254
|
+
return (
|
|
255
|
+
text.search(TEMPORAL_VARIABLE_RE) !== -1 || hasExplicitLowerBound(text)
|
|
256
|
+
);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* True if every `analytics_events` scan in `sql` has its own time bound.
|
|
261
|
+
*
|
|
262
|
+
* `hasExplicitLowerBound`/time-variable checks used to run against the whole
|
|
263
|
+
* SQL string: a multi-CTE panel with a bound ANYWHERE (e.g. only on the
|
|
264
|
+
* final SELECT, or only on one of several sibling CTEs) passed validation
|
|
265
|
+
* even though an earlier, unbounded sibling CTE still did a full-table scan
|
|
266
|
+
* — the exact shape of several production incidents (2026-07-25 org-wide
|
|
267
|
+
* audit). This only tightens the check for that specific shape: sibling
|
|
268
|
+
* top-level CTEs each need their own bound. It deliberately does NOT require
|
|
269
|
+
* every nested subquery to be independently bounded — a correlated lookup
|
|
270
|
+
* nested inside an already-bounded outer query (e.g. "has this id EVER
|
|
271
|
+
* appeared as a referrer") is a legitimate, intentionally all-time pattern,
|
|
272
|
+
* and over-flagging it would make this check untrustworthy.
|
|
273
|
+
*/
|
|
274
|
+
function everyScanIsBounded(sql: string): boolean {
|
|
275
|
+
const parsed = topLevelCtes(sql);
|
|
276
|
+
if (!parsed) {
|
|
277
|
+
// A WITH query that we cannot parse must fail closed. Falling back to a
|
|
278
|
+
// whole-query bound lets a bounded sibling hide an unbounded CTE.
|
|
279
|
+
if (TOP_LEVEL_CTE_HEAD_RE.test(sql.slice(skipSqlTrivia(sql, 0)))) {
|
|
280
|
+
return false;
|
|
281
|
+
}
|
|
282
|
+
return hasAnyTimeBound(sql);
|
|
283
|
+
}
|
|
284
|
+
const units = [
|
|
285
|
+
...parsed.ctes.map(({ bodyStart, bodyEnd }) =>
|
|
286
|
+
sql.slice(bodyStart, bodyEnd),
|
|
287
|
+
),
|
|
288
|
+
sql.slice(parsed.outerQueryStart),
|
|
289
|
+
];
|
|
290
|
+
return units.every((unit) =>
|
|
291
|
+
splitTopLevelUnionBranches(unit).every((branch) => {
|
|
292
|
+
const scans = branch.match(ANALYTICS_SCAN_RE) ?? [];
|
|
293
|
+
if (scans.length === 0) return true;
|
|
294
|
+
// A single lower bound cannot prove that every scan in a join or nested
|
|
295
|
+
// branch is bounded. Fail closed until the SQL has one scan per branch.
|
|
296
|
+
if (scans.length > 1) return false;
|
|
297
|
+
return hasAnyTimeBound(branch);
|
|
298
|
+
}),
|
|
299
|
+
);
|
|
300
|
+
}
|
|
301
|
+
|
|
67
302
|
function hasIntentionalHistoryDescription(
|
|
68
303
|
panel: DashboardPanelLike,
|
|
69
304
|
config: Record<string, unknown>,
|
|
@@ -166,5 +401,19 @@ export function validateFirstPartyDashboardTimeScope(
|
|
|
166
401
|
return `panel[${index}] ${label} reads first-party analytics without a time bound; use {{timeRange}} with a non-empty default filter, or explicitly set config.timeScope to "cohort-history" or "all-time" for intentional history scans`;
|
|
167
402
|
}
|
|
168
403
|
|
|
404
|
+
// A bound anywhere in the SQL text (checked above) is not the same as
|
|
405
|
+
// every CTE/subquery that reads analytics_events having its own bound — a
|
|
406
|
+
// multi-CTE panel can look bound overall while an earlier CTE still does a
|
|
407
|
+
// full-table scan. Skip this for "all-time" and "cohort-history": both are
|
|
408
|
+
// explicit escape hatches, and a cohort-defining CTE (e.g. "first ever
|
|
409
|
+
// active date per user") is legitimately unbounded by design.
|
|
410
|
+
if (
|
|
411
|
+
scope !== "all-time" &&
|
|
412
|
+
scope !== "cohort-history" &&
|
|
413
|
+
!everyScanIsBounded(sql)
|
|
414
|
+
) {
|
|
415
|
+
return `panel[${index}] ${label} has at least one analytics_events read (in a CTE or subquery) without its own time bound — a {{timeRange}} reference or literal bound elsewhere in the SQL does not cover it; add a bound to every analytics_events scan, or set config.timeScope to "all-time" for an intentional full-history scan`;
|
|
416
|
+
}
|
|
417
|
+
|
|
169
418
|
return null;
|
|
170
419
|
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { createHash } from "crypto";
|
|
2
|
+
|
|
3
|
+
import { getDbExec } from "@agent-native/core/db";
|
|
4
|
+
|
|
5
|
+
import type { AnalyticsQueryResult } from "./first-party-analytics.js";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* First-party dashboard panel query cache.
|
|
9
|
+
*
|
|
10
|
+
* `analytics_events` grows unbounded and several panels (rolling-window and
|
|
11
|
+
* cohort self-joins) are inherently expensive, so a page with many panels or
|
|
12
|
+
* a daily report screenshot capturing many panels at once repeatedly
|
|
13
|
+
* recomputes the same rows under concurrent load — that contention, not any
|
|
14
|
+
* single query alone, is what was pushing panels past their timeout budget.
|
|
15
|
+
* This mirrors bigquery.ts's L1 (in-process) + L2 (SQL-backed, shared across
|
|
16
|
+
* serverless invocations) cache, but keyed per scoped+interpolated SQL text
|
|
17
|
+
* (which already embeds org_id/owner_email via query args) and with a much
|
|
18
|
+
* shorter TTL since this is the app's own live data, not an immutable
|
|
19
|
+
* warehouse result.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
interface L1Entry {
|
|
23
|
+
result: AnalyticsQueryResult;
|
|
24
|
+
createdAt: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const CACHE_TTL_MS = 5 * 60 * 1000;
|
|
28
|
+
const MAX_L1_ENTRIES = 500;
|
|
29
|
+
|
|
30
|
+
const l1Cache = new Map<string, L1Entry>();
|
|
31
|
+
// De-dupes identical concurrent cache misses (e.g. the same panel query fired
|
|
32
|
+
// by several viewers, or by the report capture's 4-panel concurrent window)
|
|
33
|
+
// so only one of them actually hits the database.
|
|
34
|
+
const inFlight = new Map<string, Promise<AnalyticsQueryResult>>();
|
|
35
|
+
|
|
36
|
+
function inFlightKey(key: string, timeoutMs?: number): string {
|
|
37
|
+
return `${key}:${timeoutMs ?? "unbounded"}`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface FirstPartyCacheOptions {
|
|
41
|
+
timeoutMs?: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function firstPartyCacheKey(
|
|
45
|
+
scopedSql: string,
|
|
46
|
+
args: Array<string | null>,
|
|
47
|
+
): string {
|
|
48
|
+
return createHash("sha256")
|
|
49
|
+
.update(`${scopedSql}\n${JSON.stringify(args)}`)
|
|
50
|
+
.digest("hex");
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function getL1(key: string): AnalyticsQueryResult | null {
|
|
54
|
+
const entry = l1Cache.get(key);
|
|
55
|
+
if (!entry) return null;
|
|
56
|
+
if (Date.now() - entry.createdAt > CACHE_TTL_MS) {
|
|
57
|
+
l1Cache.delete(key);
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
return entry.result;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function setL1(key: string, result: AnalyticsQueryResult): void {
|
|
64
|
+
if (l1Cache.size >= MAX_L1_ENTRIES) {
|
|
65
|
+
const oldest = l1Cache.keys().next().value;
|
|
66
|
+
if (oldest) l1Cache.delete(oldest);
|
|
67
|
+
}
|
|
68
|
+
l1Cache.set(key, { result, createdAt: Date.now() });
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function getL2(
|
|
72
|
+
key: string,
|
|
73
|
+
timeoutMs?: number,
|
|
74
|
+
): Promise<AnalyticsQueryResult | null> {
|
|
75
|
+
try {
|
|
76
|
+
const db = getDbExec();
|
|
77
|
+
const nowIso = new Date().toISOString();
|
|
78
|
+
const { rows } = await db.execute({
|
|
79
|
+
sql: "SELECT result FROM first_party_analytics_cache WHERE key = ? AND expires_at > ?",
|
|
80
|
+
args: [key, nowIso],
|
|
81
|
+
timeoutMs,
|
|
82
|
+
});
|
|
83
|
+
if (!rows.length) return null;
|
|
84
|
+
const raw = (rows[0] as { result: string }).result;
|
|
85
|
+
return JSON.parse(raw) as AnalyticsQueryResult;
|
|
86
|
+
} catch (err) {
|
|
87
|
+
console.warn("[first-party-analytics] L2 cache read failed:", err);
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async function setL2(
|
|
93
|
+
key: string,
|
|
94
|
+
sql: string,
|
|
95
|
+
result: AnalyticsQueryResult,
|
|
96
|
+
timeoutMs?: number,
|
|
97
|
+
): Promise<void> {
|
|
98
|
+
try {
|
|
99
|
+
const db = getDbExec();
|
|
100
|
+
const now = new Date();
|
|
101
|
+
const expiresAt = new Date(now.getTime() + CACHE_TTL_MS);
|
|
102
|
+
const serialized = JSON.stringify(result);
|
|
103
|
+
// Upsert via delete+insert to stay dialect-agnostic (SQLite/Postgres).
|
|
104
|
+
await db.execute({
|
|
105
|
+
sql: "DELETE FROM first_party_analytics_cache WHERE key = ?",
|
|
106
|
+
args: [key],
|
|
107
|
+
timeoutMs,
|
|
108
|
+
});
|
|
109
|
+
await db.execute({
|
|
110
|
+
sql: "INSERT INTO first_party_analytics_cache (key, sql, result, created_at, expires_at) VALUES (?, ?, ?, ?, ?)",
|
|
111
|
+
args: [key, sql, serialized, now.toISOString(), expiresAt.toISOString()],
|
|
112
|
+
timeoutMs,
|
|
113
|
+
});
|
|
114
|
+
// Opportunistically prune expired rows so the table doesn't grow
|
|
115
|
+
// unbounded — the keyspace is effectively every distinct panel/filter
|
|
116
|
+
// combination. Run ~1% of the time to avoid thrashing on every write.
|
|
117
|
+
if (Math.random() < 0.01) {
|
|
118
|
+
await db.execute({
|
|
119
|
+
sql: "DELETE FROM first_party_analytics_cache WHERE expires_at <= ?",
|
|
120
|
+
args: [now.toISOString()],
|
|
121
|
+
timeoutMs,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
} catch (err) {
|
|
125
|
+
console.warn("[first-party-analytics] L2 cache write failed:", err);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Returns a cached result for (key, sql) if fresh, otherwise runs `compute`
|
|
131
|
+
* exactly once — even under concurrent callers with the same key — and
|
|
132
|
+
* caches the result.
|
|
133
|
+
*/
|
|
134
|
+
export async function withFirstPartyCache(
|
|
135
|
+
key: string,
|
|
136
|
+
sql: string,
|
|
137
|
+
compute: () => Promise<AnalyticsQueryResult>,
|
|
138
|
+
options: FirstPartyCacheOptions = {},
|
|
139
|
+
): Promise<AnalyticsQueryResult> {
|
|
140
|
+
const l1Hit = getL1(key);
|
|
141
|
+
if (l1Hit) return l1Hit;
|
|
142
|
+
|
|
143
|
+
// A report prewarm may have a shorter deadline than a normal panel request;
|
|
144
|
+
// never let those callers inherit one another's database timeout.
|
|
145
|
+
const requestKey = inFlightKey(key, options.timeoutMs);
|
|
146
|
+
const existing = inFlight.get(requestKey);
|
|
147
|
+
if (existing) return existing;
|
|
148
|
+
|
|
149
|
+
// Register the in-flight promise synchronously (before any `await`) so two
|
|
150
|
+
// callers racing on the same key can't both slip past the check above and
|
|
151
|
+
// both hit L2/compute.
|
|
152
|
+
const promise = (async () => {
|
|
153
|
+
const l2Hit = await getL2(key, options.timeoutMs);
|
|
154
|
+
if (l2Hit) {
|
|
155
|
+
setL1(key, l2Hit);
|
|
156
|
+
return l2Hit;
|
|
157
|
+
}
|
|
158
|
+
const result = await compute();
|
|
159
|
+
setL1(key, result);
|
|
160
|
+
await setL2(key, sql, result, options.timeoutMs);
|
|
161
|
+
return result;
|
|
162
|
+
})().finally(() => {
|
|
163
|
+
inFlight.delete(requestKey);
|
|
164
|
+
});
|
|
165
|
+
inFlight.set(requestKey, promise);
|
|
166
|
+
return promise;
|
|
167
|
+
}
|