@agent-native/core 0.141.5 → 0.141.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/templates/analytics/.agents/skills/data-querying/SKILL.md +13 -6
- package/corpus/templates/analytics/actions/get-first-party-analytics-health.ts +15 -1
- package/corpus/templates/analytics/actions/query-agent-native-analytics.ts +2 -2
- package/corpus/templates/analytics/app/i18n-data.ts +328 -0
- package/corpus/templates/analytics/app/pages/DataSources.tsx +66 -30
- package/corpus/templates/analytics/changelog/2026-08-05-historical-analytics-rollups-now-repair-automatically-after-.md +6 -0
- package/corpus/templates/analytics/netlify.toml +4 -0
- package/corpus/templates/analytics/scripts/emit-netlify-dashboard-report-cron.ts +109 -0
- package/corpus/templates/analytics/server/jobs/analytics-rollup-backfill.ts +342 -0
- package/corpus/templates/analytics/server/lib/analytics-rollup-lock.ts +8 -0
- package/corpus/templates/analytics/server/lib/dashboard-catalog.ts +11 -29
- package/corpus/templates/analytics/server/lib/dashboards-store.ts +55 -0
- package/corpus/templates/analytics/server/lib/first-party-analytics-health.ts +117 -31
- package/corpus/templates/analytics/server/lib/first-party-analytics-rollups.ts +15 -9
- package/corpus/templates/analytics/server/lib/first-party-analytics.ts +5 -16
- package/corpus/templates/analytics/server/plugins/analytics-rollup-backfill-jobs.ts +61 -0
- package/corpus/templates/analytics/server/plugins/db.ts +58 -187
- package/corpus/templates/analytics/server/routes/api/analytics-rollups/backfill.post.ts +53 -0
- package/corpus/templates/clips/changelog/2026-08-05-clips-overlays-now-use-clean-surfaces-without-clipped-shadow.md +6 -0
- package/corpus/templates/clips/desktop/src/overlays/meeting-notification.tsx +2 -2
- package/corpus/templates/clips/desktop/src/styles.css +3 -54
- package/corpus/templates/clips/desktop/src-tauri/src/notifications.rs +3 -3
- package/corpus/templates/design/actions/get-design-system.ts +3 -1
- package/corpus/templates/design/changelog/2026-08-05-missing-design-systems-now-return-a-clear-not-found-response.md +6 -0
- package/corpus/templates/slides/app/components/deck/DeckCard.tsx +2 -2
- package/corpus/templates/slides/app/components/editor/SlideEditor.tsx +1 -1
- package/corpus/templates/slides/app/i18n/en-US.ts +5 -5
- package/corpus/templates/slides/changelog/2026-08-05-new-presentation-prompts-now-clearly-describe-presentation-g.md +6 -0
- package/dist/client/AssistantChat.js +1 -1
- package/dist/client/chat/run-recovery.d.ts +2 -1
- package/dist/client/chat/run-recovery.js +10 -3
- package/dist/client/error-format.d.ts +1 -0
- package/dist/client/error-format.js +4 -2
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/observability/routes.d.ts +3 -3
- package/dist/progress/routes.d.ts +1 -1
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/secrets/routes.d.ts +3 -3
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/package.json +1 -1
- package/corpus/templates/analytics/changelog/2026-08-05-historical-analytics-rollups-stay-complete-and-atomic.md +0 -6
- package/corpus/templates/analytics/changelog/2026-08-05-historical-rollups-stay-complete-while-live-events-arrive.md +0 -6
|
@@ -120,11 +120,19 @@ interface GitHubOAuthStatus {
|
|
|
120
120
|
|
|
121
121
|
interface FirstPartyAnalyticsHealthResponse {
|
|
122
122
|
status: "healthy" | "monitor" | "recommend_bigquery" | "unavailable";
|
|
123
|
+
externalBackendRecommendation?: "none" | "connect" | "use" | "unknown";
|
|
123
124
|
metrics: {
|
|
124
125
|
eventCount: number;
|
|
125
126
|
slowQueryCount24h: number;
|
|
126
127
|
maxQueryDurationMs24h: number;
|
|
127
128
|
};
|
|
129
|
+
externalBackends?: Array<{
|
|
130
|
+
id: "bigquery" | "amplitude";
|
|
131
|
+
label: string;
|
|
132
|
+
role: "warehouse" | "product-analytics";
|
|
133
|
+
configured: boolean | null;
|
|
134
|
+
setupLink: string;
|
|
135
|
+
}>;
|
|
128
136
|
bigQuery: {
|
|
129
137
|
configured: boolean | null;
|
|
130
138
|
};
|
|
@@ -1615,23 +1623,25 @@ function FirstPartyAnalyticsCard() {
|
|
|
1615
1623
|
});
|
|
1616
1624
|
const health = rawHealth as FirstPartyAnalyticsHealthResponse | undefined;
|
|
1617
1625
|
const healthStatus = isHealthError ? "unavailable" : health?.status;
|
|
1618
|
-
const
|
|
1619
|
-
const
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1626
|
+
const externalBackends = health?.externalBackends ?? [];
|
|
1627
|
+
const externalBackendConfigured = externalBackends.some(
|
|
1628
|
+
(backend) => backend.configured === true,
|
|
1629
|
+
);
|
|
1630
|
+
const recommendsExternalBackend = healthStatus === "recommend_bigquery";
|
|
1631
|
+
const healthTitleKey = recommendsExternalBackend
|
|
1632
|
+
? externalBackendConfigured
|
|
1633
|
+
? "analyticsBackend.connectedTitle"
|
|
1634
|
+
: "analyticsBackend.recommendationTitle"
|
|
1635
|
+
: null;
|
|
1636
|
+
const healthDescriptionKey = recommendsExternalBackend
|
|
1637
|
+
? externalBackendConfigured
|
|
1638
|
+
? "analyticsBackend.connectedDescription"
|
|
1639
|
+
: "analyticsBackend.recommendationDescription"
|
|
1640
|
+
: healthStatus === "monitor"
|
|
1641
|
+
? "analyticsBackend.monitorDescription"
|
|
1642
|
+
: healthStatus === "healthy"
|
|
1643
|
+
? "analyticsBackend.healthyDescription"
|
|
1644
|
+
: "analyticsBackend.unavailableDescription";
|
|
1635
1645
|
|
|
1636
1646
|
const createKey = useActionMutation("create-analytics-public-key", {
|
|
1637
1647
|
onSuccess: (result: any) => {
|
|
@@ -1681,7 +1691,7 @@ function FirstPartyAnalyticsCard() {
|
|
|
1681
1691
|
<div className="flex shrink-0 items-center gap-2">
|
|
1682
1692
|
{isLoading ? (
|
|
1683
1693
|
<Skeleton className="h-4 w-20 rounded-full" />
|
|
1684
|
-
) :
|
|
1694
|
+
) : recommendsExternalBackend ? (
|
|
1685
1695
|
<span
|
|
1686
1696
|
className="flex min-w-0 items-center gap-1.5 text-xs font-medium text-amber-500"
|
|
1687
1697
|
title={healthTitleKey ? t(healthTitleKey) : undefined}
|
|
@@ -1690,13 +1700,13 @@ function FirstPartyAnalyticsCard() {
|
|
|
1690
1700
|
<span className="max-w-[12rem] truncate">
|
|
1691
1701
|
{healthTitleKey
|
|
1692
1702
|
? t(healthTitleKey)
|
|
1693
|
-
: t("
|
|
1703
|
+
: t("analyticsBackend.recommendationTitle")}
|
|
1694
1704
|
</span>
|
|
1695
1705
|
</span>
|
|
1696
1706
|
) : connected ? (
|
|
1697
1707
|
<span className="flex items-center gap-1.5 text-xs text-emerald-500 font-medium whitespace-nowrap">
|
|
1698
1708
|
<IconCheck className="h-3.5 w-3.5" />
|
|
1699
|
-
{t("
|
|
1709
|
+
{t("analyticsBackend.configured")}
|
|
1700
1710
|
</span>
|
|
1701
1711
|
) : (
|
|
1702
1712
|
<span className="flex items-center gap-1.5 text-xs text-muted-foreground whitespace-nowrap">
|
|
@@ -1722,7 +1732,7 @@ function FirstPartyAnalyticsCard() {
|
|
|
1722
1732
|
) : (
|
|
1723
1733
|
<div
|
|
1724
1734
|
className={`rounded-md border p-3 text-xs ${
|
|
1725
|
-
|
|
1735
|
+
recommendsExternalBackend
|
|
1726
1736
|
? "border-amber-500/30 bg-amber-500/10"
|
|
1727
1737
|
: "border-border/50 bg-muted/20"
|
|
1728
1738
|
}`}
|
|
@@ -1731,7 +1741,7 @@ function FirstPartyAnalyticsCard() {
|
|
|
1731
1741
|
<div className="flex min-w-0 items-start gap-2">
|
|
1732
1742
|
<IconAlertCircle
|
|
1733
1743
|
className={`mt-px h-3.5 w-3.5 shrink-0 ${
|
|
1734
|
-
|
|
1744
|
+
recommendsExternalBackend
|
|
1735
1745
|
? "text-amber-500"
|
|
1736
1746
|
: "text-muted-foreground"
|
|
1737
1747
|
}`}
|
|
@@ -1747,15 +1757,41 @@ function FirstPartyAnalyticsCard() {
|
|
|
1747
1757
|
</p>
|
|
1748
1758
|
</div>
|
|
1749
1759
|
</div>
|
|
1750
|
-
{healthStatus === "recommend_bigquery" &&
|
|
1751
|
-
!bigQueryConfigured ? (
|
|
1752
|
-
<Button asChild size="sm" className="shrink-0 text-xs">
|
|
1753
|
-
<Link to="/data-sources?source=bigquery">
|
|
1754
|
-
{t("dataSources.bigQuerySetup")}
|
|
1755
|
-
</Link>
|
|
1756
|
-
</Button>
|
|
1757
|
-
) : null}
|
|
1758
1760
|
</div>
|
|
1761
|
+
{externalBackends.length > 0 && (
|
|
1762
|
+
<div className="mt-3 border-t border-border/30 pt-3">
|
|
1763
|
+
<div className="mb-2 text-muted-foreground">
|
|
1764
|
+
{t("analyticsBackend.options")}
|
|
1765
|
+
</div>
|
|
1766
|
+
<div className="flex flex-wrap gap-2">
|
|
1767
|
+
{externalBackends.map((backend) => {
|
|
1768
|
+
const statusLabel =
|
|
1769
|
+
backend.configured === true
|
|
1770
|
+
? t("analyticsBackend.configured")
|
|
1771
|
+
: backend.configured === false
|
|
1772
|
+
? t("analyticsBackend.setUp")
|
|
1773
|
+
: t("dataSources.statusUnknown");
|
|
1774
|
+
return (
|
|
1775
|
+
<Button
|
|
1776
|
+
asChild
|
|
1777
|
+
key={backend.id}
|
|
1778
|
+
size="sm"
|
|
1779
|
+
variant={
|
|
1780
|
+
backend.configured === true
|
|
1781
|
+
? "outline"
|
|
1782
|
+
: "default"
|
|
1783
|
+
}
|
|
1784
|
+
className="text-xs"
|
|
1785
|
+
>
|
|
1786
|
+
<Link to={backend.setupLink}>
|
|
1787
|
+
{backend.label} · {statusLabel}
|
|
1788
|
+
</Link>
|
|
1789
|
+
</Button>
|
|
1790
|
+
);
|
|
1791
|
+
})}
|
|
1792
|
+
</div>
|
|
1793
|
+
</div>
|
|
1794
|
+
)}
|
|
1759
1795
|
{health && healthStatus !== "unavailable" && (
|
|
1760
1796
|
<div className="mt-3 grid grid-cols-3 gap-2 border-t border-border/30 pt-3">
|
|
1761
1797
|
<div>
|
|
@@ -22,6 +22,10 @@ const UPTIME_SCHEDULED_NAME = "uptime-monitor-cron";
|
|
|
22
22
|
const UPTIME_WORKER_NAME = "uptime-monitor-sweep-background";
|
|
23
23
|
const UPTIME_ROUTE_PATH = "/api/uptime-monitors/run";
|
|
24
24
|
const UPTIME_SCHEDULE = "* * * * *";
|
|
25
|
+
const ROLLUP_SCHEDULED_NAME = "analytics-rollup-backfill-cron";
|
|
26
|
+
const ROLLUP_WORKER_NAME = "analytics-rollup-backfill-background";
|
|
27
|
+
const ROLLUP_ROUTE_PATH = "/api/analytics-rollups/backfill";
|
|
28
|
+
const ROLLUP_SCHEDULE = "*/15 * * * *";
|
|
25
29
|
|
|
26
30
|
function ensureDir(dir: string) {
|
|
27
31
|
mkdirSync(dir, { recursive: true });
|
|
@@ -388,6 +392,105 @@ export const config = {
|
|
|
388
392
|
writeFileSync(path.join(dest, `${UPTIME_WORKER_NAME}.mjs`), source);
|
|
389
393
|
}
|
|
390
394
|
|
|
395
|
+
function emitRollupScheduledTrigger(token: string) {
|
|
396
|
+
const dest = path.join(FUNCTIONS_DIR, ROLLUP_SCHEDULED_NAME);
|
|
397
|
+
rmSync(dest, { recursive: true, force: true });
|
|
398
|
+
ensureDir(dest);
|
|
399
|
+
|
|
400
|
+
const source = `const WORKER_PATH = "/.netlify/functions/${ROLLUP_WORKER_NAME}";
|
|
401
|
+
const CRON_TOKEN = ${JSON.stringify(token)};
|
|
402
|
+
|
|
403
|
+
function siteOrigin(request) {
|
|
404
|
+
const configured = process.env.URL || process.env.DEPLOY_URL;
|
|
405
|
+
if (configured) return configured;
|
|
406
|
+
const url = new URL(request.url);
|
|
407
|
+
return url.origin;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
export default async function handler(request) {
|
|
411
|
+
const url = new URL(WORKER_PATH, siteOrigin(request));
|
|
412
|
+
const response = await fetch(url.toString(), {
|
|
413
|
+
method: "POST",
|
|
414
|
+
headers: {
|
|
415
|
+
"content-type": "application/json",
|
|
416
|
+
"x-agent-native-analytics-rollup-cron": CRON_TOKEN,
|
|
417
|
+
},
|
|
418
|
+
body: JSON.stringify({ scheduled: true }),
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
if (!response.ok && response.status !== 202) {
|
|
422
|
+
console.error(
|
|
423
|
+
"[analytics-rollup-backfill-cron] Background backfill trigger failed:",
|
|
424
|
+
response.status,
|
|
425
|
+
);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
export const config = {
|
|
430
|
+
name: "analytics rollup backfill cron trigger",
|
|
431
|
+
generator: "agent-native analytics build",
|
|
432
|
+
schedule: ${JSON.stringify(ROLLUP_SCHEDULE)},
|
|
433
|
+
};
|
|
434
|
+
`;
|
|
435
|
+
|
|
436
|
+
writeFileSync(path.join(dest, `${ROLLUP_SCHEDULED_NAME}.mjs`), source);
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
function emitRollupBackgroundWorker(token: string) {
|
|
440
|
+
const dest = path.join(FUNCTIONS_DIR, ROLLUP_WORKER_NAME);
|
|
441
|
+
rmSync(dest, { recursive: true, force: true });
|
|
442
|
+
cloneServerBundleForFunction(SERVER_DIR, dest);
|
|
443
|
+
rmSync(path.join(dest, "server.mjs"), { force: true });
|
|
444
|
+
|
|
445
|
+
const source = `globalThis.__AGENT_NATIVE_BACKGROUND_RUNTIME__ = true;
|
|
446
|
+
globalThis.__AGENT_NATIVE_ANALYTICS_ROLLUP_BACKFILL_SCHEDULED_RUNTIME__ = true;
|
|
447
|
+
|
|
448
|
+
const CRON_TOKEN = ${JSON.stringify(token)};
|
|
449
|
+
const ROUTE_PATH = ${JSON.stringify(ROLLUP_ROUTE_PATH)};
|
|
450
|
+
let cachedHandler;
|
|
451
|
+
|
|
452
|
+
function timingSafeEquals(a, b) {
|
|
453
|
+
if (a.length !== b.length) return false;
|
|
454
|
+
let diff = 0;
|
|
455
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
456
|
+
diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
457
|
+
}
|
|
458
|
+
return diff === 0;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
export default async function handler(request, context) {
|
|
462
|
+
const token = request.headers.get("x-agent-native-analytics-rollup-cron") || "";
|
|
463
|
+
if (!timingSafeEquals(token, CRON_TOKEN)) {
|
|
464
|
+
return new Response("Unauthorized", { status: 401 });
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
cachedHandler ??= (await import("./main.mjs")).default;
|
|
468
|
+
const url = new URL(request.url);
|
|
469
|
+
url.pathname = ROUTE_PATH;
|
|
470
|
+
url.search = "";
|
|
471
|
+
|
|
472
|
+
const rewritten = new Request(url.toString(), {
|
|
473
|
+
method: "POST",
|
|
474
|
+
headers: new Headers({ "content-type": "application/json" }),
|
|
475
|
+
body: JSON.stringify({ scheduled: true }),
|
|
476
|
+
});
|
|
477
|
+
|
|
478
|
+
return await cachedHandler(rewritten, context);
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
export const config = {
|
|
482
|
+
name: "analytics rollup backfill background sweep",
|
|
483
|
+
generator: "agent-native analytics build",
|
|
484
|
+
background: true,
|
|
485
|
+
nodeBundler: "none",
|
|
486
|
+
includedFiles: ["**"],
|
|
487
|
+
preferStatic: false,
|
|
488
|
+
};
|
|
489
|
+
`;
|
|
490
|
+
|
|
491
|
+
writeFileSync(path.join(dest, `${ROLLUP_WORKER_NAME}.mjs`), source);
|
|
492
|
+
}
|
|
493
|
+
|
|
391
494
|
function isDirectRun(): boolean {
|
|
392
495
|
const entrypoint = process.argv[1];
|
|
393
496
|
return Boolean(
|
|
@@ -407,12 +510,15 @@ function main(): void {
|
|
|
407
510
|
const token = randomBytes(32).toString("hex");
|
|
408
511
|
const alertToken = randomBytes(32).toString("hex");
|
|
409
512
|
const uptimeToken = randomBytes(32).toString("hex");
|
|
513
|
+
const rollupToken = randomBytes(32).toString("hex");
|
|
410
514
|
emitScheduledTrigger(token);
|
|
411
515
|
emitBackgroundWorker(token);
|
|
412
516
|
emitAlertScheduledTrigger(alertToken);
|
|
413
517
|
emitAlertBackgroundWorker(alertToken);
|
|
414
518
|
emitUptimeScheduledTrigger(uptimeToken);
|
|
415
519
|
emitUptimeBackgroundWorker(uptimeToken);
|
|
520
|
+
emitRollupScheduledTrigger(rollupToken);
|
|
521
|
+
emitRollupBackgroundWorker(rollupToken);
|
|
416
522
|
console.log(
|
|
417
523
|
`[dashboard-report-cron] Emitted Netlify scheduled trigger "${SCHEDULED_NAME}" (${SCHEDULE}) and background worker "${WORKER_NAME}".`,
|
|
418
524
|
);
|
|
@@ -422,6 +528,9 @@ function main(): void {
|
|
|
422
528
|
console.log(
|
|
423
529
|
`[uptime-monitor-cron] Emitted Netlify scheduled trigger "${UPTIME_SCHEDULED_NAME}" (${UPTIME_SCHEDULE}) and background worker "${UPTIME_WORKER_NAME}".`,
|
|
424
530
|
);
|
|
531
|
+
console.log(
|
|
532
|
+
`[analytics-rollup-backfill-cron] Emitted Netlify scheduled trigger "${ROLLUP_SCHEDULED_NAME}" (${ROLLUP_SCHEDULE}) and background worker "${ROLLUP_WORKER_NAME}".`,
|
|
533
|
+
);
|
|
425
534
|
}
|
|
426
535
|
|
|
427
536
|
if (isDirectRun()) {
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
import { getDbExec, getDialect, isPostgres } from "@agent-native/core/db";
|
|
2
|
+
|
|
3
|
+
import { FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY } from "../lib/analytics-rollup-lock";
|
|
4
|
+
|
|
5
|
+
const BACKFILL_STATE_ID = "historical-v1";
|
|
6
|
+
const BACKFILL_LEASE_MINUTES = 15;
|
|
7
|
+
|
|
8
|
+
const POSTGRES_BACKFILL_STATEMENTS = [
|
|
9
|
+
`
|
|
10
|
+
INSERT INTO analytics_event_daily_rollups (
|
|
11
|
+
id, tenant_key, owner_email, org_id, event_date, event_name,
|
|
12
|
+
app, template, event_count
|
|
13
|
+
)
|
|
14
|
+
SELECT
|
|
15
|
+
md5(random()::text || clock_timestamp()::text), tenant_key,
|
|
16
|
+
MIN(owner_email), MIN(org_id), event_date, event_name, app, template,
|
|
17
|
+
COUNT(*)::INTEGER
|
|
18
|
+
FROM (
|
|
19
|
+
SELECT
|
|
20
|
+
CASE
|
|
21
|
+
WHEN org_id IS NOT NULL AND org_id <> '' THEN 'org:' || org_id
|
|
22
|
+
ELSE 'user:' || owner_email
|
|
23
|
+
END AS tenant_key,
|
|
24
|
+
owner_email,
|
|
25
|
+
org_id,
|
|
26
|
+
COALESCE(NULLIF(event_date, ''), substr(timestamp, 1, 10)) AS event_date,
|
|
27
|
+
event_name,
|
|
28
|
+
COALESCE(app, '') AS app,
|
|
29
|
+
COALESCE(template, '') AS template
|
|
30
|
+
FROM analytics_events
|
|
31
|
+
) AS historical_events
|
|
32
|
+
WHERE event_date <> ''
|
|
33
|
+
GROUP BY tenant_key, event_date, event_name, app, template
|
|
34
|
+
ON CONFLICT (tenant_key, event_date, event_name, app, template)
|
|
35
|
+
DO UPDATE SET event_count = GREATEST(
|
|
36
|
+
analytics_event_daily_rollups.event_count,
|
|
37
|
+
EXCLUDED.event_count
|
|
38
|
+
)
|
|
39
|
+
`,
|
|
40
|
+
`
|
|
41
|
+
INSERT INTO analytics_user_days (
|
|
42
|
+
id, tenant_key, owner_email, org_id, event_date, user_key
|
|
43
|
+
)
|
|
44
|
+
SELECT
|
|
45
|
+
md5(random()::text || clock_timestamp()::text), tenant_key,
|
|
46
|
+
owner_email, org_id, event_date, user_key
|
|
47
|
+
FROM (
|
|
48
|
+
SELECT DISTINCT
|
|
49
|
+
CASE
|
|
50
|
+
WHEN org_id IS NOT NULL AND org_id <> '' THEN 'org:' || org_id
|
|
51
|
+
ELSE 'user:' || owner_email
|
|
52
|
+
END AS tenant_key,
|
|
53
|
+
owner_email,
|
|
54
|
+
org_id,
|
|
55
|
+
COALESCE(NULLIF(event_date, ''), substr(timestamp, 1, 10)) AS event_date,
|
|
56
|
+
COALESCE(
|
|
57
|
+
NULLIF(user_key, ''),
|
|
58
|
+
NULLIF(user_id, ''),
|
|
59
|
+
NULLIF(anonymous_id, '')
|
|
60
|
+
) AS user_key
|
|
61
|
+
FROM analytics_events
|
|
62
|
+
WHERE COALESCE(
|
|
63
|
+
NULLIF(user_key, ''),
|
|
64
|
+
NULLIF(user_id, ''),
|
|
65
|
+
NULLIF(anonymous_id, '')
|
|
66
|
+
) IS NOT NULL
|
|
67
|
+
) AS historical_user_days
|
|
68
|
+
WHERE event_date <> '' AND user_key <> ''
|
|
69
|
+
ON CONFLICT (tenant_key, event_date, user_key) DO NOTHING
|
|
70
|
+
`,
|
|
71
|
+
] as const;
|
|
72
|
+
|
|
73
|
+
const SQLITE_BACKFILL_STATEMENTS = [
|
|
74
|
+
`
|
|
75
|
+
INSERT INTO analytics_event_daily_rollups (
|
|
76
|
+
id, tenant_key, owner_email, org_id, event_date, event_name,
|
|
77
|
+
app, template, event_count
|
|
78
|
+
)
|
|
79
|
+
SELECT
|
|
80
|
+
lower(hex(randomblob(16))), tenant_key, MIN(owner_email), MIN(org_id),
|
|
81
|
+
event_date, event_name, app, template, COUNT(*)
|
|
82
|
+
FROM (
|
|
83
|
+
SELECT
|
|
84
|
+
CASE
|
|
85
|
+
WHEN org_id IS NOT NULL AND org_id <> '' THEN 'org:' || org_id
|
|
86
|
+
ELSE 'user:' || owner_email
|
|
87
|
+
END AS tenant_key,
|
|
88
|
+
owner_email,
|
|
89
|
+
org_id,
|
|
90
|
+
COALESCE(NULLIF(event_date, ''), substr(timestamp, 1, 10)) AS event_date,
|
|
91
|
+
event_name,
|
|
92
|
+
COALESCE(app, '') AS app,
|
|
93
|
+
COALESCE(template, '') AS template
|
|
94
|
+
FROM analytics_events
|
|
95
|
+
) AS historical_events
|
|
96
|
+
WHERE event_date <> ''
|
|
97
|
+
GROUP BY tenant_key, event_date, event_name, app, template
|
|
98
|
+
ON CONFLICT (tenant_key, event_date, event_name, app, template)
|
|
99
|
+
DO UPDATE SET event_count = MAX(
|
|
100
|
+
analytics_event_daily_rollups.event_count,
|
|
101
|
+
excluded.event_count
|
|
102
|
+
)
|
|
103
|
+
`,
|
|
104
|
+
`
|
|
105
|
+
INSERT INTO analytics_user_days (
|
|
106
|
+
id, tenant_key, owner_email, org_id, event_date, user_key
|
|
107
|
+
)
|
|
108
|
+
SELECT
|
|
109
|
+
lower(hex(randomblob(16))), tenant_key, owner_email, org_id,
|
|
110
|
+
event_date, user_key
|
|
111
|
+
FROM (
|
|
112
|
+
SELECT DISTINCT
|
|
113
|
+
CASE
|
|
114
|
+
WHEN org_id IS NOT NULL AND org_id <> '' THEN 'org:' || org_id
|
|
115
|
+
ELSE 'user:' || owner_email
|
|
116
|
+
END AS tenant_key,
|
|
117
|
+
owner_email,
|
|
118
|
+
org_id,
|
|
119
|
+
COALESCE(NULLIF(event_date, ''), substr(timestamp, 1, 10)) AS event_date,
|
|
120
|
+
COALESCE(
|
|
121
|
+
NULLIF(user_key, ''),
|
|
122
|
+
NULLIF(user_id, ''),
|
|
123
|
+
NULLIF(anonymous_id, '')
|
|
124
|
+
) AS user_key
|
|
125
|
+
FROM analytics_events
|
|
126
|
+
WHERE COALESCE(
|
|
127
|
+
NULLIF(user_key, ''),
|
|
128
|
+
NULLIF(user_id, ''),
|
|
129
|
+
NULLIF(anonymous_id, '')
|
|
130
|
+
) IS NOT NULL
|
|
131
|
+
) AS historical_user_days
|
|
132
|
+
WHERE event_date <> '' AND user_key <> ''
|
|
133
|
+
ON CONFLICT (tenant_key, event_date, user_key) DO NOTHING
|
|
134
|
+
`,
|
|
135
|
+
] as const;
|
|
136
|
+
|
|
137
|
+
type BackfillStatus =
|
|
138
|
+
| "completed"
|
|
139
|
+
| "already-complete"
|
|
140
|
+
| "skipped-lock"
|
|
141
|
+
| "already-running";
|
|
142
|
+
|
|
143
|
+
export interface AnalyticsRollupBackfillResult {
|
|
144
|
+
status: BackfillStatus;
|
|
145
|
+
remaining: number;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
let running = false;
|
|
149
|
+
|
|
150
|
+
function nowSql(): string {
|
|
151
|
+
return isPostgres() ? "now()::text" : "datetime('now')";
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function ensureStateSql(): string {
|
|
155
|
+
return isPostgres()
|
|
156
|
+
? `INSERT INTO analytics_rollup_backfill_state (id, status, updated_at)
|
|
157
|
+
VALUES (?, 'pending', ${nowSql()})
|
|
158
|
+
ON CONFLICT (id) DO NOTHING`
|
|
159
|
+
: `INSERT OR IGNORE INTO analytics_rollup_backfill_state
|
|
160
|
+
(id, status, updated_at)
|
|
161
|
+
VALUES (?, 'pending', ${nowSql()})`;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function createBackfillLeaseToken(): string {
|
|
165
|
+
if (typeof globalThis.crypto?.randomUUID === "function") {
|
|
166
|
+
return globalThis.crypto.randomUUID();
|
|
167
|
+
}
|
|
168
|
+
return `analytics-rollup-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function claimStateSql(): string {
|
|
172
|
+
return `UPDATE analytics_rollup_backfill_state
|
|
173
|
+
SET status = 'running', lease_token = ?,
|
|
174
|
+
lease_expires_at = datetime('now', '+${BACKFILL_LEASE_MINUTES} minutes'),
|
|
175
|
+
updated_at = datetime('now')
|
|
176
|
+
WHERE id = ?
|
|
177
|
+
AND (
|
|
178
|
+
status = 'pending'
|
|
179
|
+
OR (
|
|
180
|
+
status = 'running'
|
|
181
|
+
AND (
|
|
182
|
+
lease_expires_at IS NULL
|
|
183
|
+
OR lease_expires_at <= datetime('now')
|
|
184
|
+
)
|
|
185
|
+
)
|
|
186
|
+
)`;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function releaseStateLeaseSql(): string {
|
|
190
|
+
return `UPDATE analytics_rollup_backfill_state
|
|
191
|
+
SET status = 'pending', lease_token = NULL,
|
|
192
|
+
lease_expires_at = NULL, updated_at = datetime('now')
|
|
193
|
+
WHERE id = ? AND lease_token = ?`;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async function stateStatus(db: {
|
|
197
|
+
execute: (query: { sql: string; args: unknown[] }) => Promise<{
|
|
198
|
+
rows: any[];
|
|
199
|
+
rowsAffected: number;
|
|
200
|
+
}>;
|
|
201
|
+
}): Promise<string | null> {
|
|
202
|
+
const result = await db.execute({
|
|
203
|
+
sql: "SELECT status FROM analytics_rollup_backfill_state WHERE id = ?",
|
|
204
|
+
args: [BACKFILL_STATE_ID],
|
|
205
|
+
});
|
|
206
|
+
const status = result.rows[0]?.status;
|
|
207
|
+
return typeof status === "string" ? status : null;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async function completeState(tx: {
|
|
211
|
+
execute: (query: { sql: string; args: unknown[] }) => Promise<unknown>;
|
|
212
|
+
}): Promise<void> {
|
|
213
|
+
await tx.execute({
|
|
214
|
+
sql: `UPDATE analytics_rollup_backfill_state
|
|
215
|
+
SET status = 'completed', completed_at = ${nowSql()}, updated_at = ${nowSql()}
|
|
216
|
+
WHERE id = ?`,
|
|
217
|
+
args: [BACKFILL_STATE_ID],
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
async function runBackfillStatements(tx: {
|
|
222
|
+
execute: (query: string) => Promise<unknown>;
|
|
223
|
+
}): Promise<void> {
|
|
224
|
+
const statements = isPostgres()
|
|
225
|
+
? POSTGRES_BACKFILL_STATEMENTS
|
|
226
|
+
: SQLITE_BACKFILL_STATEMENTS;
|
|
227
|
+
for (const statement of statements) await tx.execute(statement);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export async function isHistoricalAnalyticsRollupBackfillComplete(): Promise<boolean> {
|
|
231
|
+
return (await stateStatus(getDbExec())) === "completed";
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function runTransactionalBackfill(
|
|
235
|
+
db: ReturnType<typeof getDbExec>,
|
|
236
|
+
): Promise<BackfillStatus> {
|
|
237
|
+
if (!db.transaction) {
|
|
238
|
+
throw new Error(
|
|
239
|
+
"Analytics rollup backfill requires a database transaction",
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
return db.transaction(async (tx) => {
|
|
244
|
+
if (isPostgres()) {
|
|
245
|
+
const lockResult = await tx.execute({
|
|
246
|
+
sql: "SELECT pg_try_advisory_xact_lock(hashtextextended(?, 0::bigint)) AS acquired",
|
|
247
|
+
args: [FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY],
|
|
248
|
+
});
|
|
249
|
+
const acquired = lockResult.rows[0]?.acquired;
|
|
250
|
+
if (acquired !== true && acquired !== "t") return "skipped-lock";
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
await tx.execute({ sql: ensureStateSql(), args: [BACKFILL_STATE_ID] });
|
|
254
|
+
if ((await stateStatus(tx)) === "completed") return "already-complete";
|
|
255
|
+
|
|
256
|
+
await runBackfillStatements(tx);
|
|
257
|
+
await completeState(tx);
|
|
258
|
+
return "completed";
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async function runD1Backfill(
|
|
263
|
+
db: ReturnType<typeof getDbExec>,
|
|
264
|
+
): Promise<BackfillStatus> {
|
|
265
|
+
if (!db.atomicBatch) {
|
|
266
|
+
throw new Error("D1 Analytics rollup backfill requires an atomic batch");
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
await db.execute({ sql: ensureStateSql(), args: [BACKFILL_STATE_ID] });
|
|
270
|
+
const leaseToken = createBackfillLeaseToken();
|
|
271
|
+
const claim = await db.execute({
|
|
272
|
+
sql: claimStateSql(),
|
|
273
|
+
args: [leaseToken, BACKFILL_STATE_ID],
|
|
274
|
+
});
|
|
275
|
+
if (claim.rowsAffected !== 1) {
|
|
276
|
+
return (await stateStatus(db)) === "completed"
|
|
277
|
+
? "already-complete"
|
|
278
|
+
: "skipped-lock";
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
try {
|
|
282
|
+
const results = await db.atomicBatch([
|
|
283
|
+
...SQLITE_BACKFILL_STATEMENTS,
|
|
284
|
+
{
|
|
285
|
+
sql: `UPDATE analytics_rollup_backfill_state
|
|
286
|
+
SET status = 'completed', completed_at = ${nowSql()},
|
|
287
|
+
lease_token = NULL, lease_expires_at = NULL,
|
|
288
|
+
updated_at = ${nowSql()}
|
|
289
|
+
WHERE id = ? AND status = 'running' AND lease_token = ?`,
|
|
290
|
+
args: [BACKFILL_STATE_ID, leaseToken],
|
|
291
|
+
},
|
|
292
|
+
]);
|
|
293
|
+
if (results[results.length - 1]?.rowsAffected !== 1) {
|
|
294
|
+
throw new Error(
|
|
295
|
+
"Analytics rollup backfill lost its D1 lease before completion",
|
|
296
|
+
);
|
|
297
|
+
}
|
|
298
|
+
return "completed";
|
|
299
|
+
} catch (error) {
|
|
300
|
+
try {
|
|
301
|
+
await db.execute({
|
|
302
|
+
sql: releaseStateLeaseSql(),
|
|
303
|
+
args: [BACKFILL_STATE_ID, leaseToken],
|
|
304
|
+
});
|
|
305
|
+
} catch (releaseError) {
|
|
306
|
+
console.warn(
|
|
307
|
+
"[analytics-rollup-backfill] Failed to release D1 lease after backfill error:",
|
|
308
|
+
releaseError instanceof Error ? releaseError.message : releaseError,
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
throw error;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Rebuild historical compact rollups outside the server boot path. The state
|
|
317
|
+
* row is completed in the same transaction as both aggregates, so a timeout
|
|
318
|
+
* or failed write leaves the job pending and a later scheduled run retries it.
|
|
319
|
+
* D1 claims a short lease before its atomic batch so concurrent isolates do not
|
|
320
|
+
* each scan the full event history.
|
|
321
|
+
*/
|
|
322
|
+
export async function runAnalyticsRollupBackfillOnce(): Promise<AnalyticsRollupBackfillResult> {
|
|
323
|
+
if (running) {
|
|
324
|
+
return { status: "already-running", remaining: 1 };
|
|
325
|
+
}
|
|
326
|
+
running = true;
|
|
327
|
+
|
|
328
|
+
try {
|
|
329
|
+
const db = getDbExec();
|
|
330
|
+
const status =
|
|
331
|
+
getDialect() === "d1"
|
|
332
|
+
? await runD1Backfill(db)
|
|
333
|
+
: await runTransactionalBackfill(db);
|
|
334
|
+
return {
|
|
335
|
+
status,
|
|
336
|
+
remaining:
|
|
337
|
+
status === "completed" || status === "already-complete" ? 0 : 1,
|
|
338
|
+
};
|
|
339
|
+
} finally {
|
|
340
|
+
running = false;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared with the historical backfill so live ingest and migration work hold
|
|
3
|
+
* the same transaction-scoped Postgres lock while they update rollups.
|
|
4
|
+
*/
|
|
5
|
+
export const FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY =
|
|
6
|
+
"agent-native:analytics-rollup-backfill";
|
|
7
|
+
export const FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL =
|
|
8
|
+
"SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))";
|