@agent-native/core 0.145.7 → 0.146.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/corpus/README.md +1 -1
  2. package/corpus/templates/analytics/.agents/skills/frontend-design/SKILL.md +5 -1
  3. package/corpus/templates/analytics/.agents/skills/turn-into-app/SKILL.md +37 -11
  4. package/corpus/templates/analytics/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  5. package/corpus/templates/analytics/actions/get-sql-dashboard.ts +20 -4
  6. package/corpus/templates/analytics/actions/migrate-first-party-analytics-to-bigquery.ts +59 -24
  7. package/corpus/templates/analytics/app/pages/settings/AlertRulesSettingsCard.tsx +3 -2
  8. package/corpus/templates/analytics/changelog/2026-08-07-analytics-migrations-now-use-bounded-pressure-aware-backgrou.md +6 -0
  9. package/corpus/templates/analytics/changelog/2026-08-07-settings-skeleton-placeholders.md +6 -0
  10. package/corpus/templates/analytics/changelog/2026-08-07-template-activity-charts-now-show-only-core-agent-native-app.md +6 -0
  11. package/corpus/templates/analytics/netlify.toml +1 -1
  12. package/corpus/templates/analytics/package.json +1 -0
  13. package/corpus/templates/analytics/scripts/emit-netlify-dashboard-report-cron.ts +110 -0
  14. package/corpus/templates/analytics/scripts/migrate-production.ts +24 -0
  15. package/corpus/templates/analytics/seeds/dashboards/agent-native-templates-first-party.json +13 -13
  16. package/corpus/templates/analytics/server/jobs/analytics-bigquery-backfill.ts +448 -0
  17. package/corpus/templates/analytics/server/jobs/analytics-rollup-backfill.ts +11 -2
  18. package/corpus/templates/analytics/server/lib/analytics-rollup-lock.ts +4 -5
  19. package/corpus/templates/analytics/server/lib/canonical-first-party-dashboard-repair.ts +35 -0
  20. package/corpus/templates/analytics/server/lib/dashboard-catalog.ts +3 -5
  21. package/corpus/templates/analytics/server/lib/first-party-analytics-backend.ts +10 -4
  22. package/corpus/templates/analytics/server/lib/first-party-analytics-rollups.ts +4 -11
  23. package/corpus/templates/analytics/server/lib/first-party-metric-catalog.ts +74 -28
  24. package/corpus/templates/analytics/server/plugins/analytics-rollup-backfill-jobs.ts +2 -5
  25. package/corpus/templates/analytics/server/plugins/db.ts +56 -3
  26. package/corpus/templates/analytics/server/routes/api/analytics-bigquery/backfill.post.ts +55 -0
  27. package/corpus/templates/analytics/server/routes/api/dashboard-agent-context.json.get.ts +22 -4
  28. package/corpus/templates/assets/.agents/skills/frontend-design/SKILL.md +5 -1
  29. package/corpus/templates/assets/.agents/skills/turn-into-app/SKILL.md +37 -11
  30. package/corpus/templates/assets/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  31. package/corpus/templates/brain/.agents/skills/frontend-design/SKILL.md +5 -1
  32. package/corpus/templates/brain/.agents/skills/turn-into-app/SKILL.md +37 -11
  33. package/corpus/templates/brain/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  34. package/corpus/templates/brain/server/lib/connectors.ts +38 -14
  35. package/corpus/templates/calendar/.agents/skills/frontend-design/SKILL.md +5 -1
  36. package/corpus/templates/calendar/.agents/skills/turn-into-app/SKILL.md +37 -11
  37. package/corpus/templates/calendar/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  38. package/corpus/templates/chat/.agents/skills/frontend-design/SKILL.md +5 -1
  39. package/corpus/templates/chat/.agents/skills/turn-into-app/SKILL.md +37 -11
  40. package/corpus/templates/chat/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  41. package/corpus/templates/chat/AGENTS.md +3 -0
  42. package/corpus/templates/clips/.agents/skills/frontend-design/SKILL.md +5 -1
  43. package/corpus/templates/clips/.agents/skills/turn-into-app/SKILL.md +37 -11
  44. package/corpus/templates/clips/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  45. package/corpus/templates/content/.agents/skills/frontend-design/SKILL.md +5 -1
  46. package/corpus/templates/content/.agents/skills/turn-into-app/SKILL.md +37 -11
  47. package/corpus/templates/content/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  48. package/corpus/templates/crm/.agents/skills/turn-into-app/SKILL.md +37 -11
  49. package/corpus/templates/crm/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  50. package/corpus/templates/crm/app/components/crm/IntelligenceSettings.tsx +21 -3
  51. package/corpus/templates/crm/app/components/crm/settings/ConnectionSettings.tsx +23 -5
  52. package/corpus/templates/crm/app/components/crm/settings/FieldsSettings.tsx +24 -5
  53. package/corpus/templates/crm/app/components/crm/settings/ListsSettings.tsx +23 -3
  54. package/corpus/templates/crm/changelog/2026-08-07-settings-loading-skeletons.md +6 -0
  55. package/corpus/templates/design/.agents/skills/frontend-design/SKILL.md +5 -1
  56. package/corpus/templates/design/.agents/skills/turn-into-app/SKILL.md +37 -11
  57. package/corpus/templates/design/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  58. package/corpus/templates/dispatch/.agents/skills/frontend-design/SKILL.md +5 -1
  59. package/corpus/templates/dispatch/.agents/skills/turn-into-app/SKILL.md +37 -11
  60. package/corpus/templates/dispatch/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  61. package/corpus/templates/factory/.agents/skills/frontend-design/SKILL.md +5 -1
  62. package/corpus/templates/factory/.agents/skills/turn-into-app/SKILL.md +37 -11
  63. package/corpus/templates/factory/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  64. package/corpus/templates/forms/.agents/skills/frontend-design/SKILL.md +5 -1
  65. package/corpus/templates/forms/.agents/skills/turn-into-app/SKILL.md +37 -11
  66. package/corpus/templates/forms/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  67. package/corpus/templates/macros/.agents/skills/frontend-design/SKILL.md +5 -1
  68. package/corpus/templates/macros/.agents/skills/turn-into-app/SKILL.md +37 -11
  69. package/corpus/templates/macros/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  70. package/corpus/templates/mail/.agents/skills/frontend-design/SKILL.md +5 -1
  71. package/corpus/templates/mail/.agents/skills/turn-into-app/SKILL.md +37 -11
  72. package/corpus/templates/mail/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  73. package/corpus/templates/plan/.agents/skills/frontend-design/SKILL.md +5 -1
  74. package/corpus/templates/plan/.agents/skills/turn-into-app/SKILL.md +37 -11
  75. package/corpus/templates/plan/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  76. package/corpus/templates/slides/.agents/skills/frontend-design/SKILL.md +5 -1
  77. package/corpus/templates/slides/.agents/skills/turn-into-app/SKILL.md +37 -11
  78. package/corpus/templates/slides/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  79. package/corpus/templates/slides/app/components/editor/EditorSidebar.tsx +13 -0
  80. package/corpus/templates/slides/app/components/editor/GeneratingSlidePreview.tsx +5 -0
  81. package/corpus/templates/slides/app/components/presentation/PresentationView.tsx +5 -0
  82. package/corpus/templates/slides/app/components/presentation/PresenterView.tsx +5 -0
  83. package/corpus/templates/slides/app/pages/DeckEditor.tsx +3 -0
  84. package/corpus/templates/slides/app/pages/Presentation.tsx +3 -0
  85. package/corpus/templates/slides/app/pages/SharedPresentation.tsx +5 -0
  86. package/corpus/templates/slides/app/routes/slide.tsx +10 -5
  87. package/corpus/templates/slides/changelog/2026-08-07-slide-previews-and-present-now-keep-deck-colors-consistent.md +6 -0
  88. package/corpus/templates/slides/server/db/schema.ts +1 -0
  89. package/corpus/templates/slides/server/handlers/share.ts +51 -3
  90. package/corpus/templates/slides/server/plugins/db.ts +5 -0
  91. package/corpus/templates/slides/shared/api.ts +2 -0
  92. package/corpus/templates/tasks/.agents/skills/frontend-design/SKILL.md +5 -1
  93. package/corpus/templates/tasks/.agents/skills/turn-into-app/SKILL.md +37 -11
  94. package/corpus/templates/tasks/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  95. package/dist/cli/code-agent-executor.js +8 -3
  96. package/dist/client/ConnectBuilderCard.d.ts +7 -7
  97. package/dist/client/ConnectBuilderCard.js +45 -20
  98. package/dist/client/components/ApiKeySettings.js +2 -5
  99. package/dist/client/org/TeamPage.js +4 -2
  100. package/dist/client/settings/AccountSettingsCard.js +2 -1
  101. package/dist/client/settings/AgentsSection.js +2 -1
  102. package/dist/client/settings/SecretsSection.js +3 -2
  103. package/dist/client/settings/SettingsPanel.js +6 -12
  104. package/dist/client/settings/SettingsSkeleton.d.ts +12 -0
  105. package/dist/client/settings/SettingsSkeleton.js +12 -0
  106. package/dist/client/settings/VoiceTranscriptionSection.js +5 -4
  107. package/dist/client/settings/index.d.ts +1 -0
  108. package/dist/client/settings/index.js +1 -0
  109. package/dist/config.d.ts +9 -2
  110. package/dist/config.js +38 -2
  111. package/dist/db/client.d.ts +26 -8
  112. package/dist/db/client.js +200 -9
  113. package/dist/db/create-get-db.js +2 -2
  114. package/dist/db/ddl-guard.js +3 -1
  115. package/dist/db/ensure-additive-columns.d.ts +2 -0
  116. package/dist/db/ensure-additive-columns.js +5 -2
  117. package/dist/db/index.d.ts +2 -2
  118. package/dist/db/index.js +2 -2
  119. package/dist/db/migrations.d.ts +11 -54
  120. package/dist/db/migrations.js +70 -0
  121. package/dist/deploy/build.js +5 -1
  122. package/dist/jobs/index.d.ts +1 -0
  123. package/dist/jobs/index.js +1 -0
  124. package/dist/jobs/run-history.d.ts +4 -0
  125. package/dist/jobs/run-history.js +44 -0
  126. package/dist/provider-api/actions/provider-api.d.ts +4 -4
  127. package/dist/scripts/docs/search.js +3 -2
  128. package/dist/secrets/routes.d.ts +6 -6
  129. package/dist/server/agent-chat/prompt-resources.js +4 -3
  130. package/dist/server/agent-chat-plugin.js +6 -4
  131. package/dist/server/agents-bundle.d.ts +20 -3
  132. package/dist/server/agents-bundle.js +57 -30
  133. package/dist/server/auth-plugin.js +5 -0
  134. package/dist/server/better-auth-instance.js +2 -63
  135. package/dist/server/better-auth-migrations.d.ts +8 -0
  136. package/dist/server/better-auth-migrations.js +179 -0
  137. package/dist/server/edge.d.ts +2 -0
  138. package/dist/server/edge.js +2 -0
  139. package/dist/server/index.d.ts +2 -0
  140. package/dist/server/index.js +2 -0
  141. package/dist/server/release-migrations.d.ts +8 -0
  142. package/dist/server/release-migrations.js +24 -0
  143. package/dist/server/sweep-runtime.d.ts +1 -1
  144. package/dist/server/sweep-runtime.js +15 -0
  145. package/dist/shared/runtime-config.js +1 -1
  146. package/dist/templates/chat/.agents/skills/frontend-design/SKILL.md +5 -1
  147. package/dist/templates/chat/.agents/skills/turn-into-app/SKILL.md +37 -11
  148. package/dist/templates/chat/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  149. package/dist/templates/chat/AGENTS.md +3 -0
  150. package/dist/templates/default/.agents/skills/frontend-design/SKILL.md +5 -1
  151. package/dist/templates/default/.agents/skills/turn-into-app/SKILL.md +37 -11
  152. package/dist/templates/default/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  153. package/dist/templates/default/AGENTS.md +3 -0
  154. package/dist/templates/headless/.agents/skills/turn-into-app/SKILL.md +37 -11
  155. package/dist/templates/headless/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  156. package/dist/templates/workspace-core/.agents/skills/frontend-design/SKILL.md +5 -1
  157. package/dist/templates/workspace-core/.agents/skills/turn-into-app/SKILL.md +37 -11
  158. package/dist/templates/workspace-core/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  159. package/dist/templates/workspace-core/AGENTS.md +3 -0
  160. package/dist/templates/workspace-root/AGENTS.md +3 -0
  161. package/dist/vite/agent-native-config-loader.d.ts +10 -0
  162. package/dist/vite/agent-native-config-loader.js +56 -0
  163. package/dist/vite/agents-bundle-plugin.d.ts +5 -1
  164. package/dist/vite/agents-bundle-plugin.js +29 -6
  165. package/dist/vite/client.d.ts +4 -2
  166. package/dist/vite/client.js +4 -49
  167. package/dist/vite/index.d.ts +2 -2
  168. package/dist/vite/index.js +1 -1
  169. package/docs/content/agent-native-config.mdx +43 -17
  170. package/docs/content/environment-variables.mdx +1 -1
  171. package/package.json +3 -3
  172. package/src/templates/chat/.agents/skills/frontend-design/SKILL.md +5 -1
  173. package/src/templates/chat/.agents/skills/turn-into-app/SKILL.md +37 -11
  174. package/src/templates/chat/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  175. package/src/templates/chat/AGENTS.md +3 -0
  176. package/src/templates/default/.agents/skills/frontend-design/SKILL.md +5 -1
  177. package/src/templates/default/.agents/skills/turn-into-app/SKILL.md +37 -11
  178. package/src/templates/default/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  179. package/src/templates/default/AGENTS.md +3 -0
  180. package/src/templates/headless/.agents/skills/turn-into-app/SKILL.md +37 -11
  181. package/src/templates/headless/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  182. package/src/templates/workspace-core/.agents/skills/frontend-design/SKILL.md +5 -1
  183. package/src/templates/workspace-core/.agents/skills/turn-into-app/SKILL.md +37 -11
  184. package/src/templates/workspace-core/.agents/skills/turn-into-app/references/fresh-project.md +13 -2
  185. package/src/templates/workspace-core/AGENTS.md +3 -0
  186. package/src/templates/workspace-root/AGENTS.md +3 -0
@@ -0,0 +1,448 @@
1
+ import { randomUUID } from "node:crypto";
2
+
3
+ import { getDbExec, isPostgres } from "@agent-native/core/db";
4
+ import { runWithRequestContext } from "@agent-native/core/server";
5
+
6
+ import {
7
+ backfillFirstPartyAnalyticsBatch,
8
+ type FirstPartyAnalyticsScope,
9
+ } from "../lib/first-party-analytics-backend.js";
10
+
11
+ const JOB_TABLE = "analytics_bigquery_backfill_jobs";
12
+ const LEASE_MS = 5 * 60 * 1000;
13
+ const ERROR_RETRY_MS = 5 * 60 * 1000;
14
+ const DEFAULT_BATCH_SIZE = 250;
15
+ const MAX_BATCH_SIZE = 750;
16
+ const DEFAULT_MAX_BATCHES_PER_SWEEP = 2;
17
+ const MAX_BATCHES_PER_SWEEP = 4;
18
+ const DEFAULT_MAX_ACTIVE_SESSIONS = 80;
19
+ const DEFAULT_MAX_TOTAL_SESSIONS = 250;
20
+
21
+ type Query =
22
+ | string
23
+ | {
24
+ sql: string;
25
+ args?: unknown[];
26
+ timeoutMs?: number;
27
+ maxAttempts?: number;
28
+ };
29
+
30
+ interface QueryResult {
31
+ rows?: unknown[];
32
+ rowsAffected?: number;
33
+ }
34
+
35
+ interface Executor {
36
+ execute(query: Query): Promise<QueryResult>;
37
+ transaction?<T>(fn: (tx: Executor) => Promise<T>): Promise<T>;
38
+ }
39
+
40
+ export type BigQueryBackfillJobStatus = "pending" | "running" | "completed";
41
+
42
+ export interface BigQueryBackfillJob {
43
+ id: string;
44
+ orgId: string;
45
+ ownerEmail: string;
46
+ table: string;
47
+ batchSize: number;
48
+ cursor: string | null;
49
+ status: BigQueryBackfillJobStatus;
50
+ copied: number;
51
+ leaseToken: string | null;
52
+ leaseExpiresAt: string | null;
53
+ nextRunAt: string;
54
+ lastError: string | null;
55
+ completedAt: string | null;
56
+ updatedAt: string;
57
+ }
58
+
59
+ export interface BigQueryBackfillSweepResult {
60
+ status:
61
+ | "disabled"
62
+ | "idle"
63
+ | "paused-pressure"
64
+ | "progress"
65
+ | "completed"
66
+ | "retry-scheduled";
67
+ batches: number;
68
+ copied: number;
69
+ remaining: number;
70
+ reason?: string;
71
+ error?: string;
72
+ }
73
+
74
+ function executor(): Executor {
75
+ return getDbExec() as unknown as Executor;
76
+ }
77
+
78
+ function jobId(orgId: string): string {
79
+ return `first-party-analytics:${orgId}`;
80
+ }
81
+
82
+ function boundedBatchSize(value: number | undefined): number {
83
+ if (!Number.isFinite(value)) return DEFAULT_BATCH_SIZE;
84
+ return Math.min(Math.max(Math.floor(value as number), 1), MAX_BATCH_SIZE);
85
+ }
86
+
87
+ function maxBatchesPerSweep(): number {
88
+ const raw = process.env.ANALYTICS_BIGQUERY_BACKFILL_SWEEP_LIMIT?.trim();
89
+ if (!raw) return DEFAULT_MAX_BATCHES_PER_SWEEP;
90
+ const parsed = Number.parseInt(raw, 10);
91
+ return Number.isFinite(parsed) && parsed > 0
92
+ ? Math.min(parsed, MAX_BATCHES_PER_SWEEP)
93
+ : DEFAULT_MAX_BATCHES_PER_SWEEP;
94
+ }
95
+
96
+ function positiveEnvNumber(name: string, fallback: number): number {
97
+ const raw = process.env[name]?.trim();
98
+ if (!raw) return fallback;
99
+ const parsed = Number.parseInt(raw, 10);
100
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
101
+ }
102
+
103
+ function stringValue(row: Record<string, unknown>, ...keys: string[]): string {
104
+ for (const key of keys) {
105
+ const value = row[key];
106
+ if (typeof value === "string") return value;
107
+ }
108
+ return "";
109
+ }
110
+
111
+ function nullableStringValue(
112
+ row: Record<string, unknown>,
113
+ ...keys: string[]
114
+ ): string | null {
115
+ const value = stringValue(row, ...keys);
116
+ return value || null;
117
+ }
118
+
119
+ function numberValue(row: Record<string, unknown>, ...keys: string[]): number {
120
+ for (const key of keys) {
121
+ const value = row[key];
122
+ const parsed = typeof value === "number" ? value : Number(value);
123
+ if (Number.isFinite(parsed)) return parsed;
124
+ }
125
+ return 0;
126
+ }
127
+
128
+ function rowToJob(row: Record<string, unknown>): BigQueryBackfillJob {
129
+ const status = stringValue(row, "status");
130
+ if (status !== "pending" && status !== "running" && status !== "completed") {
131
+ throw new Error(
132
+ `Unknown BigQuery backfill job status: ${status || "empty"}`,
133
+ );
134
+ }
135
+ return {
136
+ id: stringValue(row, "id"),
137
+ orgId: stringValue(row, "org_id", "orgId"),
138
+ ownerEmail: stringValue(row, "owner_email", "ownerEmail"),
139
+ table: stringValue(row, "table_ref", "tableRef"),
140
+ batchSize: boundedBatchSize(numberValue(row, "batch_size", "batchSize")),
141
+ cursor: nullableStringValue(row, "backfill_cursor", "backfillCursor"),
142
+ status,
143
+ copied: numberValue(row, "copied_count", "copiedCount"),
144
+ leaseToken: nullableStringValue(row, "lease_token", "leaseToken"),
145
+ leaseExpiresAt: nullableStringValue(
146
+ row,
147
+ "lease_expires_at",
148
+ "leaseExpiresAt",
149
+ ),
150
+ nextRunAt: stringValue(row, "next_run_at", "nextRunAt"),
151
+ lastError: nullableStringValue(row, "last_error", "lastError"),
152
+ completedAt: nullableStringValue(row, "completed_at", "completedAt"),
153
+ updatedAt: stringValue(row, "updated_at", "updatedAt"),
154
+ };
155
+ }
156
+
157
+ function rowFromResult(result: QueryResult): Record<string, unknown> | null {
158
+ const row = result.rows?.[0];
159
+ return row && typeof row === "object"
160
+ ? (row as Record<string, unknown>)
161
+ : null;
162
+ }
163
+
164
+ export async function getFirstPartyAnalyticsBigQueryBackfillJob(
165
+ scope: FirstPartyAnalyticsScope,
166
+ ): Promise<BigQueryBackfillJob | null> {
167
+ const result = await executor().execute({
168
+ sql: `SELECT id, org_id, owner_email, table_ref, batch_size,
169
+ backfill_cursor, status, copied_count, lease_token,
170
+ lease_expires_at, next_run_at, last_error, completed_at,
171
+ updated_at
172
+ FROM ${JOB_TABLE}
173
+ WHERE id = ?
174
+ LIMIT 1`,
175
+ args: [jobId(scope.orgId ?? "")],
176
+ timeoutMs: 3_000,
177
+ maxAttempts: 1,
178
+ });
179
+ const row = rowFromResult(result);
180
+ return row ? rowToJob(row) : null;
181
+ }
182
+
183
+ export async function queueFirstPartyAnalyticsBigQueryBackfill(
184
+ scope: FirstPartyAnalyticsScope,
185
+ table: string,
186
+ requestedBatchSize?: number,
187
+ ): Promise<BigQueryBackfillJob> {
188
+ const now = new Date().toISOString();
189
+ const id = jobId(scope.orgId ?? "");
190
+ if (!scope.orgId)
191
+ throw new Error("BigQuery backfill requires an organization");
192
+ await executor().execute({
193
+ sql: `INSERT INTO ${JOB_TABLE} (
194
+ id, org_id, owner_email, table_ref, batch_size, backfill_cursor,
195
+ status, copied_count, lease_token, lease_expires_at, next_run_at,
196
+ last_error, completed_at, updated_at
197
+ ) VALUES (?, ?, ?, ?, ?, NULL, 'pending', 0, NULL, NULL, ?, NULL, NULL, ?)
198
+ ON CONFLICT (id) DO NOTHING`,
199
+ args: [
200
+ id,
201
+ scope.orgId,
202
+ scope.userEmail,
203
+ table,
204
+ boundedBatchSize(requestedBatchSize),
205
+ now,
206
+ now,
207
+ ],
208
+ timeoutMs: 5_000,
209
+ maxAttempts: 1,
210
+ });
211
+ const job = await getFirstPartyAnalyticsBigQueryBackfillJob(scope);
212
+ if (!job) throw new Error("BigQuery backfill job was not persisted");
213
+ if (job.table !== table) {
214
+ throw new Error(
215
+ `BigQuery backfill already targets ${job.table}; prepare the existing migration before changing tables`,
216
+ );
217
+ }
218
+ return job;
219
+ }
220
+
221
+ async function claimNextJob(
222
+ db: Executor,
223
+ now: string,
224
+ ): Promise<BigQueryBackfillJob | null> {
225
+ if (!db.transaction) {
226
+ throw new Error("BigQuery backfill requires a database transaction");
227
+ }
228
+ const token = randomUUID();
229
+ const leaseExpiresAt = new Date(Date.now() + LEASE_MS).toISOString();
230
+ return db.transaction(async (tx) => {
231
+ const candidateResult = await tx.execute({
232
+ sql: `SELECT id, org_id, owner_email, table_ref, batch_size,
233
+ backfill_cursor, status, copied_count, lease_token,
234
+ lease_expires_at, next_run_at, last_error, completed_at,
235
+ updated_at
236
+ FROM ${JOB_TABLE}
237
+ WHERE next_run_at <= ?
238
+ AND (
239
+ status = 'pending'
240
+ OR (status = 'running' AND lease_expires_at IS NOT NULL
241
+ AND lease_expires_at <= ?)
242
+ )
243
+ ORDER BY updated_at ASC
244
+ LIMIT 1`,
245
+ args: [now, now],
246
+ timeoutMs: 3_000,
247
+ maxAttempts: 1,
248
+ });
249
+ const candidate = rowFromResult(candidateResult);
250
+ if (!candidate) return null;
251
+
252
+ const updated = await tx.execute({
253
+ sql: `UPDATE ${JOB_TABLE}
254
+ SET status = 'running', lease_token = ?, lease_expires_at = ?,
255
+ updated_at = ?
256
+ WHERE id = ?
257
+ AND next_run_at <= ?
258
+ AND (
259
+ status = 'pending'
260
+ OR (status = 'running' AND lease_expires_at IS NOT NULL
261
+ AND lease_expires_at <= ?)
262
+ )`,
263
+ args: [token, leaseExpiresAt, now, candidate.id, now, now],
264
+ timeoutMs: 3_000,
265
+ maxAttempts: 1,
266
+ });
267
+ if (updated.rowsAffected !== 1) return null;
268
+ return rowToJob({
269
+ ...candidate,
270
+ status: "running",
271
+ lease_token: token,
272
+ lease_expires_at: leaseExpiresAt,
273
+ updated_at: now,
274
+ });
275
+ });
276
+ }
277
+
278
+ async function finishJob(
279
+ db: Executor,
280
+ job: BigQueryBackfillJob,
281
+ result: { nextCursor: string | null; copied: number; complete: boolean },
282
+ now: string,
283
+ ): Promise<void> {
284
+ const completedAt = result.complete ? now : null;
285
+ const updated = await db.execute({
286
+ sql: `UPDATE ${JOB_TABLE}
287
+ SET status = ?, backfill_cursor = ?, copied_count = copied_count + ?,
288
+ lease_token = NULL, lease_expires_at = NULL,
289
+ next_run_at = ?, last_error = NULL, completed_at = ?,
290
+ updated_at = ?
291
+ WHERE id = ? AND lease_token = ?`,
292
+ args: [
293
+ result.complete ? "completed" : "pending",
294
+ result.nextCursor,
295
+ result.copied,
296
+ now,
297
+ completedAt,
298
+ now,
299
+ job.id,
300
+ job.leaseToken,
301
+ ],
302
+ timeoutMs: 5_000,
303
+ maxAttempts: 1,
304
+ });
305
+ if (updated.rowsAffected !== 1) {
306
+ throw new Error("BigQuery backfill lost its lease before saving progress");
307
+ }
308
+ }
309
+
310
+ async function scheduleRetry(
311
+ db: Executor,
312
+ job: BigQueryBackfillJob,
313
+ error: string,
314
+ now: string,
315
+ ): Promise<void> {
316
+ const retryAt = new Date(Date.now() + ERROR_RETRY_MS).toISOString();
317
+ await db.execute({
318
+ sql: `UPDATE ${JOB_TABLE}
319
+ SET status = 'pending', lease_token = NULL, lease_expires_at = NULL,
320
+ next_run_at = ?, last_error = ?, updated_at = ?
321
+ WHERE id = ? AND lease_token = ?`,
322
+ args: [retryAt, error.slice(0, 1_000), now, job.id, job.leaseToken],
323
+ timeoutMs: 5_000,
324
+ maxAttempts: 1,
325
+ });
326
+ }
327
+
328
+ async function pressureSnapshot(
329
+ db: Executor,
330
+ ): Promise<{ paused: boolean; reason?: string }> {
331
+ if (!isPostgres()) return { paused: false };
332
+ try {
333
+ const result = await db.execute({
334
+ sql: `SELECT
335
+ COUNT(*) AS total_sessions,
336
+ SUM(CASE WHEN state = 'active' THEN 1 ELSE 0 END) AS active_sessions,
337
+ SUM(CASE WHEN wait_event_type IS NOT NULL THEN 1 ELSE 0 END) AS waiting_sessions,
338
+ SUM(CASE WHEN wait_event_type = 'Lock' THEN 1 ELSE 0 END) AS lock_waiters
339
+ FROM pg_stat_activity
340
+ WHERE pid <> pg_backend_pid()`,
341
+ timeoutMs: 2_000,
342
+ maxAttempts: 1,
343
+ });
344
+ const row = rowFromResult(result);
345
+ if (!row) return { paused: true, reason: "pressure probe returned no row" };
346
+ const total = numberValue(row, "total_sessions");
347
+ const active = numberValue(row, "active_sessions");
348
+ const waiting = numberValue(row, "waiting_sessions");
349
+ const lockWaiters = numberValue(row, "lock_waiters");
350
+ const maxActive = positiveEnvNumber(
351
+ "ANALYTICS_BIGQUERY_BACKFILL_MAX_ACTIVE_SESSIONS",
352
+ DEFAULT_MAX_ACTIVE_SESSIONS,
353
+ );
354
+ const maxTotal = positiveEnvNumber(
355
+ "ANALYTICS_BIGQUERY_BACKFILL_MAX_TOTAL_SESSIONS",
356
+ DEFAULT_MAX_TOTAL_SESSIONS,
357
+ );
358
+ if (lockWaiters > 0 || active >= maxActive || total >= maxTotal) {
359
+ return {
360
+ paused: true,
361
+ reason: `database pressure: total=${total}, active=${active}, waiting=${waiting}, lockWaiters=${lockWaiters}`,
362
+ };
363
+ }
364
+ return { paused: false };
365
+ } catch (error) {
366
+ return {
367
+ paused: true,
368
+ reason: `pressure probe failed: ${error instanceof Error ? error.message : String(error)}`,
369
+ };
370
+ }
371
+ }
372
+
373
+ export async function runFirstPartyAnalyticsBigQueryBackfillOnce(): Promise<BigQueryBackfillSweepResult> {
374
+ if (process.env.ANALYTICS_BIGQUERY_BACKFILL_JOBS?.trim() === "0") {
375
+ return { status: "disabled", batches: 0, copied: 0, remaining: 0 };
376
+ }
377
+
378
+ const db = executor();
379
+ const initialPressure = await pressureSnapshot(db);
380
+ if (initialPressure.paused) {
381
+ return {
382
+ status: "paused-pressure",
383
+ batches: 0,
384
+ copied: 0,
385
+ remaining: 1,
386
+ reason: initialPressure.reason,
387
+ };
388
+ }
389
+
390
+ let batches = 0;
391
+ let copied = 0;
392
+ for (let index = 0; index < maxBatchesPerSweep(); index += 1) {
393
+ const pressure = await pressureSnapshot(db);
394
+ if (pressure.paused) {
395
+ return {
396
+ status: "paused-pressure",
397
+ batches,
398
+ copied,
399
+ remaining: 1,
400
+ reason: pressure.reason,
401
+ };
402
+ }
403
+
404
+ const job = await claimNextJob(db, new Date().toISOString());
405
+ if (!job) {
406
+ return {
407
+ status: batches ? "progress" : "idle",
408
+ batches,
409
+ copied,
410
+ remaining: batches ? 1 : 0,
411
+ };
412
+ }
413
+
414
+ try {
415
+ const batch = await runWithRequestContext(
416
+ { userEmail: job.ownerEmail, orgId: job.orgId },
417
+ () =>
418
+ backfillFirstPartyAnalyticsBatch(
419
+ { userEmail: job.ownerEmail, orgId: job.orgId },
420
+ job.cursor,
421
+ job.batchSize,
422
+ job.table,
423
+ ),
424
+ );
425
+ const now = new Date().toISOString();
426
+ // The lease row is the progress source of truth; the backend setting is
427
+ // only the migration configuration and cutover marker.
428
+ await finishJob(db, job, batch, now);
429
+ batches += 1;
430
+ copied += batch.copied;
431
+ if (batch.complete) {
432
+ return { status: "completed", batches, copied, remaining: 0 };
433
+ }
434
+ } catch (error) {
435
+ const message = error instanceof Error ? error.message : String(error);
436
+ await scheduleRetry(db, job, message, new Date().toISOString());
437
+ return {
438
+ status: "retry-scheduled",
439
+ batches,
440
+ copied,
441
+ remaining: 1,
442
+ error: message,
443
+ };
444
+ }
445
+ }
446
+
447
+ return { status: "progress", batches, copied, remaining: 1 };
448
+ }
@@ -1,6 +1,6 @@
1
1
  import { getDbExec, getDialect, isPostgres } from "@agent-native/core/db";
2
2
 
3
- import { FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY } from "../lib/analytics-rollup-lock";
3
+ import { FIRST_PARTY_ANALYTICS_ROLLUP_BACKFILL_LOCK_KEY } from "../lib/analytics-rollup-lock";
4
4
 
5
5
  const BACKFILL_STATE_ID = "historical-v1";
6
6
  const BACKFILL_LEASE_MINUTES = 15;
@@ -243,9 +243,18 @@ async function runTransactionalBackfill(
243
243
 
244
244
  return db.transaction(async (tx) => {
245
245
  if (isPostgres()) {
246
+ // This scans all of analytics_events (41 GB in production) in one
247
+ // GROUP BY, and the lease budgets 15 minutes for it. A role-level
248
+ // statement_timeout — the thing that stops a runaway dashboard query
249
+ // from holding every pooler connection — would otherwise kill this
250
+ // legitimate job partway through. SET LOCAL is scoped to this
251
+ // transaction and reverts on commit or rollback.
252
+ await tx.execute(
253
+ `SET LOCAL statement_timeout = '${BACKFILL_LEASE_MINUTES}min'`,
254
+ );
246
255
  const lockResult = await tx.execute({
247
256
  sql: "SELECT pg_try_advisory_xact_lock(hashtextextended(?, 0::bigint)) AS acquired",
248
- args: [FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY],
257
+ args: [FIRST_PARTY_ANALYTICS_ROLLUP_BACKFILL_LOCK_KEY],
249
258
  });
250
259
  const acquired = lockResult.rows[0]?.acquired;
251
260
  if (acquired !== true && acquired !== "t") return "skipped-lock";
@@ -1,8 +1,7 @@
1
1
  /**
2
- * Shared with the historical backfill so live ingest and migration work hold
3
- * the same transaction-scoped Postgres lock while they update rollups.
2
+ * Prevents concurrent historical backfills. Foreground ingest deliberately
3
+ * does not take this lock: a rebuild must never make request traffic queue
4
+ * behind a long-running transaction.
4
5
  */
5
- export const FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY =
6
+ export const FIRST_PARTY_ANALYTICS_ROLLUP_BACKFILL_LOCK_KEY =
6
7
  "agent-native:analytics-rollup-backfill";
7
- export const FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_SQL =
8
- "SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))";
@@ -1,7 +1,17 @@
1
1
  import {
2
+ CLI_COPIES_OVER_TIME_SQL,
3
+ DEMO_CLICKS_OVER_TIME_SQL,
4
+ LEGACY_CLI_COPIES_OVER_TIME_SQL,
5
+ LEGACY_DEMO_CLICKS_OVER_TIME_SQL,
6
+ LEGACY_PAGEVIEWS_OVER_TIME_SQL,
7
+ LEGACY_SIGNUPS_BY_TEMPLATE_SQL,
8
+ LEGACY_TEMPLATE_INTEREST_OVER_TIME_SQL,
9
+ PAGEVIEWS_OVER_TIME_SQL,
10
+ SIGNUPS_BY_TEMPLATE_SQL,
2
11
  LEGACY_SIGNUPS_OVER_TIME_SQL,
3
12
  LEGACY_SEED_SIGNUPS_OVER_TIME_SQL,
4
13
  SIGNUPS_OVER_TIME_SQL,
14
+ TEMPLATE_INTEREST_OVER_TIME_SQL,
5
15
  type ExactFirstPartyPanelReplacement,
6
16
  repairFirstPartyObservedRetentionPanels,
7
17
  } from "./first-party-metric-catalog";
@@ -33,6 +43,31 @@ const CANONICAL_CUSTOM_PANEL_REPLACEMENTS: readonly ExactFirstPartyPanelReplacem
33
43
  ],
34
44
  sql: SIGNUPS_OVER_TIME_SQL,
35
45
  },
46
+ {
47
+ id: "signups-by-template",
48
+ legacySql: [LEGACY_SIGNUPS_BY_TEMPLATE_SQL],
49
+ sql: SIGNUPS_BY_TEMPLATE_SQL,
50
+ },
51
+ {
52
+ id: "template-interest-over-time",
53
+ legacySql: [LEGACY_TEMPLATE_INTEREST_OVER_TIME_SQL],
54
+ sql: TEMPLATE_INTEREST_OVER_TIME_SQL,
55
+ },
56
+ {
57
+ id: "demo-clicks-over-time",
58
+ legacySql: [LEGACY_DEMO_CLICKS_OVER_TIME_SQL],
59
+ sql: DEMO_CLICKS_OVER_TIME_SQL,
60
+ },
61
+ {
62
+ id: "cli-copies-over-time",
63
+ legacySql: [LEGACY_CLI_COPIES_OVER_TIME_SQL],
64
+ sql: CLI_COPIES_OVER_TIME_SQL,
65
+ },
66
+ {
67
+ id: "pageviews-over-time",
68
+ legacySql: [LEGACY_PAGEVIEWS_OVER_TIME_SQL],
69
+ sql: PAGEVIEWS_OVER_TIME_SQL,
70
+ },
36
71
  {
37
72
  id: "new-vs-recurring-users",
38
73
  legacySql: [
@@ -1,14 +1,12 @@
1
1
  import type { SqlDashboardConfig } from "../../app/pages/adhoc/sql-dashboard/types";
2
+ import { repairCanonicalFirstPartyDashboardQueries } from "./canonical-first-party-dashboard-repair";
2
3
  import { loadDashboardSeed } from "./dashboard-seeds";
3
4
  import {
4
5
  listDashboardSummaries,
5
6
  type DashboardRecord,
6
7
  type DashboardSummaryRecord,
7
8
  } from "./dashboards-store";
8
- import {
9
- FIRST_PARTY_DASHBOARD_ID,
10
- repairFirstPartyObservedRetentionPanels,
11
- } from "./first-party-metric-catalog";
9
+ import { FIRST_PARTY_DASHBOARD_ID } from "./first-party-metric-catalog";
12
10
 
13
11
  export type DashboardTemplateCategory =
14
12
  | "Acquisition"
@@ -68,7 +66,7 @@ function seedConfig(id: string): SqlDashboardConfig {
68
66
  if (id !== FIRST_PARTY_DASHBOARD_ID) {
69
67
  return seed as unknown as SqlDashboardConfig;
70
68
  }
71
- return repairFirstPartyObservedRetentionPanels(seed)
69
+ return repairCanonicalFirstPartyDashboardQueries(seed)
72
70
  .config as unknown as SqlDashboardConfig;
73
71
  }
74
72
 
@@ -73,7 +73,7 @@ const FIRST_PARTY_QUERY_TABLES = [
73
73
  "analytics_user_days",
74
74
  ] as const;
75
75
  const BACKEND_CONFIG_CACHE_TTL_MS = 30_000;
76
- const MAX_BACKFILL_BATCH_SIZE = 5_000;
76
+ const MAX_BACKFILL_BATCH_SIZE = 750;
77
77
  const MAX_INSERT_BATCH_SIZE = 200;
78
78
 
79
79
  const backendConfigCache = new Map<
@@ -1153,9 +1153,15 @@ export async function backfillFirstPartyAnalyticsBatch(
1153
1153
  const hydratedById = new Map(
1154
1154
  hydratedRows.map((row) => [backfillRowCursor(row).id, row]),
1155
1155
  );
1156
- const selectedEvents = selectedIds
1157
- .map((id) => hydratedById.get(id))
1158
- .filter((row): row is Record<string, unknown> => row !== undefined);
1156
+ const selectedEvents = selectedIds.map((id) => {
1157
+ const row = hydratedById.get(id);
1158
+ if (!row) {
1159
+ throw new Error(
1160
+ `First-party analytics backfill row ${id} disappeared before hydration`,
1161
+ );
1162
+ }
1163
+ return row;
1164
+ });
1159
1165
  await insertFirstPartyAnalyticsRows(selectedEvents, configuredTable);
1160
1166
  const lastCursor = backfillRowCursor(selectedRows[selectedRows.length - 1]!);
1161
1167
  return {
@@ -1,8 +1,6 @@
1
- import { isPostgres } from "@agent-native/core/db";
2
1
  import { sql } from "@agent-native/core/db/schema";
3
2
 
4
3
  import { getDb, schema } from "../db/index.js";
5
- import { FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY } from "./analytics-rollup-lock.js";
6
4
 
7
5
  /**
8
6
  * The normalized subset emitted by first-party analytics ingest. Keeping this
@@ -143,15 +141,10 @@ export async function upsertFirstPartyAnalyticsRollups(
143
141
  }
144
142
 
145
143
  const writeRollups = async (tx: any) => {
146
- if (isPostgres()) {
147
- // The historical backfill takes this lock before its raw-event
148
- // snapshot. Holding it through ingest's rollup increment prevents the
149
- // backfill's monotonic upsert from losing a concurrently committed
150
- // event.
151
- await tx.execute(
152
- sql`SELECT pg_advisory_xact_lock(hashtextextended(${FIRST_PARTY_ANALYTICS_ROLLUP_LOCK_KEY}, 0::bigint))`,
153
- );
154
- }
144
+ // Do not take the historical backfill advisory lock here. Foreground
145
+ // ingest must not wait behind a long-running rebuild; the incremental
146
+ // conflict update and the backfill's GREATEST upsert are both monotonic,
147
+ // and Postgres serializes the conflicting row updates itself.
155
148
  const dailyRows = [...dailyRollups.values()];
156
149
  await tx
157
150
  .insert(schema.analyticsEventDailyRollups)