@agent-native/core 0.147.0 → 0.148.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/templates/analytics/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/analytics/DEVELOPING.md +50 -3
- package/corpus/templates/analytics/actions/purge-first-party-analytics-postgres.ts +190 -0
- package/corpus/templates/analytics/app/i18n/index.ts +14 -29
- package/corpus/templates/analytics/app/i18n-data.ts +10 -10
- package/corpus/templates/analytics/app/pages/Settings.tsx +5 -21
- package/corpus/templates/analytics/app/pages/sessions/SessionsPage.tsx +1 -3
- package/corpus/templates/analytics/scripts/first-party-analytics-bigquery-backfill.ts +197 -18
- package/corpus/templates/analytics/server/db/schema.ts +23 -0
- package/corpus/templates/analytics/server/handlers/first-party-analytics.ts +3 -1
- package/corpus/templates/analytics/server/jobs/analytics-bigquery-backfill.ts +1 -1
- package/corpus/templates/analytics/server/lib/first-party-analytics-backend.ts +113 -13
- package/corpus/templates/analytics/server/lib/first-party-analytics-purge.ts +140 -0
- package/corpus/templates/analytics/server/lib/first-party-analytics-volume.ts +207 -0
- package/corpus/templates/analytics/server/lib/first-party-analytics.ts +40 -9
- package/corpus/templates/analytics/server/lib/gcloud.ts +148 -20
- package/corpus/templates/analytics/server/plugins/db.ts +42 -0
- package/corpus/templates/assets/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/assets/app/i18n/index.ts +14 -29
- package/corpus/templates/brain/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/brain/app/i18n/index.ts +14 -29
- package/corpus/templates/calendar/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/chat/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/chat/app/i18n/index.ts +14 -29
- package/corpus/templates/clips/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/clips/app/components/player/transcript-panel.tsx +2 -7
- package/corpus/templates/clips/app/components/recorder/storage-setup-card.tsx +18 -13
- package/corpus/templates/clips/app/components/settings/ai-setup-section.tsx +87 -370
- package/corpus/templates/clips/app/components/settings/slack-section.tsx +59 -101
- package/corpus/templates/clips/app/components/settings/video-storage-section.tsx +136 -213
- package/corpus/templates/clips/app/i18n/en-US.ts +11 -8
- package/corpus/templates/clips/desktop/src/app.tsx +1 -1
- package/corpus/templates/content/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/content/app/i18n/index.ts +14 -29
- package/corpus/templates/crm/app/i18n/index.ts +14 -29
- package/corpus/templates/design/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/design/actions/analyze-brand-assets.ts +15 -15
- package/corpus/templates/design/actions/import-code.ts +2 -43
- package/corpus/templates/design/app/i18n/index.ts +14 -29
- package/corpus/templates/dispatch/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/dispatch/app/i18n-data.ts +1 -1
- package/corpus/templates/factory/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/factory/AGENTS.md +40 -49
- package/corpus/templates/factory/actions/babysit-agent-native-pull-request.ts +66 -0
- package/corpus/templates/factory/actions/evaluate-triage-item.ts +37 -0
- package/corpus/templates/factory/actions/get-slack-feedback-context.ts +20 -0
- package/corpus/templates/factory/actions/get-triage-item.ts +25 -4
- package/corpus/templates/factory/actions/govern-agent-native-pull-request.ts +55 -0
- package/corpus/templates/factory/actions/ingest-github-observation.ts +21 -0
- package/corpus/templates/factory/actions/list-factory-audit.ts +118 -0
- package/corpus/templates/factory/actions/list-factory-automations.ts +2 -1
- package/corpus/templates/factory/actions/list-triage-items.ts +24 -2
- package/corpus/templates/factory/actions/poll-github-sources.ts +60 -0
- package/corpus/templates/factory/actions/poll-sentry-errors.ts +39 -0
- package/corpus/templates/factory/actions/poll-slack-channel.ts +45 -1
- package/corpus/templates/factory/actions/run-factory-automation.ts +1 -1
- package/corpus/templates/factory/actions/save-factory-automation.ts +2 -2
- package/corpus/templates/factory/actions/start-builder-for-item.ts +51 -0
- package/corpus/templates/factory/app/components/factory/FactoryAuditView.tsx +685 -0
- package/corpus/templates/factory/app/hooks/use-navigation-state.ts +6 -0
- package/corpus/templates/factory/app/i18n/en-US.ts +25 -0
- package/corpus/templates/factory/app/routes/factory.tsx +95 -42
- package/corpus/templates/factory/server/db/schema.ts +37 -0
- package/corpus/templates/factory/server/lib/require-factory-automation.ts +5 -1
- package/corpus/templates/factory/server/plugins/factory-migrations.ts +29 -0
- package/corpus/templates/factory/server/triage/audit.ts +79 -0
- package/corpus/templates/forms/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/forms/app/i18n/index.ts +14 -29
- package/corpus/templates/macros/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/macros/app/i18n/index.ts +14 -29
- package/corpus/templates/mail/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/mail/app/i18n/index.ts +14 -29
- package/corpus/templates/plan/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/plan/app/components/plan/wireframe/Wireframe.tsx +7 -7
- package/corpus/templates/plan/app/i18n/index.ts +14 -29
- package/corpus/templates/slides/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/slides/actions/analyze-brand-assets.ts +15 -15
- package/corpus/templates/slides/actions/import-code.ts +2 -43
- package/corpus/templates/slides/app/i18n/index.ts +14 -29
- package/corpus/templates/slides/server/routes/api/preview-images.get.ts +4 -1
- package/corpus/templates/tasks/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/tasks/app/components/layout/HeaderActions.tsx +1 -84
- package/corpus/templates/tasks/app/hooks/use-mobile.tsx +1 -21
- package/corpus/templates/tasks/app/i18n/index.ts +14 -29
- package/dist/agent/engine/ai-sdk-engine.js +2 -1
- package/dist/agent/engine/openai-compatible-endpoint.d.ts +2 -0
- package/dist/agent/engine/openai-compatible-endpoint.js +5 -1
- package/dist/agent/engine/provider-endpoint-validation.d.ts +9 -0
- package/dist/agent/engine/provider-endpoint-validation.js +26 -0
- package/dist/agent/engine/provider-env-vars.d.ts +1 -1
- package/dist/agent/engine/provider-env-vars.js +1 -1
- package/dist/agent/engine/registry.d.ts +8 -6
- package/dist/agent/engine/registry.js +37 -24
- package/dist/agent/tool-search.d.ts +1 -0
- package/dist/agent/tool-search.js +17 -1
- package/dist/automations/service.js +4 -10
- package/dist/brand-kit/brand-signals.d.ts +13 -0
- package/dist/brand-kit/brand-signals.js +15 -0
- package/dist/brand-kit/index.d.ts +2 -2
- package/dist/brand-kit/index.js +1 -1
- package/dist/cli/plan-blocks.d.ts +3 -18
- package/dist/cli/plan-blocks.js +2 -95
- package/dist/cli/plan-publish-store.d.ts +2 -61
- package/dist/cli/plan-publish-store.js +2 -127
- package/dist/cli/skills-content/connection.d.ts +1 -1
- package/dist/cli/skills-content/connection.js +1 -52
- package/dist/cli/skills-content/visual-recap-skill.d.ts +1 -1
- package/dist/cli/skills-content/visual-recap-skill.js +1 -553
- package/dist/client/agent-engine-key.d.ts +14 -4
- package/dist/client/agent-engine-key.js +51 -5
- package/dist/client/agent-provider-catalog.d.ts +20 -0
- package/dist/client/agent-provider-catalog.js +100 -0
- package/dist/client/blocks/index.d.ts +1 -0
- package/dist/client/blocks/index.js +1 -0
- package/dist/client/chat/run-recovery.js +4 -48
- package/dist/client/chat-first.js +4 -3
- package/dist/client/i18n.d.ts +14 -0
- package/dist/client/i18n.js +18 -0
- package/dist/client/settings/ProviderSetupForm.d.ts +21 -0
- package/dist/client/settings/ProviderSetupForm.js +145 -0
- package/dist/client/settings/SettingsPanel.js +52 -49
- package/dist/client/settings/index.d.ts +2 -0
- package/dist/client/settings/index.js +2 -0
- package/dist/coding-tools/run-code.d.ts +5 -0
- package/dist/coding-tools/run-code.js +196 -36
- package/dist/coding-tools/tool-orchestration.d.ts +10 -0
- package/dist/coding-tools/tool-orchestration.js +107 -0
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/jobs/frontmatter.d.ts +9 -0
- package/dist/jobs/frontmatter.js +16 -0
- package/dist/jobs/scheduler.js +8 -8
- package/dist/jobs/tools.d.ts +1 -1
- package/dist/jobs/tools.js +2 -7
- package/dist/localization/default-messages.d.ts +20 -0
- package/dist/localization/default-messages.js +25 -5
- package/dist/observability/routes.d.ts +3 -3
- package/dist/onboarding/app-profile.js +1 -1
- package/dist/progress/routes.d.ts +1 -1
- package/dist/provider-api/actions/custom-provider-registration.d.ts +8 -8
- package/dist/provider-api/actions/provider-api.d.ts +11 -11
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/scripts/agent-engines/test-agent-engine.js +34 -10
- package/dist/secrets/routes.d.ts +9 -9
- package/dist/server/agent-chat/run-code-tools.d.ts +1 -1
- package/dist/server/agent-chat/run-code-tools.js +3 -1
- package/dist/server/agent-chat-plugin.js +6 -6
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/dist/server/agent-engine-api-key-route.js +43 -12
- package/dist/server/agents-bundle.js +2 -1
- package/dist/server/auth.js +21 -5
- package/dist/server/credential-provider.js +1 -0
- package/dist/server/design-token-utils.d.ts +18 -0
- package/dist/server/design-token-utils.js +31 -0
- package/dist/server/onboarding-html.js +22 -2
- package/dist/server/realtime-token.d.ts +1 -1
- package/dist/templates/chat/.agents/skills/frontend-design/SKILL.md +115 -68
- package/dist/templates/chat/app/i18n/index.ts +14 -29
- package/dist/templates/default/.agents/skills/frontend-design/SKILL.md +115 -68
- package/dist/templates/default/app/i18n/index.ts +14 -29
- package/dist/templates/workspace-core/.agents/skills/address-feedback/SKILL.md +22 -9
- package/dist/templates/workspace-core/.agents/skills/frontend-design/SKILL.md +115 -68
- package/dist/triggers/dispatcher.js +1 -8
- package/docs/content/environment-variables.mdx +39 -38
- package/package.json +4 -4
- package/src/templates/chat/.agents/skills/frontend-design/SKILL.md +115 -68
- package/src/templates/chat/app/i18n/index.ts +14 -29
- package/src/templates/default/.agents/skills/frontend-design/SKILL.md +115 -68
- package/src/templates/default/app/i18n/index.ts +14 -29
- package/src/templates/workspace-core/.agents/skills/address-feedback/SKILL.md +22 -9
- package/src/templates/workspace-core/.agents/skills/frontend-design/SKILL.md +115 -68
- package/corpus/templates/plan/app/components/plan/wireframe/sanitize-html.ts +0 -343
|
@@ -1314,7 +1314,7 @@ const enUS = {
|
|
|
1314
1314
|
storageSetupDescription:
|
|
1315
1315
|
"Session replay recordings need storage before chunks can be saved. Connect Builder.io for free-tier storage, or configure your own S3-compatible bucket.",
|
|
1316
1316
|
storageConnected: "Storage connected",
|
|
1317
|
-
connectBuilder: "Use Builder.io
|
|
1317
|
+
connectBuilder: "Use Builder.io",
|
|
1318
1318
|
configureS3: "Configure S3 storage",
|
|
1319
1319
|
devtools: "Dev Tools",
|
|
1320
1320
|
devtoolsResize: "Resize Dev Tools",
|
|
@@ -10157,7 +10157,7 @@ const translatedSessionReplayTranslations = {
|
|
|
10157
10157
|
storageSetupDescription:
|
|
10158
10158
|
"保存会话回放录制片段前需要先配置存储。可连接 Builder.io 使用免费层存储,或配置您自己的 S3 兼容存储桶。",
|
|
10159
10159
|
storageConnected: "存储已连接",
|
|
10160
|
-
connectBuilder: "使用 Builder.io
|
|
10160
|
+
connectBuilder: "使用 Builder.io",
|
|
10161
10161
|
configureS3: "配置 S3 存储",
|
|
10162
10162
|
title: "会话",
|
|
10163
10163
|
description:
|
|
@@ -10279,7 +10279,7 @@ const translatedSessionReplayTranslations = {
|
|
|
10279
10279
|
storageSetupDescription:
|
|
10280
10280
|
"Las grabaciones de repetición de sesión necesitan almacenamiento antes de poder guardar los fragmentos. Conecta Builder.io para usar almacenamiento de nivel gratuito o configura tu propio bucket compatible con S3.",
|
|
10281
10281
|
storageConnected: "Almacenamiento conectado",
|
|
10282
|
-
connectBuilder: "Usar Builder.io
|
|
10282
|
+
connectBuilder: "Usar Builder.io",
|
|
10283
10283
|
configureS3: "Configurar almacenamiento S3",
|
|
10284
10284
|
title: "Sesiones",
|
|
10285
10285
|
description:
|
|
@@ -10405,7 +10405,7 @@ const translatedSessionReplayTranslations = {
|
|
|
10405
10405
|
storageSetupDescription:
|
|
10406
10406
|
"Les enregistrements de relecture de session nécessitent un espace de stockage avant de pouvoir sauvegarder les fragments. Connectez Builder.io pour un stockage en formule gratuite, ou configurez votre propre bucket compatible S3.",
|
|
10407
10407
|
storageConnected: "Stockage connecté",
|
|
10408
|
-
connectBuilder: "Utiliser Builder.io
|
|
10408
|
+
connectBuilder: "Utiliser Builder.io",
|
|
10409
10409
|
configureS3: "Configurer le stockage S3",
|
|
10410
10410
|
title: "Sessions",
|
|
10411
10411
|
description:
|
|
@@ -10534,7 +10534,7 @@ const translatedSessionReplayTranslations = {
|
|
|
10534
10534
|
storageSetupDescription:
|
|
10535
10535
|
"Aufzeichnungen der Sitzungswiedergabe benötigen einen Speicher, bevor Chunks gesichert werden können. Verbinden Sie Builder.io für Speicher im kostenlosen Kontingent oder konfigurieren Sie Ihren eigenen S3-kompatiblen Bucket.",
|
|
10536
10536
|
storageConnected: "Speicher verbunden",
|
|
10537
|
-
connectBuilder: "Builder.io verwenden
|
|
10537
|
+
connectBuilder: "Builder.io verwenden",
|
|
10538
10538
|
configureS3: "S3-Speicher konfigurieren",
|
|
10539
10539
|
title: "Sitzungen",
|
|
10540
10540
|
description:
|
|
@@ -10659,7 +10659,7 @@ const translatedSessionReplayTranslations = {
|
|
|
10659
10659
|
storageSetupDescription:
|
|
10660
10660
|
"セッションリプレイの録画を保存するには、まずストレージが必要です。Builder.io の無料枠ストレージを接続するか、独自の S3 互換バケットを設定してください。",
|
|
10661
10661
|
storageConnected: "ストレージ接続済み",
|
|
10662
|
-
connectBuilder: "Builder.io
|
|
10662
|
+
connectBuilder: "Builder.io を使う",
|
|
10663
10663
|
configureS3: "S3 ストレージを設定",
|
|
10664
10664
|
title: "セッション",
|
|
10665
10665
|
description:
|
|
@@ -10785,7 +10785,7 @@ const translatedSessionReplayTranslations = {
|
|
|
10785
10785
|
storageSetupDescription:
|
|
10786
10786
|
"세션 재생 녹화를 저장하려면 먼저 저장소가 필요합니다. 무료 등급 저장소를 사용하려면 Builder.io를 연결하거나, 직접 S3 호환 버킷을 구성하세요.",
|
|
10787
10787
|
storageConnected: "저장소 연결됨",
|
|
10788
|
-
connectBuilder: "Builder.io 사용
|
|
10788
|
+
connectBuilder: "Builder.io 사용",
|
|
10789
10789
|
configureS3: "S3 저장소 구성",
|
|
10790
10790
|
title: "세션",
|
|
10791
10791
|
description:
|
|
@@ -10910,7 +10910,7 @@ const translatedSessionReplayTranslations = {
|
|
|
10910
10910
|
storageSetupDescription:
|
|
10911
10911
|
"As gravações de replay de sessão precisam de armazenamento antes que os fragmentos possam ser salvos. Conecte o Builder.io para armazenamento no plano gratuito ou configure seu próprio bucket compatível com S3.",
|
|
10912
10912
|
storageConnected: "Armazenamento conectado",
|
|
10913
|
-
connectBuilder: "Usar Builder.io
|
|
10913
|
+
connectBuilder: "Usar Builder.io",
|
|
10914
10914
|
configureS3: "Configurar armazenamento S3",
|
|
10915
10915
|
title: "Sessões",
|
|
10916
10916
|
description:
|
|
@@ -11034,7 +11034,7 @@ const translatedSessionReplayTranslations = {
|
|
|
11034
11034
|
storageSetupDescription:
|
|
11035
11035
|
"चंक्स सेव करने से पहले सेशन रीप्ले रिकॉर्डिंग के लिए स्टोरेज ज़रूरी है। फ्री-टियर स्टोरेज के लिए Builder.io कनेक्ट करें, या अपना खुद का S3-संगत बकेट कॉन्फ़िगर करें।",
|
|
11036
11036
|
storageConnected: "स्टोरेज कनेक्ट हो गया",
|
|
11037
|
-
connectBuilder: "Builder.io इस्तेमाल करें
|
|
11037
|
+
connectBuilder: "Builder.io इस्तेमाल करें",
|
|
11038
11038
|
configureS3: "S3 स्टोरेज कॉन्फ़िगर करें",
|
|
11039
11039
|
title: "सत्र",
|
|
11040
11040
|
description:
|
|
@@ -11155,7 +11155,7 @@ const translatedSessionReplayTranslations = {
|
|
|
11155
11155
|
storageSetupDescription:
|
|
11156
11156
|
"تحتاج تسجيلات إعادة عرض الجلسات إلى مساحة تخزين قبل أن يتسنى حفظ الأجزاء. اربط Builder.io للحصول على تخزين بالباقة المجانية، أو هيّئ حاوية متوافقة مع S3 خاصة بك.",
|
|
11157
11157
|
storageConnected: "تم ربط مساحة التخزين",
|
|
11158
|
-
connectBuilder: "استخدام Builder.io
|
|
11158
|
+
connectBuilder: "استخدام Builder.io",
|
|
11159
11159
|
configureS3: "تهيئة تخزين S3",
|
|
11160
11160
|
title: "الجلسات",
|
|
11161
11161
|
description:
|
|
@@ -20,13 +20,6 @@ import { Link } from "react-router";
|
|
|
20
20
|
import { toast } from "sonner";
|
|
21
21
|
|
|
22
22
|
import { Button } from "@/components/ui/button";
|
|
23
|
-
import {
|
|
24
|
-
Card,
|
|
25
|
-
CardContent,
|
|
26
|
-
CardDescription,
|
|
27
|
-
CardHeader,
|
|
28
|
-
CardTitle,
|
|
29
|
-
} from "@/components/ui/card";
|
|
30
23
|
import { Label } from "@/components/ui/label";
|
|
31
24
|
import { Switch } from "@/components/ui/switch";
|
|
32
25
|
|
|
@@ -218,22 +211,13 @@ export default function Settings() {
|
|
|
218
211
|
</SettingsGroup>
|
|
219
212
|
|
|
220
213
|
{replayStorageStatus.data?.configured ? (
|
|
221
|
-
<
|
|
214
|
+
<SettingsGroup
|
|
222
215
|
id="replay-storage"
|
|
223
|
-
|
|
216
|
+
title={t("sessions.storageSetupTitle")}
|
|
217
|
+
description={t("sessions.storageSetupDescription")}
|
|
224
218
|
>
|
|
225
|
-
<
|
|
226
|
-
|
|
227
|
-
{t("sessions.storageSetupTitle")}
|
|
228
|
-
</CardTitle>
|
|
229
|
-
<CardDescription>
|
|
230
|
-
{t("sessions.storageSetupDescription")}
|
|
231
|
-
</CardDescription>
|
|
232
|
-
</CardHeader>
|
|
233
|
-
<CardContent>
|
|
234
|
-
<ReplayStorageHint embedded />
|
|
235
|
-
</CardContent>
|
|
236
|
-
</Card>
|
|
219
|
+
<ReplayStorageHint embedded />
|
|
220
|
+
</SettingsGroup>
|
|
237
221
|
) : null}
|
|
238
222
|
</div>
|
|
239
223
|
}
|
|
@@ -621,9 +621,7 @@ export function ReplayStorageHint({
|
|
|
621
621
|
<IconLoader2 className="h-4 w-4 animate-spin" />
|
|
622
622
|
) : builderConnected ? (
|
|
623
623
|
<IconCheck className="h-4 w-4" />
|
|
624
|
-
) :
|
|
625
|
-
<IconExternalLink className="h-4 w-4" />
|
|
626
|
-
)}
|
|
624
|
+
) : null}
|
|
627
625
|
{builderConnected
|
|
628
626
|
? t("sessions.storageConnected")
|
|
629
627
|
: t("sessions.connectBuilder")}
|
|
@@ -36,6 +36,11 @@ const MAX_CONCURRENCY = 4;
|
|
|
36
36
|
const DEFAULT_MAX_BATCHES = 1_000;
|
|
37
37
|
const MAX_MAX_BATCHES = 10_000;
|
|
38
38
|
const DEFAULT_CHECKPOINT_PATH = ".analytics-first-party-bigquery-backfill.json";
|
|
39
|
+
const DEFAULT_LOOKBACK_DAYS = 60;
|
|
40
|
+
const MIN_LOOKBACK_DAYS = 30;
|
|
41
|
+
const MAX_LOOKBACK_DAYS = 60;
|
|
42
|
+
const DEFAULT_EXCLUDED_EVENT_NAMES = ["http.response"];
|
|
43
|
+
const MAX_EXCLUDED_EVENT_NAMES = 32;
|
|
39
44
|
const QUERY_TIMEOUT_MS = 30_000;
|
|
40
45
|
const MAX_HYDRATION_IDS = 500;
|
|
41
46
|
const JOB_TABLE = "analytics_bigquery_backfill_jobs";
|
|
@@ -59,13 +64,16 @@ export interface BranchCheckpoint {
|
|
|
59
64
|
}
|
|
60
65
|
|
|
61
66
|
export interface DedicatedBackfillCheckpoint {
|
|
62
|
-
version: 1;
|
|
67
|
+
version: 1 | 2;
|
|
63
68
|
scope: {
|
|
64
69
|
userEmail: string;
|
|
65
70
|
orgId: string;
|
|
66
71
|
};
|
|
67
72
|
table: string | null;
|
|
68
73
|
branches: Record<BranchName, BranchCheckpoint>;
|
|
74
|
+
lookbackDays?: number;
|
|
75
|
+
lookbackStart?: string | null;
|
|
76
|
+
excludedEventNames?: string[];
|
|
69
77
|
updatedAt: string;
|
|
70
78
|
}
|
|
71
79
|
|
|
@@ -76,6 +84,8 @@ export interface DedicatedBackfillOptions {
|
|
|
76
84
|
concurrency: number;
|
|
77
85
|
maxBatches: number;
|
|
78
86
|
checkpointPath: string;
|
|
87
|
+
lookbackDays?: number;
|
|
88
|
+
excludedEventNames?: string[];
|
|
79
89
|
}
|
|
80
90
|
|
|
81
91
|
export interface FinalizeResult {
|
|
@@ -127,6 +137,8 @@ interface BranchSpec {
|
|
|
127
137
|
name: BranchName;
|
|
128
138
|
predicate: string;
|
|
129
139
|
predicateArgs: string[];
|
|
140
|
+
lookbackStart?: string | null;
|
|
141
|
+
excludedEventNames?: string[];
|
|
130
142
|
}
|
|
131
143
|
|
|
132
144
|
const PROJECTED_COLUMNS = FIRST_PARTY_ANALYTICS_BACKFILL_COLUMNS.join(", ");
|
|
@@ -145,6 +157,49 @@ function positiveInteger(
|
|
|
145
157
|
return Math.min(value, maximum);
|
|
146
158
|
}
|
|
147
159
|
|
|
160
|
+
function parseLookbackDays(raw: string | undefined): number {
|
|
161
|
+
if (raw === undefined) return DEFAULT_LOOKBACK_DAYS;
|
|
162
|
+
const value = Number(raw);
|
|
163
|
+
if (
|
|
164
|
+
!Number.isInteger(value) ||
|
|
165
|
+
value < MIN_LOOKBACK_DAYS ||
|
|
166
|
+
value > MAX_LOOKBACK_DAYS
|
|
167
|
+
) {
|
|
168
|
+
throw new Error(
|
|
169
|
+
`--lookback-days must be an integer between ${MIN_LOOKBACK_DAYS} and ${MAX_LOOKBACK_DAYS}`,
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
return value;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function parseExcludedEventNames(raw: string | undefined): string[] {
|
|
176
|
+
if (raw === undefined) return [...DEFAULT_EXCLUDED_EVENT_NAMES];
|
|
177
|
+
const names = [...new Set(raw.split(",").map((name) => name.trim()))].filter(
|
|
178
|
+
Boolean,
|
|
179
|
+
);
|
|
180
|
+
if (names.length > MAX_EXCLUDED_EVENT_NAMES) {
|
|
181
|
+
throw new Error(
|
|
182
|
+
`--skip-events supports at most ${MAX_EXCLUDED_EVENT_NAMES} event names`,
|
|
183
|
+
);
|
|
184
|
+
}
|
|
185
|
+
return names;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function lookbackStart(now: string, days: number): string {
|
|
189
|
+
const date = new Date(now);
|
|
190
|
+
if (Number.isNaN(date.getTime())) {
|
|
191
|
+
throw new Error(`Invalid backfill clock value: ${now}`);
|
|
192
|
+
}
|
|
193
|
+
date.setUTCDate(date.getUTCDate() - days);
|
|
194
|
+
return date.toISOString();
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
interface BackfillQueryConfig {
|
|
198
|
+
lookbackDays: number;
|
|
199
|
+
lookbackStart: string | null;
|
|
200
|
+
excludedEventNames: string[];
|
|
201
|
+
}
|
|
202
|
+
|
|
148
203
|
function requiredArg(args: Record<string, string>, name: string): string {
|
|
149
204
|
const value = args[name]?.trim();
|
|
150
205
|
if (!value) throw new Error(`--${name} is required`);
|
|
@@ -178,6 +233,8 @@ export function parseDedicatedBackfillOptions(
|
|
|
178
233
|
MAX_MAX_BATCHES,
|
|
179
234
|
),
|
|
180
235
|
checkpointPath: resolve(args.checkpoint || DEFAULT_CHECKPOINT_PATH),
|
|
236
|
+
lookbackDays: parseLookbackDays(args["lookback-days"]),
|
|
237
|
+
excludedEventNames: parseExcludedEventNames(args["skip-events"]),
|
|
181
238
|
execute: args.execute === "true",
|
|
182
239
|
finalize: args.finalize === "true",
|
|
183
240
|
};
|
|
@@ -185,29 +242,64 @@ export function parseDedicatedBackfillOptions(
|
|
|
185
242
|
|
|
186
243
|
function branchSpecs(
|
|
187
244
|
scope: FirstPartyAnalyticsScope & { orgId: string },
|
|
245
|
+
queryConfig: BackfillQueryConfig = {
|
|
246
|
+
lookbackDays: DEFAULT_LOOKBACK_DAYS,
|
|
247
|
+
lookbackStart: null,
|
|
248
|
+
excludedEventNames: [],
|
|
249
|
+
},
|
|
188
250
|
): BranchSpec[] {
|
|
189
251
|
return [
|
|
190
252
|
{
|
|
191
253
|
name: "org",
|
|
192
254
|
predicate: "org_id = ?",
|
|
193
255
|
predicateArgs: [scope.orgId],
|
|
256
|
+
...queryConfig,
|
|
194
257
|
},
|
|
195
258
|
{
|
|
196
259
|
name: "legacy",
|
|
197
260
|
predicate: "org_id IS NULL AND owner_email = ?",
|
|
198
261
|
predicateArgs: [scope.userEmail],
|
|
262
|
+
...queryConfig,
|
|
199
263
|
},
|
|
200
264
|
];
|
|
201
265
|
}
|
|
202
266
|
|
|
267
|
+
function sourceFilter(spec: BranchSpec): {
|
|
268
|
+
clauses: string[];
|
|
269
|
+
args: string[];
|
|
270
|
+
} {
|
|
271
|
+
const clauses: string[] = [];
|
|
272
|
+
const args: string[] = [];
|
|
273
|
+
if (spec.lookbackStart) {
|
|
274
|
+
clauses.push("received_at >= ?");
|
|
275
|
+
args.push(spec.lookbackStart);
|
|
276
|
+
}
|
|
277
|
+
const excluded = spec.excludedEventNames ?? [];
|
|
278
|
+
if (excluded.length) {
|
|
279
|
+
if (
|
|
280
|
+
excluded.length === DEFAULT_EXCLUDED_EVENT_NAMES.length &&
|
|
281
|
+
excluded[0] === DEFAULT_EXCLUDED_EVENT_NAMES[0]
|
|
282
|
+
) {
|
|
283
|
+
clauses.push("event_name IS DISTINCT FROM 'http.response'");
|
|
284
|
+
} else {
|
|
285
|
+
clauses.push(
|
|
286
|
+
`(event_name IS NULL OR event_name NOT IN (${excluded.map(() => "?").join(", ")}))`,
|
|
287
|
+
);
|
|
288
|
+
args.push(...excluded);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
return { clauses, args };
|
|
292
|
+
}
|
|
293
|
+
|
|
203
294
|
export function buildHighWaterMarkQuery(spec: BranchSpec): DbQuery {
|
|
295
|
+
const filters = sourceFilter(spec);
|
|
204
296
|
return {
|
|
205
297
|
sql: `SELECT received_at, id
|
|
206
298
|
FROM analytics_events
|
|
207
|
-
WHERE ${spec.predicate}
|
|
299
|
+
WHERE ${spec.predicate}${filters.clauses.length ? `\n AND ${filters.clauses.join("\n AND ")}` : ""}
|
|
208
300
|
ORDER BY received_at DESC, id DESC
|
|
209
301
|
LIMIT 1`,
|
|
210
|
-
args: spec.predicateArgs,
|
|
302
|
+
args: [...spec.predicateArgs, ...filters.args],
|
|
211
303
|
timeoutMs: QUERY_TIMEOUT_MS,
|
|
212
304
|
maxAttempts: 1,
|
|
213
305
|
};
|
|
@@ -221,27 +313,25 @@ export function buildPageKeyQuery(
|
|
|
221
313
|
if (!state.cutoff) {
|
|
222
314
|
throw new Error(`Cannot page ${spec.name} without a high-water mark`);
|
|
223
315
|
}
|
|
316
|
+
const filters = sourceFilter(spec);
|
|
224
317
|
const args: unknown[] = [
|
|
225
318
|
...spec.predicateArgs,
|
|
226
|
-
|
|
319
|
+
...filters.args,
|
|
227
320
|
state.cutoff.receivedAt,
|
|
228
321
|
state.cutoff.id,
|
|
229
322
|
];
|
|
230
323
|
let cursorClause = "";
|
|
231
324
|
if (state.cursor) {
|
|
232
|
-
cursorClause = " AND (received_at >
|
|
233
|
-
args.push(
|
|
234
|
-
state.cursor.receivedAt,
|
|
235
|
-
state.cursor.receivedAt,
|
|
236
|
-
state.cursor.id,
|
|
237
|
-
);
|
|
325
|
+
cursorClause = " AND (received_at, id) > (?, ?)";
|
|
326
|
+
args.push(state.cursor.receivedAt, state.cursor.id);
|
|
238
327
|
}
|
|
239
328
|
args.push(batchSize);
|
|
240
329
|
return {
|
|
241
330
|
sql: `SELECT received_at, id
|
|
242
331
|
FROM analytics_events
|
|
243
332
|
WHERE ${spec.predicate}
|
|
244
|
-
|
|
333
|
+
${filters.clauses.map((clause) => `AND ${clause}`).join("\n ")}
|
|
334
|
+
AND (received_at, id) <= (?, ?)${cursorClause}
|
|
245
335
|
ORDER BY received_at ASC, id ASC
|
|
246
336
|
LIMIT ?`,
|
|
247
337
|
args,
|
|
@@ -305,9 +395,10 @@ function emptyBranchCheckpoint(): BranchCheckpoint {
|
|
|
305
395
|
function newCheckpoint(
|
|
306
396
|
options: DedicatedBackfillOptions,
|
|
307
397
|
now: string,
|
|
398
|
+
queryConfig: BackfillQueryConfig,
|
|
308
399
|
): DedicatedBackfillCheckpoint {
|
|
309
400
|
return {
|
|
310
|
-
version:
|
|
401
|
+
version: 2,
|
|
311
402
|
scope: {
|
|
312
403
|
userEmail: options.scope.userEmail,
|
|
313
404
|
orgId: options.scope.orgId,
|
|
@@ -317,15 +408,54 @@ function newCheckpoint(
|
|
|
317
408
|
org: emptyBranchCheckpoint(),
|
|
318
409
|
legacy: emptyBranchCheckpoint(),
|
|
319
410
|
},
|
|
411
|
+
lookbackDays: queryConfig.lookbackDays,
|
|
412
|
+
lookbackStart: queryConfig.lookbackStart ?? undefined,
|
|
413
|
+
excludedEventNames: queryConfig.excludedEventNames,
|
|
320
414
|
updatedAt: now,
|
|
321
415
|
};
|
|
322
416
|
}
|
|
323
417
|
|
|
418
|
+
function checkpointQueryConfig(
|
|
419
|
+
checkpoint: DedicatedBackfillCheckpoint,
|
|
420
|
+
): BackfillQueryConfig {
|
|
421
|
+
return {
|
|
422
|
+
lookbackDays:
|
|
423
|
+
checkpoint.version === 2
|
|
424
|
+
? (checkpoint.lookbackDays ?? DEFAULT_LOOKBACK_DAYS)
|
|
425
|
+
: DEFAULT_LOOKBACK_DAYS,
|
|
426
|
+
lookbackStart:
|
|
427
|
+
checkpoint.version === 2 ? (checkpoint.lookbackStart ?? null) : null,
|
|
428
|
+
excludedEventNames:
|
|
429
|
+
checkpoint.version === 2 ? (checkpoint.excludedEventNames ?? []) : [],
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function upgradeCheckpoint(
|
|
434
|
+
checkpoint: DedicatedBackfillCheckpoint,
|
|
435
|
+
queryConfig: BackfillQueryConfig,
|
|
436
|
+
): DedicatedBackfillCheckpoint {
|
|
437
|
+
if (checkpoint.version === 2) {
|
|
438
|
+
return checkpoint.lookbackDays === undefined
|
|
439
|
+
? { ...checkpoint, lookbackDays: queryConfig.lookbackDays }
|
|
440
|
+
: checkpoint;
|
|
441
|
+
}
|
|
442
|
+
for (const branch of ["org", "legacy"] as const) {
|
|
443
|
+
checkpoint.branches[branch].complete = false;
|
|
444
|
+
}
|
|
445
|
+
return {
|
|
446
|
+
...checkpoint,
|
|
447
|
+
version: 2,
|
|
448
|
+
lookbackDays: queryConfig.lookbackDays,
|
|
449
|
+
lookbackStart: queryConfig.lookbackStart ?? undefined,
|
|
450
|
+
excludedEventNames: queryConfig.excludedEventNames,
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
|
|
324
454
|
function assertCheckpointMatches(
|
|
325
455
|
checkpoint: DedicatedBackfillCheckpoint,
|
|
326
456
|
options: DedicatedBackfillOptions,
|
|
327
457
|
): void {
|
|
328
|
-
if (checkpoint.version !== 1) {
|
|
458
|
+
if (checkpoint.version !== 1 && checkpoint.version !== 2) {
|
|
329
459
|
throw new Error(
|
|
330
460
|
`Unsupported backfill checkpoint version: ${checkpoint.version}`,
|
|
331
461
|
);
|
|
@@ -343,6 +473,23 @@ function assertCheckpointMatches(
|
|
|
343
473
|
"Backfill checkpoint table does not match the current BigQuery table",
|
|
344
474
|
);
|
|
345
475
|
}
|
|
476
|
+
if (checkpoint.version === 2) {
|
|
477
|
+
const lookbackDays = checkpoint.lookbackDays;
|
|
478
|
+
if (
|
|
479
|
+
(checkpoint.lookbackStart !== undefined &&
|
|
480
|
+
typeof checkpoint.lookbackStart !== "string") ||
|
|
481
|
+
typeof lookbackDays !== "number" ||
|
|
482
|
+
!Number.isInteger(lookbackDays) ||
|
|
483
|
+
lookbackDays < MIN_LOOKBACK_DAYS ||
|
|
484
|
+
lookbackDays > MAX_LOOKBACK_DAYS ||
|
|
485
|
+
!Array.isArray(checkpoint.excludedEventNames) ||
|
|
486
|
+
checkpoint.excludedEventNames.some((name) => typeof name !== "string")
|
|
487
|
+
) {
|
|
488
|
+
throw new Error(
|
|
489
|
+
"Backfill checkpoint has invalid lookback or event filter metadata",
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
346
493
|
for (const branch of ["org", "legacy"] as const) {
|
|
347
494
|
const state = checkpoint.branches[branch];
|
|
348
495
|
if (!state)
|
|
@@ -424,7 +571,7 @@ export async function assertSourceAtCheckpoint(
|
|
|
424
571
|
checkpoint: DedicatedBackfillCheckpoint,
|
|
425
572
|
db: DbExecutor = getDbExec(),
|
|
426
573
|
): Promise<void> {
|
|
427
|
-
for (const spec of branchSpecs(scope)) {
|
|
574
|
+
for (const spec of branchSpecs(scope, checkpointQueryConfig(checkpoint))) {
|
|
428
575
|
const result = await db.execute(buildHighWaterMarkQuery(spec));
|
|
429
576
|
const latestRow = result.rows[0];
|
|
430
577
|
if (!latestRow) continue;
|
|
@@ -532,16 +679,44 @@ export async function runDedicatedBackfill(
|
|
|
532
679
|
dependencies: DedicatedBackfillDependencies,
|
|
533
680
|
): Promise<DedicatedBackfillResult> {
|
|
534
681
|
const now = dependencies.now ?? (() => new Date().toISOString());
|
|
535
|
-
const
|
|
536
|
-
|
|
537
|
-
|
|
682
|
+
const checkpointNow = now();
|
|
683
|
+
const loadedCheckpoint = await dependencies.checkpointStore.load();
|
|
684
|
+
const requestedLookbackDays = options.lookbackDays ?? DEFAULT_LOOKBACK_DAYS;
|
|
685
|
+
const requestedExcludedEventNames = options.excludedEventNames ?? [
|
|
686
|
+
...DEFAULT_EXCLUDED_EVENT_NAMES,
|
|
687
|
+
];
|
|
688
|
+
const queryConfig: BackfillQueryConfig = loadedCheckpoint
|
|
689
|
+
? checkpointQueryConfig(loadedCheckpoint)
|
|
690
|
+
: {
|
|
691
|
+
lookbackDays: requestedLookbackDays,
|
|
692
|
+
lookbackStart: lookbackStart(checkpointNow, requestedLookbackDays),
|
|
693
|
+
excludedEventNames: requestedExcludedEventNames,
|
|
694
|
+
};
|
|
695
|
+
const checkpoint = loadedCheckpoint
|
|
696
|
+
? upgradeCheckpoint(loadedCheckpoint, queryConfig)
|
|
697
|
+
: newCheckpoint(options, checkpointNow, queryConfig);
|
|
538
698
|
assertCheckpointMatches(checkpoint, options);
|
|
699
|
+
if (checkpoint.version === 2) {
|
|
700
|
+
if (checkpoint.lookbackDays !== requestedLookbackDays) {
|
|
701
|
+
throw new Error(
|
|
702
|
+
"Backfill checkpoint lookback window does not match the current --lookback-days setting",
|
|
703
|
+
);
|
|
704
|
+
}
|
|
705
|
+
if (
|
|
706
|
+
JSON.stringify(checkpoint.excludedEventNames) !==
|
|
707
|
+
JSON.stringify(requestedExcludedEventNames)
|
|
708
|
+
) {
|
|
709
|
+
throw new Error(
|
|
710
|
+
"Backfill checkpoint event filter does not match the current --skip-events setting",
|
|
711
|
+
);
|
|
712
|
+
}
|
|
713
|
+
}
|
|
539
714
|
await dependencies.checkpointStore.save(checkpoint);
|
|
540
715
|
|
|
541
716
|
let batches = 0;
|
|
542
717
|
let copiedThisRun = 0;
|
|
543
718
|
let stop = false;
|
|
544
|
-
for (const spec of branchSpecs(options.scope)) {
|
|
719
|
+
for (const spec of branchSpecs(options.scope, queryConfig)) {
|
|
545
720
|
const state = checkpoint.branches[spec.name];
|
|
546
721
|
if (state.complete) continue;
|
|
547
722
|
|
|
@@ -619,6 +794,8 @@ Options:
|
|
|
619
794
|
--concurrency=<n> BigQuery requests in flight, default 2, max 4
|
|
620
795
|
--max-batches=<n> Work budget per invocation, default 1000
|
|
621
796
|
--checkpoint=<path> Atomic local checkpoint path
|
|
797
|
+
--lookback-days=<n> Source window, 30-60 days, default 60
|
|
798
|
+
--skip-events=<csv> Event names to skip, default http.response
|
|
622
799
|
|
|
623
800
|
After the checkpoint is complete, reconcile it into the shipped durable job
|
|
624
801
|
after quiescing source event writes and checking the final high-water marks:
|
|
@@ -848,6 +1025,8 @@ async function main(): Promise<void> {
|
|
|
848
1025
|
batchSize: options.batchSize,
|
|
849
1026
|
concurrency: options.concurrency,
|
|
850
1027
|
maxBatches: options.maxBatches,
|
|
1028
|
+
lookbackDays: options.lookbackDays,
|
|
1029
|
+
excludedEventNames: options.excludedEventNames,
|
|
851
1030
|
checkpointPath: options.checkpointPath,
|
|
852
1031
|
message:
|
|
853
1032
|
"No data was copied. Pass --execute and AGENT_NATIVE_ANALYTICS_BIGQUERY_BACKFILL_ALLOW=1 after production approval.",
|
|
@@ -311,6 +311,29 @@ export const analyticsUserDays = table("analytics_user_days", {
|
|
|
311
311
|
userKey: text("user_key").notNull(),
|
|
312
312
|
});
|
|
313
313
|
|
|
314
|
+
/** Atomic per-tenant event reservations used to cap future Postgres growth. */
|
|
315
|
+
export const analyticsEventVolumeUsage = table(
|
|
316
|
+
"analytics_event_volume_usage",
|
|
317
|
+
{
|
|
318
|
+
id: text("id").primaryKey(),
|
|
319
|
+
tenantKey: text("tenant_key").notNull(),
|
|
320
|
+
ownerEmail: text("owner_email").notNull(),
|
|
321
|
+
orgId: text("org_id"),
|
|
322
|
+
windowStart: text("window_start").notNull(),
|
|
323
|
+
eventCount: integer("event_count").notNull().default(0),
|
|
324
|
+
eventLimit: integer("event_limit").notNull(),
|
|
325
|
+
updatedAt: text("updated_at").notNull().default(now()),
|
|
326
|
+
},
|
|
327
|
+
(t) => ({
|
|
328
|
+
tenantWindowUnique: uniqueIndex(
|
|
329
|
+
"analytics_event_volume_usage_tenant_window_idx",
|
|
330
|
+
).on(t.tenantKey, t.windowStart),
|
|
331
|
+
updatedAtIdx: index("analytics_event_volume_usage_updated_at_idx").on(
|
|
332
|
+
t.updatedAt,
|
|
333
|
+
),
|
|
334
|
+
}),
|
|
335
|
+
);
|
|
336
|
+
|
|
314
337
|
/**
|
|
315
338
|
* Compact pressure signals for first-party queries that are already slow or
|
|
316
339
|
* failing. Successful fast queries never write here, so the diagnostic path
|
|
@@ -49,7 +49,9 @@ export const handleAnalyticsTrack = defineEventHandler(async (event) => {
|
|
|
49
49
|
} catch (err: any) {
|
|
50
50
|
const message = err?.message || String(err);
|
|
51
51
|
const invalidKey = /invalid analytics public key/i.test(message);
|
|
52
|
-
|
|
52
|
+
const statusCode =
|
|
53
|
+
typeof err?.statusCode === "number" ? err.statusCode : undefined;
|
|
54
|
+
setResponseStatus(event, invalidKey ? 401 : (statusCode ?? 400));
|
|
53
55
|
return { error: message };
|
|
54
56
|
}
|
|
55
57
|
});
|
|
@@ -248,7 +248,7 @@ export async function acquireDedicatedFirstPartyAnalyticsBackfillLease(
|
|
|
248
248
|
SET status = 'pending', lease_token = NULL,
|
|
249
249
|
lease_expires_at = NULL, next_run_at = ?, updated_at = ?
|
|
250
250
|
WHERE id = ? AND lease_token = ?`,
|
|
251
|
-
args: [
|
|
251
|
+
args: [releasedAt, releasedAt, id, token],
|
|
252
252
|
timeoutMs: 5_000,
|
|
253
253
|
maxAttempts: 1,
|
|
254
254
|
});
|