@agent-native/core-corpus 0.212.0 → 0.213.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/corpus/README.md +1 -1
  2. package/corpus/templates/analytics/.agents/skills/onboarding-journeys/SKILL.md +29 -14
  3. package/corpus/templates/analytics/CHANGELOG.md +11 -0
  4. package/corpus/templates/analytics/actions/estimate-onboarding-journey-cost.ts +23 -6
  5. package/corpus/templates/analytics/actions/get-onboarding-journey.ts +20 -5
  6. package/corpus/templates/analytics/actions/provider-api-register.ts +2 -7
  7. package/corpus/templates/analytics/changelog/2026-10-21-analytics-recognizes-questions-about-connected-data-product-.md +6 -0
  8. package/corpus/templates/analytics/changelog/2026-10-21-onboarding-journeys-now-honor-larger-event-row-budgets.md +6 -0
  9. package/corpus/templates/analytics/changelog/2026-10-21-preserve-unknown-replay-availability-in-partial-journeys.md +6 -0
  10. package/corpus/templates/analytics/package.json +6 -0
  11. package/corpus/templates/analytics/scripts/journey-capture-plan.ts +18 -7
  12. package/corpus/templates/analytics/scripts/journey-capture.ts +3 -1
  13. package/corpus/templates/analytics/server/lib/db-admin-connections.ts +19 -12
  14. package/corpus/templates/analytics/server/lib/journey-tree.ts +24 -9
  15. package/corpus/templates/analytics/server/lib/onboarding-journey-cost.ts +163 -94
  16. package/corpus/templates/analytics/server/lib/onboarding-journey.ts +162 -74
  17. package/corpus/templates/assets/server/lib/org-admin.ts +20 -16
  18. package/corpus/templates/brain/server/lib/zoom-governance.ts +11 -6
  19. package/corpus/templates/clips/CHANGELOG.md +1 -0
  20. package/corpus/templates/clips/changelog/2026-10-16-rewind-status-says-when-it-remembers-audio.md +6 -0
  21. package/corpus/templates/clips/desktop/src/app.tsx +7 -4
  22. package/corpus/templates/clips/desktop/src/components/RewindCaptureStatus.tsx +25 -0
  23. package/corpus/templates/clips/desktop/src/lib/rewind-status.ts +6 -1
  24. package/corpus/templates/clips/desktop/src/shared/config.ts +3 -0
  25. package/corpus/templates/clips/desktop/src-tauri/src/screen_memory.rs +2 -0
  26. package/corpus/templates/clips/server/lib/recordings.ts +4 -19
  27. package/corpus/templates/community/qr/package.json +1 -1
  28. package/corpus/templates/content/CHANGELOG.md +8 -0
  29. package/corpus/templates/content/actions/_content-space-access.ts +95 -6
  30. package/corpus/templates/content/actions/_database-utils.ts +20 -16
  31. package/corpus/templates/content/actions/_github-source-freshness.ts +72 -17
  32. package/corpus/templates/content/actions/_github-source-metadata.ts +63 -17
  33. package/corpus/templates/content/actions/_github-source.ts +141 -77
  34. package/corpus/templates/content/actions/check-github-connection.ts +15 -0
  35. package/corpus/templates/content/actions/create-github-collection.ts +180 -0
  36. package/corpus/templates/content/actions/disconnect-content-database-source.ts +5 -0
  37. package/corpus/templates/content/actions/link-github-folder.ts +104 -36
  38. package/corpus/templates/content/actions/refresh-content-database-source.ts +37 -4
  39. package/corpus/templates/content/actions/sync-github-folder.ts +23 -5
  40. package/corpus/templates/content/app/components/editor/database/DatabaseView.tsx +172 -19
  41. package/corpus/templates/content/app/components/editor/database/GitHubLinkDialog.tsx +938 -0
  42. package/corpus/templates/content/app/components/editor/database/GitHubSourcePanel.tsx +228 -0
  43. package/corpus/templates/content/app/components/editor/database/GitHubSourceStatusLine.tsx +561 -0
  44. package/corpus/templates/content/app/components/editor/database/GitHubUnlinkDialog.tsx +96 -0
  45. package/corpus/templates/content/app/components/editor/database/github-availability.ts +147 -0
  46. package/corpus/templates/content/app/components/editor/database/github-source-text.ts +280 -0
  47. package/corpus/templates/content/app/components/layout/Layout.tsx +2 -0
  48. package/corpus/templates/content/app/hooks/content-action-refresh.ts +6 -0
  49. package/corpus/templates/content/app/hooks/use-content-database.ts +3 -1
  50. package/corpus/templates/content/app/hooks/use-github-folder.ts +199 -0
  51. package/corpus/templates/content/app/i18n-data.ts +154 -1
  52. package/corpus/templates/content/app/lib/github-link-dialog.ts +86 -0
  53. package/corpus/templates/content/changelog/2026-10-18-connected-ai-assistants-can-link-a-new-collection-to-a-github.md +1 -1
  54. package/corpus/templates/content/changelog/2026-10-18-link-a-new-collection-to-a-github-folder-from-its-sources.md +6 -0
  55. package/corpus/templates/content/changelog/2026-10-21-organization-favorites-include-documents-shared-with-the-cur.md +6 -0
  56. package/corpus/templates/content/docs/product/capabilities/content.source.sync-policy.md +32 -7
  57. package/corpus/templates/content/docs/product/encyclopedia.md +3 -3
  58. package/corpus/templates/content/docs/product/features/content.feature.trust-your-connected-sources.md +2 -2
  59. package/corpus/templates/content/docs/product/roadmap.md +1 -1
  60. package/corpus/templates/content/parity/matrix.md +42 -42
  61. package/corpus/templates/content/parity/matrix.ts +14 -0
  62. package/corpus/templates/content/server/lib/github-source-client.ts +50 -1
  63. package/corpus/templates/content/shared/api.ts +3 -0
  64. package/corpus/templates/content/shared/github-source.ts +68 -0
  65. package/corpus/templates/design/CHANGELOG.md +1 -0
  66. package/corpus/templates/design/actions/connect-assets-mcp.ts +5 -9
  67. package/corpus/templates/design/changelog/2026-10-21-journey-canvases-accept-partial-analytics-samples-when-repla.md +6 -0
  68. package/corpus/templates/design/shared/journey-canvas.ts +6 -4
  69. package/corpus/templates/dispatch/.agents/skills/a2a-protocol/SKILL.md +16 -13
  70. package/corpus/templates/dispatch/CHANGELOG.md +10 -0
  71. package/corpus/templates/dispatch/changelog/2026-10-21-customer-ai-credit-questions-route-to-account-health-instead.md +6 -0
  72. package/corpus/templates/dispatch/changelog/2026-10-21-workspace-apps-load-reliably-with-organization-sign-in-and-f.md +6 -0
  73. package/corpus/templates/factory/server/lib/require-workspace-member.ts +5 -17
  74. package/corpus/templates/mail/server/lib/queued-drafts.ts +9 -4
  75. package/corpus/templates/slides/server/workspace-defaults.ts +12 -15
  76. package/package.json +1 -1
@@ -17,8 +17,11 @@ import type {
17
17
  } from "./first-party-metric-catalog.js";
18
18
  import {
19
19
  buildOnboardingJourneyEventPageRequest,
20
- MAX_ONBOARDING_EVENT_READ_PAGES,
20
+ getOnboardingJourneyEventReadPageBudget,
21
+ getOnboardingJourneyEventReadPageCount,
22
+ ONBOARDING_EVENT_READ_DEADLINE_MS,
21
23
  ONBOARDING_EVENT_PAGE_ROWS,
24
+ ONBOARDING_EVENTS_DEFAULT_MAX_TOTAL_BYTES_BILLED,
22
25
  ONBOARDING_EVENTS_MAX_BYTES_BILLED,
23
26
  } from "./onboarding-journey.js";
24
27
 
@@ -37,9 +40,16 @@ export class OnboardingJourneyCostError extends Error {
37
40
  }
38
41
 
39
42
  export interface OnboardingJourneyCostEstimate {
40
- status: "within_cap" | "over_cap";
43
+ status: "within_cap" | "over_cap" | "incomplete";
41
44
  maxBytesBilledPerQuery: number;
42
- possiblePagesEstimatedBytes: number;
45
+ maxTotalBytesBilled: number;
46
+ requestedPageCount: number;
47
+ allowedPageCount: number;
48
+ pageBudgetLimited: boolean;
49
+ estimatedPagesBytes: number;
50
+ possiblePagesEstimatedBytes: number | null;
51
+ unestimatedPageCount: number;
52
+ incompleteReason?: "deadline" | "page_timeout";
43
53
  pages: Array<{
44
54
  page: number;
45
55
  offset: number;
@@ -54,118 +64,177 @@ export async function estimateOnboardingJourneyEventQueryCost(
54
64
  observation: OnboardingJourneyObservationWindow,
55
65
  freezeReceivedAt: boolean,
56
66
  signal?: AbortSignal,
67
+ maxTotalBytesBilled = ONBOARDING_EVENTS_DEFAULT_MAX_TOTAL_BYTES_BILLED,
57
68
  ): Promise<OnboardingJourneyCostEstimate> {
58
- let backend: Awaited<ReturnType<typeof getFirstPartyAnalyticsBackend>>;
59
- try {
60
- backend = await getFirstPartyAnalyticsBackend(scope, signal);
61
- } catch (error) {
62
- throwPreparationFailure(error, signal);
63
- }
64
- if (backend.sink !== "bigquery") {
65
- throw new OnboardingJourneyCostError("unsupported_backend");
66
- }
67
- let table: Awaited<ReturnType<typeof getFirstPartyAnalyticsTable>>;
68
- try {
69
- table = await getFirstPartyAnalyticsTable(backend.table, signal);
70
- } catch (error) {
71
- throwPreparationFailure(error, signal);
72
- }
69
+ const requestedPageCount =
70
+ getOnboardingJourneyEventReadPageCount(maxEventRows);
71
+ const allowedPageCount = Math.min(
72
+ requestedPageCount,
73
+ getOnboardingJourneyEventReadPageBudget(maxTotalBytesBilled),
74
+ );
73
75
  const pageOffsets = Array.from(
74
- { length: MAX_ONBOARDING_EVENT_READ_PAGES },
76
+ { length: allowedPageCount },
75
77
  (_unused, index) => index * ONBOARDING_EVENT_PAGE_ROWS,
76
- ).filter((offset) => offset <= maxEventRows);
78
+ );
77
79
  const pages: OnboardingJourneyCostEstimate["pages"] = [];
80
+ let incompleteReason: OnboardingJourneyCostEstimate["incompleteReason"];
81
+ const deadline = new AbortController();
82
+ const deadlineTimer = setTimeout(
83
+ () =>
84
+ deadline.abort(
85
+ new DOMException(
86
+ "Onboarding cost estimate deadline exceeded",
87
+ "AbortError",
88
+ ),
89
+ ),
90
+ ONBOARDING_EVENT_READ_DEADLINE_MS,
91
+ );
92
+ const estimateSignal = signal
93
+ ? AbortSignal.any([signal, deadline.signal])
94
+ : deadline.signal;
78
95
 
79
- for (const [index, offset] of pageOffsets.entries()) {
80
- if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
81
- let renderedSql: string;
96
+ try {
97
+ let backend: Awaited<
98
+ ReturnType<typeof getFirstPartyAnalyticsBackend>
99
+ > | null = null;
82
100
  try {
83
- const request = buildOnboardingJourneyEventPageRequest(
84
- filters,
85
- maxEventRows,
86
- offset,
87
- observation,
88
- freezeReceivedAt,
89
- );
90
- const scoped = scopedAnalyticsSql(request.sql, scope, undefined, {
91
- scopedEventsSingleScan: true,
92
- scopedEventsProjection: "onboarding_journey",
93
- scopedEventsSourceProjections: {
94
- e: "onboarding_journey",
95
- r: "onboarding_journey_response_identity",
96
- },
97
- });
98
- renderedSql = renderFirstPartyAnalyticsBigQueryRequestSql(
99
- scoped.sql,
100
- scoped.args,
101
- table,
102
- {
103
- eventDateRange: request.eventDateRange,
104
- scopedEventsSingleScan: true,
105
- scopedEventsProjection: "onboarding_journey",
106
- scopedEventsSourceProjections: {
107
- e: "onboarding_journey",
108
- r: "onboarding_journey_response_identity",
109
- },
110
- },
111
- );
101
+ backend = await getFirstPartyAnalyticsBackend(scope, estimateSignal);
112
102
  } catch (error) {
113
- throwPreparationFailure(error, signal);
103
+ if (signal?.aborted) throw error;
104
+ if (!deadline.signal.aborted) throwPreparationFailure(error, signal);
114
105
  }
115
- let dryRun: Awaited<ReturnType<typeof dryRunQuerySchema>>;
116
- try {
117
- dryRun = await dryRunQuerySchema(renderedSql, {
118
- signal,
119
- wrapPreparationErrors: true,
120
- });
121
- } catch (error) {
122
- if (
123
- signal?.aborted ||
124
- (error instanceof Error && error.name === "AbortError")
125
- ) {
126
- throw error;
127
- }
128
- throw new OnboardingJourneyCostError(
129
- error instanceof BigQueryDryRunPreparationError
130
- ? "preparation_failed"
131
- : "dry_run_failed",
132
- );
106
+ if (backend && backend.sink !== "bigquery") {
107
+ throw new OnboardingJourneyCostError("unsupported_backend");
133
108
  }
134
- if (dryRun.error !== null) {
135
- throw new OnboardingJourneyCostError(
136
- dryRun.timedOut ? "dry_run_timeout" : "dry_run_failed",
137
- );
109
+ let table: Awaited<ReturnType<typeof getFirstPartyAnalyticsTable>> | null =
110
+ null;
111
+ if (backend) {
112
+ try {
113
+ table = await getFirstPartyAnalyticsTable(
114
+ backend.table,
115
+ estimateSignal,
116
+ );
117
+ } catch (error) {
118
+ if (signal?.aborted) throw error;
119
+ if (!deadline.signal.aborted) throwPreparationFailure(error, signal);
120
+ }
138
121
  }
139
- const estimatedBytes = dryRun.totalBytesProcessed;
140
- if (
141
- typeof estimatedBytes !== "number" ||
142
- !Number.isSafeInteger(estimatedBytes) ||
143
- estimatedBytes < 0
144
- ) {
145
- throw new OnboardingJourneyCostError("estimate_unavailable");
122
+ if (table) {
123
+ for (const [index, offset] of pageOffsets.entries()) {
124
+ if (signal?.aborted) {
125
+ throw new DOMException("Aborted", "AbortError");
126
+ }
127
+ if (deadline.signal.aborted) {
128
+ incompleteReason = "deadline";
129
+ break;
130
+ }
131
+ let renderedSql: string;
132
+ try {
133
+ const request = buildOnboardingJourneyEventPageRequest(
134
+ filters,
135
+ maxEventRows,
136
+ offset,
137
+ observation,
138
+ freezeReceivedAt,
139
+ );
140
+ const scoped = scopedAnalyticsSql(request.sql, scope, undefined, {
141
+ scopedEventsSingleScan: true,
142
+ scopedEventsProjection: "onboarding_journey",
143
+ scopedEventsSourceProjections: {
144
+ e: "onboarding_journey",
145
+ r: "onboarding_journey_response_identity",
146
+ },
147
+ });
148
+ renderedSql = renderFirstPartyAnalyticsBigQueryRequestSql(
149
+ scoped.sql,
150
+ scoped.args,
151
+ table,
152
+ {
153
+ eventDateRange: request.eventDateRange,
154
+ scopedEventsSingleScan: true,
155
+ scopedEventsProjection: "onboarding_journey",
156
+ scopedEventsSourceProjections: {
157
+ e: "onboarding_journey",
158
+ r: "onboarding_journey_response_identity",
159
+ },
160
+ },
161
+ );
162
+ } catch (error) {
163
+ throwPreparationFailure(error, signal);
164
+ }
165
+ let dryRun: Awaited<ReturnType<typeof dryRunQuerySchema>>;
166
+ try {
167
+ dryRun = await dryRunQuerySchema(renderedSql, {
168
+ signal: estimateSignal,
169
+ wrapPreparationErrors: true,
170
+ });
171
+ } catch (error) {
172
+ if (signal?.aborted) throw error;
173
+ if (deadline.signal.aborted) {
174
+ incompleteReason = "deadline";
175
+ break;
176
+ }
177
+ throw new OnboardingJourneyCostError(
178
+ error instanceof BigQueryDryRunPreparationError
179
+ ? "preparation_failed"
180
+ : "dry_run_failed",
181
+ );
182
+ }
183
+ if (dryRun.timedOut) {
184
+ incompleteReason = "page_timeout";
185
+ break;
186
+ }
187
+ if (dryRun.error !== null) {
188
+ throw new OnboardingJourneyCostError("dry_run_failed");
189
+ }
190
+ const estimatedBytes = dryRun.totalBytesProcessed;
191
+ if (
192
+ typeof estimatedBytes !== "number" ||
193
+ !Number.isSafeInteger(estimatedBytes) ||
194
+ estimatedBytes < 0
195
+ ) {
196
+ throw new OnboardingJourneyCostError("estimate_unavailable");
197
+ }
198
+ pages.push({
199
+ page: index + 1,
200
+ offset,
201
+ estimatedBytes,
202
+ });
203
+ }
204
+ } else {
205
+ incompleteReason = "deadline";
146
206
  }
147
- pages.push({
148
- page: index + 1,
149
- offset,
150
- estimatedBytes,
151
- });
207
+ } finally {
208
+ clearTimeout(deadlineTimer);
152
209
  }
153
210
 
154
- const possiblePagesEstimatedBytes = pages.reduce(
211
+ const estimatedPagesBytes = pages.reduce(
155
212
  (total, page) => total + page.estimatedBytes,
156
213
  0,
157
214
  );
158
- if (!Number.isSafeInteger(possiblePagesEstimatedBytes)) {
215
+ if (!Number.isSafeInteger(estimatedPagesBytes)) {
159
216
  throw new OnboardingJourneyCostError("estimate_unavailable");
160
217
  }
218
+ const unestimatedPageCount = allowedPageCount - pages.length;
219
+ const overCap = pages.some(
220
+ (page) => page.estimatedBytes > ONBOARDING_EVENTS_MAX_BYTES_BILLED,
221
+ );
161
222
  return {
162
- status: pages.some(
163
- (page) => page.estimatedBytes > ONBOARDING_EVENTS_MAX_BYTES_BILLED,
164
- )
223
+ status: overCap
165
224
  ? "over_cap"
166
- : "within_cap",
225
+ : unestimatedPageCount > 0
226
+ ? "incomplete"
227
+ : "within_cap",
167
228
  maxBytesBilledPerQuery: ONBOARDING_EVENTS_MAX_BYTES_BILLED,
168
- possiblePagesEstimatedBytes,
229
+ maxTotalBytesBilled,
230
+ requestedPageCount: getOnboardingJourneyEventReadPageCount(maxEventRows),
231
+ allowedPageCount,
232
+ pageBudgetLimited: allowedPageCount < requestedPageCount,
233
+ estimatedPagesBytes,
234
+ possiblePagesEstimatedBytes:
235
+ unestimatedPageCount === 0 ? estimatedPagesBytes : null,
236
+ unestimatedPageCount,
237
+ ...(incompleteReason ? { incompleteReason } : {}),
169
238
  pages,
170
239
  };
171
240
  }
@@ -7,7 +7,10 @@ import {
7
7
  BigQueryQueryTimeoutError,
8
8
  type BigQueryBackendReason,
9
9
  } from "./bigquery.js";
10
- import { getFirstPartyAnalyticsBackend } from "./first-party-analytics-backend.js";
10
+ import {
11
+ getFirstPartyAnalyticsBackend,
12
+ type FirstPartyAnalyticsSink,
13
+ } from "./first-party-analytics-backend.js";
11
14
  import {
12
15
  queryFirstPartyAnalytics,
13
16
  FirstPartyAnalyticsQueryTimeoutError,
@@ -50,15 +53,33 @@ import {
50
53
  // Both backends cap a query result at 5,000 rows; stay under it so a full page
51
54
  // is never mistaken for a cut one.
52
55
  export const ONBOARDING_EVENT_PAGE_ROWS = 4_000;
53
- export const MAX_ONBOARDING_EVENT_READ_PAGES = 2;
56
+ export const DEFAULT_ONBOARDING_EVENT_ROW_BUDGET = 40_000;
57
+ export const MAX_ONBOARDING_EVENT_ROW_BUDGET = 200_000;
54
58
  const MAX_ONBOARDING_JOURNEY_FOLLOWUP_TERMINALS = 2_000;
55
59
  const MAX_FOLLOWUP_QUERY_CHARS = 800_000;
56
60
  const MAX_FOLLOWUP_QUERY_TOKENS = 50_000;
57
61
  const ONBOARDING_QUERY_TIMEOUT_MS = 20_000;
62
+ export const ONBOARDING_EVENT_READ_DEADLINE_MS = 45_000;
58
63
  export const ONBOARDING_EVENTS_MAX_BYTES_BILLED = 25_000_000_000;
64
+ export const ONBOARDING_EVENTS_DEFAULT_MAX_TOTAL_BYTES_BILLED =
65
+ ONBOARDING_EVENTS_MAX_BYTES_BILLED * 11;
66
+ export const ONBOARDING_EVENTS_MAX_TOTAL_BYTES_BILLED =
67
+ ONBOARDING_EVENTS_MAX_BYTES_BILLED * 51;
59
68
  const ONBOARDING_FOLLOWUP_MAX_BYTES_BILLED = 10_000_000_000;
60
69
  const DAY_MS = 24 * 60 * 60 * 1000;
61
70
 
71
+ export function getOnboardingJourneyEventReadPageCount(
72
+ maxEventRows: number,
73
+ ): number {
74
+ return Math.ceil((maxEventRows + 1) / ONBOARDING_EVENT_PAGE_ROWS);
75
+ }
76
+
77
+ export function getOnboardingJourneyEventReadPageBudget(
78
+ maxTotalBytesBilled: number,
79
+ ): number {
80
+ return Math.floor(maxTotalBytesBilled / ONBOARDING_EVENTS_MAX_BYTES_BILLED);
81
+ }
82
+
62
83
  function throwIfAborted(signal?: AbortSignal): void {
63
84
  if (signal?.aborted) {
64
85
  throw new DOMException("The operation was aborted", "AbortError");
@@ -185,6 +206,7 @@ export interface OnboardingJourneyArgs extends OnboardingJourneyEventsFilters {
185
206
  minNodeSessions: number;
186
207
  examplesPerNode: number;
187
208
  maxEventRows: number;
209
+ maxTotalBytesBilled: number;
188
210
  maxNodes: number;
189
211
  settleMs: number;
190
212
  recency: "newest" | "none";
@@ -199,8 +221,8 @@ export interface JourneyTree {
199
221
  coverage: {
200
222
  /** Sessions in the tree: entered onboarding in the window and produced a step. */
201
223
  sessionsWithEvents: number;
202
- /** Sessions with a playable recording the caller can open. */
203
- sessionsWithReplay: number;
224
+ /** Null when a truncated event sample causes replay reads to be skipped. */
225
+ sessionsWithReplay: number | null;
204
226
  /** Counts are a partial sample: the event read or the node list was cut. */
205
227
  truncated: boolean;
206
228
  };
@@ -212,7 +234,7 @@ export interface JourneyTree {
212
234
  rootN: number;
213
235
  coverage: {
214
236
  sessionsWithEvents: number;
215
- sessionsWithReplay: number;
237
+ sessionsWithReplay: number | null;
216
238
  truncated: boolean;
217
239
  };
218
240
  nodes: JourneyNode[];
@@ -228,7 +250,7 @@ export interface JourneySummary {
228
250
  rootN: number;
229
251
  coverage: {
230
252
  sessionsWithEvents: number;
231
- /** Null for summaries that skip replay reads or when a replay read fails. */
253
+ /** Null when replay reads are skipped for summaries or truncated samples. */
232
254
  sessionsWithReplay: number | null;
233
255
  truncated: boolean;
234
256
  };
@@ -472,7 +494,12 @@ interface EventRead {
472
494
  lastSessionDroppedFor: JourneyEventRow["journeyKind"] | null;
473
495
  pages: number;
474
496
  paginationConsistency: "stable" | "may_have_shifted";
475
- truncationReason: "max_event_rows" | "backend_result" | "page_budget" | null;
497
+ truncationReason:
498
+ | "max_event_rows"
499
+ | "backend_result"
500
+ | "page_budget"
501
+ | "read_deadline"
502
+ | null;
476
503
  }
477
504
 
478
505
  export interface OnboardingJourneyEventPageRequest {
@@ -513,15 +540,13 @@ async function readJourneyEvents(
513
540
  scope: AnalyticsScope,
514
541
  filters: OnboardingJourneyEventsFilters,
515
542
  maxEventRows: number,
543
+ maxTotalBytesBilled: number,
516
544
  observation: OnboardingJourneyObservationWindow,
517
545
  freezeReceivedAt: boolean,
518
546
  signal?: AbortSignal,
519
547
  ): Promise<EventRead> {
520
- const { sink } = await getFirstPartyAnalyticsBackend(scope, signal);
521
- const maxPages =
522
- sink === "bigquery"
523
- ? MAX_ONBOARDING_EVENT_READ_PAGES
524
- : Math.ceil((maxEventRows + 1) / ONBOARDING_EVENT_PAGE_ROWS);
548
+ const requestedMaxPages =
549
+ getOnboardingJourneyEventReadPageCount(maxEventRows);
525
550
  const raw: Record<string, unknown>[] = [];
526
551
  let truncated = false;
527
552
  let truncationReason: EventRead["truncationReason"] = null;
@@ -529,61 +554,109 @@ async function readJourneyEvents(
529
554
  let overflowSessionId: string | null = null;
530
555
  let pages = 0;
531
556
  let rowsFetched = 0;
532
- for (;;) {
557
+ const deadline = new AbortController();
558
+ const deadlineTimer = setTimeout(
559
+ () =>
560
+ deadline.abort(
561
+ new DOMException(
562
+ "Onboarding event read deadline exceeded",
563
+ "AbortError",
564
+ ),
565
+ ),
566
+ ONBOARDING_EVENT_READ_DEADLINE_MS,
567
+ );
568
+ const querySignal = signal
569
+ ? AbortSignal.any([signal, deadline.signal])
570
+ : deadline.signal;
571
+ try {
533
572
  throwIfAborted(signal);
534
- // One row past the budget tells a full read from a cut one.
535
- const request = buildOnboardingJourneyEventPageRequest(
536
- filters,
537
- maxEventRows,
538
- raw.length,
539
- observation,
540
- freezeReceivedAt,
541
- );
542
- let page: Awaited<ReturnType<typeof queryFirstPartyAnalytics>>;
573
+ let backendSink: FirstPartyAnalyticsSink | null = null;
543
574
  try {
544
- page = await queryFirstPartyAnalytics(request.sql, scope, {
545
- cache: true,
546
- timeoutMs: ONBOARDING_QUERY_TIMEOUT_MS,
547
- maxBytesBilled: ONBOARDING_EVENTS_MAX_BYTES_BILLED,
548
- eventDateRange: request.eventDateRange,
549
- scopedEventsSingleScan: true,
550
- scopedEventsProjection: "onboarding_journey",
551
- scopedEventsSourceProjections: {
552
- e: "onboarding_journey",
553
- r: "onboarding_journey_response_identity",
554
- },
555
- signal,
556
- });
575
+ const backend = await getFirstPartyAnalyticsBackend(scope, querySignal);
576
+ backendSink = backend.sink;
557
577
  } catch (error) {
558
- if (signal?.aborted || isAbortError(error)) throw error;
559
- throw new OnboardingJourneyReadError(error, pages + 1);
560
- }
561
- pages += 1;
562
- raw.push(...page.rows);
563
- rowsFetched += page.rows.length;
564
- if (page.truncated) {
565
- truncated = true;
566
- truncationReason = "backend_result";
567
- break;
568
- }
569
- if (page.rows.length < request.limit) break;
570
- if (raw.length > maxEventRows) {
578
+ if (signal?.aborted) throw error;
579
+ if (!deadline.signal.aborted) throw error;
571
580
  truncated = true;
572
- truncationReason = "max_event_rows";
573
- const overflowKind = raw[maxEventRows]?.journey_kind;
574
- truncatedAt =
575
- overflowKind === "onboarding" || overflowKind === "standalone_setup"
576
- ? overflowKind
577
- : null;
578
- overflowSessionId = text(raw[maxEventRows]?.session_id);
579
- raw.length = maxEventRows;
580
- break;
581
+ truncationReason = "read_deadline";
581
582
  }
582
- if (pages >= maxPages) {
583
- truncated = true;
584
- truncationReason = "page_budget";
585
- break;
583
+ const maxPages =
584
+ backendSink === "bigquery"
585
+ ? Math.min(
586
+ requestedMaxPages,
587
+ getOnboardingJourneyEventReadPageBudget(maxTotalBytesBilled),
588
+ )
589
+ : requestedMaxPages;
590
+
591
+ while (!truncated) {
592
+ throwIfAborted(signal);
593
+ if (deadline.signal.aborted) {
594
+ truncated = true;
595
+ truncationReason = "read_deadline";
596
+ break;
597
+ }
598
+ // One row past the budget tells a full read from a cut one.
599
+ const request = buildOnboardingJourneyEventPageRequest(
600
+ filters,
601
+ maxEventRows,
602
+ raw.length,
603
+ observation,
604
+ freezeReceivedAt,
605
+ );
606
+ let page: Awaited<ReturnType<typeof queryFirstPartyAnalytics>>;
607
+ try {
608
+ page = await queryFirstPartyAnalytics(request.sql, scope, {
609
+ cache: true,
610
+ timeoutMs: ONBOARDING_QUERY_TIMEOUT_MS,
611
+ maxBytesBilled: ONBOARDING_EVENTS_MAX_BYTES_BILLED,
612
+ eventDateRange: request.eventDateRange,
613
+ scopedEventsSingleScan: true,
614
+ scopedEventsProjection: "onboarding_journey",
615
+ scopedEventsSourceProjections: {
616
+ e: "onboarding_journey",
617
+ r: "onboarding_journey_response_identity",
618
+ },
619
+ signal: querySignal,
620
+ });
621
+ } catch (error) {
622
+ if (signal?.aborted) throw error;
623
+ if (deadline.signal.aborted) {
624
+ truncated = true;
625
+ truncationReason = "read_deadline";
626
+ break;
627
+ }
628
+ if (isAbortError(error)) throw error;
629
+ throw new OnboardingJourneyReadError(error, pages + 1);
630
+ }
631
+ pages += 1;
632
+ raw.push(...page.rows);
633
+ rowsFetched += page.rows.length;
634
+ if (page.truncated) {
635
+ truncated = true;
636
+ truncationReason = "backend_result";
637
+ break;
638
+ }
639
+ if (page.rows.length < request.limit) break;
640
+ if (raw.length > maxEventRows) {
641
+ truncated = true;
642
+ truncationReason = "max_event_rows";
643
+ const overflowKind = raw[maxEventRows]?.journey_kind;
644
+ truncatedAt =
645
+ overflowKind === "onboarding" || overflowKind === "standalone_setup"
646
+ ? overflowKind
647
+ : null;
648
+ overflowSessionId = text(raw[maxEventRows]?.session_id);
649
+ raw.length = maxEventRows;
650
+ break;
651
+ }
652
+ if (pages >= maxPages) {
653
+ truncated = true;
654
+ truncationReason = "page_budget";
655
+ break;
656
+ }
586
657
  }
658
+ } finally {
659
+ clearTimeout(deadlineTimer);
587
660
  }
588
661
 
589
662
  const byId = new Map<string, JourneyEventRow>();
@@ -1577,6 +1650,7 @@ export async function getOnboardingJourney(
1577
1650
  scope,
1578
1651
  args,
1579
1652
  args.maxEventRows,
1653
+ args.maxTotalBytesBilled,
1580
1654
  observation,
1581
1655
  args.followUpMode === "person",
1582
1656
  signal,
@@ -1647,7 +1721,8 @@ export async function getOnboardingJourney(
1647
1721
 
1648
1722
  let recordings: JourneyRecording[] | null;
1649
1723
  throwIfAborted(signal);
1650
- if (args.format === "summary") {
1724
+ const replayReadSkipped = args.format === "tree" && read.truncated;
1725
+ if (args.format === "summary" || replayReadSkipped) {
1651
1726
  recordings = null;
1652
1727
  } else {
1653
1728
  recordings = await readRecordings(
@@ -1663,8 +1738,9 @@ export async function getOnboardingJourney(
1663
1738
  if (list) list.push(recording);
1664
1739
  else bySession.set(recording.sessionId, [recording]);
1665
1740
  }
1741
+ const recordingsBySession = recordings === null ? null : bySession;
1666
1742
 
1667
- const built = buildJourneyTree(sessions, bySession, {
1743
+ const built = buildJourneyTree(sessions, recordingsBySession, {
1668
1744
  maxDepth: args.maxDepth,
1669
1745
  minNodeSessions: args.minNodeSessions,
1670
1746
  examplesPerNode: args.format === "tree" ? args.examplesPerNode : 0,
@@ -1677,7 +1753,7 @@ export async function getOnboardingJourney(
1677
1753
  const hasStandaloneResult =
1678
1754
  standalone.sessions.length > 0 || read.standaloneSetupTruncated;
1679
1755
  const standaloneBuilt = hasStandaloneResult
1680
- ? buildJourneyTree(standalone.sessions, bySession, {
1756
+ ? buildJourneyTree(standalone.sessions, recordingsBySession, {
1681
1757
  maxDepth: args.maxDepth,
1682
1758
  minNodeSessions: args.minNodeSessions,
1683
1759
  examplesPerNode: args.format === "tree" ? args.examplesPerNode : 0,
@@ -1694,10 +1770,12 @@ export async function getOnboardingJourney(
1694
1770
  const notes: string[] = [];
1695
1771
  const eventReadLimit =
1696
1772
  read.truncationReason === "page_budget"
1697
- ? `Event read stopped after ${MAX_ONBOARDING_EVENT_READ_PAGES} BigQuery pages to bound query cost`
1698
- : read.truncationReason === "backend_result"
1699
- ? "BigQuery returned an incomplete event page"
1700
- : `Event read stopped at maxEventRows=${args.maxEventRows}`;
1773
+ ? `Event read reached its aggregate maxTotalBytesBilled=${args.maxTotalBytesBilled} bytes budget before confirming whether maxEventRows=${args.maxEventRows} was exceeded`
1774
+ : read.truncationReason === "read_deadline"
1775
+ ? `Event read reached its ${ONBOARDING_EVENT_READ_DEADLINE_MS / 1000}-second deadline before confirming whether maxEventRows=${args.maxEventRows} was exceeded`
1776
+ : read.truncationReason === "backend_result"
1777
+ ? "BigQuery returned an incomplete event page"
1778
+ : `Event read stopped at maxEventRows=${args.maxEventRows}`;
1701
1779
  if (read.onboardingTruncated) {
1702
1780
  notes.push(
1703
1781
  `${eventReadLimit} while reading onboarding events; onboarding counts are a partial sample${read.lastSessionDroppedFor === "onboarding" ? " and the last onboarding session read was left out" : ""}.`,
@@ -1734,6 +1812,11 @@ export async function getOnboardingJourney(
1734
1812
  `${read.invalidRows} event rows had no id, session, or readable timestamp and were not counted.`,
1735
1813
  );
1736
1814
  }
1815
+ if (replayReadSkipped) {
1816
+ notes.push(
1817
+ "Replay availability is unknown because the event sample was truncated; replay reads were skipped.",
1818
+ );
1819
+ }
1737
1820
  if (sessionsWithoutSteps + standalone.sessionsWithoutSteps) {
1738
1821
  notes.push(
1739
1822
  `${sessionsWithoutSteps + standalone.sessionsWithoutSteps} sessions had events but no step with a meaning (for example a pageview without a path) and are not in a tree.`,
@@ -1793,9 +1876,11 @@ export async function getOnboardingJourney(
1793
1876
  ...head,
1794
1877
  coverage: {
1795
1878
  sessionsWithEvents: sessions.length,
1796
- sessionsWithReplay: sessions.filter((session) =>
1797
- bySession.has(session.sessionId),
1798
- ).length,
1879
+ sessionsWithReplay:
1880
+ recordings === null
1881
+ ? null
1882
+ : sessions.filter((session) => bySession.has(session.sessionId))
1883
+ .length,
1799
1884
  truncated,
1800
1885
  },
1801
1886
  nodes: capped.nodes,
@@ -1807,9 +1892,12 @@ export async function getOnboardingJourney(
1807
1892
  rootN: standaloneBuilt.rootN,
1808
1893
  coverage: {
1809
1894
  sessionsWithEvents: standalone.sessions.length,
1810
- sessionsWithReplay: standalone.sessions.filter((session) =>
1811
- bySession.has(session.sessionId),
1812
- ).length,
1895
+ sessionsWithReplay:
1896
+ recordings === null
1897
+ ? null
1898
+ : standalone.sessions.filter((session) =>
1899
+ bySession.has(session.sessionId),
1900
+ ).length,
1813
1901
  truncated:
1814
1902
  read.standaloneSetupTruncated ||
1815
1903
  standaloneDepthTruncated ||