@agent-native/core-corpus 0.212.1 → 0.213.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/templates/analytics/.agents/skills/onboarding-journeys/SKILL.md +29 -14
- package/corpus/templates/analytics/CHANGELOG.md +11 -0
- package/corpus/templates/analytics/actions/estimate-onboarding-journey-cost.ts +23 -6
- package/corpus/templates/analytics/actions/get-onboarding-journey.ts +20 -5
- package/corpus/templates/analytics/actions/provider-api-register.ts +2 -7
- package/corpus/templates/analytics/changelog/2026-10-21-analytics-recognizes-questions-about-connected-data-product-.md +6 -0
- package/corpus/templates/analytics/changelog/2026-10-21-onboarding-journeys-now-honor-larger-event-row-budgets.md +6 -0
- package/corpus/templates/analytics/changelog/2026-10-21-preserve-unknown-replay-availability-in-partial-journeys.md +6 -0
- package/corpus/templates/analytics/package.json +6 -0
- package/corpus/templates/analytics/scripts/journey-capture-plan.ts +18 -7
- package/corpus/templates/analytics/scripts/journey-capture.ts +3 -1
- package/corpus/templates/analytics/server/lib/db-admin-connections.ts +19 -12
- package/corpus/templates/analytics/server/lib/journey-tree.ts +24 -9
- package/corpus/templates/analytics/server/lib/onboarding-journey-cost.ts +163 -94
- package/corpus/templates/analytics/server/lib/onboarding-journey.ts +162 -74
- package/corpus/templates/assets/server/lib/org-admin.ts +20 -16
- package/corpus/templates/brain/server/lib/zoom-governance.ts +11 -6
- package/corpus/templates/clips/CHANGELOG.md +1 -0
- package/corpus/templates/clips/changelog/2026-10-16-rewind-status-says-when-it-remembers-audio.md +6 -0
- package/corpus/templates/clips/desktop/src/app.tsx +7 -4
- package/corpus/templates/clips/desktop/src/components/RewindCaptureStatus.tsx +25 -0
- package/corpus/templates/clips/desktop/src/lib/rewind-status.ts +6 -1
- package/corpus/templates/clips/desktop/src/shared/config.ts +3 -0
- package/corpus/templates/clips/desktop/src-tauri/src/screen_memory.rs +2 -0
- package/corpus/templates/clips/server/lib/recordings.ts +4 -19
- package/corpus/templates/community/qr/package.json +1 -1
- package/corpus/templates/content/CHANGELOG.md +8 -0
- package/corpus/templates/content/actions/_content-space-access.ts +95 -6
- package/corpus/templates/content/actions/_database-utils.ts +20 -16
- package/corpus/templates/content/actions/_github-source-freshness.ts +72 -17
- package/corpus/templates/content/actions/_github-source-metadata.ts +63 -17
- package/corpus/templates/content/actions/_github-source.ts +141 -77
- package/corpus/templates/content/actions/check-github-connection.ts +15 -0
- package/corpus/templates/content/actions/create-github-collection.ts +180 -0
- package/corpus/templates/content/actions/disconnect-content-database-source.ts +5 -0
- package/corpus/templates/content/actions/link-github-folder.ts +104 -36
- package/corpus/templates/content/actions/refresh-content-database-source.ts +37 -4
- package/corpus/templates/content/actions/sync-github-folder.ts +23 -5
- package/corpus/templates/content/app/components/editor/database/DatabaseView.tsx +172 -19
- package/corpus/templates/content/app/components/editor/database/GitHubLinkDialog.tsx +938 -0
- package/corpus/templates/content/app/components/editor/database/GitHubSourcePanel.tsx +228 -0
- package/corpus/templates/content/app/components/editor/database/GitHubSourceStatusLine.tsx +561 -0
- package/corpus/templates/content/app/components/editor/database/GitHubUnlinkDialog.tsx +96 -0
- package/corpus/templates/content/app/components/editor/database/github-availability.ts +147 -0
- package/corpus/templates/content/app/components/editor/database/github-source-text.ts +280 -0
- package/corpus/templates/content/app/components/layout/Layout.tsx +2 -0
- package/corpus/templates/content/app/hooks/content-action-refresh.ts +6 -0
- package/corpus/templates/content/app/hooks/use-content-database.ts +3 -1
- package/corpus/templates/content/app/hooks/use-github-folder.ts +199 -0
- package/corpus/templates/content/app/i18n-data.ts +154 -1
- package/corpus/templates/content/app/lib/github-link-dialog.ts +86 -0
- package/corpus/templates/content/changelog/2026-10-18-connected-ai-assistants-can-link-a-new-collection-to-a-github.md +1 -1
- package/corpus/templates/content/changelog/2026-10-18-link-a-new-collection-to-a-github-folder-from-its-sources.md +6 -0
- package/corpus/templates/content/changelog/2026-10-21-organization-favorites-include-documents-shared-with-the-cur.md +6 -0
- package/corpus/templates/content/docs/product/capabilities/content.source.sync-policy.md +32 -7
- package/corpus/templates/content/docs/product/encyclopedia.md +3 -3
- package/corpus/templates/content/docs/product/features/content.feature.trust-your-connected-sources.md +2 -2
- package/corpus/templates/content/docs/product/roadmap.md +1 -1
- package/corpus/templates/content/parity/matrix.md +42 -42
- package/corpus/templates/content/parity/matrix.ts +14 -0
- package/corpus/templates/content/server/lib/github-source-client.ts +50 -1
- package/corpus/templates/content/shared/api.ts +3 -0
- package/corpus/templates/content/shared/github-source.ts +68 -0
- package/corpus/templates/design/CHANGELOG.md +1 -0
- package/corpus/templates/design/actions/connect-assets-mcp.ts +5 -9
- package/corpus/templates/design/changelog/2026-10-21-journey-canvases-accept-partial-analytics-samples-when-repla.md +6 -0
- package/corpus/templates/design/shared/journey-canvas.ts +6 -4
- package/corpus/templates/dispatch/.agents/skills/a2a-protocol/SKILL.md +16 -13
- package/corpus/templates/dispatch/CHANGELOG.md +4 -0
- package/corpus/templates/dispatch/changelog/2026-10-21-customer-ai-credit-questions-route-to-account-health-instead.md +6 -0
- package/corpus/templates/factory/server/lib/require-workspace-member.ts +5 -17
- package/corpus/templates/mail/server/lib/queued-drafts.ts +9 -4
- package/corpus/templates/slides/server/workspace-defaults.ts +12 -15
- package/package.json +1 -1
|
@@ -17,8 +17,11 @@ import type {
|
|
|
17
17
|
} from "./first-party-metric-catalog.js";
|
|
18
18
|
import {
|
|
19
19
|
buildOnboardingJourneyEventPageRequest,
|
|
20
|
-
|
|
20
|
+
getOnboardingJourneyEventReadPageBudget,
|
|
21
|
+
getOnboardingJourneyEventReadPageCount,
|
|
22
|
+
ONBOARDING_EVENT_READ_DEADLINE_MS,
|
|
21
23
|
ONBOARDING_EVENT_PAGE_ROWS,
|
|
24
|
+
ONBOARDING_EVENTS_DEFAULT_MAX_TOTAL_BYTES_BILLED,
|
|
22
25
|
ONBOARDING_EVENTS_MAX_BYTES_BILLED,
|
|
23
26
|
} from "./onboarding-journey.js";
|
|
24
27
|
|
|
@@ -37,9 +40,16 @@ export class OnboardingJourneyCostError extends Error {
|
|
|
37
40
|
}
|
|
38
41
|
|
|
39
42
|
export interface OnboardingJourneyCostEstimate {
|
|
40
|
-
status: "within_cap" | "over_cap";
|
|
43
|
+
status: "within_cap" | "over_cap" | "incomplete";
|
|
41
44
|
maxBytesBilledPerQuery: number;
|
|
42
|
-
|
|
45
|
+
maxTotalBytesBilled: number;
|
|
46
|
+
requestedPageCount: number;
|
|
47
|
+
allowedPageCount: number;
|
|
48
|
+
pageBudgetLimited: boolean;
|
|
49
|
+
estimatedPagesBytes: number;
|
|
50
|
+
possiblePagesEstimatedBytes: number | null;
|
|
51
|
+
unestimatedPageCount: number;
|
|
52
|
+
incompleteReason?: "deadline" | "page_timeout";
|
|
43
53
|
pages: Array<{
|
|
44
54
|
page: number;
|
|
45
55
|
offset: number;
|
|
@@ -54,118 +64,177 @@ export async function estimateOnboardingJourneyEventQueryCost(
|
|
|
54
64
|
observation: OnboardingJourneyObservationWindow,
|
|
55
65
|
freezeReceivedAt: boolean,
|
|
56
66
|
signal?: AbortSignal,
|
|
67
|
+
maxTotalBytesBilled = ONBOARDING_EVENTS_DEFAULT_MAX_TOTAL_BYTES_BILLED,
|
|
57
68
|
): Promise<OnboardingJourneyCostEstimate> {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if (backend.sink !== "bigquery") {
|
|
65
|
-
throw new OnboardingJourneyCostError("unsupported_backend");
|
|
66
|
-
}
|
|
67
|
-
let table: Awaited<ReturnType<typeof getFirstPartyAnalyticsTable>>;
|
|
68
|
-
try {
|
|
69
|
-
table = await getFirstPartyAnalyticsTable(backend.table, signal);
|
|
70
|
-
} catch (error) {
|
|
71
|
-
throwPreparationFailure(error, signal);
|
|
72
|
-
}
|
|
69
|
+
const requestedPageCount =
|
|
70
|
+
getOnboardingJourneyEventReadPageCount(maxEventRows);
|
|
71
|
+
const allowedPageCount = Math.min(
|
|
72
|
+
requestedPageCount,
|
|
73
|
+
getOnboardingJourneyEventReadPageBudget(maxTotalBytesBilled),
|
|
74
|
+
);
|
|
73
75
|
const pageOffsets = Array.from(
|
|
74
|
-
{ length:
|
|
76
|
+
{ length: allowedPageCount },
|
|
75
77
|
(_unused, index) => index * ONBOARDING_EVENT_PAGE_ROWS,
|
|
76
|
-
)
|
|
78
|
+
);
|
|
77
79
|
const pages: OnboardingJourneyCostEstimate["pages"] = [];
|
|
80
|
+
let incompleteReason: OnboardingJourneyCostEstimate["incompleteReason"];
|
|
81
|
+
const deadline = new AbortController();
|
|
82
|
+
const deadlineTimer = setTimeout(
|
|
83
|
+
() =>
|
|
84
|
+
deadline.abort(
|
|
85
|
+
new DOMException(
|
|
86
|
+
"Onboarding cost estimate deadline exceeded",
|
|
87
|
+
"AbortError",
|
|
88
|
+
),
|
|
89
|
+
),
|
|
90
|
+
ONBOARDING_EVENT_READ_DEADLINE_MS,
|
|
91
|
+
);
|
|
92
|
+
const estimateSignal = signal
|
|
93
|
+
? AbortSignal.any([signal, deadline.signal])
|
|
94
|
+
: deadline.signal;
|
|
78
95
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
96
|
+
try {
|
|
97
|
+
let backend: Awaited<
|
|
98
|
+
ReturnType<typeof getFirstPartyAnalyticsBackend>
|
|
99
|
+
> | null = null;
|
|
82
100
|
try {
|
|
83
|
-
|
|
84
|
-
filters,
|
|
85
|
-
maxEventRows,
|
|
86
|
-
offset,
|
|
87
|
-
observation,
|
|
88
|
-
freezeReceivedAt,
|
|
89
|
-
);
|
|
90
|
-
const scoped = scopedAnalyticsSql(request.sql, scope, undefined, {
|
|
91
|
-
scopedEventsSingleScan: true,
|
|
92
|
-
scopedEventsProjection: "onboarding_journey",
|
|
93
|
-
scopedEventsSourceProjections: {
|
|
94
|
-
e: "onboarding_journey",
|
|
95
|
-
r: "onboarding_journey_response_identity",
|
|
96
|
-
},
|
|
97
|
-
});
|
|
98
|
-
renderedSql = renderFirstPartyAnalyticsBigQueryRequestSql(
|
|
99
|
-
scoped.sql,
|
|
100
|
-
scoped.args,
|
|
101
|
-
table,
|
|
102
|
-
{
|
|
103
|
-
eventDateRange: request.eventDateRange,
|
|
104
|
-
scopedEventsSingleScan: true,
|
|
105
|
-
scopedEventsProjection: "onboarding_journey",
|
|
106
|
-
scopedEventsSourceProjections: {
|
|
107
|
-
e: "onboarding_journey",
|
|
108
|
-
r: "onboarding_journey_response_identity",
|
|
109
|
-
},
|
|
110
|
-
},
|
|
111
|
-
);
|
|
101
|
+
backend = await getFirstPartyAnalyticsBackend(scope, estimateSignal);
|
|
112
102
|
} catch (error) {
|
|
113
|
-
|
|
103
|
+
if (signal?.aborted) throw error;
|
|
104
|
+
if (!deadline.signal.aborted) throwPreparationFailure(error, signal);
|
|
114
105
|
}
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
dryRun = await dryRunQuerySchema(renderedSql, {
|
|
118
|
-
signal,
|
|
119
|
-
wrapPreparationErrors: true,
|
|
120
|
-
});
|
|
121
|
-
} catch (error) {
|
|
122
|
-
if (
|
|
123
|
-
signal?.aborted ||
|
|
124
|
-
(error instanceof Error && error.name === "AbortError")
|
|
125
|
-
) {
|
|
126
|
-
throw error;
|
|
127
|
-
}
|
|
128
|
-
throw new OnboardingJourneyCostError(
|
|
129
|
-
error instanceof BigQueryDryRunPreparationError
|
|
130
|
-
? "preparation_failed"
|
|
131
|
-
: "dry_run_failed",
|
|
132
|
-
);
|
|
106
|
+
if (backend && backend.sink !== "bigquery") {
|
|
107
|
+
throw new OnboardingJourneyCostError("unsupported_backend");
|
|
133
108
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
109
|
+
let table: Awaited<ReturnType<typeof getFirstPartyAnalyticsTable>> | null =
|
|
110
|
+
null;
|
|
111
|
+
if (backend) {
|
|
112
|
+
try {
|
|
113
|
+
table = await getFirstPartyAnalyticsTable(
|
|
114
|
+
backend.table,
|
|
115
|
+
estimateSignal,
|
|
116
|
+
);
|
|
117
|
+
} catch (error) {
|
|
118
|
+
if (signal?.aborted) throw error;
|
|
119
|
+
if (!deadline.signal.aborted) throwPreparationFailure(error, signal);
|
|
120
|
+
}
|
|
138
121
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
122
|
+
if (table) {
|
|
123
|
+
for (const [index, offset] of pageOffsets.entries()) {
|
|
124
|
+
if (signal?.aborted) {
|
|
125
|
+
throw new DOMException("Aborted", "AbortError");
|
|
126
|
+
}
|
|
127
|
+
if (deadline.signal.aborted) {
|
|
128
|
+
incompleteReason = "deadline";
|
|
129
|
+
break;
|
|
130
|
+
}
|
|
131
|
+
let renderedSql: string;
|
|
132
|
+
try {
|
|
133
|
+
const request = buildOnboardingJourneyEventPageRequest(
|
|
134
|
+
filters,
|
|
135
|
+
maxEventRows,
|
|
136
|
+
offset,
|
|
137
|
+
observation,
|
|
138
|
+
freezeReceivedAt,
|
|
139
|
+
);
|
|
140
|
+
const scoped = scopedAnalyticsSql(request.sql, scope, undefined, {
|
|
141
|
+
scopedEventsSingleScan: true,
|
|
142
|
+
scopedEventsProjection: "onboarding_journey",
|
|
143
|
+
scopedEventsSourceProjections: {
|
|
144
|
+
e: "onboarding_journey",
|
|
145
|
+
r: "onboarding_journey_response_identity",
|
|
146
|
+
},
|
|
147
|
+
});
|
|
148
|
+
renderedSql = renderFirstPartyAnalyticsBigQueryRequestSql(
|
|
149
|
+
scoped.sql,
|
|
150
|
+
scoped.args,
|
|
151
|
+
table,
|
|
152
|
+
{
|
|
153
|
+
eventDateRange: request.eventDateRange,
|
|
154
|
+
scopedEventsSingleScan: true,
|
|
155
|
+
scopedEventsProjection: "onboarding_journey",
|
|
156
|
+
scopedEventsSourceProjections: {
|
|
157
|
+
e: "onboarding_journey",
|
|
158
|
+
r: "onboarding_journey_response_identity",
|
|
159
|
+
},
|
|
160
|
+
},
|
|
161
|
+
);
|
|
162
|
+
} catch (error) {
|
|
163
|
+
throwPreparationFailure(error, signal);
|
|
164
|
+
}
|
|
165
|
+
let dryRun: Awaited<ReturnType<typeof dryRunQuerySchema>>;
|
|
166
|
+
try {
|
|
167
|
+
dryRun = await dryRunQuerySchema(renderedSql, {
|
|
168
|
+
signal: estimateSignal,
|
|
169
|
+
wrapPreparationErrors: true,
|
|
170
|
+
});
|
|
171
|
+
} catch (error) {
|
|
172
|
+
if (signal?.aborted) throw error;
|
|
173
|
+
if (deadline.signal.aborted) {
|
|
174
|
+
incompleteReason = "deadline";
|
|
175
|
+
break;
|
|
176
|
+
}
|
|
177
|
+
throw new OnboardingJourneyCostError(
|
|
178
|
+
error instanceof BigQueryDryRunPreparationError
|
|
179
|
+
? "preparation_failed"
|
|
180
|
+
: "dry_run_failed",
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
if (dryRun.timedOut) {
|
|
184
|
+
incompleteReason = "page_timeout";
|
|
185
|
+
break;
|
|
186
|
+
}
|
|
187
|
+
if (dryRun.error !== null) {
|
|
188
|
+
throw new OnboardingJourneyCostError("dry_run_failed");
|
|
189
|
+
}
|
|
190
|
+
const estimatedBytes = dryRun.totalBytesProcessed;
|
|
191
|
+
if (
|
|
192
|
+
typeof estimatedBytes !== "number" ||
|
|
193
|
+
!Number.isSafeInteger(estimatedBytes) ||
|
|
194
|
+
estimatedBytes < 0
|
|
195
|
+
) {
|
|
196
|
+
throw new OnboardingJourneyCostError("estimate_unavailable");
|
|
197
|
+
}
|
|
198
|
+
pages.push({
|
|
199
|
+
page: index + 1,
|
|
200
|
+
offset,
|
|
201
|
+
estimatedBytes,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
} else {
|
|
205
|
+
incompleteReason = "deadline";
|
|
146
206
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
offset,
|
|
150
|
-
estimatedBytes,
|
|
151
|
-
});
|
|
207
|
+
} finally {
|
|
208
|
+
clearTimeout(deadlineTimer);
|
|
152
209
|
}
|
|
153
210
|
|
|
154
|
-
const
|
|
211
|
+
const estimatedPagesBytes = pages.reduce(
|
|
155
212
|
(total, page) => total + page.estimatedBytes,
|
|
156
213
|
0,
|
|
157
214
|
);
|
|
158
|
-
if (!Number.isSafeInteger(
|
|
215
|
+
if (!Number.isSafeInteger(estimatedPagesBytes)) {
|
|
159
216
|
throw new OnboardingJourneyCostError("estimate_unavailable");
|
|
160
217
|
}
|
|
218
|
+
const unestimatedPageCount = allowedPageCount - pages.length;
|
|
219
|
+
const overCap = pages.some(
|
|
220
|
+
(page) => page.estimatedBytes > ONBOARDING_EVENTS_MAX_BYTES_BILLED,
|
|
221
|
+
);
|
|
161
222
|
return {
|
|
162
|
-
status:
|
|
163
|
-
(page) => page.estimatedBytes > ONBOARDING_EVENTS_MAX_BYTES_BILLED,
|
|
164
|
-
)
|
|
223
|
+
status: overCap
|
|
165
224
|
? "over_cap"
|
|
166
|
-
:
|
|
225
|
+
: unestimatedPageCount > 0
|
|
226
|
+
? "incomplete"
|
|
227
|
+
: "within_cap",
|
|
167
228
|
maxBytesBilledPerQuery: ONBOARDING_EVENTS_MAX_BYTES_BILLED,
|
|
168
|
-
|
|
229
|
+
maxTotalBytesBilled,
|
|
230
|
+
requestedPageCount: getOnboardingJourneyEventReadPageCount(maxEventRows),
|
|
231
|
+
allowedPageCount,
|
|
232
|
+
pageBudgetLimited: allowedPageCount < requestedPageCount,
|
|
233
|
+
estimatedPagesBytes,
|
|
234
|
+
possiblePagesEstimatedBytes:
|
|
235
|
+
unestimatedPageCount === 0 ? estimatedPagesBytes : null,
|
|
236
|
+
unestimatedPageCount,
|
|
237
|
+
...(incompleteReason ? { incompleteReason } : {}),
|
|
169
238
|
pages,
|
|
170
239
|
};
|
|
171
240
|
}
|
|
@@ -7,7 +7,10 @@ import {
|
|
|
7
7
|
BigQueryQueryTimeoutError,
|
|
8
8
|
type BigQueryBackendReason,
|
|
9
9
|
} from "./bigquery.js";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
getFirstPartyAnalyticsBackend,
|
|
12
|
+
type FirstPartyAnalyticsSink,
|
|
13
|
+
} from "./first-party-analytics-backend.js";
|
|
11
14
|
import {
|
|
12
15
|
queryFirstPartyAnalytics,
|
|
13
16
|
FirstPartyAnalyticsQueryTimeoutError,
|
|
@@ -50,15 +53,33 @@ import {
|
|
|
50
53
|
// Both backends cap a query result at 5,000 rows; stay under it so a full page
|
|
51
54
|
// is never mistaken for a cut one.
|
|
52
55
|
export const ONBOARDING_EVENT_PAGE_ROWS = 4_000;
|
|
53
|
-
export const
|
|
56
|
+
export const DEFAULT_ONBOARDING_EVENT_ROW_BUDGET = 40_000;
|
|
57
|
+
export const MAX_ONBOARDING_EVENT_ROW_BUDGET = 200_000;
|
|
54
58
|
const MAX_ONBOARDING_JOURNEY_FOLLOWUP_TERMINALS = 2_000;
|
|
55
59
|
const MAX_FOLLOWUP_QUERY_CHARS = 800_000;
|
|
56
60
|
const MAX_FOLLOWUP_QUERY_TOKENS = 50_000;
|
|
57
61
|
const ONBOARDING_QUERY_TIMEOUT_MS = 20_000;
|
|
62
|
+
export const ONBOARDING_EVENT_READ_DEADLINE_MS = 45_000;
|
|
58
63
|
export const ONBOARDING_EVENTS_MAX_BYTES_BILLED = 25_000_000_000;
|
|
64
|
+
export const ONBOARDING_EVENTS_DEFAULT_MAX_TOTAL_BYTES_BILLED =
|
|
65
|
+
ONBOARDING_EVENTS_MAX_BYTES_BILLED * 11;
|
|
66
|
+
export const ONBOARDING_EVENTS_MAX_TOTAL_BYTES_BILLED =
|
|
67
|
+
ONBOARDING_EVENTS_MAX_BYTES_BILLED * 51;
|
|
59
68
|
const ONBOARDING_FOLLOWUP_MAX_BYTES_BILLED = 10_000_000_000;
|
|
60
69
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
61
70
|
|
|
71
|
+
export function getOnboardingJourneyEventReadPageCount(
|
|
72
|
+
maxEventRows: number,
|
|
73
|
+
): number {
|
|
74
|
+
return Math.ceil((maxEventRows + 1) / ONBOARDING_EVENT_PAGE_ROWS);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function getOnboardingJourneyEventReadPageBudget(
|
|
78
|
+
maxTotalBytesBilled: number,
|
|
79
|
+
): number {
|
|
80
|
+
return Math.floor(maxTotalBytesBilled / ONBOARDING_EVENTS_MAX_BYTES_BILLED);
|
|
81
|
+
}
|
|
82
|
+
|
|
62
83
|
function throwIfAborted(signal?: AbortSignal): void {
|
|
63
84
|
if (signal?.aborted) {
|
|
64
85
|
throw new DOMException("The operation was aborted", "AbortError");
|
|
@@ -185,6 +206,7 @@ export interface OnboardingJourneyArgs extends OnboardingJourneyEventsFilters {
|
|
|
185
206
|
minNodeSessions: number;
|
|
186
207
|
examplesPerNode: number;
|
|
187
208
|
maxEventRows: number;
|
|
209
|
+
maxTotalBytesBilled: number;
|
|
188
210
|
maxNodes: number;
|
|
189
211
|
settleMs: number;
|
|
190
212
|
recency: "newest" | "none";
|
|
@@ -199,8 +221,8 @@ export interface JourneyTree {
|
|
|
199
221
|
coverage: {
|
|
200
222
|
/** Sessions in the tree: entered onboarding in the window and produced a step. */
|
|
201
223
|
sessionsWithEvents: number;
|
|
202
|
-
/**
|
|
203
|
-
sessionsWithReplay: number;
|
|
224
|
+
/** Null when a truncated event sample causes replay reads to be skipped. */
|
|
225
|
+
sessionsWithReplay: number | null;
|
|
204
226
|
/** Counts are a partial sample: the event read or the node list was cut. */
|
|
205
227
|
truncated: boolean;
|
|
206
228
|
};
|
|
@@ -212,7 +234,7 @@ export interface JourneyTree {
|
|
|
212
234
|
rootN: number;
|
|
213
235
|
coverage: {
|
|
214
236
|
sessionsWithEvents: number;
|
|
215
|
-
sessionsWithReplay: number;
|
|
237
|
+
sessionsWithReplay: number | null;
|
|
216
238
|
truncated: boolean;
|
|
217
239
|
};
|
|
218
240
|
nodes: JourneyNode[];
|
|
@@ -228,7 +250,7 @@ export interface JourneySummary {
|
|
|
228
250
|
rootN: number;
|
|
229
251
|
coverage: {
|
|
230
252
|
sessionsWithEvents: number;
|
|
231
|
-
/** Null
|
|
253
|
+
/** Null when replay reads are skipped for summaries or truncated samples. */
|
|
232
254
|
sessionsWithReplay: number | null;
|
|
233
255
|
truncated: boolean;
|
|
234
256
|
};
|
|
@@ -472,7 +494,12 @@ interface EventRead {
|
|
|
472
494
|
lastSessionDroppedFor: JourneyEventRow["journeyKind"] | null;
|
|
473
495
|
pages: number;
|
|
474
496
|
paginationConsistency: "stable" | "may_have_shifted";
|
|
475
|
-
truncationReason:
|
|
497
|
+
truncationReason:
|
|
498
|
+
| "max_event_rows"
|
|
499
|
+
| "backend_result"
|
|
500
|
+
| "page_budget"
|
|
501
|
+
| "read_deadline"
|
|
502
|
+
| null;
|
|
476
503
|
}
|
|
477
504
|
|
|
478
505
|
export interface OnboardingJourneyEventPageRequest {
|
|
@@ -513,15 +540,13 @@ async function readJourneyEvents(
|
|
|
513
540
|
scope: AnalyticsScope,
|
|
514
541
|
filters: OnboardingJourneyEventsFilters,
|
|
515
542
|
maxEventRows: number,
|
|
543
|
+
maxTotalBytesBilled: number,
|
|
516
544
|
observation: OnboardingJourneyObservationWindow,
|
|
517
545
|
freezeReceivedAt: boolean,
|
|
518
546
|
signal?: AbortSignal,
|
|
519
547
|
): Promise<EventRead> {
|
|
520
|
-
const
|
|
521
|
-
|
|
522
|
-
sink === "bigquery"
|
|
523
|
-
? MAX_ONBOARDING_EVENT_READ_PAGES
|
|
524
|
-
: Math.ceil((maxEventRows + 1) / ONBOARDING_EVENT_PAGE_ROWS);
|
|
548
|
+
const requestedMaxPages =
|
|
549
|
+
getOnboardingJourneyEventReadPageCount(maxEventRows);
|
|
525
550
|
const raw: Record<string, unknown>[] = [];
|
|
526
551
|
let truncated = false;
|
|
527
552
|
let truncationReason: EventRead["truncationReason"] = null;
|
|
@@ -529,61 +554,109 @@ async function readJourneyEvents(
|
|
|
529
554
|
let overflowSessionId: string | null = null;
|
|
530
555
|
let pages = 0;
|
|
531
556
|
let rowsFetched = 0;
|
|
532
|
-
|
|
557
|
+
const deadline = new AbortController();
|
|
558
|
+
const deadlineTimer = setTimeout(
|
|
559
|
+
() =>
|
|
560
|
+
deadline.abort(
|
|
561
|
+
new DOMException(
|
|
562
|
+
"Onboarding event read deadline exceeded",
|
|
563
|
+
"AbortError",
|
|
564
|
+
),
|
|
565
|
+
),
|
|
566
|
+
ONBOARDING_EVENT_READ_DEADLINE_MS,
|
|
567
|
+
);
|
|
568
|
+
const querySignal = signal
|
|
569
|
+
? AbortSignal.any([signal, deadline.signal])
|
|
570
|
+
: deadline.signal;
|
|
571
|
+
try {
|
|
533
572
|
throwIfAborted(signal);
|
|
534
|
-
|
|
535
|
-
const request = buildOnboardingJourneyEventPageRequest(
|
|
536
|
-
filters,
|
|
537
|
-
maxEventRows,
|
|
538
|
-
raw.length,
|
|
539
|
-
observation,
|
|
540
|
-
freezeReceivedAt,
|
|
541
|
-
);
|
|
542
|
-
let page: Awaited<ReturnType<typeof queryFirstPartyAnalytics>>;
|
|
573
|
+
let backendSink: FirstPartyAnalyticsSink | null = null;
|
|
543
574
|
try {
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
timeoutMs: ONBOARDING_QUERY_TIMEOUT_MS,
|
|
547
|
-
maxBytesBilled: ONBOARDING_EVENTS_MAX_BYTES_BILLED,
|
|
548
|
-
eventDateRange: request.eventDateRange,
|
|
549
|
-
scopedEventsSingleScan: true,
|
|
550
|
-
scopedEventsProjection: "onboarding_journey",
|
|
551
|
-
scopedEventsSourceProjections: {
|
|
552
|
-
e: "onboarding_journey",
|
|
553
|
-
r: "onboarding_journey_response_identity",
|
|
554
|
-
},
|
|
555
|
-
signal,
|
|
556
|
-
});
|
|
575
|
+
const backend = await getFirstPartyAnalyticsBackend(scope, querySignal);
|
|
576
|
+
backendSink = backend.sink;
|
|
557
577
|
} catch (error) {
|
|
558
|
-
if (signal?.aborted
|
|
559
|
-
|
|
560
|
-
}
|
|
561
|
-
pages += 1;
|
|
562
|
-
raw.push(...page.rows);
|
|
563
|
-
rowsFetched += page.rows.length;
|
|
564
|
-
if (page.truncated) {
|
|
565
|
-
truncated = true;
|
|
566
|
-
truncationReason = "backend_result";
|
|
567
|
-
break;
|
|
568
|
-
}
|
|
569
|
-
if (page.rows.length < request.limit) break;
|
|
570
|
-
if (raw.length > maxEventRows) {
|
|
578
|
+
if (signal?.aborted) throw error;
|
|
579
|
+
if (!deadline.signal.aborted) throw error;
|
|
571
580
|
truncated = true;
|
|
572
|
-
truncationReason = "
|
|
573
|
-
const overflowKind = raw[maxEventRows]?.journey_kind;
|
|
574
|
-
truncatedAt =
|
|
575
|
-
overflowKind === "onboarding" || overflowKind === "standalone_setup"
|
|
576
|
-
? overflowKind
|
|
577
|
-
: null;
|
|
578
|
-
overflowSessionId = text(raw[maxEventRows]?.session_id);
|
|
579
|
-
raw.length = maxEventRows;
|
|
580
|
-
break;
|
|
581
|
+
truncationReason = "read_deadline";
|
|
581
582
|
}
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
583
|
+
const maxPages =
|
|
584
|
+
backendSink === "bigquery"
|
|
585
|
+
? Math.min(
|
|
586
|
+
requestedMaxPages,
|
|
587
|
+
getOnboardingJourneyEventReadPageBudget(maxTotalBytesBilled),
|
|
588
|
+
)
|
|
589
|
+
: requestedMaxPages;
|
|
590
|
+
|
|
591
|
+
while (!truncated) {
|
|
592
|
+
throwIfAborted(signal);
|
|
593
|
+
if (deadline.signal.aborted) {
|
|
594
|
+
truncated = true;
|
|
595
|
+
truncationReason = "read_deadline";
|
|
596
|
+
break;
|
|
597
|
+
}
|
|
598
|
+
// One row past the budget tells a full read from a cut one.
|
|
599
|
+
const request = buildOnboardingJourneyEventPageRequest(
|
|
600
|
+
filters,
|
|
601
|
+
maxEventRows,
|
|
602
|
+
raw.length,
|
|
603
|
+
observation,
|
|
604
|
+
freezeReceivedAt,
|
|
605
|
+
);
|
|
606
|
+
let page: Awaited<ReturnType<typeof queryFirstPartyAnalytics>>;
|
|
607
|
+
try {
|
|
608
|
+
page = await queryFirstPartyAnalytics(request.sql, scope, {
|
|
609
|
+
cache: true,
|
|
610
|
+
timeoutMs: ONBOARDING_QUERY_TIMEOUT_MS,
|
|
611
|
+
maxBytesBilled: ONBOARDING_EVENTS_MAX_BYTES_BILLED,
|
|
612
|
+
eventDateRange: request.eventDateRange,
|
|
613
|
+
scopedEventsSingleScan: true,
|
|
614
|
+
scopedEventsProjection: "onboarding_journey",
|
|
615
|
+
scopedEventsSourceProjections: {
|
|
616
|
+
e: "onboarding_journey",
|
|
617
|
+
r: "onboarding_journey_response_identity",
|
|
618
|
+
},
|
|
619
|
+
signal: querySignal,
|
|
620
|
+
});
|
|
621
|
+
} catch (error) {
|
|
622
|
+
if (signal?.aborted) throw error;
|
|
623
|
+
if (deadline.signal.aborted) {
|
|
624
|
+
truncated = true;
|
|
625
|
+
truncationReason = "read_deadline";
|
|
626
|
+
break;
|
|
627
|
+
}
|
|
628
|
+
if (isAbortError(error)) throw error;
|
|
629
|
+
throw new OnboardingJourneyReadError(error, pages + 1);
|
|
630
|
+
}
|
|
631
|
+
pages += 1;
|
|
632
|
+
raw.push(...page.rows);
|
|
633
|
+
rowsFetched += page.rows.length;
|
|
634
|
+
if (page.truncated) {
|
|
635
|
+
truncated = true;
|
|
636
|
+
truncationReason = "backend_result";
|
|
637
|
+
break;
|
|
638
|
+
}
|
|
639
|
+
if (page.rows.length < request.limit) break;
|
|
640
|
+
if (raw.length > maxEventRows) {
|
|
641
|
+
truncated = true;
|
|
642
|
+
truncationReason = "max_event_rows";
|
|
643
|
+
const overflowKind = raw[maxEventRows]?.journey_kind;
|
|
644
|
+
truncatedAt =
|
|
645
|
+
overflowKind === "onboarding" || overflowKind === "standalone_setup"
|
|
646
|
+
? overflowKind
|
|
647
|
+
: null;
|
|
648
|
+
overflowSessionId = text(raw[maxEventRows]?.session_id);
|
|
649
|
+
raw.length = maxEventRows;
|
|
650
|
+
break;
|
|
651
|
+
}
|
|
652
|
+
if (pages >= maxPages) {
|
|
653
|
+
truncated = true;
|
|
654
|
+
truncationReason = "page_budget";
|
|
655
|
+
break;
|
|
656
|
+
}
|
|
586
657
|
}
|
|
658
|
+
} finally {
|
|
659
|
+
clearTimeout(deadlineTimer);
|
|
587
660
|
}
|
|
588
661
|
|
|
589
662
|
const byId = new Map<string, JourneyEventRow>();
|
|
@@ -1577,6 +1650,7 @@ export async function getOnboardingJourney(
|
|
|
1577
1650
|
scope,
|
|
1578
1651
|
args,
|
|
1579
1652
|
args.maxEventRows,
|
|
1653
|
+
args.maxTotalBytesBilled,
|
|
1580
1654
|
observation,
|
|
1581
1655
|
args.followUpMode === "person",
|
|
1582
1656
|
signal,
|
|
@@ -1647,7 +1721,8 @@ export async function getOnboardingJourney(
|
|
|
1647
1721
|
|
|
1648
1722
|
let recordings: JourneyRecording[] | null;
|
|
1649
1723
|
throwIfAborted(signal);
|
|
1650
|
-
|
|
1724
|
+
const replayReadSkipped = args.format === "tree" && read.truncated;
|
|
1725
|
+
if (args.format === "summary" || replayReadSkipped) {
|
|
1651
1726
|
recordings = null;
|
|
1652
1727
|
} else {
|
|
1653
1728
|
recordings = await readRecordings(
|
|
@@ -1663,8 +1738,9 @@ export async function getOnboardingJourney(
|
|
|
1663
1738
|
if (list) list.push(recording);
|
|
1664
1739
|
else bySession.set(recording.sessionId, [recording]);
|
|
1665
1740
|
}
|
|
1741
|
+
const recordingsBySession = recordings === null ? null : bySession;
|
|
1666
1742
|
|
|
1667
|
-
const built = buildJourneyTree(sessions,
|
|
1743
|
+
const built = buildJourneyTree(sessions, recordingsBySession, {
|
|
1668
1744
|
maxDepth: args.maxDepth,
|
|
1669
1745
|
minNodeSessions: args.minNodeSessions,
|
|
1670
1746
|
examplesPerNode: args.format === "tree" ? args.examplesPerNode : 0,
|
|
@@ -1677,7 +1753,7 @@ export async function getOnboardingJourney(
|
|
|
1677
1753
|
const hasStandaloneResult =
|
|
1678
1754
|
standalone.sessions.length > 0 || read.standaloneSetupTruncated;
|
|
1679
1755
|
const standaloneBuilt = hasStandaloneResult
|
|
1680
|
-
? buildJourneyTree(standalone.sessions,
|
|
1756
|
+
? buildJourneyTree(standalone.sessions, recordingsBySession, {
|
|
1681
1757
|
maxDepth: args.maxDepth,
|
|
1682
1758
|
minNodeSessions: args.minNodeSessions,
|
|
1683
1759
|
examplesPerNode: args.format === "tree" ? args.examplesPerNode : 0,
|
|
@@ -1694,10 +1770,12 @@ export async function getOnboardingJourney(
|
|
|
1694
1770
|
const notes: string[] = [];
|
|
1695
1771
|
const eventReadLimit =
|
|
1696
1772
|
read.truncationReason === "page_budget"
|
|
1697
|
-
? `Event read
|
|
1698
|
-
: read.truncationReason === "
|
|
1699
|
-
?
|
|
1700
|
-
:
|
|
1773
|
+
? `Event read reached its aggregate maxTotalBytesBilled=${args.maxTotalBytesBilled} bytes budget before confirming whether maxEventRows=${args.maxEventRows} was exceeded`
|
|
1774
|
+
: read.truncationReason === "read_deadline"
|
|
1775
|
+
? `Event read reached its ${ONBOARDING_EVENT_READ_DEADLINE_MS / 1000}-second deadline before confirming whether maxEventRows=${args.maxEventRows} was exceeded`
|
|
1776
|
+
: read.truncationReason === "backend_result"
|
|
1777
|
+
? "BigQuery returned an incomplete event page"
|
|
1778
|
+
: `Event read stopped at maxEventRows=${args.maxEventRows}`;
|
|
1701
1779
|
if (read.onboardingTruncated) {
|
|
1702
1780
|
notes.push(
|
|
1703
1781
|
`${eventReadLimit} while reading onboarding events; onboarding counts are a partial sample${read.lastSessionDroppedFor === "onboarding" ? " and the last onboarding session read was left out" : ""}.`,
|
|
@@ -1734,6 +1812,11 @@ export async function getOnboardingJourney(
|
|
|
1734
1812
|
`${read.invalidRows} event rows had no id, session, or readable timestamp and were not counted.`,
|
|
1735
1813
|
);
|
|
1736
1814
|
}
|
|
1815
|
+
if (replayReadSkipped) {
|
|
1816
|
+
notes.push(
|
|
1817
|
+
"Replay availability is unknown because the event sample was truncated; replay reads were skipped.",
|
|
1818
|
+
);
|
|
1819
|
+
}
|
|
1737
1820
|
if (sessionsWithoutSteps + standalone.sessionsWithoutSteps) {
|
|
1738
1821
|
notes.push(
|
|
1739
1822
|
`${sessionsWithoutSteps + standalone.sessionsWithoutSteps} sessions had events but no step with a meaning (for example a pageview without a path) and are not in a tree.`,
|
|
@@ -1793,9 +1876,11 @@ export async function getOnboardingJourney(
|
|
|
1793
1876
|
...head,
|
|
1794
1877
|
coverage: {
|
|
1795
1878
|
sessionsWithEvents: sessions.length,
|
|
1796
|
-
sessionsWithReplay:
|
|
1797
|
-
|
|
1798
|
-
|
|
1879
|
+
sessionsWithReplay:
|
|
1880
|
+
recordings === null
|
|
1881
|
+
? null
|
|
1882
|
+
: sessions.filter((session) => bySession.has(session.sessionId))
|
|
1883
|
+
.length,
|
|
1799
1884
|
truncated,
|
|
1800
1885
|
},
|
|
1801
1886
|
nodes: capped.nodes,
|
|
@@ -1807,9 +1892,12 @@ export async function getOnboardingJourney(
|
|
|
1807
1892
|
rootN: standaloneBuilt.rootN,
|
|
1808
1893
|
coverage: {
|
|
1809
1894
|
sessionsWithEvents: standalone.sessions.length,
|
|
1810
|
-
sessionsWithReplay:
|
|
1811
|
-
|
|
1812
|
-
|
|
1895
|
+
sessionsWithReplay:
|
|
1896
|
+
recordings === null
|
|
1897
|
+
? null
|
|
1898
|
+
: standalone.sessions.filter((session) =>
|
|
1899
|
+
bySession.has(session.sessionId),
|
|
1900
|
+
).length,
|
|
1813
1901
|
truncated:
|
|
1814
1902
|
read.standaloneSetupTruncated ||
|
|
1815
1903
|
standaloneDepthTruncated ||
|