@agent-native/core-corpus 0.229.0 → 0.229.1-nightly-20261116212341
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/templates/analytics/.agents/skills/onboarding-journeys/SKILL.md +12 -9
- package/corpus/templates/analytics/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/analytics/actions/get-onboarding-journey.ts +2 -2
- package/corpus/templates/analytics/scripts/journey-snapshot-plan.ts +42 -0
- package/corpus/templates/analytics/scripts/journey-snapshot.ts +26 -7
- package/corpus/templates/analytics/server/lib/bigquery.ts +269 -79
- package/corpus/templates/analytics/server/lib/first-party-analytics-backend.ts +7 -0
- package/corpus/templates/analytics/server/lib/first-party-analytics.ts +15 -0
- package/corpus/templates/analytics/server/lib/onboarding-journey.ts +67 -11
- package/corpus/templates/assets/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/brain/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/calendar/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/chat/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/clips/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/clips/changelog/2026-11-16-agent-frame-requests-on-large-recordings-no-longer-fail.md +6 -0
- package/corpus/templates/clips/server/lib/agent-frame-range.ts +5 -3
- package/corpus/templates/content/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/crm/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/design/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/design/.generated/bridge/native-builtin-metadata.generated.ts +326 -0
- package/corpus/templates/design/.generated/bridge/native-worker-source.generated.ts +6 -0
- package/corpus/templates/dispatch/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/forms/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/mail/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/plan/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/corpus/templates/slides/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
- package/package.json +1 -1
package/corpus/README.md
CHANGED
|
@@ -154,16 +154,19 @@ top-level tree to `journey:capture`; the capture CLI reads top-level `nodes`.
|
|
|
154
154
|
`format: "summary"` for long windows. Pages after the first run up
|
|
155
155
|
to ten at a time. If the aggregate budget or time budget cuts a read,
|
|
156
156
|
`coverage.truncated` and `notes` identify why; read them before interpreting
|
|
157
|
-
counts. When
|
|
158
|
-
`observationWatermark`: call again
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
157
|
+
counts. When a time limit stopped it, a note gives an
|
|
158
|
+
`observationWatermark`: call again within a few minutes with the same
|
|
159
|
+
arguments plus that value. Pages and follow-ups that finished come from the
|
|
160
|
+
query cache (writing to it is best-effort), and the BigQuery jobs that were
|
|
161
|
+
still running keep running, so the next call collects them instead of
|
|
162
|
+
starting over, even a query that needs longer than one call. Each call gets
|
|
163
|
+
further; repeat until the note is gone. A job is collected only within an
|
|
164
|
+
hour of when it started; after that, a call runs the query again. A
|
|
165
|
+
PostgreSQL-backed app has no jobs to collect, so a follow-up that ran out of
|
|
166
|
+
its full 20 seconds gets a note to read a shorter range instead. In
|
|
162
167
|
session mode the watermark pins event time but not receive time, so an event
|
|
163
168
|
that arrives between calls can still change pages; the read then reports
|
|
164
|
-
`may_have_shifted`. A
|
|
165
|
-
note that a follow-up needed more than the 20 seconds one query may run means a
|
|
166
|
-
retry will not finish it; ask for a shorter date range instead. A truncated event sample skips replay reads and
|
|
169
|
+
`may_have_shifted`. A truncated event sample skips replay reads and
|
|
167
170
|
returns `sessionsWithReplay: null` because replay availability is unknown. If
|
|
168
171
|
a read truncates, page boundaries may have shifted,
|
|
169
172
|
or the terminal cohort cannot be covered in one query under the
|
|
@@ -368,7 +371,7 @@ pnpm --filter analytics journey:snapshot --week 2026-10-19 --apps clips,design,s
|
|
|
368
371
|
pnpm --filter analytics journey:snapshot --week 2026-10-19 --apps design --experiment first_run_builder_credits
|
|
369
372
|
```
|
|
370
373
|
|
|
371
|
-
`--week` is a closed UTC Monday week. The query starts the previous day to include sessions already underway. Without `--week`, the CLI selects the last full week. It runs the pinned cost preflight and reads one tree per missing app/week/cohort from deployed Analytics
|
|
374
|
+
`--week` is a closed UTC Monday week. The query starts the previous day to include sessions already underway. Without `--week`, the CLI selects the last full week. It runs the pinned cost preflight and reads one tree per missing app/week/cohort from deployed Analytics. When a time limit stops a read, it calls again with the returned `continueWith.observationWatermark`, up to 8 calls 10 seconds apart, collecting the queries each call left running, and publishes nothing if the read is still stopped after the last one. Complete trees are cached by app, dates, preset, app URL, and authenticated principal. `--dry-run` prints the request without network access. `--refresh` bypasses existing tree, screenshot, and Design caches and should be used only for an intentional refresh.
|
|
372
375
|
|
|
373
376
|
The output lives under `.tmp/onboarding-journey-snapshots/runs/<week>/`: each app with captured session frames has a Design canvas, and each app has a per-app tree cache and screenshot manifests; the shared `summary.md` includes the cohort trees. Each canvas has a 36 screenshot-backed frame ceiling allocated across its cohort bands. Multi-band canvases place independent trees side by side and keep each band's own root denominator. Same-app cohort trees use `app::cohort::step` node keys so the source app and band denominator stay distinct. Canvas cache keys include the experiment and cohort layout so different runs cannot overwrite one another. The optional experiment accepts `first_run_builder_credits` and `first_run_builder_credits_v2`; control and credits share one canvas, and `summary.md` reports per-arm forks and matching-step week-over-week changes from the full Analytics trees, including cohorts with zero screenshot frames. The canvas contains only screenshot-backed cohorts. Clips first output is `recording_ready`.
|
|
374
377
|
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// AUTO-GENERATED by @agent-native/core deploy build
|
|
2
|
+
import { markDefaultPluginProvided } from "@agent-native/core/server/edge";
|
|
3
|
+
|
|
4
|
+
const pluginStems = ["agent-chat","analytics-alert-jobs","analytics-rollup-backfill-jobs","auth","collab","config","content-metrics-collection-jobs","core-routes","creative-context","dashboard-report-jobs","db","feature-flags","labs","onboarding","otel","session-replay-retention-jobs","transactional-emails","uptime-monitor-jobs"];
|
|
5
|
+
|
|
6
|
+
export default function markBuildDiscoveredPlugins(nitroApp) {
|
|
7
|
+
for (const stem of pluginStems) {
|
|
8
|
+
markDefaultPluginProvided(nitroApp, stem);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
@@ -42,7 +42,7 @@ const isoDate = z
|
|
|
42
42
|
|
|
43
43
|
export default defineAction({
|
|
44
44
|
description:
|
|
45
|
-
'Return an access-scoped per-session onboarding tree through explicit saved-output events, preserving session counts and direct-parent denominators. When experimentId or splitDesignFirstTouch is supplied, return independent trees for each cohort; Design first-touch cohorts are crossed with experiment arms, and sessionless MCP OAuth signups are counted as events without replay. An aggregated `other` node includes up to 20 root-to-branch label paths, source step keys for identical paths, session counts, and direct-parent percentages; branch summaries are capped at 200 per tree and 64 KiB of serialized detail. `otherBranchCount` gives the full number of branches, and `otherBranchSummariesPartial: true` marks summaries omitted by those bounds. Labels/path segments longer than 300 characters and keys longer than 2,048 characters are visibly shortened with stable hash suffixes and truncation flags so the tree stays within the Design input contract. Journey counts stay bounded by the 200,000-event read cap. BigQuery pages have a 25 GB per-query cap, and maxTotalBytesBilled bounds how many pages one event read may run; it defaults to 275 GB. Event and follow-up queries stop inside a 22-second time budget (15 seconds for the event read, follow-ups get the rest), leaving room in a hosted call before its gateway gives up; tree-format replay lookups run after that budget, so prefer format "summary" for long windows; coverage.truncated and notes identify partial counts, and a
|
|
45
|
+
'Return an access-scoped per-session onboarding tree through explicit saved-output events, preserving session counts and direct-parent denominators. When experimentId or splitDesignFirstTouch is supplied, return independent trees for each cohort; Design first-touch cohorts are crossed with experiment arms, and sessionless MCP OAuth signups are counted as events without replay. An aggregated `other` node includes up to 20 root-to-branch label paths, source step keys for identical paths, session counts, and direct-parent percentages; branch summaries are capped at 200 per tree and 64 KiB of serialized detail. `otherBranchCount` gives the full number of branches, and `otherBranchSummariesPartial: true` marks summaries omitted by those bounds. Labels/path segments longer than 300 characters and keys longer than 2,048 characters are visibly shortened with stable hash suffixes and truncation flags so the tree stays within the Design input contract. Journey counts stay bounded by the 200,000-event read cap. BigQuery pages have a 25 GB per-query cap, and maxTotalBytesBilled bounds how many pages one event read may run; it defaults to 275 GB. Event and follow-up queries stop inside a 22-second time budget (15 seconds for the event read, follow-ups get the rest), leaving room in a hosted call before its gateway gives up; tree-format replay lookups run after that budget, so prefer format "summary" for long windows; coverage.truncated and notes identify partial counts, and when a time limit stopped the call, continueWith.observationWatermark lets the same call continue from the queries already cached or still running. Raise maxTotalBytesBilled deliberately to read more pages, up to 1.275 TB. When event coverage is truncated, replay reads are skipped, coverage.sessionsWithReplay is null, and examples omit recordingId with viewportReason "recording_unknown"; recordingId null means a completed lookup found no replay. followUpMode "session" keeps existing bounded same-session counts. "person" adds aggregate-only activity joined into people by agreed session emails and direct canonical auth_user_id, matching later activity by any email or ID of that person (emails compared as hashes computed in the warehouse), across first-party sessions and apps over a fixed 30-day horizon, with a 25 GB bytes-billed cap per follow-up query; identity coverage says how many sessions matched each way. It separates selected-session activity, outside-session/app activity and their overlap, no activity after a fully observed horizon, right-censoring, unknown identity, identity coverage, and read completeness. Its minute-aligned observationWatermark freezes event and receive time. Unknown identity and sessions without a selected step never count as inactive; no-activity is window-bounded evidence, not permanent churn. Output and attempt IDs are never returned. Incomplete follow-up counts are null. A query that fails for a backend, cost, or per-query timeout reason returns a stage-specific error with safe BigQuery status/reason when available; an internal event-read deadline instead returns completed pages with truncated coverage. An unreadable query never becomes zero counts or an empty-tree substitute.',
|
|
46
46
|
schema: z
|
|
47
47
|
.object({
|
|
48
48
|
dateFrom: isoDate.describe(
|
|
@@ -84,7 +84,7 @@ export default defineAction({
|
|
|
84
84
|
.regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:00\.000Z$/)
|
|
85
85
|
.optional()
|
|
86
86
|
.describe(
|
|
87
|
-
'A past UTC minute, such as 2026-11-10T22:38:00.000Z, to read as of instead of now. Pass
|
|
87
|
+
'A past UTC minute, such as 2026-11-10T22:38:00.000Z, to read as of instead of now. Pass continueWith.observationWatermark from a response a time limit stopped: the queries it finished are cached and the BigQuery queries it stopped keep running, so the same request with this value, sent within a few minutes, continues from them. It pins event time in both follow-up modes; only followUpMode "person" also pins receive time, so in session mode an event that arrives between calls can change pages, which coverage reports as may_have_shifted.',
|
|
88
88
|
),
|
|
89
89
|
format: z
|
|
90
90
|
.enum(["tree", "summary"])
|
|
@@ -147,6 +147,48 @@ export function validateSnapshotCostEstimate(
|
|
|
147
147
|
return estimatedBytes as number;
|
|
148
148
|
}
|
|
149
149
|
|
|
150
|
+
// A read too slow for one hosted call finishes over several: each call
|
|
151
|
+
// collects the queries the last one left running and starts the next ones.
|
|
152
|
+
export const SNAPSHOT_JOURNEY_CALLS = 8;
|
|
153
|
+
export const SNAPSHOT_JOURNEY_CALL_DELAY_MS = 10_000;
|
|
154
|
+
|
|
155
|
+
/** The watermark a journey response asks the next call to send, if any. */
|
|
156
|
+
export function snapshotContinueWatermark(value: unknown): string | null {
|
|
157
|
+
if (typeof value !== "object" || value === null) return null;
|
|
158
|
+
const continueWith = (value as { continueWith?: unknown }).continueWith;
|
|
159
|
+
if (typeof continueWith !== "object" || continueWith === null) return null;
|
|
160
|
+
const watermark = (continueWith as { observationWatermark?: unknown })
|
|
161
|
+
.observationWatermark;
|
|
162
|
+
return typeof watermark === "string" ? watermark : null;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Calls the journey action again while a time limit stopped it, sending the
|
|
167
|
+
* first stopped call's watermark so every later call reads the same queries.
|
|
168
|
+
*/
|
|
169
|
+
export async function callJourneyUntilComplete(
|
|
170
|
+
call: (observationWatermark: string | undefined) => Promise<unknown>,
|
|
171
|
+
wait: (ms: number) => Promise<void> = (ms) =>
|
|
172
|
+
new Promise((resolve) => setTimeout(resolve, ms)),
|
|
173
|
+
): Promise<{
|
|
174
|
+
value: unknown;
|
|
175
|
+
calls: number;
|
|
176
|
+
observationWatermark: string | undefined;
|
|
177
|
+
}> {
|
|
178
|
+
let value = await call(undefined);
|
|
179
|
+
let calls = 1;
|
|
180
|
+
let observationWatermark: string | undefined;
|
|
181
|
+
while (calls < SNAPSHOT_JOURNEY_CALLS) {
|
|
182
|
+
const next = snapshotContinueWatermark(value);
|
|
183
|
+
if (!next) break;
|
|
184
|
+
observationWatermark ??= next;
|
|
185
|
+
await wait(SNAPSHOT_JOURNEY_CALL_DELAY_MS);
|
|
186
|
+
value = await call(observationWatermark);
|
|
187
|
+
calls += 1;
|
|
188
|
+
}
|
|
189
|
+
return { value, calls, observationWatermark };
|
|
190
|
+
}
|
|
191
|
+
|
|
150
192
|
export function isBuilderCdnUrl(value: unknown): value is string {
|
|
151
193
|
if (typeof value !== "string" || !URL.canParse(value)) return false;
|
|
152
194
|
const url = new URL(value);
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
import {
|
|
17
17
|
arrayItemAt,
|
|
18
18
|
allocateFrameBudgets,
|
|
19
|
+
callJourneyUntilComplete,
|
|
19
20
|
combineSnapshotBands,
|
|
20
21
|
compareTrees,
|
|
21
22
|
defaultSnapshotWeek,
|
|
@@ -27,6 +28,7 @@ import {
|
|
|
27
28
|
SNAPSHOT_MAX_FRAMES,
|
|
28
29
|
SNAPSHOT_PRESET,
|
|
29
30
|
snapshotCanvasCacheKey,
|
|
31
|
+
snapshotContinueWatermark,
|
|
30
32
|
snapshotPopulationUnit,
|
|
31
33
|
snapshotTreeCacheKey,
|
|
32
34
|
snapshotPresetHash,
|
|
@@ -595,6 +597,7 @@ async function getTreeResult(input: {
|
|
|
595
597
|
refresh: boolean;
|
|
596
598
|
}): Promise<SnapshotTree | CohortResponse> {
|
|
597
599
|
const request = actionRequest(input);
|
|
600
|
+
let sentRequest: Record<string, unknown> = request;
|
|
598
601
|
const expected = {
|
|
599
602
|
...(input.experimentId ? { experimentId: input.experimentId } : {}),
|
|
600
603
|
splitDesignFirstTouch: input.app === "design",
|
|
@@ -665,13 +668,29 @@ async function getTreeResult(input: {
|
|
|
665
668
|
`${input.app} cost estimate failed; no journey query was run: ${error instanceof Error ? error.message : "invalid estimate"}.`,
|
|
666
669
|
);
|
|
667
670
|
}
|
|
668
|
-
const
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
671
|
+
const tree = await callJourneyUntilComplete((observationWatermark) =>
|
|
672
|
+
callAction(
|
|
673
|
+
input.analyticsUrl,
|
|
674
|
+
input.analyticsToken,
|
|
675
|
+
"get-onboarding-journey",
|
|
676
|
+
{
|
|
677
|
+
...request,
|
|
678
|
+
...(observationWatermark ? { observationWatermark } : {}),
|
|
679
|
+
},
|
|
680
|
+
),
|
|
673
681
|
);
|
|
674
|
-
|
|
682
|
+
if (tree.observationWatermark) {
|
|
683
|
+
sentRequest = {
|
|
684
|
+
...request,
|
|
685
|
+
observationWatermark: tree.observationWatermark,
|
|
686
|
+
};
|
|
687
|
+
}
|
|
688
|
+
if (snapshotContinueWatermark(tree.value)) {
|
|
689
|
+
throw checkFailure(
|
|
690
|
+
`${input.app} journey still stopped at its time limit after ${tree.calls} calls; no canvas was published.`,
|
|
691
|
+
);
|
|
692
|
+
}
|
|
693
|
+
result = parseTreeResult(tree.value, input.app, expected);
|
|
675
694
|
}
|
|
676
695
|
assertCompleteResult(result, input.app);
|
|
677
696
|
if (!cacheHit) {
|
|
@@ -679,7 +698,7 @@ async function getTreeResult(input: {
|
|
|
679
698
|
presetHash,
|
|
680
699
|
sourceUrl: input.analyticsUrl,
|
|
681
700
|
principalScopeHash: input.principalScopeHash,
|
|
682
|
-
request,
|
|
701
|
+
request: sentRequest,
|
|
683
702
|
result,
|
|
684
703
|
});
|
|
685
704
|
}
|
|
@@ -1003,10 +1003,24 @@ export interface RunQueryOptions {
|
|
|
1003
1003
|
* is stored with the result as read.
|
|
1004
1004
|
*/
|
|
1005
1005
|
sourceGeneration?: SourceGenerationResult;
|
|
1006
|
+
/**
|
|
1007
|
+
* Lets a later identical call collect this query's job instead of running
|
|
1008
|
+
* it again. The job is named after the query and its caller, so a repeat
|
|
1009
|
+
* attaches to it while it started within the last hour; and once this signal
|
|
1010
|
+
* has aborted, an unfinished job is left running (still capped by
|
|
1011
|
+
* maxBytesBilled) instead of cancelled. Ignored on a forced refresh.
|
|
1012
|
+
*/
|
|
1013
|
+
resumeAfter?: AbortSignal;
|
|
1006
1014
|
}
|
|
1007
1015
|
|
|
1008
1016
|
/** How long a result read while its source generation was unknown is reused. */
|
|
1009
1017
|
export const UNKNOWN_SOURCE_GENERATION_REUSE_MS = 60 * 60_000;
|
|
1018
|
+
// A resumable job is collected only while this young, so an attached result
|
|
1019
|
+
// is never older than the app cache would serve.
|
|
1020
|
+
const RESUMABLE_JOB_MAX_AGE_MS = UNKNOWN_SOURCE_GENERATION_REUSE_MS;
|
|
1021
|
+
// A query whose resumable job can't be collected runs under the next of these
|
|
1022
|
+
// IDs, so its replacement can still be collected; past the last, a random one.
|
|
1023
|
+
const RESUMABLE_JOB_IDS = 3;
|
|
1010
1024
|
|
|
1011
1025
|
function sourceGenerationCacheKey(result: SourceGenerationResult): string {
|
|
1012
1026
|
return result.status === "ok"
|
|
@@ -1094,6 +1108,109 @@ async function cancelQueryJob(
|
|
|
1094
1108
|
return response.ok;
|
|
1095
1109
|
}
|
|
1096
1110
|
|
|
1111
|
+
function resumableJobId(
|
|
1112
|
+
cacheKey: string,
|
|
1113
|
+
ownerEmail: string,
|
|
1114
|
+
maxBytesBilled: number,
|
|
1115
|
+
index: number,
|
|
1116
|
+
): string {
|
|
1117
|
+
return `agent_native_r_${createHash("sha256")
|
|
1118
|
+
.update(`${cacheKey}\n${ownerEmail}\n${maxBytesBilled}\n${index}`)
|
|
1119
|
+
.digest("hex")}`;
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
function isDuplicateJobRejection(status: number, body: string): boolean {
|
|
1123
|
+
if (status !== 409) return false;
|
|
1124
|
+
try {
|
|
1125
|
+
const parsed = JSON.parse(body) as {
|
|
1126
|
+
error?: { errors?: Array<{ reason?: unknown }> };
|
|
1127
|
+
};
|
|
1128
|
+
return (
|
|
1129
|
+
parsed.error?.errors?.some((entry) => entry.reason === "duplicate") ===
|
|
1130
|
+
true
|
|
1131
|
+
);
|
|
1132
|
+
} catch {
|
|
1133
|
+
// coercion-ok: an unparseable 409 is not a known duplicate; the caller throws it as a backend error.
|
|
1134
|
+
return false;
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
/** BigQuery names a duplicate job as `project:LOCATION.jobId`. */
|
|
1139
|
+
function duplicateJobLocation(body: string, jobId: string): string | undefined {
|
|
1140
|
+
return new RegExp(`:([A-Za-z0-9-]+)\\.${jobId}(?![A-Za-z0-9_-])`).exec(
|
|
1141
|
+
body,
|
|
1142
|
+
)?.[1];
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
/**
|
|
1146
|
+
* Whether the job a resumable submission collided with can be collected: it
|
|
1147
|
+
* ran this same SQL, started within RESUMABLE_JOB_MAX_AGE_MS, and has not
|
|
1148
|
+
* failed or been cancelled. Throws when the job can't be read, since that
|
|
1149
|
+
* doesn't show it can't be collected.
|
|
1150
|
+
*/
|
|
1151
|
+
async function resumableJobState(
|
|
1152
|
+
projectId: string,
|
|
1153
|
+
jobId: string,
|
|
1154
|
+
token: string,
|
|
1155
|
+
sql: string,
|
|
1156
|
+
knownLocation: string | undefined,
|
|
1157
|
+
signal?: AbortSignal,
|
|
1158
|
+
): Promise<{ collectable: boolean; location: string | undefined }> {
|
|
1159
|
+
const createdAfter = Date.now() - RESUMABLE_JOB_MAX_AGE_MS;
|
|
1160
|
+
try {
|
|
1161
|
+
const location =
|
|
1162
|
+
knownLocation ??
|
|
1163
|
+
(await findQueryJobLocation(projectId, jobId, token, createdAfter));
|
|
1164
|
+
// Every job of the last hour is listed, so this one is older.
|
|
1165
|
+
if (!location) return { collectable: false, location: undefined };
|
|
1166
|
+
const params = new URLSearchParams({
|
|
1167
|
+
fields:
|
|
1168
|
+
"jobReference,configuration(query(query)),status,statistics(creationTime)",
|
|
1169
|
+
location,
|
|
1170
|
+
});
|
|
1171
|
+
const timeout = AbortSignal.timeout(CACHE_CANCEL_TIMEOUT_MS);
|
|
1172
|
+
const response = await fetch(
|
|
1173
|
+
`https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/jobs/${jobId}?${params}`,
|
|
1174
|
+
{
|
|
1175
|
+
signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
|
|
1176
|
+
headers: {
|
|
1177
|
+
Authorization: `Bearer ${token}`,
|
|
1178
|
+
"Content-Type": "application/json",
|
|
1179
|
+
},
|
|
1180
|
+
},
|
|
1181
|
+
);
|
|
1182
|
+
if (!response.ok) {
|
|
1183
|
+
const text = await response.text();
|
|
1184
|
+
throw new BigQueryBackendError(
|
|
1185
|
+
"job",
|
|
1186
|
+
response.status,
|
|
1187
|
+
bigQueryReasonFromResponse(text),
|
|
1188
|
+
bigQueryProviderDetailFromResponse(text),
|
|
1189
|
+
);
|
|
1190
|
+
}
|
|
1191
|
+
const job = (await response.json()) as {
|
|
1192
|
+
jobReference?: { jobId?: string; location?: string };
|
|
1193
|
+
configuration?: { query?: { query?: string } };
|
|
1194
|
+
status?: { errorResult?: unknown };
|
|
1195
|
+
statistics?: { creationTime?: string };
|
|
1196
|
+
};
|
|
1197
|
+
const createdAt = Number(job.statistics?.creationTime);
|
|
1198
|
+
if (job.jobReference?.jobId !== jobId || !Number.isFinite(createdAt)) {
|
|
1199
|
+
throw new BigQueryBackendError("job", null, "other");
|
|
1200
|
+
}
|
|
1201
|
+
return {
|
|
1202
|
+
collectable:
|
|
1203
|
+
job.configuration?.query?.query === sql &&
|
|
1204
|
+
!job.status?.errorResult &&
|
|
1205
|
+
createdAt >= createdAfter,
|
|
1206
|
+
location: job.jobReference.location ?? location,
|
|
1207
|
+
};
|
|
1208
|
+
} catch (error) {
|
|
1209
|
+
throwIfAborted(signal);
|
|
1210
|
+
throw error;
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1097
1214
|
async function findQueryJobLocation(
|
|
1098
1215
|
projectId: string,
|
|
1099
1216
|
jobId: string,
|
|
@@ -1504,97 +1621,165 @@ ${sourceGenerationCacheKey(options.sourceGeneration)}`
|
|
|
1504
1621
|
let jobSubmissionResponseReceived = false;
|
|
1505
1622
|
let token: string | null = null;
|
|
1506
1623
|
let cancelJob = false;
|
|
1624
|
+
let attachedJob = false;
|
|
1625
|
+
// Only where the app cache could also serve the result, so the SQL means
|
|
1626
|
+
// the same thing whenever in its window the job ran.
|
|
1627
|
+
const resumable =
|
|
1628
|
+
options.resumeAfter !== undefined &&
|
|
1629
|
+
!forceRefresh &&
|
|
1630
|
+
!preserveCurrentDate &&
|
|
1631
|
+
!snapshot.hasUnresolvedCall;
|
|
1632
|
+
let resumableId: string | null = null;
|
|
1507
1633
|
try {
|
|
1508
1634
|
token = await getBigQueryAccessToken({ signal });
|
|
1509
1635
|
assertDateUnchanged();
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
usage = newBigQueryJobUsage(
|
|
1513
|
-
projectId,
|
|
1514
|
-
jobId,
|
|
1515
|
-
requireRequestCredentialContext("BIGQUERY_PROJECT_ID"),
|
|
1636
|
+
const credentialContext = requireRequestCredentialContext(
|
|
1637
|
+
"BIGQUERY_PROJECT_ID",
|
|
1516
1638
|
);
|
|
1517
|
-
|
|
1518
|
-
const
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1639
|
+
let resumableIndex = 0;
|
|
1640
|
+
const nextResumableId = () =>
|
|
1641
|
+
resumable && resumableIndex < RESUMABLE_JOB_IDS
|
|
1642
|
+
? resumableJobId(
|
|
1643
|
+
cacheKey,
|
|
1644
|
+
credentialContext.userEmail,
|
|
1645
|
+
maxBytesBilled,
|
|
1646
|
+
resumableIndex,
|
|
1647
|
+
)
|
|
1648
|
+
: null;
|
|
1649
|
+
resumableId = nextResumableId();
|
|
1650
|
+
// A resumable submission whose job already exists collects that job; when
|
|
1651
|
+
// it can't be collected, the query runs again under the next ID.
|
|
1652
|
+
for (;;) {
|
|
1653
|
+
jobId = resumableId ?? `agent_native_${randomUUID().replace(/-/g, "")}`;
|
|
1654
|
+
jobCreatedAfter = resumableId
|
|
1655
|
+
? Date.now() - RESUMABLE_JOB_MAX_AGE_MS
|
|
1656
|
+
: Date.now();
|
|
1657
|
+
usage = newBigQueryJobUsage(projectId, jobId, credentialContext);
|
|
1658
|
+
usageStarted = await persistBigQueryJobUsage(usage);
|
|
1659
|
+
const url = `https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/jobs`;
|
|
1660
|
+
const submissionTimeoutController = new AbortController();
|
|
1661
|
+
const submissionTimeoutAbortSignal = submissionTimeoutController.signal;
|
|
1662
|
+
const submissionTimeoutError = new BigQueryQueryTimeoutError();
|
|
1663
|
+
const submissionTimeout = setTimeout(() => {
|
|
1664
|
+
submissionTimeoutController.abort(submissionTimeoutError);
|
|
1665
|
+
}, JOB_SUBMISSION_TIMEOUT_MS);
|
|
1666
|
+
|
|
1667
|
+
let insertedJob:
|
|
1668
|
+
| { jobReference?: { jobId?: string; location?: string } }
|
|
1669
|
+
| undefined;
|
|
1670
|
+
let duplicateResponse: string | undefined;
|
|
1671
|
+
try {
|
|
1672
|
+
const res = await fetch(url, {
|
|
1673
|
+
method: "POST",
|
|
1674
|
+
signal: signal
|
|
1675
|
+
? AbortSignal.any([signal, submissionTimeoutAbortSignal])
|
|
1676
|
+
: submissionTimeoutAbortSignal,
|
|
1677
|
+
headers: {
|
|
1678
|
+
Authorization: `Bearer ${token}`,
|
|
1679
|
+
"Content-Type": "application/json",
|
|
1548
1680
|
},
|
|
1549
|
-
|
|
1550
|
-
|
|
1681
|
+
body: JSON.stringify({
|
|
1682
|
+
jobReference: { projectId, jobId },
|
|
1683
|
+
configuration: {
|
|
1684
|
+
query: {
|
|
1685
|
+
query: cacheableSql,
|
|
1686
|
+
useLegacySql: false,
|
|
1687
|
+
maximumBytesBilled: String(maxBytesBilled),
|
|
1688
|
+
...(forceRefresh ? { useQueryCache: false } : {}),
|
|
1689
|
+
},
|
|
1690
|
+
// A job left running stops once it is too old to collect, so
|
|
1691
|
+
// it never runs beside its replacement.
|
|
1692
|
+
...(jobId === resumableId
|
|
1693
|
+
? { jobTimeoutMs: String(RESUMABLE_JOB_MAX_AGE_MS) }
|
|
1694
|
+
: {}),
|
|
1695
|
+
},
|
|
1696
|
+
}),
|
|
1697
|
+
});
|
|
1551
1698
|
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1699
|
+
if (!res.ok) {
|
|
1700
|
+
usage.submissionState = "unknown";
|
|
1701
|
+
const text = await res.text();
|
|
1702
|
+
if (
|
|
1703
|
+
jobId === resumableId &&
|
|
1704
|
+
isDuplicateJobRejection(res.status, text)
|
|
1705
|
+
) {
|
|
1706
|
+
duplicateResponse = text;
|
|
1707
|
+
} else {
|
|
1708
|
+
if (isMaximumBytesBilledError(res.status, text)) {
|
|
1709
|
+
usage.submissionState = "rejected";
|
|
1710
|
+
jobId = null;
|
|
1711
|
+
throw new BigQueryMaximumBytesBilledError(res.status);
|
|
1712
|
+
}
|
|
1713
|
+
if (isDefiniteJobSubmissionRejection(res.status, text)) {
|
|
1714
|
+
usage.submissionState = "rejected";
|
|
1715
|
+
jobId = null;
|
|
1716
|
+
}
|
|
1717
|
+
throw new BigQueryBackendError(
|
|
1718
|
+
"submit",
|
|
1719
|
+
res.status,
|
|
1720
|
+
bigQueryReasonFromResponse(text),
|
|
1721
|
+
bigQueryProviderDetailFromResponse(text),
|
|
1722
|
+
);
|
|
1723
|
+
}
|
|
1724
|
+
} else {
|
|
1725
|
+
insertedJob = (await res.json()) as {
|
|
1726
|
+
jobReference?: { jobId?: string; location?: string };
|
|
1727
|
+
};
|
|
1559
1728
|
}
|
|
1560
|
-
if (
|
|
1561
|
-
|
|
1562
|
-
jobId = null;
|
|
1729
|
+
if (submissionTimeoutController.signal.aborted) {
|
|
1730
|
+
throw submissionTimeoutError;
|
|
1563
1731
|
}
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1732
|
+
} catch (error) {
|
|
1733
|
+
if (
|
|
1734
|
+
submissionTimeoutController.signal.aborted &&
|
|
1735
|
+
submissionTimeoutController.signal.reason ===
|
|
1736
|
+
submissionTimeoutError &&
|
|
1737
|
+
(error === submissionTimeoutError ||
|
|
1738
|
+
(error instanceof Error && error.name === "AbortError"))
|
|
1739
|
+
) {
|
|
1740
|
+
throw submissionTimeoutError;
|
|
1741
|
+
}
|
|
1742
|
+
throw error;
|
|
1743
|
+
} finally {
|
|
1744
|
+
clearTimeout(submissionTimeout);
|
|
1570
1745
|
}
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1746
|
+
if (duplicateResponse === undefined) {
|
|
1747
|
+
if (
|
|
1748
|
+
insertedJob?.jobReference?.jobId &&
|
|
1749
|
+
insertedJob.jobReference.jobId !== jobId
|
|
1750
|
+
) {
|
|
1751
|
+
throw new Error("BigQuery did not accept the requested job ID");
|
|
1752
|
+
}
|
|
1753
|
+
jobLocation = insertedJob?.jobReference?.location;
|
|
1754
|
+
break;
|
|
1577
1755
|
}
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1756
|
+
const knownLocation = duplicateJobLocation(duplicateResponse, jobId);
|
|
1757
|
+
let existing: Awaited<ReturnType<typeof resumableJobState>>;
|
|
1758
|
+
try {
|
|
1759
|
+
existing = await resumableJobState(
|
|
1760
|
+
projectId,
|
|
1761
|
+
jobId,
|
|
1762
|
+
token,
|
|
1763
|
+
cacheableSql,
|
|
1764
|
+
knownLocation,
|
|
1765
|
+
signal,
|
|
1766
|
+
);
|
|
1767
|
+
} catch (error) {
|
|
1768
|
+
// Another call's job: it keeps running for a later call to collect.
|
|
1769
|
+
jobLocation = knownLocation;
|
|
1770
|
+
jobId = null;
|
|
1771
|
+
throw error;
|
|
1586
1772
|
}
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1773
|
+
if (existing.collectable) {
|
|
1774
|
+
jobLocation = existing.location;
|
|
1775
|
+
attachedJob = true;
|
|
1776
|
+
break;
|
|
1777
|
+
}
|
|
1778
|
+
usage.location = existing.location;
|
|
1779
|
+
await finalizeBigQueryJobUsage(usage, token, usageStarted, signal);
|
|
1780
|
+
resumableIndex += 1;
|
|
1781
|
+
resumableId = nextResumableId();
|
|
1596
1782
|
}
|
|
1597
|
-
jobLocation = insertedJob.jobReference?.location;
|
|
1598
1783
|
jobSubmissionResponseReceived = true;
|
|
1599
1784
|
usage.submissionState = "accepted";
|
|
1600
1785
|
usage.location = jobLocation;
|
|
@@ -1743,6 +1928,11 @@ ${sourceGenerationCacheKey(options.sourceGeneration)}`
|
|
|
1743
1928
|
completedResult = { ...result, cached: false };
|
|
1744
1929
|
} catch (error) {
|
|
1745
1930
|
if (jobId) cancelJob = true;
|
|
1931
|
+
// Stopped for time, a resumable job keeps running for the next call.
|
|
1932
|
+
if (jobId !== null && jobId === resumableId && options.resumeAfter?.aborted)
|
|
1933
|
+
cancelJob = false;
|
|
1934
|
+
// Another call started a job this one attached to, and may collect it too.
|
|
1935
|
+
if (attachedJob) cancelJob = false;
|
|
1746
1936
|
if (cancelJob && jobId && token) {
|
|
1747
1937
|
if (!jobSubmissionResponseReceived) {
|
|
1748
1938
|
try {
|
|
@@ -2101,6 +2101,8 @@ export async function queryFirstPartyAnalyticsInBigQuery(
|
|
|
2101
2101
|
maxBytesBilled?: number;
|
|
2102
2102
|
timeoutMs?: number;
|
|
2103
2103
|
signal?: AbortSignal;
|
|
2104
|
+
/** Stopped by this or by `timeoutMs`, the job is left for a later call. */
|
|
2105
|
+
resumeAfter?: AbortSignal;
|
|
2104
2106
|
} = {},
|
|
2105
2107
|
): Promise<AnalyticsQueryResult> {
|
|
2106
2108
|
const timeoutMs = options.timeoutMs;
|
|
@@ -2118,6 +2120,10 @@ export async function queryFirstPartyAnalyticsInBigQuery(
|
|
|
2118
2120
|
options.signal && timeoutSignal
|
|
2119
2121
|
? AbortSignal.any([options.signal, timeoutSignal])
|
|
2120
2122
|
: (options.signal ?? timeoutSignal);
|
|
2123
|
+
const resumeAfter =
|
|
2124
|
+
options.resumeAfter && timeoutSignal
|
|
2125
|
+
? AbortSignal.any([options.resumeAfter, timeoutSignal])
|
|
2126
|
+
: options.resumeAfter;
|
|
2121
2127
|
let eventsSource = options.eventsSource;
|
|
2122
2128
|
let result: Awaited<ReturnType<typeof runQuery>>;
|
|
2123
2129
|
try {
|
|
@@ -2139,6 +2145,7 @@ export async function queryFirstPartyAnalyticsInBigQuery(
|
|
|
2139
2145
|
{
|
|
2140
2146
|
maxBytesBilled: options.maxBytesBilled,
|
|
2141
2147
|
...(signal ? { signal } : {}),
|
|
2148
|
+
...(resumeAfter ? { resumeAfter } : {}),
|
|
2142
2149
|
...(eventsSource?.source === "model" && sourceGeneration
|
|
2143
2150
|
? { sourceGeneration }
|
|
2144
2151
|
: {}),
|