@agent-native/core-corpus 0.229.0 → 0.229.1-nightly-20261116212341

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/corpus/README.md +1 -1
  2. package/corpus/templates/analytics/.agents/skills/onboarding-journeys/SKILL.md +12 -9
  3. package/corpus/templates/analytics/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  4. package/corpus/templates/analytics/actions/get-onboarding-journey.ts +2 -2
  5. package/corpus/templates/analytics/scripts/journey-snapshot-plan.ts +42 -0
  6. package/corpus/templates/analytics/scripts/journey-snapshot.ts +26 -7
  7. package/corpus/templates/analytics/server/lib/bigquery.ts +269 -79
  8. package/corpus/templates/analytics/server/lib/first-party-analytics-backend.ts +7 -0
  9. package/corpus/templates/analytics/server/lib/first-party-analytics.ts +15 -0
  10. package/corpus/templates/analytics/server/lib/onboarding-journey.ts +67 -11
  11. package/corpus/templates/assets/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  12. package/corpus/templates/brain/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  13. package/corpus/templates/calendar/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  14. package/corpus/templates/chat/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  15. package/corpus/templates/clips/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  16. package/corpus/templates/clips/changelog/2026-11-16-agent-frame-requests-on-large-recordings-no-longer-fail.md +6 -0
  17. package/corpus/templates/clips/server/lib/agent-frame-range.ts +5 -3
  18. package/corpus/templates/content/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  19. package/corpus/templates/crm/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  20. package/corpus/templates/design/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  21. package/corpus/templates/design/.generated/bridge/native-builtin-metadata.generated.ts +326 -0
  22. package/corpus/templates/design/.generated/bridge/native-worker-source.generated.ts +6 -0
  23. package/corpus/templates/dispatch/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  24. package/corpus/templates/forms/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  25. package/corpus/templates/mail/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  26. package/corpus/templates/plan/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  27. package/corpus/templates/slides/.deploy-tmp/agent-native-provided-plugins.mjs +10 -0
  28. package/package.json +1 -1
package/corpus/README.md CHANGED
@@ -32,4 +32,4 @@ rg -n "defineAction|useActionQuery" node_modules/@agent-native/core-corpus/corpu
32
32
 
33
33
  ## Generated Counts
34
34
 
35
- - template files: 14996
35
+ - template files: 15047
@@ -154,16 +154,19 @@ top-level tree to `journey:capture`; the capture CLI reads top-level `nodes`.
154
154
  `format: "summary"` for long windows. Pages after the first run up
155
155
  to ten at a time. If the aggregate budget or time budget cuts a read,
156
156
  `coverage.truncated` and `notes` identify why; read them before interpreting
157
- counts. When the time budget stopped it, a note gives an
158
- `observationWatermark`: call again with the same arguments plus that value
159
- within 5 minutes, and the pages and follow-ups that finished normally come
160
- from the query cache (writing to it is best-effort), so each call gets
161
- further. Repeat until the note is gone. In
157
+ counts. When a time limit stopped it, a note gives an
158
+ `observationWatermark`: call again within a few minutes with the same
159
+ arguments plus that value. Pages and follow-ups that finished come from the
160
+ query cache (writing to it is best-effort), and the BigQuery jobs that were
161
+ still running keep running, so the next call collects them instead of
162
+ starting over, even a query that needs longer than one call. Each call gets
163
+ further; repeat until the note is gone. A job is collected only within an
164
+ hour of when it started; after that, a call runs the query again. A
165
+ PostgreSQL-backed app has no jobs to collect, so a follow-up that ran out of
166
+ its full 20 seconds gets a note to read a shorter range instead. In
162
167
  session mode the watermark pins event time but not receive time, so an event
163
168
  that arrives between calls can still change pages; the read then reports
164
- `may_have_shifted`. A
165
- note that a follow-up needed more than the 20 seconds one query may run means a
166
- retry will not finish it; ask for a shorter date range instead. A truncated event sample skips replay reads and
169
+ `may_have_shifted`. A truncated event sample skips replay reads and
167
170
  returns `sessionsWithReplay: null` because replay availability is unknown. If
168
171
  a read truncates, page boundaries may have shifted,
169
172
  or the terminal cohort cannot be covered in one query under the
@@ -368,7 +371,7 @@ pnpm --filter analytics journey:snapshot --week 2026-10-19 --apps clips,design,s
368
371
  pnpm --filter analytics journey:snapshot --week 2026-10-19 --apps design --experiment first_run_builder_credits
369
372
  ```
370
373
 
371
- `--week` is a closed UTC Monday week. The query starts the previous day to include sessions already underway. Without `--week`, the CLI selects the last full week. It runs the pinned cost preflight and reads one tree per missing app/week/cohort from deployed Analytics; complete trees are cached by app, dates, preset, app URL, and authenticated principal. `--dry-run` prints the request without network access. `--refresh` bypasses existing tree, screenshot, and Design caches and should be used only for an intentional refresh.
374
+ `--week` is a closed UTC Monday week. The query starts the previous day to include sessions already underway. Without `--week`, the CLI selects the last full week. It runs the pinned cost preflight and reads one tree per missing app/week/cohort from deployed Analytics. When a time limit stops a read, it calls again with the returned `continueWith.observationWatermark`, up to 8 calls 10 seconds apart, collecting the queries each call left running, and publishes nothing if the read is still stopped after the last one. Complete trees are cached by app, dates, preset, app URL, and authenticated principal. `--dry-run` prints the request without network access. `--refresh` bypasses existing tree, screenshot, and Design caches and should be used only for an intentional refresh.
372
375
 
373
376
  The output lives under `.tmp/onboarding-journey-snapshots/runs/<week>/`: each app with captured session frames has a Design canvas, and each app has a per-app tree cache and screenshot manifests; the shared `summary.md` includes the cohort trees. Each canvas has a 36 screenshot-backed frame ceiling allocated across its cohort bands. Multi-band canvases place independent trees side by side and keep each band's own root denominator. Same-app cohort trees use `app::cohort::step` node keys so the source app and band denominator stay distinct. Canvas cache keys include the experiment and cohort layout so different runs cannot overwrite one another. The optional experiment accepts `first_run_builder_credits` and `first_run_builder_credits_v2`; control and credits share one canvas, and `summary.md` reports per-arm forks and matching-step week-over-week changes from the full Analytics trees, including cohorts with zero screenshot frames. The canvas contains only screenshot-backed cohorts. Clips first output is `recording_ready`.
374
377
 
@@ -0,0 +1,10 @@
1
+ // AUTO-GENERATED by @agent-native/core deploy build
2
+ import { markDefaultPluginProvided } from "@agent-native/core/server/edge";
3
+
4
+ const pluginStems = ["agent-chat","analytics-alert-jobs","analytics-rollup-backfill-jobs","auth","collab","config","content-metrics-collection-jobs","core-routes","creative-context","dashboard-report-jobs","db","feature-flags","labs","onboarding","otel","session-replay-retention-jobs","transactional-emails","uptime-monitor-jobs"];
5
+
6
+ export default function markBuildDiscoveredPlugins(nitroApp) {
7
+ for (const stem of pluginStems) {
8
+ markDefaultPluginProvided(nitroApp, stem);
9
+ }
10
+ }
@@ -42,7 +42,7 @@ const isoDate = z
42
42
 
43
43
  export default defineAction({
44
44
  description:
45
- 'Return an access-scoped per-session onboarding tree through explicit saved-output events, preserving session counts and direct-parent denominators. When experimentId or splitDesignFirstTouch is supplied, return independent trees for each cohort; Design first-touch cohorts are crossed with experiment arms, and sessionless MCP OAuth signups are counted as events without replay. An aggregated `other` node includes up to 20 root-to-branch label paths, source step keys for identical paths, session counts, and direct-parent percentages; branch summaries are capped at 200 per tree and 64 KiB of serialized detail. `otherBranchCount` gives the full number of branches, and `otherBranchSummariesPartial: true` marks summaries omitted by those bounds. Labels/path segments longer than 300 characters and keys longer than 2,048 characters are visibly shortened with stable hash suffixes and truncation flags so the tree stays within the Design input contract. Journey counts stay bounded by the 200,000-event read cap. BigQuery pages have a 25 GB per-query cap, and maxTotalBytesBilled bounds how many pages one event read may run; it defaults to 275 GB. Event and follow-up queries stop inside a 22-second time budget (15 seconds for the event read, follow-ups get the rest), leaving room in a hosted call before its gateway gives up; tree-format replay lookups run after that budget, so prefer format "summary" for long windows; coverage.truncated and notes identify partial counts, and a note gives the observationWatermark that lets the same call continue from the queries already cached. Raise maxTotalBytesBilled deliberately to read more pages, up to 1.275 TB. When event coverage is truncated, replay reads are skipped, coverage.sessionsWithReplay is null, and examples omit recordingId with viewportReason "recording_unknown"; recordingId null means a completed lookup found no replay. followUpMode "session" keeps existing bounded same-session counts. "person" adds aggregate-only activity joined into people by agreed session emails and direct canonical auth_user_id, matching later activity by any email or ID of that person (emails compared as hashes computed in the warehouse), across first-party sessions and apps over a fixed 30-day horizon, with a 25 GB bytes-billed cap per follow-up query; identity coverage says how many sessions matched each way. It separates selected-session activity, outside-session/app activity and their overlap, no activity after a fully observed horizon, right-censoring, unknown identity, identity coverage, and read completeness. Its minute-aligned observationWatermark freezes event and receive time. Unknown identity and sessions without a selected step never count as inactive; no-activity is window-bounded evidence, not permanent churn. Output and attempt IDs are never returned. Incomplete follow-up counts are null. A query that fails for a backend, cost, or per-query timeout reason returns a stage-specific error with safe BigQuery status/reason when available; an internal event-read deadline instead returns completed pages with truncated coverage. An unreadable query never becomes zero counts or an empty-tree substitute.',
45
+ 'Return an access-scoped per-session onboarding tree through explicit saved-output events, preserving session counts and direct-parent denominators. When experimentId or splitDesignFirstTouch is supplied, return independent trees for each cohort; Design first-touch cohorts are crossed with experiment arms, and sessionless MCP OAuth signups are counted as events without replay. An aggregated `other` node includes up to 20 root-to-branch label paths, source step keys for identical paths, session counts, and direct-parent percentages; branch summaries are capped at 200 per tree and 64 KiB of serialized detail. `otherBranchCount` gives the full number of branches, and `otherBranchSummariesPartial: true` marks summaries omitted by those bounds. Labels/path segments longer than 300 characters and keys longer than 2,048 characters are visibly shortened with stable hash suffixes and truncation flags so the tree stays within the Design input contract. Journey counts stay bounded by the 200,000-event read cap. BigQuery pages have a 25 GB per-query cap, and maxTotalBytesBilled bounds how many pages one event read may run; it defaults to 275 GB. Event and follow-up queries stop inside a 22-second time budget (15 seconds for the event read, follow-ups get the rest), leaving room in a hosted call before its gateway gives up; tree-format replay lookups run after that budget, so prefer format "summary" for long windows; coverage.truncated and notes identify partial counts, and when a time limit stopped the call, continueWith.observationWatermark lets the same call continue from the queries already cached or still running. Raise maxTotalBytesBilled deliberately to read more pages, up to 1.275 TB. When event coverage is truncated, replay reads are skipped, coverage.sessionsWithReplay is null, and examples omit recordingId with viewportReason "recording_unknown"; recordingId null means a completed lookup found no replay. followUpMode "session" keeps existing bounded same-session counts. "person" adds aggregate-only activity joined into people by agreed session emails and direct canonical auth_user_id, matching later activity by any email or ID of that person (emails compared as hashes computed in the warehouse), across first-party sessions and apps over a fixed 30-day horizon, with a 25 GB bytes-billed cap per follow-up query; identity coverage says how many sessions matched each way. It separates selected-session activity, outside-session/app activity and their overlap, no activity after a fully observed horizon, right-censoring, unknown identity, identity coverage, and read completeness. Its minute-aligned observationWatermark freezes event and receive time. Unknown identity and sessions without a selected step never count as inactive; no-activity is window-bounded evidence, not permanent churn. Output and attempt IDs are never returned. Incomplete follow-up counts are null. A query that fails for a backend, cost, or per-query timeout reason returns a stage-specific error with safe BigQuery status/reason when available; an internal event-read deadline instead returns completed pages with truncated coverage. An unreadable query never becomes zero counts or an empty-tree substitute.',
46
46
  schema: z
47
47
  .object({
48
48
  dateFrom: isoDate.describe(
@@ -84,7 +84,7 @@ export default defineAction({
84
84
  .regex(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:00\.000Z$/)
85
85
  .optional()
86
86
  .describe(
87
- 'A past UTC minute, such as 2026-11-10T22:38:00.000Z, to read as of instead of now. Pass the observationWatermark from a response whose notes say it stopped at its time budget: the queries it finished are normally cached for at least 5 minutes, so the same request with this value usually continues from them. It pins event time in both follow-up modes; only followUpMode "person" also pins receive time, so in session mode an event that arrives between calls can change pages, which coverage reports as may_have_shifted.',
87
+ 'A past UTC minute, such as 2026-11-10T22:38:00.000Z, to read as of instead of now. Pass continueWith.observationWatermark from a response a time limit stopped: the queries it finished are cached and the BigQuery queries it stopped keep running, so the same request with this value, sent within a few minutes, continues from them. It pins event time in both follow-up modes; only followUpMode "person" also pins receive time, so in session mode an event that arrives between calls can change pages, which coverage reports as may_have_shifted.',
88
88
  ),
89
89
  format: z
90
90
  .enum(["tree", "summary"])
@@ -147,6 +147,48 @@ export function validateSnapshotCostEstimate(
147
147
  return estimatedBytes as number;
148
148
  }
149
149
 
150
+ // A read too slow for one hosted call finishes over several: each call
151
+ // collects the queries the last one left running and starts the next ones.
152
+ export const SNAPSHOT_JOURNEY_CALLS = 8;
153
+ export const SNAPSHOT_JOURNEY_CALL_DELAY_MS = 10_000;
154
+
155
+ /** The watermark a journey response asks the next call to send, if any. */
156
+ export function snapshotContinueWatermark(value: unknown): string | null {
157
+ if (typeof value !== "object" || value === null) return null;
158
+ const continueWith = (value as { continueWith?: unknown }).continueWith;
159
+ if (typeof continueWith !== "object" || continueWith === null) return null;
160
+ const watermark = (continueWith as { observationWatermark?: unknown })
161
+ .observationWatermark;
162
+ return typeof watermark === "string" ? watermark : null;
163
+ }
164
+
165
+ /**
166
+ * Calls the journey action again while a time limit stopped it, sending the
167
+ * first stopped call's watermark so every later call reads the same queries.
168
+ */
169
+ export async function callJourneyUntilComplete(
170
+ call: (observationWatermark: string | undefined) => Promise<unknown>,
171
+ wait: (ms: number) => Promise<void> = (ms) =>
172
+ new Promise((resolve) => setTimeout(resolve, ms)),
173
+ ): Promise<{
174
+ value: unknown;
175
+ calls: number;
176
+ observationWatermark: string | undefined;
177
+ }> {
178
+ let value = await call(undefined);
179
+ let calls = 1;
180
+ let observationWatermark: string | undefined;
181
+ while (calls < SNAPSHOT_JOURNEY_CALLS) {
182
+ const next = snapshotContinueWatermark(value);
183
+ if (!next) break;
184
+ observationWatermark ??= next;
185
+ await wait(SNAPSHOT_JOURNEY_CALL_DELAY_MS);
186
+ value = await call(observationWatermark);
187
+ calls += 1;
188
+ }
189
+ return { value, calls, observationWatermark };
190
+ }
191
+
150
192
  export function isBuilderCdnUrl(value: unknown): value is string {
151
193
  if (typeof value !== "string" || !URL.canParse(value)) return false;
152
194
  const url = new URL(value);
@@ -16,6 +16,7 @@ import {
16
16
  import {
17
17
  arrayItemAt,
18
18
  allocateFrameBudgets,
19
+ callJourneyUntilComplete,
19
20
  combineSnapshotBands,
20
21
  compareTrees,
21
22
  defaultSnapshotWeek,
@@ -27,6 +28,7 @@ import {
27
28
  SNAPSHOT_MAX_FRAMES,
28
29
  SNAPSHOT_PRESET,
29
30
  snapshotCanvasCacheKey,
31
+ snapshotContinueWatermark,
30
32
  snapshotPopulationUnit,
31
33
  snapshotTreeCacheKey,
32
34
  snapshotPresetHash,
@@ -595,6 +597,7 @@ async function getTreeResult(input: {
595
597
  refresh: boolean;
596
598
  }): Promise<SnapshotTree | CohortResponse> {
597
599
  const request = actionRequest(input);
600
+ let sentRequest: Record<string, unknown> = request;
598
601
  const expected = {
599
602
  ...(input.experimentId ? { experimentId: input.experimentId } : {}),
600
603
  splitDesignFirstTouch: input.app === "design",
@@ -665,13 +668,29 @@ async function getTreeResult(input: {
665
668
  `${input.app} cost estimate failed; no journey query was run: ${error instanceof Error ? error.message : "invalid estimate"}.`,
666
669
  );
667
670
  }
668
- const treeValue = await callAction(
669
- input.analyticsUrl,
670
- input.analyticsToken,
671
- "get-onboarding-journey",
672
- request,
671
+ const tree = await callJourneyUntilComplete((observationWatermark) =>
672
+ callAction(
673
+ input.analyticsUrl,
674
+ input.analyticsToken,
675
+ "get-onboarding-journey",
676
+ {
677
+ ...request,
678
+ ...(observationWatermark ? { observationWatermark } : {}),
679
+ },
680
+ ),
673
681
  );
674
- result = parseTreeResult(treeValue, input.app, expected);
682
+ if (tree.observationWatermark) {
683
+ sentRequest = {
684
+ ...request,
685
+ observationWatermark: tree.observationWatermark,
686
+ };
687
+ }
688
+ if (snapshotContinueWatermark(tree.value)) {
689
+ throw checkFailure(
690
+ `${input.app} journey still stopped at its time limit after ${tree.calls} calls; no canvas was published.`,
691
+ );
692
+ }
693
+ result = parseTreeResult(tree.value, input.app, expected);
675
694
  }
676
695
  assertCompleteResult(result, input.app);
677
696
  if (!cacheHit) {
@@ -679,7 +698,7 @@ async function getTreeResult(input: {
679
698
  presetHash,
680
699
  sourceUrl: input.analyticsUrl,
681
700
  principalScopeHash: input.principalScopeHash,
682
- request,
701
+ request: sentRequest,
683
702
  result,
684
703
  });
685
704
  }
@@ -1003,10 +1003,24 @@ export interface RunQueryOptions {
1003
1003
  * is stored with the result as read.
1004
1004
  */
1005
1005
  sourceGeneration?: SourceGenerationResult;
1006
+ /**
1007
+ * Lets a later identical call collect this query's job instead of running
1008
+ * it again. The job is named after the query and its caller, so a repeat
1009
+ * attaches to it while it started within the last hour; and once this signal
1010
+ * has aborted, an unfinished job is left running (still capped by
1011
+ * maxBytesBilled) instead of cancelled. Ignored on a forced refresh.
1012
+ */
1013
+ resumeAfter?: AbortSignal;
1006
1014
  }
1007
1015
 
1008
1016
  /** How long a result read while its source generation was unknown is reused. */
1009
1017
  export const UNKNOWN_SOURCE_GENERATION_REUSE_MS = 60 * 60_000;
1018
+ // A resumable job is collected only while this young, so an attached result
1019
+ // is never older than the app cache would serve.
1020
+ const RESUMABLE_JOB_MAX_AGE_MS = UNKNOWN_SOURCE_GENERATION_REUSE_MS;
1021
+ // A query whose resumable job can't be collected runs under the next of these
1022
+ // IDs, so its replacement can still be collected; past the last, a random one.
1023
+ const RESUMABLE_JOB_IDS = 3;
1010
1024
 
1011
1025
  function sourceGenerationCacheKey(result: SourceGenerationResult): string {
1012
1026
  return result.status === "ok"
@@ -1094,6 +1108,109 @@ async function cancelQueryJob(
1094
1108
  return response.ok;
1095
1109
  }
1096
1110
 
1111
+ function resumableJobId(
1112
+ cacheKey: string,
1113
+ ownerEmail: string,
1114
+ maxBytesBilled: number,
1115
+ index: number,
1116
+ ): string {
1117
+ return `agent_native_r_${createHash("sha256")
1118
+ .update(`${cacheKey}\n${ownerEmail}\n${maxBytesBilled}\n${index}`)
1119
+ .digest("hex")}`;
1120
+ }
1121
+
1122
+ function isDuplicateJobRejection(status: number, body: string): boolean {
1123
+ if (status !== 409) return false;
1124
+ try {
1125
+ const parsed = JSON.parse(body) as {
1126
+ error?: { errors?: Array<{ reason?: unknown }> };
1127
+ };
1128
+ return (
1129
+ parsed.error?.errors?.some((entry) => entry.reason === "duplicate") ===
1130
+ true
1131
+ );
1132
+ } catch {
1133
+ // coercion-ok: an unparseable 409 is not a known duplicate; the caller throws it as a backend error.
1134
+ return false;
1135
+ }
1136
+ }
1137
+
1138
+ /** BigQuery names a duplicate job as `project:LOCATION.jobId`. */
1139
+ function duplicateJobLocation(body: string, jobId: string): string | undefined {
1140
+ return new RegExp(`:([A-Za-z0-9-]+)\\.${jobId}(?![A-Za-z0-9_-])`).exec(
1141
+ body,
1142
+ )?.[1];
1143
+ }
1144
+
1145
+ /**
1146
+ * Whether the job a resumable submission collided with can be collected: it
1147
+ * ran this same SQL, started within RESUMABLE_JOB_MAX_AGE_MS, and has not
1148
+ * failed or been cancelled. Throws when the job can't be read, since that
1149
+ * doesn't show it can't be collected.
1150
+ */
1151
+ async function resumableJobState(
1152
+ projectId: string,
1153
+ jobId: string,
1154
+ token: string,
1155
+ sql: string,
1156
+ knownLocation: string | undefined,
1157
+ signal?: AbortSignal,
1158
+ ): Promise<{ collectable: boolean; location: string | undefined }> {
1159
+ const createdAfter = Date.now() - RESUMABLE_JOB_MAX_AGE_MS;
1160
+ try {
1161
+ const location =
1162
+ knownLocation ??
1163
+ (await findQueryJobLocation(projectId, jobId, token, createdAfter));
1164
+ // Every job of the last hour is listed, so this one is older.
1165
+ if (!location) return { collectable: false, location: undefined };
1166
+ const params = new URLSearchParams({
1167
+ fields:
1168
+ "jobReference,configuration(query(query)),status,statistics(creationTime)",
1169
+ location,
1170
+ });
1171
+ const timeout = AbortSignal.timeout(CACHE_CANCEL_TIMEOUT_MS);
1172
+ const response = await fetch(
1173
+ `https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/jobs/${jobId}?${params}`,
1174
+ {
1175
+ signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
1176
+ headers: {
1177
+ Authorization: `Bearer ${token}`,
1178
+ "Content-Type": "application/json",
1179
+ },
1180
+ },
1181
+ );
1182
+ if (!response.ok) {
1183
+ const text = await response.text();
1184
+ throw new BigQueryBackendError(
1185
+ "job",
1186
+ response.status,
1187
+ bigQueryReasonFromResponse(text),
1188
+ bigQueryProviderDetailFromResponse(text),
1189
+ );
1190
+ }
1191
+ const job = (await response.json()) as {
1192
+ jobReference?: { jobId?: string; location?: string };
1193
+ configuration?: { query?: { query?: string } };
1194
+ status?: { errorResult?: unknown };
1195
+ statistics?: { creationTime?: string };
1196
+ };
1197
+ const createdAt = Number(job.statistics?.creationTime);
1198
+ if (job.jobReference?.jobId !== jobId || !Number.isFinite(createdAt)) {
1199
+ throw new BigQueryBackendError("job", null, "other");
1200
+ }
1201
+ return {
1202
+ collectable:
1203
+ job.configuration?.query?.query === sql &&
1204
+ !job.status?.errorResult &&
1205
+ createdAt >= createdAfter,
1206
+ location: job.jobReference.location ?? location,
1207
+ };
1208
+ } catch (error) {
1209
+ throwIfAborted(signal);
1210
+ throw error;
1211
+ }
1212
+ }
1213
+
1097
1214
  async function findQueryJobLocation(
1098
1215
  projectId: string,
1099
1216
  jobId: string,
@@ -1504,97 +1621,165 @@ ${sourceGenerationCacheKey(options.sourceGeneration)}`
1504
1621
  let jobSubmissionResponseReceived = false;
1505
1622
  let token: string | null = null;
1506
1623
  let cancelJob = false;
1624
+ let attachedJob = false;
1625
+ // Only where the app cache could also serve the result, so the SQL means
1626
+ // the same thing whenever in its window the job ran.
1627
+ const resumable =
1628
+ options.resumeAfter !== undefined &&
1629
+ !forceRefresh &&
1630
+ !preserveCurrentDate &&
1631
+ !snapshot.hasUnresolvedCall;
1632
+ let resumableId: string | null = null;
1507
1633
  try {
1508
1634
  token = await getBigQueryAccessToken({ signal });
1509
1635
  assertDateUnchanged();
1510
- jobId = `agent_native_${randomUUID().replace(/-/g, "")}`;
1511
- jobCreatedAfter = Date.now();
1512
- usage = newBigQueryJobUsage(
1513
- projectId,
1514
- jobId,
1515
- requireRequestCredentialContext("BIGQUERY_PROJECT_ID"),
1636
+ const credentialContext = requireRequestCredentialContext(
1637
+ "BIGQUERY_PROJECT_ID",
1516
1638
  );
1517
- usageStarted = await persistBigQueryJobUsage(usage);
1518
- const url = `https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/jobs`;
1519
- const submissionTimeoutController = new AbortController();
1520
- const submissionTimeoutAbortSignal = submissionTimeoutController.signal;
1521
- const submissionTimeoutError = new BigQueryQueryTimeoutError();
1522
- const submissionTimeout = setTimeout(() => {
1523
- submissionTimeoutController.abort(submissionTimeoutError);
1524
- }, JOB_SUBMISSION_TIMEOUT_MS);
1525
-
1526
- let insertedJob: {
1527
- jobReference?: { jobId?: string; location?: string };
1528
- };
1529
- try {
1530
- const res = await fetch(url, {
1531
- method: "POST",
1532
- signal: signal
1533
- ? AbortSignal.any([signal, submissionTimeoutAbortSignal])
1534
- : submissionTimeoutAbortSignal,
1535
- headers: {
1536
- Authorization: `Bearer ${token}`,
1537
- "Content-Type": "application/json",
1538
- },
1539
- body: JSON.stringify({
1540
- jobReference: { projectId, jobId },
1541
- configuration: {
1542
- query: {
1543
- query: cacheableSql,
1544
- useLegacySql: false,
1545
- maximumBytesBilled: String(maxBytesBilled),
1546
- ...(forceRefresh ? { useQueryCache: false } : {}),
1547
- },
1639
+ let resumableIndex = 0;
1640
+ const nextResumableId = () =>
1641
+ resumable && resumableIndex < RESUMABLE_JOB_IDS
1642
+ ? resumableJobId(
1643
+ cacheKey,
1644
+ credentialContext.userEmail,
1645
+ maxBytesBilled,
1646
+ resumableIndex,
1647
+ )
1648
+ : null;
1649
+ resumableId = nextResumableId();
1650
+ // A resumable submission whose job already exists collects that job; when
1651
+ // it can't be collected, the query runs again under the next ID.
1652
+ for (;;) {
1653
+ jobId = resumableId ?? `agent_native_${randomUUID().replace(/-/g, "")}`;
1654
+ jobCreatedAfter = resumableId
1655
+ ? Date.now() - RESUMABLE_JOB_MAX_AGE_MS
1656
+ : Date.now();
1657
+ usage = newBigQueryJobUsage(projectId, jobId, credentialContext);
1658
+ usageStarted = await persistBigQueryJobUsage(usage);
1659
+ const url = `https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/jobs`;
1660
+ const submissionTimeoutController = new AbortController();
1661
+ const submissionTimeoutAbortSignal = submissionTimeoutController.signal;
1662
+ const submissionTimeoutError = new BigQueryQueryTimeoutError();
1663
+ const submissionTimeout = setTimeout(() => {
1664
+ submissionTimeoutController.abort(submissionTimeoutError);
1665
+ }, JOB_SUBMISSION_TIMEOUT_MS);
1666
+
1667
+ let insertedJob:
1668
+ | { jobReference?: { jobId?: string; location?: string } }
1669
+ | undefined;
1670
+ let duplicateResponse: string | undefined;
1671
+ try {
1672
+ const res = await fetch(url, {
1673
+ method: "POST",
1674
+ signal: signal
1675
+ ? AbortSignal.any([signal, submissionTimeoutAbortSignal])
1676
+ : submissionTimeoutAbortSignal,
1677
+ headers: {
1678
+ Authorization: `Bearer ${token}`,
1679
+ "Content-Type": "application/json",
1548
1680
  },
1549
- }),
1550
- });
1681
+ body: JSON.stringify({
1682
+ jobReference: { projectId, jobId },
1683
+ configuration: {
1684
+ query: {
1685
+ query: cacheableSql,
1686
+ useLegacySql: false,
1687
+ maximumBytesBilled: String(maxBytesBilled),
1688
+ ...(forceRefresh ? { useQueryCache: false } : {}),
1689
+ },
1690
+ // A job left running stops once it is too old to collect, so
1691
+ // it never runs beside its replacement.
1692
+ ...(jobId === resumableId
1693
+ ? { jobTimeoutMs: String(RESUMABLE_JOB_MAX_AGE_MS) }
1694
+ : {}),
1695
+ },
1696
+ }),
1697
+ });
1551
1698
 
1552
- if (!res.ok) {
1553
- usage.submissionState = "unknown";
1554
- const text = await res.text();
1555
- if (isMaximumBytesBilledError(res.status, text)) {
1556
- usage.submissionState = "rejected";
1557
- jobId = null;
1558
- throw new BigQueryMaximumBytesBilledError(res.status);
1699
+ if (!res.ok) {
1700
+ usage.submissionState = "unknown";
1701
+ const text = await res.text();
1702
+ if (
1703
+ jobId === resumableId &&
1704
+ isDuplicateJobRejection(res.status, text)
1705
+ ) {
1706
+ duplicateResponse = text;
1707
+ } else {
1708
+ if (isMaximumBytesBilledError(res.status, text)) {
1709
+ usage.submissionState = "rejected";
1710
+ jobId = null;
1711
+ throw new BigQueryMaximumBytesBilledError(res.status);
1712
+ }
1713
+ if (isDefiniteJobSubmissionRejection(res.status, text)) {
1714
+ usage.submissionState = "rejected";
1715
+ jobId = null;
1716
+ }
1717
+ throw new BigQueryBackendError(
1718
+ "submit",
1719
+ res.status,
1720
+ bigQueryReasonFromResponse(text),
1721
+ bigQueryProviderDetailFromResponse(text),
1722
+ );
1723
+ }
1724
+ } else {
1725
+ insertedJob = (await res.json()) as {
1726
+ jobReference?: { jobId?: string; location?: string };
1727
+ };
1559
1728
  }
1560
- if (isDefiniteJobSubmissionRejection(res.status, text)) {
1561
- usage.submissionState = "rejected";
1562
- jobId = null;
1729
+ if (submissionTimeoutController.signal.aborted) {
1730
+ throw submissionTimeoutError;
1563
1731
  }
1564
- throw new BigQueryBackendError(
1565
- "submit",
1566
- res.status,
1567
- bigQueryReasonFromResponse(text),
1568
- bigQueryProviderDetailFromResponse(text),
1569
- );
1732
+ } catch (error) {
1733
+ if (
1734
+ submissionTimeoutController.signal.aborted &&
1735
+ submissionTimeoutController.signal.reason ===
1736
+ submissionTimeoutError &&
1737
+ (error === submissionTimeoutError ||
1738
+ (error instanceof Error && error.name === "AbortError"))
1739
+ ) {
1740
+ throw submissionTimeoutError;
1741
+ }
1742
+ throw error;
1743
+ } finally {
1744
+ clearTimeout(submissionTimeout);
1570
1745
  }
1571
-
1572
- insertedJob = (await res.json()) as {
1573
- jobReference?: { jobId?: string; location?: string };
1574
- };
1575
- if (submissionTimeoutController.signal.aborted) {
1576
- throw submissionTimeoutError;
1746
+ if (duplicateResponse === undefined) {
1747
+ if (
1748
+ insertedJob?.jobReference?.jobId &&
1749
+ insertedJob.jobReference.jobId !== jobId
1750
+ ) {
1751
+ throw new Error("BigQuery did not accept the requested job ID");
1752
+ }
1753
+ jobLocation = insertedJob?.jobReference?.location;
1754
+ break;
1577
1755
  }
1578
- } catch (error) {
1579
- if (
1580
- submissionTimeoutController.signal.aborted &&
1581
- submissionTimeoutController.signal.reason === submissionTimeoutError &&
1582
- (error === submissionTimeoutError ||
1583
- (error instanceof Error && error.name === "AbortError"))
1584
- ) {
1585
- throw submissionTimeoutError;
1756
+ const knownLocation = duplicateJobLocation(duplicateResponse, jobId);
1757
+ let existing: Awaited<ReturnType<typeof resumableJobState>>;
1758
+ try {
1759
+ existing = await resumableJobState(
1760
+ projectId,
1761
+ jobId,
1762
+ token,
1763
+ cacheableSql,
1764
+ knownLocation,
1765
+ signal,
1766
+ );
1767
+ } catch (error) {
1768
+ // Another call's job: it keeps running for a later call to collect.
1769
+ jobLocation = knownLocation;
1770
+ jobId = null;
1771
+ throw error;
1586
1772
  }
1587
- throw error;
1588
- } finally {
1589
- clearTimeout(submissionTimeout);
1590
- }
1591
- if (
1592
- insertedJob.jobReference?.jobId &&
1593
- insertedJob.jobReference.jobId !== jobId
1594
- ) {
1595
- throw new Error("BigQuery did not accept the requested job ID");
1773
+ if (existing.collectable) {
1774
+ jobLocation = existing.location;
1775
+ attachedJob = true;
1776
+ break;
1777
+ }
1778
+ usage.location = existing.location;
1779
+ await finalizeBigQueryJobUsage(usage, token, usageStarted, signal);
1780
+ resumableIndex += 1;
1781
+ resumableId = nextResumableId();
1596
1782
  }
1597
- jobLocation = insertedJob.jobReference?.location;
1598
1783
  jobSubmissionResponseReceived = true;
1599
1784
  usage.submissionState = "accepted";
1600
1785
  usage.location = jobLocation;
@@ -1743,6 +1928,11 @@ ${sourceGenerationCacheKey(options.sourceGeneration)}`
1743
1928
  completedResult = { ...result, cached: false };
1744
1929
  } catch (error) {
1745
1930
  if (jobId) cancelJob = true;
1931
+ // Stopped for time, a resumable job keeps running for the next call.
1932
+ if (jobId !== null && jobId === resumableId && options.resumeAfter?.aborted)
1933
+ cancelJob = false;
1934
+ // Another call started a job this one attached to, and may collect it too.
1935
+ if (attachedJob) cancelJob = false;
1746
1936
  if (cancelJob && jobId && token) {
1747
1937
  if (!jobSubmissionResponseReceived) {
1748
1938
  try {
@@ -2101,6 +2101,8 @@ export async function queryFirstPartyAnalyticsInBigQuery(
2101
2101
  maxBytesBilled?: number;
2102
2102
  timeoutMs?: number;
2103
2103
  signal?: AbortSignal;
2104
+ /** Stopped by this or by `timeoutMs`, the job is left for a later call. */
2105
+ resumeAfter?: AbortSignal;
2104
2106
  } = {},
2105
2107
  ): Promise<AnalyticsQueryResult> {
2106
2108
  const timeoutMs = options.timeoutMs;
@@ -2118,6 +2120,10 @@ export async function queryFirstPartyAnalyticsInBigQuery(
2118
2120
  options.signal && timeoutSignal
2119
2121
  ? AbortSignal.any([options.signal, timeoutSignal])
2120
2122
  : (options.signal ?? timeoutSignal);
2123
+ const resumeAfter =
2124
+ options.resumeAfter && timeoutSignal
2125
+ ? AbortSignal.any([options.resumeAfter, timeoutSignal])
2126
+ : options.resumeAfter;
2121
2127
  let eventsSource = options.eventsSource;
2122
2128
  let result: Awaited<ReturnType<typeof runQuery>>;
2123
2129
  try {
@@ -2139,6 +2145,7 @@ export async function queryFirstPartyAnalyticsInBigQuery(
2139
2145
  {
2140
2146
  maxBytesBilled: options.maxBytesBilled,
2141
2147
  ...(signal ? { signal } : {}),
2148
+ ...(resumeAfter ? { resumeAfter } : {}),
2142
2149
  ...(eventsSource?.source === "model" && sourceGeneration
2143
2150
  ? { sourceGeneration }
2144
2151
  : {}),