@dalgo/bigquery 0.0.0-stage → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +472 -2
  3. package/dist/analytical/canonical.d.ts +10 -0
  4. package/dist/analytical/canonical.d.ts.map +1 -0
  5. package/dist/analytical/canonical.js +50 -0
  6. package/dist/analytical/client.d.ts +84 -0
  7. package/dist/analytical/client.d.ts.map +1 -0
  8. package/dist/analytical/client.js +730 -0
  9. package/dist/analytical/deadline.d.ts +13 -0
  10. package/dist/analytical/deadline.d.ts.map +1 -0
  11. package/dist/analytical/deadline.js +23 -0
  12. package/dist/analytical/google-identity.d.ts +40 -0
  13. package/dist/analytical/google-identity.d.ts.map +1 -0
  14. package/dist/analytical/google-identity.js +158 -0
  15. package/dist/analytical/ledger.d.ts +80 -0
  16. package/dist/analytical/ledger.d.ts.map +1 -0
  17. package/dist/analytical/ledger.js +89 -0
  18. package/dist/analytical/metadata-client.d.ts +66 -0
  19. package/dist/analytical/metadata-client.d.ts.map +1 -0
  20. package/dist/analytical/metadata-client.js +247 -0
  21. package/dist/analytical/metadata-harness.d.ts +28 -0
  22. package/dist/analytical/metadata-harness.d.ts.map +1 -0
  23. package/dist/analytical/metadata-harness.js +127 -0
  24. package/dist/analytical/metadata.d.ts +8 -0
  25. package/dist/analytical/metadata.d.ts.map +1 -0
  26. package/dist/analytical/metadata.js +155 -0
  27. package/dist/analytical/protocol.d.ts +157 -0
  28. package/dist/analytical/protocol.d.ts.map +1 -0
  29. package/dist/analytical/protocol.js +320 -0
  30. package/dist/analytical/public-metadata.d.ts +29 -0
  31. package/dist/analytical/public-metadata.d.ts.map +1 -0
  32. package/dist/analytical/public-metadata.js +70 -0
  33. package/dist/analytical/transport.d.ts +39 -0
  34. package/dist/analytical/transport.d.ts.map +1 -0
  35. package/dist/analytical/transport.js +224 -0
  36. package/dist/analytical/values.d.ts +18 -0
  37. package/dist/analytical/values.d.ts.map +1 -0
  38. package/dist/analytical/values.js +157 -0
  39. package/dist/analytical/wire.d.ts +26 -0
  40. package/dist/analytical/wire.d.ts.map +1 -0
  41. package/dist/analytical/wire.js +141 -0
  42. package/dist/analytical.d.ts +27 -0
  43. package/dist/analytical.d.ts.map +1 -0
  44. package/dist/analytical.js +14 -0
  45. package/dist/database.d.ts +35 -0
  46. package/dist/database.d.ts.map +1 -0
  47. package/dist/database.js +437 -0
  48. package/dist/index.d.ts +4 -0
  49. package/dist/index.d.ts.map +1 -0
  50. package/dist/index.js +2 -0
  51. package/dist/sql.d.ts +29 -0
  52. package/dist/sql.d.ts.map +1 -0
  53. package/dist/sql.js +198 -0
  54. package/dist/types.d.ts +49 -0
  55. package/dist/types.d.ts.map +1 -0
  56. package/dist/types.js +1 -0
  57. package/package.json +58 -4
@@ -0,0 +1,730 @@
1
+ import { AnalyticalError, fail, parseJSON } from "./wire.js";
2
+ import { canonicalJSON } from "./canonical.js";
3
+ import { decodeRows } from "./values.js";
4
+ import { decodeSchema, object, same, validateDataset, validateTable } from "./metadata.js";
5
+ import { bounds, checkDecimalConstraints, cloneFrozen, compileReadPlan, digest, encoded, exactKeys, integerText, nonempty, timestamp, validateExecution, validateSource, wireParameters } from "./protocol.js";
6
+ import { opaqueID } from "./ledger.js";
7
+ import { abortable, realClock, Transport } from "./transport.js";
8
+ /** Capability minted by this client only; JSON cannot manufacture approval. */
9
+ export class Approval {
10
+ #opaque = true;
11
+ constructor() { void this.#opaque; }
12
+ static mint() { return new Approval(); }
13
+ }
14
+ /** Returns immutable page/row receipts; local stop never implies server cancel. */
15
+ export class AnalyticalRun {
16
+ #driver;
17
+ constructor(driver) { this.#driver = driver; }
18
+ receipt() { return this.#driver.receipt(); }
19
+ schema() { return this.#driver.schema(); }
20
+ nextPage(options = {}) { return this.#driver.next("page", options); }
21
+ nextRow(options = {}) { return this.#driver.next("row", options); }
22
+ close() { return this.#driver.close(); }
23
+ }
24
+ const codeOf = (error) => error instanceof AnalyticalError ? error.code : "remote_failed";
25
+ const warnings = (value) => {
26
+ if (value === undefined)
27
+ return [];
28
+ if (!Array.isArray(value))
29
+ fail("malformed_wire");
30
+ return value.map(entry => { const reason = object(entry).reason; return typeof reason === "string" && ["accessDenied", "backendError", "billingTierLimitExceeded", "invalidQuery", "notFound", "rateLimitExceeded", "resourcesExceeded", "stopped", "cancelled"].includes(reason) ? reason : "provider_warning"; });
31
+ };
32
+ async function sha(value) { const digest = await crypto.subtle.digest("SHA-256", value); return Array.from(new Uint8Array(digest), byte => byte.toString(16).padStart(2, "0")).join(""); }
33
+ function parseJob(value) {
34
+ const job = object(value);
35
+ exactKeys(job, ["projectId", "jobId", "location"], [], "malformed_wire");
36
+ for (const key of ["projectId", "jobId", "location"])
37
+ if (typeof job[key] !== "string" || job[key] === "")
38
+ fail("malformed_wire");
39
+ return job;
40
+ }
41
+ const budgetKey = (preview) => JSON.stringify([preview.execution.principal.kind, preview.execution.principal.subject, preview.execution.jobProject]);
42
+ function approvalPayload(preview) {
43
+ return {
44
+ effectivePlanDigest: preview.plan.digest, observationDigest: preview.observation.digest, policyDigest: preview.policyDigest, principal: preview.execution.principal, jobProject: preview.execution.jobProject, location: preview.observation.location, maximumBytesBilled: preview.execution.maximumBytesBilled, sessionBudgetBytes: preview.execution.sessionBudgetBytes, bounds: preview.bounds, estimatedBytes: preview.estimatedBytes
45
+ };
46
+ }
47
+ export function queryRequest(plan, execution, limits, location, dryRun) {
48
+ validateExecution(execution);
49
+ return {
50
+ query: plan.sql, useLegacySql: false, parameterMode: "NAMED", queryParameters: wireParameters(plan.parameters), location, maximumBytesBilled: execution.maximumBytesBilled, maxResults: limits.pageSize, timeoutMs: 1000, jobTimeoutMs: String(limits.wallMs), jobCreationMode: "JOB_CREATION_REQUIRED", formatOptions: {
51
+ useInt64Timestamp: true
52
+ }, dryRun
53
+ };
54
+ }
55
+ export class BigQueryAnalyticalClient {
56
+ #profiles = new Map();
57
+ #prepare;
58
+ #ledger;
59
+ #clock;
60
+ #transport;
61
+ #approvals = new WeakMap();
62
+ constructor(config) { this.#prepare = config.prepare; this.#ledger = config.ledger; this.#clock = config.clock ?? realClock; this.#transport = new Transport(config.provider, config.fetch ?? ((url, init) => globalThis.fetch(url, init)), this.#clock); }
63
+ static async create(config) {
64
+ if (!config.ledger || typeof config.prepare !== "function" || !config.provider || !Array.isArray(config.profiles) || config.profiles.length === 0)
65
+ fail("invalid_input");
66
+ const client = new BigQueryAnalyticalClient(config);
67
+ for (const input of config.profiles) {
68
+ validateSource(input);
69
+ const profile = cloneFrozen(input);
70
+ const key = await digest("SourceProfile", profile);
71
+ if (client.#profiles.has(key))
72
+ fail("invalid_input");
73
+ client.#profiles.set(key, profile);
74
+ }
75
+ return client;
76
+ }
77
+ async #prepared(signal, deadline = this.#clock.now() + 15000) {
78
+ if (signal?.aborted || this.#clock.now() >= deadline)
79
+ fail("local_stopped");
80
+ const controller = new AbortController();
81
+ const abort = () => controller.abort();
82
+ signal?.addEventListener("abort", abort, {
83
+ once: true
84
+ });
85
+ const timer = setTimeout(abort, deadline - this.#clock.now());
86
+ let prepared;
87
+ try {
88
+ prepared = await abortable(this.#prepare(controller.signal), controller.signal);
89
+ if (this.#clock.now() >= deadline)
90
+ fail("local_stopped");
91
+ }
92
+ catch (error) {
93
+ if (error instanceof AnalyticalError)
94
+ throw error;
95
+ fail("policy_denied");
96
+ }
97
+ finally {
98
+ clearTimeout(timer);
99
+ signal?.removeEventListener("abort", abort);
100
+ }
101
+ nonempty(prepared.policyDigest);
102
+ const plan = await compileReadPlan(prepared.source, prepared.query);
103
+ const source = this.#profiles.get(plan.sourceDigest);
104
+ if (source === undefined)
105
+ fail("source_ineligible");
106
+ return {
107
+ plan, policyDigest: prepared.policyDigest, source
108
+ };
109
+ }
110
+ async #reauthorize(preview, signal, deadline = this.#clock.now() + preview.bounds.httpMs) {
111
+ const prepared = await this.#prepared(signal, deadline);
112
+ if (prepared.plan.digest !== preview.plan.digest || prepared.policyDigest !== preview.policyDigest)
113
+ fail("approval_changed");
114
+ }
115
+ async #load(id) {
116
+ return this.#ledger.update(state => {
117
+ const record = state.runs[id];
118
+ if (record === undefined)
119
+ fail("cursor_invalid");
120
+ return record;
121
+ });
122
+ }
123
+ async #mutate(id, fn) {
124
+ return this.#ledger.update(state => {
125
+ const record = state.runs[id];
126
+ if (record === undefined)
127
+ fail("cursor_invalid");
128
+ return fn(record);
129
+ });
130
+ }
131
+ #scope(preview, id, options = {}, result = false) {
132
+ let bytes = 0;
133
+ const now = this.#clock.now();
134
+ const deadline = id === undefined ? now + preview.bounds.wallMs : Date.parse(preview.createdAt); // replaced by trusted run below.
135
+ let runDeadline = deadline;
136
+ let runPrincipal = preview.execution.principal;
137
+ const scope = {
138
+ bounds: preview.bounds, get executionDeadline() { return runDeadline; }, get principal() { return runPrincipal; }, ...options, ...(result && id !== undefined ? { beforeDispatch: () => this.#chargePage(id) } : {}),
139
+ remaining: async () => {
140
+ if (id === undefined)
141
+ return preview.bounds.totalResponseBytes - bytes;
142
+ const record = await this.#load(id);
143
+ runDeadline = Date.parse(record.receipt.executionDeadline);
144
+ runPrincipal = record.activePrincipal;
145
+ return record.receipt.bounds.totalResponseBytes - record.receipt.counters.bytes;
146
+ },
147
+ debit: async (length) => {
148
+ if (id === undefined) {
149
+ bytes += length;
150
+ if (bytes > preview.bounds.totalResponseBytes)
151
+ fail("response_limit");
152
+ return;
153
+ }
154
+ let exceeded = false;
155
+ await this.#mutate(id, record => {
156
+ const counters = {
157
+ ...record.receipt.counters, bytes: record.receipt.counters.bytes + length
158
+ };
159
+ exceeded = counters.bytes > record.receipt.bounds.totalResponseBytes;
160
+ record.receipt = {
161
+ ...record.receipt, counters, ...(exceeded ? {
162
+ localStopped: true, reason: "response_limit"
163
+ } : {})
164
+ };
165
+ });
166
+ if (exceeded)
167
+ fail("response_limit");
168
+ },
169
+ };
170
+ return scope;
171
+ }
172
+ async #observe(source, scope) {
173
+ const path = `projects/${source.sourceProject}/datasets/${source.datasetId}`;
174
+ const dataset = await this.#transport.call(scope, "GET", path, {});
175
+ validateDataset(dataset, source);
176
+ const table = await this.#transport.call(scope, "GET", `${path}/tables/${source.tableId}`, {});
177
+ return validateTable(table, source, this.#clock.now());
178
+ }
179
+ async #estimate(plan, execution, limits, location, scope) {
180
+ const response = object(await this.#transport.call(scope, "POST", `projects/${execution.jobProject}/queries`, {}, queryRequest(plan, execution, limits, location, true)));
181
+ if (Object.hasOwn(response, "errors") && warnings(response.errors).length > 0)
182
+ fail("remote_failed");
183
+ if (!Object.hasOwn(response, "totalBytesProcessed"))
184
+ fail("estimate_missing");
185
+ const estimate = integerText(response.totalBytesProcessed, "estimate_missing");
186
+ if (BigInt(estimate) > BigInt(execution.maximumBytesBilled))
187
+ fail("cap_exceeded");
188
+ return estimate;
189
+ }
190
+ async preview(execution, requestedBounds = {}, options = {}) {
191
+ validateExecution(execution);
192
+ const safeExecution = cloneFrozen(execution);
193
+ const limits = bounds(requestedBounds);
194
+ const prepared = await this.#prepared(options.signal, Math.min(this.#clock.now() + limits.httpMs, options.callerDeadline ?? Number.MAX_SAFE_INTEGER));
195
+ const createdAt = timestamp(this.#clock.now());
196
+ const draft = {
197
+ nonce: opaqueID(), plan: prepared.plan, observation: {
198
+ table: {
199
+ projectId: prepared.source.sourceProject, datasetId: prepared.source.datasetId, tableId: prepared.source.tableId
200
+ }, location: prepared.source.location, type: "TABLE", config: {}, schema: prepared.source.schema, digest: "", observedAt: createdAt
201
+ }, policyDigest: prepared.policyDigest, execution: safeExecution, bounds: limits, estimatedBytes: "0", createdAt, expiresAt: timestamp(Date.parse(createdAt) + 300000), approvalDigest: ""
202
+ };
203
+ const scope = this.#scope(draft, undefined, options);
204
+ const observation = await this.#observe(prepared.source, scope);
205
+ const estimatedBytes = await this.#estimate(prepared.plan, safeExecution, limits, observation.location, scope);
206
+ const result = {
207
+ ...draft, observation, estimatedBytes
208
+ };
209
+ result.approvalDigest = await digest("Approval", approvalPayload(result));
210
+ await this.#ledger.update(state => {
211
+ state.previews[result.nonce] = {
212
+ preview: result, used: false
213
+ };
214
+ });
215
+ return cloneFrozen(result);
216
+ }
217
+ async approve(preview, approvedDigest) {
218
+ const stored = await this.#ledger.update(state => state.previews[preview.nonce]);
219
+ if (stored === undefined || stored.used)
220
+ fail("approval_required");
221
+ if (approvedDigest !== stored.preview.approvalDigest || !same(preview, stored.preview) || this.#clock.now() >= Date.parse(stored.preview.expiresAt))
222
+ fail("approval_changed");
223
+ const approval = Approval.mint();
224
+ this.#approvals.set(approval, {
225
+ nonce: preview.nonce, digest: approvedDigest
226
+ });
227
+ return approval;
228
+ }
229
+ #session(id, accessPrincipal) {
230
+ return {
231
+ id, accessPrincipal: cloneFrozen(accessPrincipal), emptyDelivered: false, controller: new AbortController(), rows: [], index: 0, loaded: false, done: false, mode: "", digest: ""
232
+ };
233
+ }
234
+ #run(session) {
235
+ return new AnalyticalRun({
236
+ receipt: async () => cloneFrozen((await this.#load(session.id)).receipt), schema: async () => cloneFrozen((await this.#load(session.id)).schema), next: (mode, options) => this.#read(session, mode, options), close: async () => {
237
+ session.controller.abort();
238
+ session.rows = [];
239
+ await this.#mutate(session.id, r => {
240
+ r.receipt = {
241
+ ...r.receipt, localStopped: true, reason: "local_stopped"
242
+ };
243
+ });
244
+ }
245
+ });
246
+ }
247
+ async execute(approval, options = {}) {
248
+ const capability = this.#approvals.get(approval);
249
+ if (capability === undefined)
250
+ fail("approval_required");
251
+ const id = opaqueID();
252
+ const record = await this.#ledger.update(state => {
253
+ const stored = state.previews[capability.nonce];
254
+ if (stored === undefined || stored.used)
255
+ fail("approval_required");
256
+ const preview = stored.preview;
257
+ if (this.#clock.now() >= Date.parse(preview.expiresAt) || preview.approvalDigest !== capability.digest)
258
+ fail("approval_changed");
259
+ let reserved = 0n;
260
+ const budget = BigInt(preview.execution.sessionBudgetBytes);
261
+ for (const run of Object.values(state.runs))
262
+ if (budgetKey(run.preview) === budgetKey(preview)) {
263
+ if (run.preview.execution.sessionBudgetBytes !== preview.execution.sessionBudgetBytes)
264
+ fail("budget_exhausted");
265
+ reserved += BigInt(run.reservation);
266
+ }
267
+ if (reserved + BigInt(preview.execution.maximumBytesBilled) > budget)
268
+ fail("budget_exhausted");
269
+ const now = this.#clock.now();
270
+ stored.used = true;
271
+ const receipt = {
272
+ version: 1, runId: id, approvalDigest: capability.digest, sourceDigest: preview.plan.sourceDigest, observationDigest: preview.observation.digest, schemaDigest: "", principal: preview.execution.principal, state: "reserved", runStartedAt: timestamp(now), executionDeadline: timestamp(now + preview.bounds.wallMs), bounds: preview.bounds, counters: {
273
+ bytes: 0, rows: 0, pages: 0
274
+ }, localStopped: false, warnings: [], residualSourceReplacementRace: true
275
+ };
276
+ const record = {
277
+ receipt, preview, activePrincipal: preview.execution.principal, reservation: preview.execution.maximumBytesBilled, pageToken: null, nextToken: null, pageDigest: "", offset: 0, pageOrdinal: 0, pageDone: false, schema: [], seenTokens: [], cursor: null
278
+ };
279
+ state.runs[id] = record;
280
+ return record;
281
+ });
282
+ const session = this.#session(id, record.activePrincipal);
283
+ const run = this.#run(session);
284
+ const scope = this.#scope(record.preview, id, options);
285
+ let dispatched = false;
286
+ try {
287
+ await this.#ledger.withLease(id, options.signal, async () => {
288
+ await this.#reauthorize(record.preview, options.signal, Math.min(Date.parse(record.receipt.executionDeadline), this.#clock.now() + record.receipt.bounds.httpMs, options.callerDeadline ?? Number.MAX_SAFE_INTEGER));
289
+ const source = this.#profiles.get(record.preview.plan.sourceDigest);
290
+ let observation = await this.#observe(source, scope);
291
+ const estimate = await this.#estimate(record.preview.plan, record.preview.execution, record.preview.bounds, observation.location, scope);
292
+ if (await digest("Approval", approvalPayload({
293
+ ...record.preview, observation, estimatedBytes: estimate
294
+ })) !== capability.digest)
295
+ fail("approval_changed");
296
+ observation = await this.#observe(source, scope);
297
+ if (observation.digest !== record.preview.observation.digest)
298
+ fail("source_changed");
299
+ await this.#reauthorize(record.preview, options.signal, Math.min(Date.parse(record.receipt.executionDeadline), this.#clock.now() + record.receipt.bounds.httpMs, options.callerDeadline ?? Number.MAX_SAFE_INTEGER));
300
+ await this.#mutate(id, r => {
301
+ r.receipt = {
302
+ ...r.receipt, state: "submitting"
303
+ };
304
+ });
305
+ const raw = object(await this.#transport.call(this.#scope(record.preview, id, options, true), "POST", `projects/${record.preview.execution.jobProject}/queries`, {}, queryRequest(record.preview.plan, record.preview.execution, record.preview.bounds, observation.location, false), false, false, () => { dispatched = true; }));
306
+ const job = parseJob(raw.jobReference);
307
+ if (job.projectId !== record.preview.execution.jobProject || job.location !== observation.location)
308
+ fail("malformed_wire");
309
+ await this.#mutate(id, r => {
310
+ r.receipt = {
311
+ ...r.receipt, job, state: "running"
312
+ };
313
+ });
314
+ await this.#install(session, raw, null, false);
315
+ });
316
+ return run;
317
+ }
318
+ catch (error) {
319
+ await this.#mutate(id, r => {
320
+ r.receipt = {
321
+ ...r.receipt, ...(dispatched ? {
322
+ state: r.receipt.job === undefined ? "submission_unknown" : r.receipt.state, localStopped: true
323
+ } : {
324
+ state: "failed"
325
+ }), reason: codeOf(error)
326
+ };
327
+ if (!dispatched)
328
+ r.reservation = "0";
329
+ });
330
+ throw new ExecutionFailure(error, await run.receipt(), run);
331
+ }
332
+ }
333
+ async #chargePage(id) {
334
+ await this.#mutate(id, r => { if (r.receipt.counters.pages >= r.receipt.bounds.maxPages)
335
+ fail("response_limit"); r.receipt = { ...r.receipt, counters: { ...r.receipt.counters, pages: r.receipt.counters.pages + 1 } }; });
336
+ }
337
+ async #install(session, response, token, refetch) {
338
+ const record = await this.#load(session.id);
339
+ if (record.receipt.job === undefined)
340
+ fail("malformed_wire");
341
+ exactKeys(response, ["jobComplete"], ["kind", "etag", "jobReference", "schema", "rows", "pageToken", "totalRows", "totalBytesProcessed", "totalBytesBilled", "cacheHit", "errors", "jobCreationReason", "creationTime", "startTime", "endTime", "location", "queryId", "statementType", "pageRowCount", "totalSlotMs"], "malformed_wire");
342
+ if (Object.hasOwn(response, "jobReference") && !same(parseJob(response.jobReference), record.receipt.job))
343
+ fail("malformed_wire");
344
+ if (typeof response.jobComplete !== "boolean")
345
+ fail("malformed_wire");
346
+ let schema = record.schema;
347
+ if (Object.hasOwn(response, "schema")) {
348
+ const incoming = decodeSchema(response.schema);
349
+ const profile = this.#profiles.get(record.preview.plan.sourceDigest);
350
+ const expected = record.preview.plan.projection.map(name => profile.schema.find(field => field.name === name));
351
+ if (!same(incoming, expected) || schema.length > 0 && !same(schema, incoming))
352
+ fail("schema_changed");
353
+ schema = incoming;
354
+ }
355
+ const rawRows = Object.hasOwn(response, "rows") ? response.rows : [];
356
+ if (!Array.isArray(rawRows))
357
+ fail("malformed_wire");
358
+ const totalRows = Object.hasOwn(response, "totalRows") ? integerText(response.totalRows) : record.totalRows;
359
+ const pageStart = record.receipt.counters.rows - (refetch ? record.offset : 0);
360
+ if (pageStart < 0)
361
+ fail("cursor_invalid");
362
+ if (pageStart + rawRows.length > record.preview.plan.limit ||
363
+ totalRows !== undefined && (BigInt(totalRows) > BigInt(record.preview.plan.limit) || BigInt(pageStart + rawRows.length) > BigInt(totalRows) ||
364
+ record.totalRows !== undefined && totalRows !== record.totalRows))
365
+ fail("malformed_wire");
366
+ if (schema.length === 0) {
367
+ if (rawRows.length > 0 || response.jobComplete)
368
+ fail("malformed_wire");
369
+ session.loaded = false;
370
+ return;
371
+ }
372
+ const rows = decodeRows(encoded(rawRows), schema, record.receipt.bounds.responseBytes);
373
+ if (rows.length > record.receipt.bounds.pageSize)
374
+ fail("response_limit");
375
+ for (const row of rows)
376
+ row.forEach((cell, index) => {
377
+ const field = schema[index];
378
+ if (field !== undefined && typeof cell.value === "string" && ["NUMERIC", "BIGNUMERIC"].includes(cell.type))
379
+ checkDecimalConstraints(field, cell.value);
380
+ });
381
+ const pageDigest = await sha(canonicalJSON(encoded(rawRows)));
382
+ let next = null;
383
+ if (Object.hasOwn(response, "pageToken")) {
384
+ if (typeof response.pageToken !== "string" || response.pageToken.length === 0 || response.pageToken.length > 16384)
385
+ fail("malformed_wire");
386
+ next = response.pageToken;
387
+ }
388
+ if (next !== null && (pageStart + rows.length >= record.preview.plan.limit || totalRows !== undefined && BigInt(pageStart + rows.length) >= BigInt(totalRows)))
389
+ fail("malformed_wire");
390
+ if (response.jobComplete && next === null && totalRows !== undefined && BigInt(pageStart + rows.length) !== BigInt(totalRows))
391
+ fail("malformed_wire");
392
+ if (refetch && (pageDigest !== record.pageDigest || next !== record.nextToken || response.jobComplete !== record.pageDone))
393
+ fail("cursor_invalid");
394
+ const schemaDigest = await sha(canonicalJSON(encoded(schema)));
395
+ const processed = Object.hasOwn(response, "totalBytesProcessed") ? integerText(response.totalBytesProcessed) : undefined;
396
+ const billed = Object.hasOwn(response, "totalBytesBilled") ? integerText(response.totalBytesBilled) : undefined;
397
+ if (Object.hasOwn(response, "cacheHit") && typeof response.cacheHit !== "boolean")
398
+ fail("malformed_wire");
399
+ await this.#mutate(session.id, r => {
400
+ if (!refetch) {
401
+ if (next !== null && (r.seenTokens.includes(next) || token === next))
402
+ fail("cursor_invalid");
403
+ if (token !== null)
404
+ r.seenTokens.push(token);
405
+ r.pageOrdinal += 1;
406
+ r.offset = 0;
407
+ r.pageToken = token;
408
+ r.nextToken = next;
409
+ r.pageDigest = pageDigest;
410
+ r.pageDone = response.jobComplete;
411
+ }
412
+ r.schema = schema;
413
+ if (totalRows !== undefined)
414
+ r.totalRows = totalRows;
415
+ r.receipt = {
416
+ ...r.receipt, schemaDigest, warnings: [...r.receipt.warnings, ...warnings(response.errors)], ...(processed === undefined ? {} : {
417
+ processedBytes: processed
418
+ }), ...(billed === undefined ? {} : {
419
+ billedBytes: billed
420
+ }), ...(response.cacheHit === undefined ? {} : {
421
+ cacheHit: response.cacheHit
422
+ })
423
+ };
424
+ });
425
+ const loaded = await this.#load(session.id);
426
+ if (loaded.offset > rows.length)
427
+ fail("cursor_invalid");
428
+ session.rows = rows;
429
+ session.index = loaded.offset;
430
+ session.digest = pageDigest;
431
+ session.loaded = true;
432
+ session.done = response.jobComplete && next === null;
433
+ }
434
+ async #fetchPage(session, token, refetch, options) {
435
+ const record = await this.#load(session.id);
436
+ if (record.receipt.counters.rows >= Math.min(record.preview.plan.limit, record.receipt.bounds.maxRows) || record.receipt.counters.pages >= record.receipt.bounds.maxPages)
437
+ fail("response_limit");
438
+ await this.#reauthorize(record.preview, options.signal, Math.min(Date.parse(record.receipt.executionDeadline), this.#clock.now() + record.receipt.bounds.httpMs, options.callerDeadline ?? Number.MAX_SAFE_INTEGER));
439
+ const job = record.receipt.job;
440
+ if (job === undefined)
441
+ fail("submission_unknown");
442
+ const query = {
443
+ location: job.location, maxResults: String(record.receipt.bounds.pageSize), "formatOptions.useInt64Timestamp": "true", timeoutMs: "1000", ...(token === null ? {
444
+ startIndex: "0"
445
+ } : {
446
+ pageToken: token
447
+ })
448
+ };
449
+ const raw = object(await this.#transport.call(this.#scope(record.preview, session.id, options, true), "GET", `projects/${job.projectId}/queries/${encodeURIComponent(job.jobId)}`, query));
450
+ await this.#install(session, raw, token, refetch);
451
+ }
452
+ async #read(session, mode, options) {
453
+ if (session.mode !== "" && session.mode !== mode)
454
+ fail("invalid_input");
455
+ session.mode = mode;
456
+ try {
457
+ return await this.#ledger.withLease(session.id, options.signal, async () => {
458
+ const abort = () => session.controller.abort();
459
+ options.signal?.addEventListener("abort", abort, {
460
+ once: true
461
+ });
462
+ const safeOptions = {
463
+ ...options, signal: session.controller.signal
464
+ };
465
+ try {
466
+ const initial = await this.#load(session.id);
467
+ if (!same(initial.activePrincipal, session.accessPrincipal))
468
+ fail("approval_changed");
469
+ await this.#reauthorize(initial.preview, safeOptions.signal, Math.min(Date.parse(initial.receipt.executionDeadline), this.#clock.now() + initial.receipt.bounds.httpMs, options.callerDeadline ?? Number.MAX_SAFE_INTEGER));
470
+ await this.#transport.verify(initial.activePrincipal, Math.min(Date.parse(initial.receipt.executionDeadline), this.#clock.now() + initial.receipt.bounds.httpMs, options.callerDeadline ?? Number.MAX_SAFE_INTEGER), safeOptions.signal);
471
+ while (true) {
472
+ const record = await this.#load(session.id);
473
+ if (session.controller.signal.aborted || this.#clock.now() >= Date.parse(record.receipt.executionDeadline) || (options.callerDeadline !== undefined && this.#clock.now() >= options.callerDeadline))
474
+ fail("local_stopped");
475
+ if (session.loaded && (record.offset !== session.index || record.pageDigest !== session.digest))
476
+ fail("cursor_invalid");
477
+ if (session.loaded && session.index < session.rows.length)
478
+ break;
479
+ if (session.loaded && session.done) {
480
+ if (session.rows.length === 0 && !session.emptyDelivered) {
481
+ session.emptyDelivered = true;
482
+ const cursor = await this.#delivery(session, 0);
483
+ const current = await this.#load(session.id);
484
+ return cloneFrozen({
485
+ schema: current.schema, rows: [], cursor, receipt: current.receipt
486
+ });
487
+ }
488
+ return null;
489
+ }
490
+ if (record.receipt.counters.rows >= Math.min(record.preview.plan.limit, record.receipt.bounds.maxRows) || record.receipt.counters.pages >= record.receipt.bounds.maxPages)
491
+ fail("response_limit");
492
+ await this.#fetchPage(session, session.loaded ? record.nextToken : record.pageToken, false, safeOptions);
493
+ if (!session.loaded) {
494
+ await this.#clock.sleep(100, safeOptions.signal);
495
+ }
496
+ }
497
+ const record = await this.#load(session.id);
498
+ let amount = mode === "row" ? 1 : session.rows.length - session.index;
499
+ amount = Math.min(amount, Math.min(record.preview.plan.limit, record.receipt.bounds.maxRows) - record.receipt.counters.rows);
500
+ if (amount < 1)
501
+ fail("response_limit");
502
+ const rows = cloneFrozen(session.rows.slice(session.index, session.index + amount));
503
+ const offset = session.index + amount;
504
+ const cursor = await this.#delivery(session, amount);
505
+ session.index = offset;
506
+ const current = await this.#load(session.id);
507
+ return cloneFrozen({
508
+ schema: current.schema, rows, cursor, receipt: current.receipt
509
+ });
510
+ }
511
+ finally {
512
+ options.signal?.removeEventListener("abort", abort);
513
+ }
514
+ });
515
+ }
516
+ catch (error) {
517
+ await this.#mutate(session.id, r => {
518
+ r.receipt = {
519
+ ...r.receipt, localStopped: true, reason: codeOf(error)
520
+ };
521
+ });
522
+ throw new ExecutionFailure(error, (await this.#load(session.id)).receipt, this.#run(session));
523
+ }
524
+ }
525
+ async #delivery(session, amount) {
526
+ const offset = session.index + amount;
527
+ const cursor = await this.#mutate(session.id, r => {
528
+ if (r.receipt.counters.rows + amount > Math.min(r.preview.plan.limit, r.receipt.bounds.maxRows))
529
+ fail("response_limit");
530
+ r.offset = offset;
531
+ r.receipt = {
532
+ ...r.receipt, localStopped: false, counters: {
533
+ ...r.receipt.counters, rows: r.receipt.counters.rows + amount
534
+ }
535
+ };
536
+ const boundary = offset === session.rows.length && r.nextToken !== null;
537
+ const cursor = {
538
+ version: 1, runId: session.id, job: r.receipt.job, planDigest: r.preview.plan.digest, schemaDigest: r.receipt.schemaDigest, principal: r.activePrincipal, pageToken: boundary ? r.nextToken : r.pageToken, pageSize: r.receipt.bounds.pageSize, offset: boundary ? 0 : offset, pageDigest: boundary ? "" : r.pageDigest, ordinal: r.pageOrdinal + (boundary ? 1 : 0), counters: r.receipt.counters, ledgerRef: opaqueID()
539
+ };
540
+ r.cursor = cursor;
541
+ return cursor;
542
+ });
543
+ return encodeCursor(cursor);
544
+ }
545
+ async resume(receipt, cursor, options = {}) {
546
+ const decoded = decodeCursor(cursor);
547
+ const record = await this.#load(receipt.runId);
548
+ if (!same(decoded, record.cursor) || !receiptMatches(receipt, record.receipt))
549
+ fail("cursor_invalid");
550
+ const session = this.#session(receipt.runId, record.activePrincipal);
551
+ const run = this.#run(session);
552
+ if (this.#clock.now() >= Date.parse(record.receipt.executionDeadline)) {
553
+ await this.#mutate(session.id, r => {
554
+ r.receipt = {
555
+ ...r.receipt, localStopped: true, reason: "local_stopped"
556
+ };
557
+ });
558
+ throw new ExecutionFailure(new AnalyticalError("local_stopped"), await run.receipt(), run);
559
+ }
560
+ await this.#ledger.withLease(session.id, options.signal, async () => {
561
+ const current = await this.#load(session.id);
562
+ if (!same(decoded, current.cursor))
563
+ fail("cursor_invalid");
564
+ await this.#reauthorize(current.preview, options.signal, Math.min(Date.parse(current.receipt.executionDeadline), this.#clock.now() + current.receipt.bounds.httpMs, options.callerDeadline ?? Number.MAX_SAFE_INTEGER));
565
+ const partial = decoded.pageDigest !== "";
566
+ if (!partial)
567
+ await this.#mutate(session.id, r => { r.pageToken = decoded.pageToken; r.offset = 0; });
568
+ await this.#fetchPage(session, decoded.pageToken, partial, options);
569
+ });
570
+ return run;
571
+ }
572
+ /** Explicit reconnect of an existing known job after the trusted provider has
573
+ * verified the same stable subject. Keeps original approval, budget and deadline;
574
+ * invalidates old cursors and never submits a query. Approval principal provenance
575
+ * remains immutable; only the separately persisted access generation changes.
576
+ */
577
+ async rebind(receipt, cursor, options = {}) {
578
+ options = {
579
+ ...options, callerDeadline: Math.min(options.callerDeadline ?? Number.MAX_SAFE_INTEGER, this.#clock.now() + Math.min(15000, receipt.bounds.httpMs))
580
+ };
581
+ return this.#ledger.withLease(receipt.runId, options.signal, async () => {
582
+ const current = await this.#load(receipt.runId);
583
+ if (!receiptMatches(receipt, current.receipt) || current.receipt.job === undefined || (cursor === null ? current.cursor !== null : !same(decodeCursor(cursor), current.cursor)))
584
+ fail("cursor_invalid");
585
+ await this.#reauthorize(current.preview, options.signal, options.callerDeadline);
586
+ const principal = await this.#transport.verify(current.activePrincipal, Math.min(this.#clock.now() + 15000, options.callerDeadline ?? Number.MAX_SAFE_INTEGER), options.signal, true);
587
+ if (principal.generation === current.activePrincipal.generation)
588
+ fail("approval_changed");
589
+ const result = await this.#mutate(receipt.runId, r => {
590
+ if (!same(r.activePrincipal, current.activePrincipal) || !same(r.cursor, current.cursor))
591
+ fail("cursor_invalid");
592
+ r.activePrincipal = principal;
593
+ if (r.cursor !== null)
594
+ r.cursor = {
595
+ ...r.cursor, principal, ledgerRef: opaqueID()
596
+ };
597
+ return {
598
+ receipt: r.receipt, cursor: r.cursor === null ? null : encodeCursor(r.cursor)
599
+ };
600
+ });
601
+ return cloneFrozen(result);
602
+ });
603
+ }
604
+ async status(receipt, options = {}) {
605
+ options = {
606
+ ...options, callerDeadline: Math.min(options.callerDeadline ?? Number.MAX_SAFE_INTEGER, this.#clock.now() + Math.min(15000, receipt.bounds.httpMs))
607
+ };
608
+ return this.#ledger.withLease(receipt.runId, options.signal, async () => {
609
+ const record = await this.#load(receipt.runId);
610
+ if (!receiptMatches(receipt, record.receipt) || record.receipt.job === undefined)
611
+ fail("cursor_invalid");
612
+ await this.#reauthorize(record.preview, options.signal, options.callerDeadline);
613
+ const job = record.receipt.job;
614
+ const raw = object(await this.#transport.call(this.#scope(record.preview, receipt.runId, options), "GET", `projects/${job.projectId}/jobs/${encodeURIComponent(job.jobId)}`, {
615
+ location: job.location
616
+ }, undefined, true));
617
+ if (!same(parseJob(raw.jobReference), job))
618
+ fail("malformed_wire");
619
+ exactKeys(raw, ["jobReference", "status"], ["kind", "etag", "id", "selfLink", "user_email", "principal_subject", "configuration", "statistics", "jobCreationReason"], "malformed_wire");
620
+ const status = object(raw.status);
621
+ exactKeys(status, ["state"], ["errorResult", "errors"], "malformed_wire");
622
+ if (!["PENDING", "RUNNING", "DONE"].includes(String(status.state)))
623
+ fail("malformed_wire");
624
+ let state = "running";
625
+ const reasons = warnings(status.errors);
626
+ let billedBytes;
627
+ if (status.state === "DONE") {
628
+ state = "completed";
629
+ if (Object.hasOwn(status, "errorResult") && status.errorResult !== null) {
630
+ const error = object(status.errorResult);
631
+ if (typeof error.reason !== "string" || error.reason === "")
632
+ fail("malformed_wire");
633
+ state = error.reason === "cancelled" ? "cancelled" : "failed";
634
+ reasons.push(...warnings([error]));
635
+ }
636
+ if (raw.statistics !== undefined) {
637
+ const statistics = object(raw.statistics);
638
+ if (statistics.query !== undefined) {
639
+ const query = object(statistics.query);
640
+ if (Object.hasOwn(query, "totalBytesBilled"))
641
+ billedBytes = integerText(query.totalBytesBilled);
642
+ }
643
+ }
644
+ }
645
+ await this.#mutate(receipt.runId, r => {
646
+ if (status.state === "DONE") {
647
+ if (r.receipt.state === "cancel_requested" && state === "completed")
648
+ reasons.push("completed_before_cancel");
649
+ r.receipt = {
650
+ ...r.receipt, state, ...(billedBytes === undefined ? {} : {
651
+ billedBytes
652
+ })
653
+ };
654
+ if (billedBytes !== undefined)
655
+ r.reservation = billedBytes;
656
+ }
657
+ r.receipt = {
658
+ ...r.receipt, warnings: [...r.receipt.warnings, ...reasons]
659
+ };
660
+ });
661
+ return cloneFrozen({
662
+ job, state, warnings: reasons, ...(billedBytes === undefined ? {} : {
663
+ billedBytes
664
+ })
665
+ });
666
+ });
667
+ }
668
+ async cancel(receipt, options = {}) {
669
+ options = {
670
+ ...options, callerDeadline: Math.min(options.callerDeadline ?? Number.MAX_SAFE_INTEGER, this.#clock.now() + Math.min(15000, receipt.bounds.httpMs))
671
+ };
672
+ return this.#ledger.withLease(receipt.runId, options.signal, async () => {
673
+ const record = await this.#load(receipt.runId);
674
+ if (!receiptMatches(receipt, record.receipt) || record.receipt.job === undefined)
675
+ fail("cursor_invalid");
676
+ await this.#reauthorize(record.preview, options.signal, options.callerDeadline);
677
+ const job = record.receipt.job;
678
+ const raw = object(await this.#transport.call(this.#scope(record.preview, receipt.runId, options), "POST", `projects/${job.projectId}/jobs/${encodeURIComponent(job.jobId)}/cancel`, {
679
+ location: job.location
680
+ }, undefined, true, true));
681
+ const returned = parseJob(object(raw.job).jobReference);
682
+ if (!same(returned, job))
683
+ fail("cancellation_unknown");
684
+ await this.#mutate(receipt.runId, r => {
685
+ if (!["completed", "failed", "cancelled"].includes(r.receipt.state))
686
+ r.receipt = {
687
+ ...r.receipt, state: "cancel_requested"
688
+ };
689
+ });
690
+ return cloneFrozen({
691
+ job, state: "cancel_requested"
692
+ });
693
+ });
694
+ }
695
+ }
696
+ /** Failed operations expose the existing sanitized receipt and recoverable run;
697
+ * the message/code never includes query values, OAuth tokens or provider bodies.
698
+ */
699
+ export class ExecutionFailure extends AnalyticalError {
700
+ receipt;
701
+ run;
702
+ constructor(error, receipt, run) { super(error instanceof AnalyticalError ? error.code : "remote_failed"); this.receipt = cloneFrozen(receipt); this.run = run; }
703
+ }
704
+ function receiptMatches(a, b) { return same([a.version, a.runId, a.approvalDigest, a.sourceDigest, a.observationDigest, a.schemaDigest, a.principal, a.job, a.bounds, a.runStartedAt, a.executionDeadline], [b.version, b.runId, b.approvalDigest, b.sourceDigest, b.observationDigest, b.schemaDigest, b.principal, b.job, b.bounds, b.runStartedAt, b.executionDeadline]); }
705
+ function encodeCursor(cursor) {
706
+ const bytes = encoded(cursor);
707
+ let binary = "";
708
+ for (const byte of bytes)
709
+ binary += String.fromCharCode(byte);
710
+ return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, "");
711
+ }
712
+ function decodeCursor(value) {
713
+ if (typeof value !== "string" || value.length > 45000 || !/^[A-Za-z0-9_-]+$/u.test(value))
714
+ fail("cursor_invalid");
715
+ try {
716
+ const binary = atob(value.replace(/-/g, "+").replace(/_/g, "/"));
717
+ const bytes = Uint8Array.from(binary, char => char.charCodeAt(0));
718
+ parseJSON(bytes, 32768);
719
+ const text = new TextDecoder("utf-8", {
720
+ fatal: true
721
+ }).decode(bytes);
722
+ const cursor = JSON.parse(text);
723
+ if (encodeCursor(cursor) !== value)
724
+ fail("cursor_invalid");
725
+ return cursor;
726
+ }
727
+ catch {
728
+ return fail("cursor_invalid");
729
+ }
730
+ }