google-apis-cli 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +42 -26
  2. package/dist/bin/gapi.js +11 -135
  3. package/dist/src/apis/ads.js +59 -49
  4. package/dist/src/apis/bid-manager.js +4 -0
  5. package/dist/src/apis/bigquery-dts.js +4 -0
  6. package/dist/src/apis/business-account-management.js +5 -0
  7. package/dist/src/apis/business-information.js +5 -0
  8. package/dist/src/apis/business-lodging.js +5 -0
  9. package/dist/src/apis/business-notifications.js +5 -0
  10. package/dist/src/apis/business-performance.js +5 -0
  11. package/dist/src/apis/business-place-actions.js +5 -0
  12. package/dist/src/apis/business-v4.js +52 -0
  13. package/dist/src/apis/business-verifications.js +5 -0
  14. package/dist/src/apis/cm360.js +4 -0
  15. package/dist/src/apis/crux.js +8 -0
  16. package/dist/src/apis/data-manager.js +4 -0
  17. package/dist/src/apis/dv360.js +4 -0
  18. package/dist/src/apis/ga4-admin-alpha.js +4 -0
  19. package/dist/src/apis/ga4-admin.js +1 -288
  20. package/dist/src/apis/ga4-data-alpha.js +5 -0
  21. package/dist/src/apis/ga4-data.js +9 -69
  22. package/dist/src/apis/gsc.js +6 -58
  23. package/dist/src/apis/gtm.js +1 -189
  24. package/dist/src/apis/indexing.js +37 -15
  25. package/dist/src/apis/merchant-accounts.js +4 -0
  26. package/dist/src/apis/merchant-conversions.js +4 -0
  27. package/dist/src/apis/merchant-datasources.js +4 -0
  28. package/dist/src/apis/merchant-inventories.js +4 -0
  29. package/dist/src/apis/merchant-issueresolution.js +4 -0
  30. package/dist/src/apis/merchant-lfp.js +4 -0
  31. package/dist/src/apis/merchant-loyaltycustomers.js +4 -0
  32. package/dist/src/apis/merchant-notifications.js +4 -0
  33. package/dist/src/apis/merchant-ordertracking.js +4 -0
  34. package/dist/src/apis/merchant-products.js +4 -0
  35. package/dist/src/apis/merchant-promotions.js +4 -0
  36. package/dist/src/apis/merchant-quota.js +4 -0
  37. package/dist/src/apis/merchant-reports.js +5 -0
  38. package/dist/src/apis/merchant-reviews.js +4 -0
  39. package/dist/src/apis/pagespeed.js +8 -0
  40. package/dist/src/apis/places.js +19 -0
  41. package/dist/src/apis/postmaster.js +4 -0
  42. package/dist/src/apis/sa360-legacy.js +4 -0
  43. package/dist/src/apis/sa360.js +13 -0
  44. package/dist/src/apis/sheets.js +4 -0
  45. package/dist/src/apis/site-verification.js +4 -0
  46. package/dist/src/apis/web-risk.js +18 -0
  47. package/dist/src/apis/youtube-analytics.js +4 -0
  48. package/dist/src/apis/youtube-reporting.js +4 -0
  49. package/dist/src/apis/youtube.js +4 -0
  50. package/dist/src/catalog.js +218 -0
  51. package/dist/src/credentials.js +342 -0
  52. package/dist/src/errors.js +16 -0
  53. package/dist/src/json.js +11 -0
  54. package/dist/src/plan.js +348 -0
  55. package/dist/src/render.js +248 -0
  56. package/dist/src/responses.js +37 -0
  57. package/dist/src/run.js +374 -0
  58. package/dist/src/search.js +53 -0
  59. package/dist/src/transport/batch.js +75 -0
  60. package/dist/src/transport/download.js +21 -0
  61. package/dist/src/transport/retry.js +21 -0
  62. package/dist/src/transport/upload.js +106 -0
  63. package/dist/src/transport.js +125 -0
  64. package/dist/src/types.js +1 -13
  65. package/package.json +23 -8
  66. package/dist/src/apis/ga4-admin-v1alpha.js +0 -280
  67. package/dist/src/auth.js +0 -172
  68. package/dist/src/config.js +0 -89
  69. package/dist/src/manifest.js +0 -69
  70. package/dist/src/rest.js +0 -90
@@ -0,0 +1,374 @@
1
+ // Process io is injected so tests drive the whole CLI in-process.
2
+ import { readFile } from 'node:fs/promises';
3
+ import { join } from 'node:path';
4
+ import { parseArgs } from 'node:util';
5
+ import { openCatalog } from './catalog.js';
6
+ import { configDir, openCredentials } from './credentials.js';
7
+ import { GapiError, messageOf, reasonOf } from './errors.js';
8
+ import { isRecord } from './json.js';
9
+ import { parseJson, plan, resolve } from './plan.js';
10
+ import { AUTH_USAGE, DISCOVERY_USAGE, levelText, methodHelp, rootHelp, SCHEMA_USAGE, SEARCH_USAGE, schemaFind, schemaView, searchText, } from './render.js';
11
+ import { MAX_AGE_MS, openResponses } from './responses.js';
12
+ import { matchQuery, search } from './search.js';
13
+ import { send } from './transport.js';
14
+ const GLOBAL_OPTIONS = {
15
+ params: { type: 'string' },
16
+ body: { type: 'string' },
17
+ 'dry-run': { type: 'boolean' },
18
+ 'page-all': { type: 'boolean' },
19
+ 'page-limit': { type: 'string' },
20
+ upload: { type: 'string' },
21
+ 'upload-content-type': { type: 'string' },
22
+ 'upload-protocol': { type: 'string' },
23
+ output: { type: 'string' },
24
+ project: { type: 'string' },
25
+ 'api-key': { type: 'string' },
26
+ cache: { type: 'string' },
27
+ help: { type: 'boolean' },
28
+ };
29
+ // D10: a helper takes the global flags that do not shape the request, plus its own options.
30
+ const HELPER_OPTIONS = {
31
+ 'dry-run': GLOBAL_OPTIONS['dry-run'],
32
+ project: GLOBAL_OPTIONS.project,
33
+ 'api-key': GLOBAL_OPTIONS['api-key'],
34
+ cache: GLOBAL_OPTIONS.cache,
35
+ help: GLOBAL_OPTIONS.help,
36
+ };
37
+ const AGE_UNIT_MS = { s: 1e3, m: 60e3, h: 3600e3, d: 86400e3 };
38
+ // `--cache <n>s|m|h|d`, from 1 s to the 7-day store limit (ADR 0001).
39
+ function parseAge(age) {
40
+ const m = /^(\d+)([smhd])$/.exec(age);
41
+ const ms = m ? Number(m[1]) * AGE_UNIT_MS[m[2]] : 0;
42
+ if (ms < 1000 || ms > MAX_AGE_MS) {
43
+ throw new GapiError(3, `--cache: ${age} is not an age from 1s to 7d`);
44
+ }
45
+ return ms;
46
+ }
47
+ const isHelperName = (word) => word.startsWith('+');
48
+ // D4: an entry header flag is accepted only on the APIs whose entry declares it.
49
+ const headerOptions = (entry) => Object.fromEntries((entry?.headers ?? []).map((h) => [h.flag, { type: 'string' }]));
50
+ export async function run(argv, io) {
51
+ try {
52
+ await dispatch(argv, io);
53
+ return 0;
54
+ }
55
+ catch (e) {
56
+ const err = e instanceof GapiError ? e : new GapiError(5, messageOf(e));
57
+ const { body } = err;
58
+ const shown = isRecord(body) ? (body.error ?? body) : body;
59
+ const detail = body ? `\n${JSON.stringify(shown, null, 2)}` : '';
60
+ io.stderr.write(`gapi: ${err.message}${detail}\n`);
61
+ return err.exit;
62
+ }
63
+ }
64
+ // Strict argv parsing; an unknown flag, a missing flag value or a repeated flag (parseArgs
65
+ // would keep the last one) is a validation error.
66
+ function parse(args, options) {
67
+ try {
68
+ const parsed = parseArgs({ args, options, allowPositionals: true, strict: true, tokens: true });
69
+ const names = parsed.tokens.flatMap((t) => (t.kind === 'option' ? [t.name] : []));
70
+ const twice = names.find((n, i) => names.indexOf(n) !== i);
71
+ if (twice !== undefined)
72
+ throw new GapiError(3, `--${twice} given twice`);
73
+ return parsed;
74
+ }
75
+ catch (e) {
76
+ throw e instanceof GapiError ? e : new GapiError(3, messageOf(e));
77
+ }
78
+ }
79
+ async function runAuth(sub, values, { io, creds, catalog, responses, }) {
80
+ // --help never starts a side effect: no server, no fetch, no file write.
81
+ if (values.help) {
82
+ io.stdout.write(`${AUTH_USAGE}\n`);
83
+ return;
84
+ }
85
+ if (sub === 'login') {
86
+ const consent = await consentScopes(catalog);
87
+ await responses.clear(); // a new account starts with no stored responses
88
+ const tokens = await creds.login({ consent, manual: values.manual === true });
89
+ io.stderr.write(`gapi: authorized (${tokens.refresh_token ? 'refresh token stored' : 'no refresh token'})\n`);
90
+ return;
91
+ }
92
+ if (sub === 'setup') {
93
+ // Header prompts come from the entries' `headers` overlays, in api order.
94
+ const headers = Object.values(catalog.entries)
95
+ .toSorted((a, b) => a.api.localeCompare(b.api))
96
+ .flatMap((e) => (e.headers ?? []).map((h) => ({ ...h, api: e.api })));
97
+ io.stderr.write(`gapi: saved to ${await creds.setup(headers)}\n`);
98
+ return;
99
+ }
100
+ if (sub === 'status') {
101
+ const consent = await consentScopes(catalog);
102
+ io.stdout.write(`${JSON.stringify(await creds.status(consent), null, 2)}\n`);
103
+ return;
104
+ }
105
+ if (sub === 'logout') {
106
+ try {
107
+ await creds.logout();
108
+ }
109
+ finally {
110
+ await responses.clear(); // also when the revoke fails: no account data stays on disk
111
+ }
112
+ io.stderr.write('gapi: credentials removed\n');
113
+ return;
114
+ }
115
+ throw new GapiError(3, sub === undefined ? `usage: ${AUTH_USAGE}` : `unknown auth command: ${sub}`);
116
+ }
117
+ // D8 consent set: every entry's per-method scope plus its entry scopes (identity is added by login).
118
+ async function consentScopes(catalog) {
119
+ const models = await Promise.all(Object.keys(catalog.entries).map((api) => catalog.load(api)));
120
+ const scopes = new Set();
121
+ for (const model of models) {
122
+ for (const s of model.scopes)
123
+ scopes.add(s);
124
+ for (const m of model.byId.values())
125
+ if (m.scope)
126
+ scopes.add(m.scope);
127
+ }
128
+ return [...scopes];
129
+ }
130
+ // D9 order: a method path, else a schema name. Only a document-level method can share a
131
+ // schema's name, since method paths of nested resources contain a dot.
132
+ function schemaTarget(model, path) {
133
+ if (!model.methods.has(path) && model.schemas.has(path))
134
+ return { schema: path };
135
+ try {
136
+ const found = resolve(model, path.split('.'));
137
+ if (!found.method)
138
+ throw new GapiError(3, `${path} is a resource, not a method or schema`);
139
+ return { method: found.method };
140
+ }
141
+ catch (e) {
142
+ // resolve knows only the tree; a one-word path can also name a schema.
143
+ if (e instanceof GapiError && /^unknown /.test(e.message) && !path.includes('.')) {
144
+ throw new GapiError(3, `${e.message}, or a schema name`);
145
+ }
146
+ throw e;
147
+ }
148
+ }
149
+ async function dispatch(argv, io) {
150
+ const dir = configDir(io.env);
151
+ const warn = (message) => io.stderr.write(`gapi: warning: ${message}\n`);
152
+ const catalog = await openCatalog({ dir, fetch: io.fetch, warn, apisDir: io.apisDir });
153
+ const creds = openCredentials({
154
+ dir,
155
+ env: io.env,
156
+ warn,
157
+ stdin: io.stdin,
158
+ stderr: io.stderr,
159
+ });
160
+ const responses = openResponses(join(dir, 'responses'));
161
+ await responses.sweep();
162
+ // ADR 0001 eligibility, before any network call: the entry allows caching, the call is
163
+ // one request or pages, the method is a GET or a `reads` POST.
164
+ const cacheFor = async (call, model, values) => {
165
+ if (values.cache === undefined)
166
+ return undefined;
167
+ const maxAgeMs = parseAge(values.cache);
168
+ const { nocache } = catalog.entries[model.api];
169
+ if (nocache) {
170
+ throw new GapiError(3, `--cache: ${model.api} responses must not be cached: ${nocache.reason} (${nocache.source})`);
171
+ }
172
+ const method = model.byId.get(call.methodId);
173
+ const once = call.send === 'once' || call.send === 'pages';
174
+ if (!once || (method?.httpMethod !== 'GET' && !method?.read)) {
175
+ const command = method ? `${model.api} ${method.words.join(' ')}` : call.methodId;
176
+ throw new GapiError(3, `--cache: ${command} is not a cacheable read`);
177
+ }
178
+ return { key: responses.key(call, await creds.identity(call.auth)), maxAgeMs };
179
+ };
180
+ if (argv.length === 0 || (argv[0] === '--help' && argv.length === 1)) {
181
+ io.stdout.write(rootHelp(Object.keys(catalog.entries)));
182
+ return;
183
+ }
184
+ const [api, ...rest] = argv;
185
+ if (api === 'auth') {
186
+ const { values, positionals } = parse(rest, {
187
+ manual: { type: 'boolean' },
188
+ help: { type: 'boolean' },
189
+ });
190
+ return runAuth(positionals[0], values, { io, creds, catalog, responses });
191
+ }
192
+ if (api === 'search') {
193
+ // No option parse: a word like `-list` is query text (story 11).
194
+ if (rest.includes('--help')) {
195
+ io.stdout.write(`${SEARCH_USAGE}\n`);
196
+ return;
197
+ }
198
+ const query = rest.join(' ');
199
+ matchQuery(query); // a query with no token exits 3 before any document loads
200
+ const models = await Promise.all(Object.keys(catalog.entries).map((a) => catalog.load(a)));
201
+ const hits = search(models, query);
202
+ if (hits.length === 0)
203
+ io.stderr.write('gapi: no match\n');
204
+ io.stdout.write(searchText(hits));
205
+ return;
206
+ }
207
+ if (api === 'schema') {
208
+ const { values, positionals } = parse(rest, {
209
+ 'resolve-refs': { type: 'boolean' },
210
+ find: { type: 'string' },
211
+ help: { type: 'boolean' },
212
+ });
213
+ if (values.help) {
214
+ io.stdout.write(`${SCHEMA_USAGE}\n`);
215
+ return;
216
+ }
217
+ if (positionals.length !== 2)
218
+ throw new GapiError(3, `usage: ${SCHEMA_USAGE}`);
219
+ const [name, path] = positionals;
220
+ if (values.find === '')
221
+ throw new GapiError(3, '--find: needs a word');
222
+ const model = await catalog.load(name);
223
+ const target = schemaTarget(model, path);
224
+ if (values.find !== undefined) {
225
+ const found = schemaFind(model, target, values.find);
226
+ if (found === '')
227
+ io.stderr.write('gapi: no match\n');
228
+ io.stdout.write(found);
229
+ return;
230
+ }
231
+ const resolveRefs = values['resolve-refs'] === true;
232
+ io.stdout.write(schemaView(model, target, { resolveRefs }));
233
+ return;
234
+ }
235
+ if (api === 'discovery') {
236
+ const { values, positionals } = parse(rest, { help: { type: 'boolean' } });
237
+ if (values.help) {
238
+ io.stdout.write(`${DISCOVERY_USAGE}\n`);
239
+ return;
240
+ }
241
+ const [sub, ...apis] = positionals;
242
+ if (sub === 'path') {
243
+ if (apis.length > 0)
244
+ throw new GapiError(3, `unexpected argument '${apis[0]}'`);
245
+ io.stdout.write(`${catalog.cacheDir}\n`);
246
+ return;
247
+ }
248
+ if (sub !== 'refresh')
249
+ throw new GapiError(3, `usage: ${DISCOVERY_USAGE}`);
250
+ // Validated and deduplicated before any fetch: parallel loads of one api share a temp file.
251
+ const names = apis.length > 0 ? [...new Set(apis)] : Object.keys(catalog.entries);
252
+ for (const a of names) {
253
+ if (!Object.hasOwn(catalog.entries, a))
254
+ throw new GapiError(3, `unknown api: ${a}`);
255
+ }
256
+ await Promise.all(names.map((a) => catalog.load(a, { refresh: true })));
257
+ for (const a of names)
258
+ io.stdout.write(`refreshed ${a}\n`);
259
+ return;
260
+ }
261
+ const [word, ...helperArgs] = rest;
262
+ if (word !== undefined && isHelperName(word)) {
263
+ if (!Object.hasOwn(catalog.entries, api))
264
+ throw new GapiError(3, `unknown api: ${api}`);
265
+ const helpers = catalog.entries[api].helpers ?? {};
266
+ const helper = Object.hasOwn(helpers, word) ? helpers[word] : undefined;
267
+ if (!helper) {
268
+ const valid = Object.keys(helpers).join(', ') || 'none';
269
+ throw new GapiError(3, `unknown helper '${word}' (valid: ${valid})`);
270
+ }
271
+ const { values, positionals } = parse(helperArgs, {
272
+ ...HELPER_OPTIONS,
273
+ ...headerOptions(catalog.entries[api]),
274
+ ...helper.options,
275
+ });
276
+ if (values.help) {
277
+ io.stdout.write(`gapi ${api} ${word} ${helper.summary}\n`);
278
+ return;
279
+ }
280
+ const model = await catalog.load(api);
281
+ const call = await helper.plan({
282
+ model,
283
+ args: positionals,
284
+ flags: values,
285
+ settings: await creds.settings(values, model.headers),
286
+ read: (path) => readFile(path, 'utf8').catch((e) => {
287
+ throw new GapiError(3, `cannot read ${path}: ${reasonOf(e)}`);
288
+ }),
289
+ });
290
+ const cache = await cacheFor(call, model, values);
291
+ return perform(call, values['dry-run'] === true, io, creds.authorize, { responses, cache });
292
+ }
293
+ const entry = Object.hasOwn(catalog.entries, api) ? catalog.entries[api] : undefined;
294
+ const { values, positionals } = parse(rest, { ...GLOBAL_OPTIONS, ...headerOptions(entry) });
295
+ const model = await catalog.load(api);
296
+ const target = resolve(model, positionals);
297
+ if (!target.method) {
298
+ // A listing sends nothing: a request flag here means the method word is missing.
299
+ const flag = Object.keys(values).find((k) => k !== 'help');
300
+ if (flag)
301
+ throw new GapiError(3, `--${flag}: ${[api, ...positionals].join(' ')} is not a method`);
302
+ const helpers = positionals.length === 0 ? catalog.entries[api].helpers : undefined;
303
+ io.stdout.write(levelText(model, positionals, target.node, helpers));
304
+ return;
305
+ }
306
+ if (values.help) {
307
+ io.stdout.write(methodHelp(model, target.method));
308
+ return;
309
+ }
310
+ const dryRun = values['dry-run'] === true;
311
+ const call = plan(model, target.method.id, {
312
+ params: parseJson('--params', values.params),
313
+ body: parseJson('--body', values.body),
314
+ dryRun,
315
+ pageAll: values['page-all'],
316
+ pageLimit: values['page-limit'],
317
+ upload: values.upload,
318
+ uploadContentType: values['upload-content-type'],
319
+ uploadProtocol: values['upload-protocol'],
320
+ output: values.output,
321
+ }, await creds.settings(values, model.headers));
322
+ const cache = await cacheFor(call, model, values);
323
+ return perform(call, dryRun, io, creds.authorize, { responses, cache });
324
+ }
325
+ // Prints the plan's warnings, then the redacted request (--dry-run), the stored response
326
+ // (a --cache hit) or the live response, which a --cache miss stores after a full success.
327
+ async function perform(call, dryRun, io, authorize, { responses, cache }) {
328
+ for (const w of call.warnings)
329
+ io.stderr.write(`gapi: warning: ${w}\n`);
330
+ const mtime = cache && (await responses.mtime(cache.key));
331
+ const fresh = cache && mtime !== undefined && Date.now() - mtime < cache.maxAgeMs;
332
+ const stored = fresh ? await responses.get(cache.key) : undefined;
333
+ if (dryRun) {
334
+ const headers = { ...call.headers, ...(await authorize(call.auth, { redact: true })) };
335
+ const request = {
336
+ method: call.httpMethod,
337
+ url: call.url,
338
+ headers,
339
+ body: call.body,
340
+ upload: call.send === 'upload' ? call.upload : undefined,
341
+ output: call.send === 'download' ? call.output : undefined,
342
+ parts: call.send === 'batch'
343
+ ? call.parts.map((p) => ({
344
+ method: p.httpMethod,
345
+ url: p.url,
346
+ headers: p.headers,
347
+ body: p.body,
348
+ }))
349
+ : undefined,
350
+ cache: cache && (stored === undefined ? 'miss' : 'hit'),
351
+ };
352
+ io.stdout.write(`${JSON.stringify(request, null, 2)}\n`);
353
+ return;
354
+ }
355
+ if (stored !== undefined && mtime !== undefined) {
356
+ io.stdout.write(stored);
357
+ io.stderr.write(`gapi: cached response from ${new Date(mtime).toISOString()}\n`);
358
+ return;
359
+ }
360
+ const chunks = [];
361
+ const out = {
362
+ write: (chunk) => {
363
+ chunks.push(chunk);
364
+ return io.stdout.write(chunk);
365
+ },
366
+ };
367
+ await send(call, { fetch: io.fetch, authorize, out, sleep: io.sleep });
368
+ if (!cache)
369
+ return;
370
+ // Best effort: the response is already printed, so a failed write only warns.
371
+ await responses.put(cache.key, chunks.join('')).catch((e) => {
372
+ io.stderr.write(`gapi: warning: cache write failed: ${reasonOf(e)}\n`);
373
+ });
374
+ }
@@ -0,0 +1,53 @@
1
+ // Search: finds methods by intent across API entries with an in-memory SQLite FTS5 index (#182 §1).
2
+ // shortcut: DatabaseSync is a documentation-only deprecated alias of Database since Node v26.11.0
3
+ // (DEP0210); switch when @types/node types Database.
4
+ import { DatabaseSync } from 'node:sqlite';
5
+ import { GapiError } from './errors.js';
6
+ import { SEARCH_USAGE } from './render.js';
7
+ // `conversionActions` → `conversion Actions`, `URLInspection` → `URL Inspection`, `a-b_c.d` → `a b c d`.
8
+ const split = (text) => text
9
+ .replace(/([\p{Ll}\p{N}])(\p{Lu})/gu, '$1 $2')
10
+ .replace(/(\p{Lu})(\p{Lu}\p{Ll})/gu, '$1 $2')
11
+ .replace(/[-_.]/g, ' ');
12
+ // Tokens hold only letters and digits, so no FTS5 syntax reaches the match expression.
13
+ export function matchQuery(query) {
14
+ const tokens = split(query)
15
+ .toLowerCase()
16
+ .match(/[\p{L}\p{N}]+/gu);
17
+ if (!tokens)
18
+ throw new GapiError(3, `usage: ${SEARCH_USAGE}`);
19
+ return tokens.map((t) => `"${t}"`).join(' OR ');
20
+ }
21
+ export function search(models, query) {
22
+ const match = matchQuery(query);
23
+ const db = new DatabaseSync(':memory:');
24
+ try {
25
+ db.exec("create virtual table m using fts5(api unindexed, id unindexed, words, extra, descr, tokenize = 'porter unicode61')");
26
+ const insert = db.prepare('insert into m values (?, ?, ?, ?, ?)');
27
+ db.exec('begin');
28
+ for (const model of models) {
29
+ for (const m of model.byId.values()) {
30
+ const words = split([model.api, ...m.words].join(' '));
31
+ const extra = [
32
+ split([model.title, m.request, m.response].filter(Boolean).join(' ')),
33
+ m.upload && 'upload',
34
+ m.download && 'download',
35
+ ].filter(Boolean);
36
+ insert.run(model.api, m.id, words, extra.join(' '), m.description ?? '');
37
+ }
38
+ }
39
+ db.exec('commit');
40
+ const byApi = new Map(models.map((model) => [model.api, model]));
41
+ return db
42
+ .prepare('select api, id from m where m match ? order by bm25(m, 0, 0, 6, 2, 1) limit 20')
43
+ .all(match)
44
+ .flatMap(({ api, id }) => {
45
+ const model = byApi.get(String(api));
46
+ const method = model?.byId.get(String(id));
47
+ return model && method ? [{ model, method }] : [];
48
+ });
49
+ }
50
+ finally {
51
+ db.close();
52
+ }
53
+ }
@@ -0,0 +1,75 @@
1
+ // Batch (epic #30 D7): parts go out as one multipart/mixed request per 100, sequentially.
2
+ // Format: https://developers.google.com/search/apis/indexing-api/v3/using-api#batch
3
+ import { randomUUID } from 'node:crypto';
4
+ import { GapiError } from '../errors.js';
5
+ import { parseOrText } from '../json.js';
6
+ // Google's per-call part limit for the Indexing batch endpoint.
7
+ const MAX_PARTS = 100;
8
+ // A failed call stops the run but keeps the results of the calls before it: Google has
9
+ // already acted on those parts, so the caller must still see them.
10
+ export async function batch(plan, call) {
11
+ const results = [];
12
+ for (let start = 0; start < plan.parts.length; start += MAX_PARTS) {
13
+ try {
14
+ results.push(...(await chunk(plan, plan.parts.slice(start, start + MAX_PARTS), call)));
15
+ }
16
+ catch (error) {
17
+ return { results, error };
18
+ }
19
+ }
20
+ return { results };
21
+ }
22
+ async function chunk(plan, parts, call) {
23
+ const boundary = randomUUID();
24
+ const body = `${parts.map((p, i) => request(boundary, i + 1, p)).join('')}--${boundary}--`;
25
+ const headers = { ...plan.headers, 'Content-Type': `multipart/mixed; boundary=${boundary}` };
26
+ const res = await call(plan.url, { method: 'POST', headers, body });
27
+ // Google picks its own response boundary.
28
+ const responseBoundary = /boundary="?([^";]+)"?/i.exec(res.headers.get('Content-Type') ?? '')?.[1];
29
+ // A response gapi cannot map to its parts is Google's surprise: exit 1 with the body.
30
+ const text = await res.text();
31
+ const surprise = (message) => new GapiError(1, message, parseOrText(text));
32
+ if (!responseBoundary)
33
+ throw surprise('batch: the response has no multipart boundary');
34
+ const byId = responses(text, responseBoundary);
35
+ if (!byId)
36
+ throw surprise('batch: malformed response part');
37
+ return parts.map((_, i) => {
38
+ const result = byId.get(String(i + 1));
39
+ if (!result)
40
+ throw surprise(`batch: no response for part ${i + 1}`);
41
+ return result;
42
+ });
43
+ }
44
+ // One `application/http` part: the inner request line (path only), its headers and JSON body.
45
+ function request(boundary, id, part) {
46
+ const { pathname, search } = new URL(part.url);
47
+ const headers = Object.entries(part.headers).map(([k, v]) => `${k}: ${v}\r\n`);
48
+ return (`--${boundary}\r\nContent-Type: application/http\r\nContent-Transfer-Encoding: binary\r\n` +
49
+ `Content-ID: <${id}>\r\n\r\n${part.httpMethod} ${pathname}${search}\r\n${headers.join('')}\r\n` +
50
+ `${part.body === undefined ? '' : JSON.stringify(part.body)}\r\n`);
51
+ }
52
+ // Response parts keyed by request Content-ID: each answers `<response-n>` with an inner
53
+ // `HTTP/1.1 <status> …`, headers, blank line, body. Google may answer in any order.
54
+ // A part with no Content-ID or status line makes the whole response unreadable: undefined.
55
+ function responses(text, boundary) {
56
+ const byId = new Map();
57
+ for (const raw of text.split(`--${boundary}`).slice(1)) {
58
+ if (raw.startsWith('--'))
59
+ break;
60
+ const [head, response] = splitAtBlankLine(raw.replace(/^\r?\n/, ''));
61
+ const id = /^content-id:\s*<response-([^>]+)>\s*$/im.exec(head)?.[1];
62
+ const [statusHead, content] = splitAtBlankLine(response);
63
+ const status = Number(/^HTTP\/\S+ (\d{3})/.exec(statusHead)?.[1]);
64
+ if (id === undefined || !status)
65
+ return undefined;
66
+ byId.set(id, { status, text: content.trim() });
67
+ }
68
+ return byId;
69
+ }
70
+ function splitAtBlankLine(text) {
71
+ const blank = /\r?\n\r?\n/.exec(text);
72
+ return blank
73
+ ? [text.slice(0, blank.index), text.slice(blank.index + blank[0].length)]
74
+ : [text, ''];
75
+ }
@@ -0,0 +1,21 @@
1
+ // Media download (epic #30 D7): the response body streams to the --output file.
2
+ import { createWriteStream } from 'node:fs';
3
+ import { access, constants, rm } from 'node:fs/promises';
4
+ import { dirname } from 'node:path';
5
+ import { Readable } from 'node:stream';
6
+ import { pipeline } from 'node:stream/promises';
7
+ import { GapiError, reasonOf } from '../errors.js';
8
+ // The file is created only after a 2xx, so a Google error leaves no empty file behind.
9
+ export async function download(plan, call) {
10
+ // A bad local path is the caller's input (exit 3), found before Google sends any byte.
11
+ await access(dirname(plan.output), constants.W_OK).catch((e) => {
12
+ throw new GapiError(3, `--output: cannot write to ${dirname(plan.output)}: ${reasonOf(e)}`);
13
+ });
14
+ const res = await call(plan.url, { method: plan.httpMethod, headers: plan.headers });
15
+ const file = createWriteStream(plan.output);
16
+ await pipeline(res.body ?? Readable.from([]), file).catch(async (e) => {
17
+ await rm(plan.output, { force: true }); // a cut-off stream leaves no partial file
18
+ throw e;
19
+ });
20
+ return { path: plan.output, bytes: file.bytesWritten };
21
+ }
@@ -0,0 +1,21 @@
1
+ // Retry policy shared by Transport and the resumable upload (Google client idioms, #194).
2
+ import { GapiError } from '../errors.js';
3
+ // The request never got an HTTP answer (DNS, reset, timeout): exit 1, like a Google error.
4
+ export class NetworkError extends GapiError {
5
+ }
6
+ // Google's retry rule: 408, 429, 5xx, and a 403 for a rate limit (google-api-python-client).
7
+ export const RETRIES = 3;
8
+ export const retryable = (status, text = '') => status === 408 ||
9
+ status === 429 ||
10
+ status >= 500 ||
11
+ (status === 403 && /"reason":\s*"(user)?[rR]ateLimitExceeded"/.test(text));
12
+ // Truncated exponential backoff with jitter: min(32 s, 1 s × 2^n + 0–1 s), Google's docs.
13
+ export const backoff = (attempt) => Math.min(32_000, 1000 * 2 ** attempt + Math.random() * 1000);
14
+ // Our choice, no Google client reads it: `Retry-After` (seconds or HTTP date) is a floor.
15
+ export const retryAfter = (res) => {
16
+ const v = res.headers.get('Retry-After');
17
+ if (!v)
18
+ return 0;
19
+ const ms = /^\d+$/.test(v) ? Number(v) * 1000 : Date.parse(v) - Date.now();
20
+ return Number.isFinite(ms) ? Math.max(0, ms) : 0;
21
+ };
@@ -0,0 +1,106 @@
1
+ // Media upload (epic #30 D7): the file streams from disk as a Blob backed by it.
2
+ import { randomUUID } from 'node:crypto';
3
+ import { openAsBlob } from 'node:fs';
4
+ import { open } from 'node:fs/promises';
5
+ import { GapiError, reasonOf } from '../errors.js';
6
+ import { parseOrText } from '../json.js';
7
+ import { backoff, NetworkError, RETRIES, retryable } from './retry.js';
8
+ // Google wants chunks in multiples of 256 KiB; 8 MiB keeps the request count low.
9
+ const CHUNK = 8 * 1024 * 1024;
10
+ // Consecutive 308s that move the upload no further before gapi gives up.
11
+ const MAX_STALLS = 3;
12
+ // Returns the final response; Transport prints it like a `once` call.
13
+ export async function upload(plan, call, sleep) {
14
+ const file = await readable(plan.upload.path);
15
+ const { protocol, contentType } = plan.upload;
16
+ if (protocol === 'media') {
17
+ return call(plan.url, { method: plan.httpMethod, headers: plan.headers, body: file });
18
+ }
19
+ if (protocol === 'resumable')
20
+ return resumable(plan, file, call, sleep);
21
+ // multipart/related: the JSON metadata part, then the media part.
22
+ const boundary = randomUUID();
23
+ const body = new Blob([
24
+ `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n`,
25
+ JSON.stringify(plan.body),
26
+ `\r\n--${boundary}\r\nContent-Type: ${contentType}\r\n\r\n`,
27
+ file,
28
+ `\r\n--${boundary}--`,
29
+ ]);
30
+ const headers = { ...plan.headers, 'Content-Type': `multipart/related; boundary=${boundary}` };
31
+ return call(plan.url, { method: plan.httpMethod, headers, body });
32
+ }
33
+ // Checked before any request: the Blob reads lazily, so a missing, unreadable or non-regular
34
+ // file would otherwise fail mid-upload.
35
+ async function readable(path) {
36
+ const handle = await open(path, 'r').catch((e) => {
37
+ throw new GapiError(3, `--upload: cannot read ${path}: ${reasonOf(e)}`);
38
+ });
39
+ try {
40
+ if (!(await handle.stat()).isFile())
41
+ throw new GapiError(3, `--upload: ${path} is not a file`);
42
+ }
43
+ finally {
44
+ await handle.close();
45
+ }
46
+ return openAsBlob(path);
47
+ }
48
+ // Session request on the upload URL, then PUT chunks to the session URI; 308 = send more.
49
+ // A 5xx or a network error on a chunk asks the session what it kept (`bytes */<size>`) and
50
+ // goes on from there (google-api-python-client `next_chunk`, @google-cloud/storage).
51
+ async function resumable(plan, file, call, sleep) {
52
+ const session = await call(plan.url, {
53
+ method: plan.httpMethod,
54
+ headers: { ...plan.headers, 'X-Upload-Content-Length': String(file.size) },
55
+ body: plan.body === undefined ? undefined : JSON.stringify(plan.body),
56
+ });
57
+ const location = session.headers.get('Location');
58
+ if (!location) {
59
+ const body = parseOrText(await session.text());
60
+ throw new GapiError(1, 'resumable upload: the session response has no Location header', body);
61
+ }
62
+ // undici holds the connection until a body is read or cancelled; only headers matter here.
63
+ await session.body?.cancel();
64
+ // A retryable answer or no answer at all is undefined: the caller asks the session again.
65
+ const put = (range, body) => call(location, { method: 'PUT', headers: { 'Content-Range': range }, body }, (s) => s === 308 || retryable(s)).then(async (res) => (retryable(res.status) ? void (await res.body?.cancel()) : res), (e) => {
66
+ if (e instanceof NetworkError)
67
+ return undefined;
68
+ throw e;
69
+ });
70
+ // Every final failure names the session, so the operator sees where the upload stopped.
71
+ try {
72
+ let start = 0;
73
+ let stalls = 0;
74
+ let failures = 0;
75
+ for (;;) {
76
+ const end = Math.min(start + CHUNK, file.size);
77
+ const range = file.size === 0 ? 'bytes */0' : `bytes ${start}-${end - 1}/${file.size}`;
78
+ let res = await put(range, file.slice(start, end));
79
+ while (!res) {
80
+ if (++failures > RETRIES)
81
+ throw new GapiError(1, `resumable upload: no answer at byte ${start}`);
82
+ await sleep(backoff(failures - 1));
83
+ res = await put(`bytes */${file.size}`);
84
+ }
85
+ if (res.status !== 308)
86
+ return res;
87
+ // `Range: bytes=0-<last>` names the bytes Google kept; no Range means it kept none.
88
+ const kept = /^bytes=0-(\d+)$/.exec(res.headers.get('Range') ?? '');
89
+ const next = kept ? Number(kept[1]) + 1 : 0;
90
+ stalls = next > start ? 0 : stalls + 1;
91
+ if (next > start)
92
+ failures = 0;
93
+ if (stalls === MAX_STALLS || next >= file.size) {
94
+ const body = parseOrText(await res.text());
95
+ throw new GapiError(1, `resumable upload: 308 at byte ${next} of ${file.size}, no progress`, body);
96
+ }
97
+ await res.body?.cancel();
98
+ start = next;
99
+ }
100
+ }
101
+ catch (e) {
102
+ if (!(e instanceof GapiError))
103
+ throw e;
104
+ throw new GapiError(e.exit, `${e.message}; session ${location}`, e.body);
105
+ }
106
+ }