google-apis-cli 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -8
- package/dist/bin/gapi.js +10 -3
- package/dist/src/apis/ads.js +9 -1
- package/dist/src/apis/crux.js +4 -0
- package/dist/src/apis/ga4-data-alpha.js +1 -0
- package/dist/src/apis/ga4-data.js +8 -0
- package/dist/src/apis/gsc.js +2 -1
- package/dist/src/apis/merchant-reports.js +1 -0
- package/dist/src/apis/pagespeed.js +4 -0
- package/dist/src/apis/places.js +4 -0
- package/dist/src/apis/sa360.js +1 -0
- package/dist/src/apis/web-risk.js +4 -0
- package/dist/src/catalog.js +22 -7
- package/dist/src/credentials.js +42 -16
- package/dist/src/plan.js +36 -3
- package/dist/src/render.js +69 -4
- package/dist/src/responses.js +37 -0
- package/dist/src/run.js +129 -18
- package/dist/src/search.js +53 -0
- package/dist/src/transport/download.js +7 -1
- package/dist/src/transport/retry.js +21 -0
- package/dist/src/transport/upload.js +46 -21
- package/dist/src/transport.js +39 -13
- package/package.json +6 -6
package/README.md
CHANGED
|
@@ -14,10 +14,10 @@
|
|
|
14
14
|
- One login. One refresh token. Every scope the entries use (computed from the documents). Read and write.
|
|
15
15
|
- One runtime dependency: `google-auth-library` (Google). Nothing third-party.
|
|
16
16
|
- All network calls use native `fetch`.
|
|
17
|
-
- TypeScript 7 (strict), Node
|
|
17
|
+
- TypeScript 7 (strict), Node 26. Node runs `.ts` directly (type-stripping) — no build step in dev. Biome for lint + format. pnpm 12 (pinned via `packageManager`) for package management.
|
|
18
18
|
- It replaces the 4 fragmented Google MCPs in vmCODE.
|
|
19
19
|
|
|
20
|
-
Command shape: `gapi <api> <resource>… <method> [--params '<json>'] [--body '<json>'] [--upload <file> [--upload-content-type <type>] [--upload-protocol media|multipart|resumable]] [--output <file>] [--page-all [--page-limit <n>]] [--dry-run]`. `gapi <api> [<resource>…]` lists a level (the api level also lists helpers); `… <method> --help` shows method help. `gapi <api> +<helper> … [--dry-run] [--project <id>] [--api-key <key>] [<entry header flag>]` runs a hand command, e.g. `gapi ads +gaql "<query>" --customer <id> [--stream]`, `gapi indexing +publishBatch <file>`. `gapi schema <api> <resource>[.<sub>…].<method> | <SchemaName> [--resolve-refs]` prints a method shape or a schema as JSON. `gapi discovery refresh [<api>…]` refetches discovery documents now; `gapi discovery path` prints the cache dir. `gapi schema --help`, `gapi discovery --help` and `gapi auth [<sub>] --help` print that command's usage and run nothing.
|
|
20
|
+
Command shape: `gapi <api> <resource>… <method> [--params '<json>'] [--body '<json>'] [--upload <file> [--upload-content-type <type>] [--upload-protocol media|multipart|resumable]] [--output <file>] [--page-all [--page-limit <n>]] [--cache <age>] [--dry-run]`. `gapi <api> [<resource>…]` lists a level (the api level also lists helpers); `… <method> --help` shows method help. `gapi <api> +<helper> … [--dry-run] [--cache <age>] [--project <id>] [--api-key <key>] [<entry header flag>]` runs a hand command, e.g. `gapi ads +gaql "<query>" --customer <id> [--stream]`, `gapi indexing +publishBatch <file>`. `gapi search <words…>` lists at most 20 methods across all API entries that match the words (command, marks, HTTP method, first sentence of the description). `gapi schema <api> <resource>[.<sub>…].<method> | <SchemaName> [--resolve-refs]` prints a method shape or a schema as JSON; `--find <word>` prints instead only the field paths of the expanded schema (a method: `request.…` and `response.…`) whose path or printed description (first sentence, max 100 chars) contains the word. `gapi discovery refresh [<api>…]` refetches discovery documents now; `gapi discovery path` prints the cache dir. `gapi search --help`, `gapi schema --help`, `gapi discovery --help` and `gapi auth [<sub>] --help` print that command's usage and run nothing.
|
|
21
21
|
|
|
22
22
|
## Install
|
|
23
23
|
|
|
@@ -25,7 +25,7 @@ Command shape: `gapi <api> <resource>… <method> [--params '<json>'] [--body '<
|
|
|
25
25
|
npm i -g google-apis-cli # installs the `gapi` bin
|
|
26
26
|
```
|
|
27
27
|
|
|
28
|
-
Needs **Node ≥
|
|
28
|
+
Needs **Node ≥ 26**. The package ships `dist/` (plain `tsc` output, built at publish, no bundle); in the repo, Node runs the `.ts` sources directly.
|
|
29
29
|
|
|
30
30
|
## Start here
|
|
31
31
|
|
|
@@ -54,12 +54,14 @@ A call moves through these parts in order.
|
|
|
54
54
|
|
|
55
55
|
| Part | File | Function |
|
|
56
56
|
|---|---|---|
|
|
57
|
-
| **run** | `src/run.ts` | `run(argv, io)`: strict argv parse, routing (`--help`, `auth`, `schema`, `discovery`, `<api> …`), `GapiError` → exit code. `bin/gapi.ts` passes process io. |
|
|
57
|
+
| **run** | `src/run.ts` | `run(argv, io)`: strict argv parse, routing (`--help`, `auth`, `search`, `schema`, `discovery`, `<api> …`), `GapiError` → exit code. `bin/gapi.ts` passes process io. |
|
|
58
58
|
| **Catalog** | `src/catalog.ts` | API entries (`src/apis/*.ts`, glob, no registry) → `ApiModel`. Discovery entries: document cached 24 h. |
|
|
59
59
|
| **Planner** | `src/plan.ts` | `plan(model, words \| methodId, input, settings)` → `CallPlan`. Pure. Every `--params`/`--body` rule; validation fails exit 3 before network. |
|
|
60
|
-
| **Transport** | `src/transport.ts` | `send(plan, { fetch, authorize, out })`. `authorize` adds the credential per request; non-2xx → exit 1; 403 `ACCESS_TOKEN_SCOPE_INSUFFICIENT` → exit 2. `--page-all` follows `nextPageToken`, sending `pageToken` in the query or in the request body (methods whose request schema has `pageToken`): one JSON line per page, max `--page-limit` (default 10). `--upload` streams a file to `rootUrl` + `upload/` (media, multipart, or resumable in 8 MiB chunks); `--output` writes the `alt=media` body to a file, from `rootUrl` + `download/` when the method sets `useMediaDownloadService`. `--dry-run` prints the plan instead. |
|
|
61
|
-
| **
|
|
62
|
-
| **
|
|
60
|
+
| **Transport** | `src/transport.ts` + `src/transport/{upload,download,batch}.ts` | `send(plan, { fetch, authorize, out, sleep })`. `authorize` adds the credential per request; GET/HEAD/PUT/DELETE and `reads` POSTs retry 408/429/5xx and network errors (max 3, backoff with jitter); non-2xx or network failure → exit 1; 403 `ACCESS_TOKEN_SCOPE_INSUFFICIENT` (also in an array body or a batch part) → exit 2. `--page-all` follows `nextPageToken`, sending `pageToken` in the query or in the request body (methods whose request schema has `pageToken`): one JSON line per page, max `--page-limit` (default 10). `--upload` streams a file to `rootUrl` + `upload/` (media, multipart, or resumable in 8 MiB chunks, resumed from the session's `Range` after a 5xx or network error); `--output` writes the `alt=media` body to a file, from `rootUrl` + `download/` when the method sets `useMediaDownloadService`. `+publishBatch` sends `multipart/mixed` to the document's `batchPath`. `--dry-run` prints the plan instead. |
|
|
61
|
+
| **Search** | `src/search.ts` | `search(models, query)`: SQLite FTS5 index (`node:sqlite`) built in memory per call from every loaded ApiModel, nothing on disk. Words split at camelCase, `-`, `_`, `.`; query tokens are letters and digits only, double-quoted, joined with `OR`; best 20 by `bm25`. |
|
|
62
|
+
| **Renderers** | `src/render.ts` | Root help, level listing, method help, search hits, `schemaView` (`$ref` expansion, cycle guard), `schemaFind` (`--find` field paths), and the skill reference Markdown that `pnpm gen:skill` writes to `skills/gapi/reference/` (generated; do not edit by hand). |
|
|
63
|
+
| **Response cache** | `src/responses.ts` | `--cache <age>` (1s–7d, opt-in per call, [ADR 0001](docs/adr/0001-response-cache.md)): `run` checks eligibility before any network call (no `nocache` entry, single call or pages, `GET` or a `reads` POST), then answers a hit from `responses/` in the config dir (dir 700, files 600) with no fetch, or stores a miss after a successful send. Key holds `identity(auth)` from Credentials (refresh-token hash), so accounts never share entries. `auth login` and `auth logout` delete `responses/`. |
|
|
64
|
+
| **Credentials** | `src/credentials.ts` | Only reader and writer of `config.json` and `credentials.json` (mode 600) in the config dir (`GAPI_CONFIG_DIR` or `~/.config/gapi`; it also holds the discovery `cache/` and `responses/`). `settings`: `--project`, `--api-key` and entry header flags, each flag → env → config. `authorize`: OAuth Bearer (refreshed token saved) or `X-Goog-Api-Key`. `login`, `status` (with `missing`), `logout` (revoke, then delete), `setup`. |
|
|
63
65
|
|
|
64
66
|
CLI tests call `run` in-process (`test/run.test.ts`, `test/helpers/io.ts`). Terms: [GLOSSARY.md](GLOSSARY.md).
|
|
65
67
|
|
|
@@ -77,7 +79,8 @@ CLI tests call `run` in-process (`test/run.test.ts`, `test/helpers/io.ts`). Term
|
|
|
77
79
|
|
|
78
80
|
## Add an API or a resource
|
|
79
81
|
|
|
80
|
-
- New API = one new `src/apis/<api>.ts` file that default-exports an API entry (`satisfies ApiEntry`): the discovery URL plus overlays (`retired`, `deprecated`, `paid`, `headers`, `helpers`). The discovery document gives every resource and method. The glob finds the file; `api` must equal the file name, or the Catalog fails to load. No shared file changes.
|
|
82
|
+
- New API = one new `src/apis/<api>.ts` file that default-exports an API entry (`satisfies ApiEntry`): the discovery URL plus overlays (`retired`, `deprecated`, `paid`, `reads`, `nocache`, `headers`, `helpers`). The discovery document gives every resource and method. The glob finds the file; `api` must equal the file name, or the Catalog fails to load. No shared file changes.
|
|
83
|
+
- Check the API's caching terms; set `nocache` (`{ reason, source }`) when they forbid or bound caching ([ADR 0001](docs/adr/0001-response-cache.md)). Else list POST methods that only read (reports, searches) by id in `reads`, so `--cache` accepts them.
|
|
81
84
|
- Accept rule — `pnpm discovery:check` exits 0. It fails when an overlay names a method id that is not in the document. CI runs it weekly and on manual dispatch (`.github/workflows/discovery.yml`); `prepublishOnly` runs it before every publish.
|
|
82
85
|
- Test through `run(argv, io)`: fixture entries via `io.apisDir`, cache seeded in a temp `GAPI_CONFIG_DIR`, fake `fetch` (`test/helpers/fetch.ts`). `--dry-run` shows the request. Use no real network.
|
|
83
86
|
- Before commit: `pnpm typecheck` (tsc --noEmit) · `pnpm lint` (biome) · `pnpm test`.
|
package/dist/bin/gapi.js
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { run } from '../src/run.js';
|
|
3
3
|
const { argv, env, stdin, stdout, stderr } = process;
|
|
4
|
-
//
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
// stdout EPIPE (`gapi … | head`): the reader is gone, so stop, which also stops --page-all
|
|
5
|
+
// fetches. Any other stream error (ENOSPC, EBADF) lost output: exit 5, never a silent 0.
|
|
6
|
+
for (const s of [stdout, stderr]) {
|
|
7
|
+
s.on('error', (e) => {
|
|
8
|
+
if (e.code !== 'EPIPE')
|
|
9
|
+
process.exit(5);
|
|
10
|
+
if (s === stdout)
|
|
11
|
+
process.exit(0);
|
|
12
|
+
});
|
|
13
|
+
}
|
|
7
14
|
process.exitCode = await run(argv.slice(2), { env, fetch, stdin, stdout, stderr });
|
package/dist/src/apis/ads.js
CHANGED
|
@@ -21,7 +21,10 @@ const streamConcat = (data) => {
|
|
|
21
21
|
const message = isRecord(failed.error) ? failed.error.message : undefined;
|
|
22
22
|
throw new GapiError(1, typeof message === 'string' ? message : 'searchStream: a chunk holds an error', failed);
|
|
23
23
|
}
|
|
24
|
-
|
|
24
|
+
const results = data.flatMap((c) => (isRecord(c) && Array.isArray(c.results) ? c.results : []));
|
|
25
|
+
// SUMMARY_ROW_SETTING puts the totals in the last chunk's summaryRow; keep them.
|
|
26
|
+
const summaryRow = data.findLast((c) => isRecord(c) && c.summaryRow !== undefined)?.summaryRow;
|
|
27
|
+
return summaryRow === undefined ? results : { results, summaryRow };
|
|
25
28
|
};
|
|
26
29
|
// D10: one GAQL query; `--stream` returns every row, without it the first page as returned.
|
|
27
30
|
const gaql = {
|
|
@@ -44,6 +47,11 @@ const gaql = {
|
|
|
44
47
|
export default {
|
|
45
48
|
api: 'ads',
|
|
46
49
|
discovery: 'https://googleads.googleapis.com/$discovery/rest?version=v26',
|
|
50
|
+
reads: [
|
|
51
|
+
'googleads.customers.googleAds.search',
|
|
52
|
+
'googleads.customers.googleAds.searchStream',
|
|
53
|
+
'googleads.googleAdsFields.search',
|
|
54
|
+
],
|
|
47
55
|
headers: [
|
|
48
56
|
{
|
|
49
57
|
name: 'login-customer-id',
|
package/dist/src/apis/crux.js
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
export default {
|
|
2
2
|
api: 'crux',
|
|
3
3
|
discovery: 'https://chromeuxreport.googleapis.com/$discovery/rest?version=v1',
|
|
4
|
+
nocache: {
|
|
5
|
+
reason: 'Google-owned content; Google APIs Terms §5 bound caching to the cache header',
|
|
6
|
+
source: 'https://developers.google.com/terms',
|
|
7
|
+
},
|
|
4
8
|
};
|
|
@@ -1,4 +1,12 @@
|
|
|
1
1
|
export default {
|
|
2
2
|
api: 'ga4-data',
|
|
3
3
|
discovery: 'https://analyticsdata.googleapis.com/$discovery/rest?version=v1beta',
|
|
4
|
+
reads: [
|
|
5
|
+
'analyticsdata.properties.runReport',
|
|
6
|
+
'analyticsdata.properties.runPivotReport',
|
|
7
|
+
'analyticsdata.properties.batchRunReports',
|
|
8
|
+
'analyticsdata.properties.batchRunPivotReports',
|
|
9
|
+
'analyticsdata.properties.runRealtimeReport',
|
|
10
|
+
'analyticsdata.properties.checkCompatibility',
|
|
11
|
+
],
|
|
4
12
|
};
|
package/dist/src/apis/gsc.js
CHANGED
|
@@ -4,7 +4,8 @@ export default {
|
|
|
4
4
|
retired: {
|
|
5
5
|
'searchconsole.urlTestingTools.mobileFriendlyTest.run': {
|
|
6
6
|
reason: 'Retired by Google in December 2023',
|
|
7
|
-
source: 'https://developers.google.com/search/blog/
|
|
7
|
+
source: 'https://developers.google.com/search/blog/2023/04/page-experience-in-search',
|
|
8
8
|
},
|
|
9
9
|
},
|
|
10
|
+
reads: ['webmasters.searchanalytics.query', 'searchconsole.urlInspection.index.inspect'],
|
|
10
11
|
};
|
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
export default {
|
|
2
2
|
api: 'pagespeed',
|
|
3
3
|
discovery: 'https://pagespeedonline.googleapis.com/$discovery/rest?version=v5',
|
|
4
|
+
nocache: {
|
|
5
|
+
reason: 'Google-owned content; Google APIs Terms §5 bound caching to the cache header',
|
|
6
|
+
source: 'https://developers.google.com/terms',
|
|
7
|
+
},
|
|
4
8
|
};
|
package/dist/src/apis/places.js
CHANGED
|
@@ -5,6 +5,10 @@ const billed = {
|
|
|
5
5
|
export default {
|
|
6
6
|
api: 'places',
|
|
7
7
|
discovery: 'https://places.googleapis.com/$discovery/rest?version=v1',
|
|
8
|
+
nocache: {
|
|
9
|
+
reason: 'Places terms forbid caching API content (place_id excepted)',
|
|
10
|
+
source: 'https://developers.google.com/maps/documentation/places/web-service/policies',
|
|
11
|
+
},
|
|
8
12
|
paid: {
|
|
9
13
|
'places.places.autocomplete': billed,
|
|
10
14
|
'places.places.get': billed,
|
package/dist/src/apis/sa360.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
export default {
|
|
2
2
|
api: 'web-risk',
|
|
3
3
|
discovery: 'https://webrisk.googleapis.com/$discovery/rest?version=v1',
|
|
4
|
+
nocache: {
|
|
5
|
+
reason: 'Lookup results expire at the response expireTime, not a fixed age',
|
|
6
|
+
source: 'https://cloud.google.com/web-risk/docs/caching',
|
|
7
|
+
},
|
|
4
8
|
paid: {
|
|
5
9
|
'webrisk.hashes.search': {
|
|
6
10
|
reason: 'Paid only',
|
package/dist/src/catalog.js
CHANGED
|
@@ -8,10 +8,11 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
|
|
|
8
8
|
};
|
|
9
9
|
// Catalog: the API entries found by glob, their discovery documents behind a
|
|
10
10
|
// 24 h disk cache, and the one normalized ApiModel per API (epic #30 D2, D8, D13).
|
|
11
|
+
import { randomUUID } from 'node:crypto';
|
|
11
12
|
import { mkdir, readdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
|
|
12
13
|
import { basename, dirname, extname, join } from 'node:path';
|
|
13
14
|
import { pathToFileURL } from 'node:url';
|
|
14
|
-
import { GapiError } from './errors.js';
|
|
15
|
+
import { GapiError, reasonOf } from './errors.js';
|
|
15
16
|
import { isRecord, parseOrText } from './json.js';
|
|
16
17
|
const TTL_MS = 24 * 60 * 60 * 1000;
|
|
17
18
|
const AUTH_PREFIX = 'https://www.googleapis.com/auth/';
|
|
@@ -42,14 +43,17 @@ export async function openCatalog({ dir, fetch, warn, apisDir = join(import.meta
|
|
|
42
43
|
async function document(entry, refresh) {
|
|
43
44
|
const file = join(cacheDir, `${entry.api}.json`);
|
|
44
45
|
const mtime = await stat(file).then((s) => s.mtimeMs, () => undefined);
|
|
45
|
-
|
|
46
|
+
// A future mtime (clock skew) counts as stale, so no document freezes.
|
|
47
|
+
const age = mtime === undefined ? -1 : Date.now() - mtime;
|
|
48
|
+
if (!refresh && age >= 0 && age < TTL_MS) {
|
|
46
49
|
const doc = await cached(file);
|
|
47
50
|
if (doc)
|
|
48
51
|
return doc;
|
|
49
52
|
}
|
|
50
53
|
const got = await download(entry.discovery).catch((e) => e instanceof Error ? e : new Error(String(e)));
|
|
51
54
|
if (!(got instanceof Error)) {
|
|
52
|
-
|
|
55
|
+
// The document is already here; a read-only cache dir only costs the next fetch.
|
|
56
|
+
await writeAtomic(file, got.text).catch((e) => warn(`cannot cache the ${entry.api} document: ${reasonOf(e)}`));
|
|
53
57
|
return got.doc;
|
|
54
58
|
}
|
|
55
59
|
const stale = mtime === undefined ? undefined : await cached(file);
|
|
@@ -119,6 +123,13 @@ function toModel(doc, entry) {
|
|
|
119
123
|
overlay('paid', entry.paid, (m, v) => {
|
|
120
124
|
m.paid = v;
|
|
121
125
|
});
|
|
126
|
+
// Only a POST: a `reads` id on a write would let --cache store it and Transport resend it.
|
|
127
|
+
overlay('reads', Object.fromEntries((entry.reads ?? []).map((id) => [id, true])), (m) => {
|
|
128
|
+
if (m.httpMethod === 'POST')
|
|
129
|
+
m.read = true;
|
|
130
|
+
else
|
|
131
|
+
problems.push(`reads ${m.id}: ${m.httpMethod}, not a POST`);
|
|
132
|
+
});
|
|
122
133
|
return {
|
|
123
134
|
...root,
|
|
124
135
|
api: entry.api,
|
|
@@ -129,6 +140,7 @@ function toModel(doc, entry) {
|
|
|
129
140
|
documentationLink: doc.documentationLink,
|
|
130
141
|
rootUrl: doc.rootUrl ?? '',
|
|
131
142
|
servicePath: doc.servicePath ?? '',
|
|
143
|
+
batchPath: doc.batchPath ?? 'batch',
|
|
132
144
|
auth,
|
|
133
145
|
scopes,
|
|
134
146
|
byId,
|
|
@@ -190,14 +202,17 @@ async function loadEntries(dir) {
|
|
|
190
202
|
const name = basename(f, extname(f));
|
|
191
203
|
if (e.api !== name)
|
|
192
204
|
throw new Error(`${file}: api '${e.api}' must equal '${name}'`);
|
|
205
|
+
if (!/^[a-z0-9-]+$/.test(e.api))
|
|
206
|
+
throw new Error(`${file}: api '${e.api}' must match [a-z0-9-]+`);
|
|
193
207
|
entries[e.api] = e;
|
|
194
208
|
}
|
|
195
209
|
return entries;
|
|
196
210
|
}
|
|
197
|
-
//
|
|
198
|
-
|
|
211
|
+
// gcloud WriteFileAtomically: a new temp file (mode 600, `wx` never reuses a looser one),
|
|
212
|
+
// flushed to disk, then renamed over the target; readers see the old or the new file.
|
|
213
|
+
export async function writeAtomic(file, text) {
|
|
199
214
|
await mkdir(dirname(file), { recursive: true, mode: 0o700 });
|
|
200
|
-
const tmp = `${file}.${
|
|
201
|
-
await writeFile(tmp, text, { mode: 0o600 });
|
|
215
|
+
const tmp = `${file}.${randomUUID()}.tmp`;
|
|
216
|
+
await writeFile(tmp, text, { mode: 0o600, flag: 'wx', flush: true });
|
|
202
217
|
await rename(tmp, file);
|
|
203
218
|
}
|
package/dist/src/credentials.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
// Credentials: the only reader and writer of config.json and credentials.json (epic #30 D8, D13).
|
|
2
2
|
// Settings precedence, Bearer vs API key, token refresh + persistence, loopback/manual login.
|
|
3
|
-
import { randomBytes } from 'node:crypto';
|
|
4
|
-
import { chmod,
|
|
3
|
+
import { hash, randomBytes } from 'node:crypto';
|
|
4
|
+
import { chmod, readFile, rm } from 'node:fs/promises';
|
|
5
5
|
import { createServer } from 'node:http';
|
|
6
6
|
import { homedir } from 'node:os';
|
|
7
7
|
import { join } from 'node:path';
|
|
8
8
|
import { createInterface } from 'node:readline/promises';
|
|
9
9
|
import { Writable } from 'node:stream';
|
|
10
10
|
import { CodeChallengeMethod, OAuth2Client } from 'google-auth-library';
|
|
11
|
-
import { shortScope } from './catalog.js';
|
|
11
|
+
import { shortScope, writeAtomic } from './catalog.js';
|
|
12
12
|
import { codeOf, GapiError, messageOf } from './errors.js';
|
|
13
13
|
import { isRecord } from './json.js';
|
|
14
14
|
const LOOPBACK_HOST = '127.0.0.1';
|
|
@@ -62,12 +62,11 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
62
62
|
throw new GapiError(2, `${configFile} is not a JSON object; fix or remove it`);
|
|
63
63
|
return cfg;
|
|
64
64
|
};
|
|
65
|
-
// Owner-only:
|
|
66
|
-
//
|
|
65
|
+
// Owner-only: file 600 by temp file + rename, so a crash or a concurrent gapi never
|
|
66
|
+
// leaves a broken file. `mkdir` mode applies only on create: narrow an existing dir too.
|
|
67
67
|
const writeSecure = async (file, data) => {
|
|
68
|
-
await
|
|
69
|
-
await
|
|
70
|
-
await chmod(file, 0o600);
|
|
68
|
+
await writeAtomic(file, JSON.stringify(data, null, 2));
|
|
69
|
+
await chmod(dir, 0o700);
|
|
71
70
|
};
|
|
72
71
|
const readCredentials = async () => {
|
|
73
72
|
let creds;
|
|
@@ -95,6 +94,9 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
95
94
|
const { tokens } = await c.getToken({ code, codeVerifier }).catch((e) => {
|
|
96
95
|
throw new GapiError(2, `token exchange failed: ${messageOf(e)}`);
|
|
97
96
|
});
|
|
97
|
+
// Google: "refresh tokens are always returned for installed applications".
|
|
98
|
+
if (!tokens.refresh_token)
|
|
99
|
+
throw new GapiError(2, `no refresh token returned; ${RELOGIN}`);
|
|
98
100
|
await writeSecure(credentialsFile, tokens);
|
|
99
101
|
return tokens;
|
|
100
102
|
};
|
|
@@ -137,8 +139,10 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
137
139
|
const c = await client();
|
|
138
140
|
c.setCredentials(await readCredentials());
|
|
139
141
|
// An event listener is not awaited: keep the write so the refresh below can await it.
|
|
142
|
+
// Copy now: google-auth-library puts the old refresh token back on this object.
|
|
140
143
|
c.on('tokens', (t) => {
|
|
141
|
-
|
|
144
|
+
const fresh = { ...t };
|
|
145
|
+
saving = readCredentials().then((stored) => writeSecure(credentialsFile, { ...stored, ...fresh }));
|
|
142
146
|
});
|
|
143
147
|
tokenClient = c;
|
|
144
148
|
}
|
|
@@ -158,6 +162,14 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
158
162
|
throw new GapiError(2, `no access token; ${RELOGIN}`);
|
|
159
163
|
return { Authorization: `Bearer ${token}` };
|
|
160
164
|
};
|
|
165
|
+
// Response cache key part (ADR 0001): one per Google account, never the token itself.
|
|
166
|
+
// No stored token → undefined; entries are stored only after an authorized success.
|
|
167
|
+
async function identity(auth) {
|
|
168
|
+
if (auth.kind === 'apiKey')
|
|
169
|
+
return 'apiKey';
|
|
170
|
+
const token = (await readCredentials().catch(() => undefined))?.refresh_token;
|
|
171
|
+
return token ? hash('sha256', token) : undefined;
|
|
172
|
+
}
|
|
161
173
|
// Both flows send `state` and a PKCE S256 challenge, and redeem one redirect URL.
|
|
162
174
|
async function login({ consent, manual = false, prompt = ask, }) {
|
|
163
175
|
const c = await client(REDIRECT_URI);
|
|
@@ -171,17 +183,24 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
171
183
|
code_challenge_method: CodeChallengeMethod.S256,
|
|
172
184
|
code_challenge: codeChallenge,
|
|
173
185
|
});
|
|
174
|
-
|
|
186
|
+
// `state` first, so only Google's own redirect can end the login (CSRF).
|
|
187
|
+
const redeem = async (redirect) => {
|
|
175
188
|
const query = redirect.searchParams;
|
|
189
|
+
if (query.get('state') !== state)
|
|
190
|
+
throw new GapiError(2, 'state mismatch');
|
|
176
191
|
const error = query.get('error');
|
|
177
192
|
if (error)
|
|
178
193
|
throw new GapiError(2, `consent failed: ${error}`);
|
|
179
|
-
if (query.get('state') !== state)
|
|
180
|
-
throw new GapiError(2, 'state mismatch');
|
|
181
194
|
const code = query.get('code');
|
|
182
195
|
if (!code)
|
|
183
196
|
throw new GapiError(2, 'no code in redirect');
|
|
184
|
-
|
|
197
|
+
const tokens = await exchange(c, code, codeVerifier);
|
|
198
|
+
// Google: "Your app must verify which scopes were actually granted".
|
|
199
|
+
const granted = new Set((tokens.scope ?? '').split(' '));
|
|
200
|
+
const missing = consent.filter((s) => !granted.has(s)).map(shortScope);
|
|
201
|
+
if (missing.length > 0)
|
|
202
|
+
warn(`scopes not granted: ${missing.join(', ')}`);
|
|
203
|
+
return tokens;
|
|
185
204
|
};
|
|
186
205
|
if (manual) {
|
|
187
206
|
stderr.write(`Open this URL and grant access:\n${url}\n` +
|
|
@@ -200,6 +219,11 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
200
219
|
res.end();
|
|
201
220
|
return;
|
|
202
221
|
}
|
|
222
|
+
if (redirect.searchParams.get('state') !== state) {
|
|
223
|
+
res.statusCode = 400; // not Google's redirect: keep waiting for it
|
|
224
|
+
res.end('gapi: state mismatch');
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
203
227
|
try {
|
|
204
228
|
const tokens = await redeem(redirect);
|
|
205
229
|
res.end('gapi: authorized. You can close this tab.');
|
|
@@ -235,7 +259,8 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
235
259
|
};
|
|
236
260
|
}
|
|
237
261
|
// Revoke, then delete (the `gcloud auth revoke` pattern, #151). Revoking the refresh
|
|
238
|
-
// token ends the whole grant. Google answers 400 for a token already
|
|
262
|
+
// token ends the whole grant. Google answers 400 `invalid_token` for a token already
|
|
263
|
+
// invalid or revoked; another 400 error code is a failed request.
|
|
239
264
|
// Any other failure keeps credentials.json, so the logout can be retried.
|
|
240
265
|
// No stored token (missing or unreadable file) leaves nothing to revoke. The revoke
|
|
241
266
|
// endpoint takes only the token, so no OAuth client id or secret is needed.
|
|
@@ -245,7 +270,8 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
245
270
|
await oauth()
|
|
246
271
|
.revokeToken(token)
|
|
247
272
|
.catch((e) => {
|
|
248
|
-
|
|
273
|
+
const data = isRecord(e) && isRecord(e.response) ? e.response.data : undefined;
|
|
274
|
+
if (!isRecord(data) || data.error !== 'invalid_token')
|
|
249
275
|
throw new GapiError(1, `token revoke failed, credentials kept: ${messageOf(e)}`);
|
|
250
276
|
warn('refresh token already invalid or revoked');
|
|
251
277
|
});
|
|
@@ -277,7 +303,7 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
277
303
|
await writeSecure(configFile, cfg);
|
|
278
304
|
return configFile;
|
|
279
305
|
}
|
|
280
|
-
return { settings, authorize, login, status, logout, setup };
|
|
306
|
+
return { settings, authorize, identity, login, status, logout, setup };
|
|
281
307
|
}
|
|
282
308
|
// Decode a JWT payload without verifying (display only; it is our own token).
|
|
283
309
|
function accountFromIdToken(idToken) {
|
package/dist/src/plan.js
CHANGED
|
@@ -15,6 +15,8 @@ const OWNED = new Set([
|
|
|
15
15
|
'upload_protocol',
|
|
16
16
|
'$.xgafv',
|
|
17
17
|
]);
|
|
18
|
+
// gaxios httpMethodsToRetry, without OPTIONS: a resend of these changes nothing more.
|
|
19
|
+
const IDEMPOTENT = new Set(['GET', 'HEAD', 'PUT', 'DELETE']);
|
|
18
20
|
// The document-level parameters a caller may pass (D5).
|
|
19
21
|
const SHARED = ['fields', 'quotaUser'];
|
|
20
22
|
const PROTOCOLS = ['media', 'multipart', 'resumable'];
|
|
@@ -92,17 +94,18 @@ export function plan(model, target, input, settings) {
|
|
|
92
94
|
body,
|
|
93
95
|
auth: { kind: model.auth, scopes: method.scopes },
|
|
94
96
|
warnings,
|
|
97
|
+
...(IDEMPOTENT.has(method.httpMethod) || method.read ? { retry: true } : {}),
|
|
95
98
|
...mode,
|
|
96
99
|
};
|
|
97
100
|
}
|
|
98
|
-
// The outer CallPlan of a batch call: one POST to the
|
|
101
|
+
// The outer CallPlan of a batch call: one POST to the document's `batchPath` endpoint that
|
|
99
102
|
// carries `parts`, each planned by `plan` with its own headers (D7).
|
|
100
103
|
export function batchPlan(model, parts) {
|
|
101
104
|
const [first] = parts;
|
|
102
105
|
return {
|
|
103
106
|
methodId: first.methodId,
|
|
104
107
|
httpMethod: 'POST',
|
|
105
|
-
url: `${model.rootUrl}
|
|
108
|
+
url: `${model.rootUrl}${model.batchPath}`,
|
|
106
109
|
headers: {},
|
|
107
110
|
auth: first.auth,
|
|
108
111
|
warnings: first.warnings,
|
|
@@ -258,10 +261,25 @@ function paramValues(model, method, params) {
|
|
|
258
261
|
return String(v);
|
|
259
262
|
throw invalid(`${name} must be a string, number or boolean`);
|
|
260
263
|
});
|
|
264
|
+
// An empty list or a required '' sends nothing; a path would lose its segment.
|
|
265
|
+
if (values.length === 0)
|
|
266
|
+
throw invalid(`${name} needs at least one value`);
|
|
267
|
+
if (spec.required && values.includes(''))
|
|
268
|
+
throw invalid(`${name} needs a value`);
|
|
261
269
|
for (const v of values) {
|
|
262
270
|
if (spec.enum && !spec.enum.includes(v)) {
|
|
263
271
|
throw invalid(`${name} must be one of: ${spec.enum.join(', ')}`);
|
|
264
272
|
}
|
|
273
|
+
const type = spec.type === 'integer' || /int64$/.test(spec.format ?? '') ? 'integer' : spec.type;
|
|
274
|
+
if (type === 'integer' && !/^-?\d+$/.test(v))
|
|
275
|
+
throw invalid(`${name} must be an integer`);
|
|
276
|
+
if (type === 'boolean' && v !== 'true' && v !== 'false') {
|
|
277
|
+
throw invalid(`${name} must be true or false`);
|
|
278
|
+
}
|
|
279
|
+
// Path patterns are checked in buildUrl, after the bare-id hint (#28).
|
|
280
|
+
if (spec.location !== 'path' && spec.pattern && !matches(spec.pattern, v)) {
|
|
281
|
+
throw invalid(`${name} must match ${spec.pattern}`);
|
|
282
|
+
}
|
|
265
283
|
}
|
|
266
284
|
out.set(name, { spec, values });
|
|
267
285
|
}
|
|
@@ -280,9 +298,14 @@ function buildUrl(model, method, params, mode) {
|
|
|
280
298
|
if (!param)
|
|
281
299
|
throw invalid(`missing required param ${name}`);
|
|
282
300
|
const value = param.values[0];
|
|
283
|
-
if (param.spec.pattern && !
|
|
301
|
+
if (param.spec.pattern && !matches(param.spec.pattern, value)) {
|
|
284
302
|
throw invalid(`${name} must match ${param.spec.pattern}`);
|
|
285
303
|
}
|
|
304
|
+
// googleapis-common validateAndEncodeParams: a `.`/`..` segment would make the URL
|
|
305
|
+
// parser leave the method path; `%2e` too, since WHATWG URL reads it as a dot.
|
|
306
|
+
const dot = (reserved ? value.split('/') : [value]).find((s) => /^(?:\.|%2e){1,2}$/i.test(s));
|
|
307
|
+
if (dot !== undefined)
|
|
308
|
+
throw invalid(`${name}: '${dot}' is a dot segment; pass a real id`);
|
|
286
309
|
if (reserved)
|
|
287
310
|
return value.split('/').map(encodeURIComponent).join('/');
|
|
288
311
|
// #28: `properties/{property}` + `properties/123` would encode to properties%2F123.
|
|
@@ -310,6 +333,16 @@ function buildUrl(model, method, params, mode) {
|
|
|
310
333
|
: model.rootUrl;
|
|
311
334
|
return `${root}${model.servicePath}${path}${qs ? `?${qs}` : ''}`;
|
|
312
335
|
}
|
|
336
|
+
// shortcut: a Java-only discovery pattern (e.g. `(?i)`) is skipped, not checked; translate
|
|
337
|
+
// Java regex to JS if Google ever relies on one for a path.
|
|
338
|
+
function matches(pattern, value) {
|
|
339
|
+
try {
|
|
340
|
+
return new RegExp(pattern).test(value);
|
|
341
|
+
}
|
|
342
|
+
catch {
|
|
343
|
+
return true;
|
|
344
|
+
}
|
|
345
|
+
}
|
|
313
346
|
// google/api/http.proto: {x} percent-encodes every character except [-_.~0-9a-zA-Z];
|
|
314
347
|
// encodeURIComponent leaves !'()* as is.
|
|
315
348
|
const encodeSegment = (value) => encodeURIComponent(value).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
|
package/dist/src/render.js
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
|
-
|
|
1
|
+
// Renderers: plain text views over an ApiModel (epic #30 D4, story 8).
|
|
2
|
+
import { isRecord } from './json.js';
|
|
3
|
+
export const SCHEMA_USAGE = 'gapi schema <api> <resource>[.<sub-resource>…].<method> | <SchemaName> [--resolve-refs | --find <word>]';
|
|
2
4
|
export const DISCOVERY_USAGE = 'gapi discovery refresh [<api>…] | path';
|
|
3
5
|
export const AUTH_USAGE = 'gapi auth setup | login [--manual] | status | logout';
|
|
6
|
+
export const SEARCH_USAGE = 'gapi search <words…>';
|
|
4
7
|
const USAGE = `gapi <api> <resource> [<sub-resource>…] <method> [--params '<json>'] [--body '<json>']
|
|
5
8
|
[--upload <file> [--upload-content-type <type>] [--upload-protocol media|multipart|resumable]]
|
|
6
|
-
[--output <file>] [--page-all [--page-limit <n>]] [--
|
|
7
|
-
[<entry header flags, e.g. ads --login-customer-id <id>>]
|
|
9
|
+
[--output <file>] [--page-all [--page-limit <n>]] [--cache <age>] [--dry-run]
|
|
10
|
+
[--project <id>] [--api-key <key>] [<entry header flags, e.g. ads --login-customer-id <id>>]
|
|
8
11
|
gapi <api> [<resource>…] list sub-resources and methods
|
|
9
12
|
gapi <api> <resource>… <method> --help method help
|
|
10
|
-
gapi <api> +<helper> … [--dry-run] [--project <id>] [--api-key <key>] [<entry header flags>]
|
|
13
|
+
gapi <api> +<helper> … [--dry-run] [--cache <age>] [--project <id>] [--api-key <key>] [<entry header flags>]
|
|
14
|
+
${SEARCH_USAGE} find methods by intent, max 20 hits
|
|
11
15
|
${SCHEMA_USAGE}
|
|
12
16
|
${DISCOVERY_USAGE}
|
|
13
17
|
${AUTH_USAGE}
|
|
@@ -95,6 +99,12 @@ const oneLine = (description = '') => description
|
|
|
95
99
|
.trim()
|
|
96
100
|
.split('\n')[0]
|
|
97
101
|
.split(/(?<=\.)\s+(?=[A-Z*])/)[0];
|
|
102
|
+
// The first sentence, max 100 chars: one line of search hits and of `schema --find`.
|
|
103
|
+
const brief = (description) => {
|
|
104
|
+
const sentence = oneLine(description);
|
|
105
|
+
const chars = [...sentence]; // code points: a cut never splits a surrogate pair
|
|
106
|
+
return chars.length > 100 ? `${chars.slice(0, 99).join('')}…` : sentence;
|
|
107
|
+
};
|
|
98
108
|
// Code-point order, so the output does not depend on the machine's locale.
|
|
99
109
|
const compare = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
100
110
|
const byKey = (map) => [...map].sort(([a], [b]) => compare(a, b));
|
|
@@ -142,6 +152,15 @@ function referenceRow(model, m) {
|
|
|
142
152
|
const marked = markList(m, { reasons: true }).join('; ');
|
|
143
153
|
return `| \`gapi ${model.api} ${m.words.join(' ')}\` | ${m.httpMethod} | ${cell(oneLine(m.description))} | ${required.join(', ')} | ${m.request ? `\`${m.request}\`` : ''} | ${cell(marked)} |`;
|
|
144
154
|
}
|
|
155
|
+
// One line per hit: command, marks, HTTP method, first sentence of the description.
|
|
156
|
+
export function searchText(hits) {
|
|
157
|
+
return hits
|
|
158
|
+
.map(({ model, method: m }) => {
|
|
159
|
+
const line = `gapi ${model.api} ${m.words.join(' ')}${marks(m)} ${m.httpMethod} ${brief(m.description)}`;
|
|
160
|
+
return `${line.trimEnd()}\n`;
|
|
161
|
+
})
|
|
162
|
+
.join('');
|
|
163
|
+
}
|
|
145
164
|
// D9: a method's shape (request/response as schema names), or one document schema.
|
|
146
165
|
export function schemaView(model, target, { resolveRefs }) {
|
|
147
166
|
// A method shape holds schema names, never a $ref, so --resolve-refs leaves it as is.
|
|
@@ -165,6 +184,52 @@ export function schemaView(model, target, { resolveRefs }) {
|
|
|
165
184
|
const out = resolveRefs && schema ? expandObject(schema, model.schemas, [target.schema]) : schema;
|
|
166
185
|
return `${JSON.stringify(out, null, 2)}\n`;
|
|
167
186
|
}
|
|
187
|
+
// `--find`: one line per leaf of the expanded schema (a method: request and response) whose
|
|
188
|
+
// path or printed sentence contains `word`, case-insensitive. A $ref the cycle guard left is a leaf.
|
|
189
|
+
export function schemaFind(model, target, word) {
|
|
190
|
+
const roots = 'method' in target
|
|
191
|
+
? [
|
|
192
|
+
['request', target.method.request],
|
|
193
|
+
['response', target.method.response],
|
|
194
|
+
]
|
|
195
|
+
: [['', target.schema]];
|
|
196
|
+
const needle = word.toLowerCase();
|
|
197
|
+
const lines = [];
|
|
198
|
+
// An array or map item without a description takes its field's (Google puts it there).
|
|
199
|
+
const walk = (node, path, fieldDescription = '') => {
|
|
200
|
+
const own = typeof node.description === 'string' ? node.description : fieldDescription;
|
|
201
|
+
const props = isRecord(node.properties) ? Object.entries(node.properties) : [];
|
|
202
|
+
const children = [
|
|
203
|
+
...props.map(([key, child]) => [
|
|
204
|
+
path ? `${path}.${key}` : key,
|
|
205
|
+
child,
|
|
206
|
+
'',
|
|
207
|
+
]),
|
|
208
|
+
[`${path}[]`, node.items, own],
|
|
209
|
+
[`${path}{}`, node.additionalProperties, own],
|
|
210
|
+
];
|
|
211
|
+
let leaf = true;
|
|
212
|
+
for (const [childPath, child, description] of children) {
|
|
213
|
+
if (!isRecord(child))
|
|
214
|
+
continue;
|
|
215
|
+
leaf = false;
|
|
216
|
+
walk(child, childPath, description);
|
|
217
|
+
}
|
|
218
|
+
const ref = typeof node.$ref === 'string' ? node.$ref : undefined;
|
|
219
|
+
const text = ref ? '' : brief(own);
|
|
220
|
+
if (!leaf || !`${path}\n${text}`.toLowerCase().includes(needle))
|
|
221
|
+
return;
|
|
222
|
+
const type = ref ? `$ref ${ref}` : `${String(node.type ?? 'any')}${node.enum ? ' enum' : ''}`;
|
|
223
|
+
lines.push(`${path}: ${type}${text ? ` — ${text}` : ''}`);
|
|
224
|
+
};
|
|
225
|
+
for (const [path, name] of roots) {
|
|
226
|
+
const schema = name ? model.schemas.get(name) : undefined;
|
|
227
|
+
const expanded = name && schema ? expandObject(schema, model.schemas, [name]) : undefined;
|
|
228
|
+
if (isRecord(expanded))
|
|
229
|
+
walk(expanded, path);
|
|
230
|
+
}
|
|
231
|
+
return lines.map((line) => `${line}\n`).join('');
|
|
232
|
+
}
|
|
168
233
|
function expandRefs(value, schemas, path) {
|
|
169
234
|
if (Array.isArray(value))
|
|
170
235
|
return value.map((v) => expandRefs(v, schemas, path));
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// Response cache (ADR 0001): one file per key under <config dir>/responses/, dir 700, files
|
|
2
|
+
// 600, written by temp file + rename. Holds Google account data: login and logout clear it.
|
|
3
|
+
import { hash } from 'node:crypto';
|
|
4
|
+
import { readdir, readFile, rm, stat } from 'node:fs/promises';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { writeAtomic } from './catalog.js';
|
|
7
|
+
export const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
8
|
+
export function openResponses(dir) {
|
|
9
|
+
const file = (key) => join(dir, key);
|
|
10
|
+
// Every value that shapes the request or its output; `url` carries every query value.
|
|
11
|
+
const key = (call, identity) => hash('sha256', JSON.stringify([
|
|
12
|
+
identity,
|
|
13
|
+
call.httpMethod,
|
|
14
|
+
call.url,
|
|
15
|
+
call.headers,
|
|
16
|
+
call.body,
|
|
17
|
+
call.send,
|
|
18
|
+
call.send === 'pages' ? call.pageLimit : undefined,
|
|
19
|
+
call.decode !== undefined, // a helper's decoded output differs from the raw one
|
|
20
|
+
]));
|
|
21
|
+
// When the entry was stored (epoch ms); undefined when there is none.
|
|
22
|
+
const mtime = (k) => stat(file(k)).then((s) => s.mtimeMs, () => undefined);
|
|
23
|
+
const get = (k) => readFile(file(k), 'utf8').catch(() => undefined);
|
|
24
|
+
const put = (k, text) => writeAtomic(file(k), text);
|
|
25
|
+
// Deletes every file (temp files too) older than the max age. Every run calls it, so
|
|
26
|
+
// account data never outlives 7 days on disk (YouTube policy, ADR 0001).
|
|
27
|
+
async function sweep() {
|
|
28
|
+
for (const name of await readdir(dir).catch(() => [])) {
|
|
29
|
+
const f = join(dir, name);
|
|
30
|
+
const s = await stat(f).catch(() => undefined);
|
|
31
|
+
if (s && Date.now() - s.mtimeMs > MAX_AGE_MS)
|
|
32
|
+
await rm(f, { force: true });
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
const clear = () => rm(dir, { recursive: true, force: true });
|
|
36
|
+
return { key, mtime, get, put, sweep, clear };
|
|
37
|
+
}
|
package/dist/src/run.js
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
// Process io is injected so tests drive the whole CLI in-process.
|
|
2
2
|
import { readFile } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
3
4
|
import { parseArgs } from 'node:util';
|
|
4
5
|
import { openCatalog } from './catalog.js';
|
|
5
6
|
import { configDir, openCredentials } from './credentials.js';
|
|
6
7
|
import { GapiError, messageOf, reasonOf } from './errors.js';
|
|
7
8
|
import { isRecord } from './json.js';
|
|
8
9
|
import { parseJson, plan, resolve } from './plan.js';
|
|
9
|
-
import { AUTH_USAGE, DISCOVERY_USAGE, levelText, methodHelp, rootHelp, SCHEMA_USAGE, schemaView, } from './render.js';
|
|
10
|
+
import { AUTH_USAGE, DISCOVERY_USAGE, levelText, methodHelp, rootHelp, SCHEMA_USAGE, SEARCH_USAGE, schemaFind, schemaView, searchText, } from './render.js';
|
|
11
|
+
import { MAX_AGE_MS, openResponses } from './responses.js';
|
|
12
|
+
import { matchQuery, search } from './search.js';
|
|
10
13
|
import { send } from './transport.js';
|
|
11
14
|
const GLOBAL_OPTIONS = {
|
|
12
15
|
params: { type: 'string' },
|
|
@@ -20,6 +23,7 @@ const GLOBAL_OPTIONS = {
|
|
|
20
23
|
output: { type: 'string' },
|
|
21
24
|
project: { type: 'string' },
|
|
22
25
|
'api-key': { type: 'string' },
|
|
26
|
+
cache: { type: 'string' },
|
|
23
27
|
help: { type: 'boolean' },
|
|
24
28
|
};
|
|
25
29
|
// D10: a helper takes the global flags that do not shape the request, plus its own options.
|
|
@@ -27,8 +31,19 @@ const HELPER_OPTIONS = {
|
|
|
27
31
|
'dry-run': GLOBAL_OPTIONS['dry-run'],
|
|
28
32
|
project: GLOBAL_OPTIONS.project,
|
|
29
33
|
'api-key': GLOBAL_OPTIONS['api-key'],
|
|
34
|
+
cache: GLOBAL_OPTIONS.cache,
|
|
30
35
|
help: GLOBAL_OPTIONS.help,
|
|
31
36
|
};
|
|
37
|
+
const AGE_UNIT_MS = { s: 1e3, m: 60e3, h: 3600e3, d: 86400e3 };
|
|
38
|
+
// `--cache <n>s|m|h|d`, from 1 s to the 7-day store limit (ADR 0001).
|
|
39
|
+
function parseAge(age) {
|
|
40
|
+
const m = /^(\d+)([smhd])$/.exec(age);
|
|
41
|
+
const ms = m ? Number(m[1]) * AGE_UNIT_MS[m[2]] : 0;
|
|
42
|
+
if (ms < 1000 || ms > MAX_AGE_MS) {
|
|
43
|
+
throw new GapiError(3, `--cache: ${age} is not an age from 1s to 7d`);
|
|
44
|
+
}
|
|
45
|
+
return ms;
|
|
46
|
+
}
|
|
32
47
|
const isHelperName = (word) => word.startsWith('+');
|
|
33
48
|
// D4: an entry header flag is accepted only on the APIs whose entry declares it.
|
|
34
49
|
const headerOptions = (entry) => Object.fromEntries((entry?.headers ?? []).map((h) => [h.flag, { type: 'string' }]));
|
|
@@ -46,16 +61,22 @@ export async function run(argv, io) {
|
|
|
46
61
|
return err.exit;
|
|
47
62
|
}
|
|
48
63
|
}
|
|
49
|
-
// Strict argv parsing; an unknown flag
|
|
64
|
+
// Strict argv parsing; an unknown flag, a missing flag value or a repeated flag (parseArgs
|
|
65
|
+
// would keep the last one) is a validation error.
|
|
50
66
|
function parse(args, options) {
|
|
51
67
|
try {
|
|
52
|
-
|
|
68
|
+
const parsed = parseArgs({ args, options, allowPositionals: true, strict: true, tokens: true });
|
|
69
|
+
const names = parsed.tokens.flatMap((t) => (t.kind === 'option' ? [t.name] : []));
|
|
70
|
+
const twice = names.find((n, i) => names.indexOf(n) !== i);
|
|
71
|
+
if (twice !== undefined)
|
|
72
|
+
throw new GapiError(3, `--${twice} given twice`);
|
|
73
|
+
return parsed;
|
|
53
74
|
}
|
|
54
75
|
catch (e) {
|
|
55
|
-
throw new GapiError(3, messageOf(e));
|
|
76
|
+
throw e instanceof GapiError ? e : new GapiError(3, messageOf(e));
|
|
56
77
|
}
|
|
57
78
|
}
|
|
58
|
-
async function runAuth(sub, values, { io, creds, catalog }) {
|
|
79
|
+
async function runAuth(sub, values, { io, creds, catalog, responses, }) {
|
|
59
80
|
// --help never starts a side effect: no server, no fetch, no file write.
|
|
60
81
|
if (values.help) {
|
|
61
82
|
io.stdout.write(`${AUTH_USAGE}\n`);
|
|
@@ -63,6 +84,7 @@ async function runAuth(sub, values, { io, creds, catalog }) {
|
|
|
63
84
|
}
|
|
64
85
|
if (sub === 'login') {
|
|
65
86
|
const consent = await consentScopes(catalog);
|
|
87
|
+
await responses.clear(); // a new account starts with no stored responses
|
|
66
88
|
const tokens = await creds.login({ consent, manual: values.manual === true });
|
|
67
89
|
io.stderr.write(`gapi: authorized (${tokens.refresh_token ? 'refresh token stored' : 'no refresh token'})\n`);
|
|
68
90
|
return;
|
|
@@ -81,11 +103,16 @@ async function runAuth(sub, values, { io, creds, catalog }) {
|
|
|
81
103
|
return;
|
|
82
104
|
}
|
|
83
105
|
if (sub === 'logout') {
|
|
84
|
-
|
|
106
|
+
try {
|
|
107
|
+
await creds.logout();
|
|
108
|
+
}
|
|
109
|
+
finally {
|
|
110
|
+
await responses.clear(); // also when the revoke fails: no account data stays on disk
|
|
111
|
+
}
|
|
85
112
|
io.stderr.write('gapi: credentials removed\n');
|
|
86
113
|
return;
|
|
87
114
|
}
|
|
88
|
-
throw new GapiError(3, `unknown auth command: ${sub}`);
|
|
115
|
+
throw new GapiError(3, sub === undefined ? `usage: ${AUTH_USAGE}` : `unknown auth command: ${sub}`);
|
|
89
116
|
}
|
|
90
117
|
// D8 consent set: every entry's per-method scope plus its entry scopes (identity is added by login).
|
|
91
118
|
async function consentScopes(catalog) {
|
|
@@ -105,10 +132,19 @@ async function consentScopes(catalog) {
|
|
|
105
132
|
function schemaTarget(model, path) {
|
|
106
133
|
if (!model.methods.has(path) && model.schemas.has(path))
|
|
107
134
|
return { schema: path };
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
135
|
+
try {
|
|
136
|
+
const found = resolve(model, path.split('.'));
|
|
137
|
+
if (!found.method)
|
|
138
|
+
throw new GapiError(3, `${path} is a resource, not a method or schema`);
|
|
139
|
+
return { method: found.method };
|
|
140
|
+
}
|
|
141
|
+
catch (e) {
|
|
142
|
+
// resolve knows only the tree; a one-word path can also name a schema.
|
|
143
|
+
if (e instanceof GapiError && /^unknown /.test(e.message) && !path.includes('.')) {
|
|
144
|
+
throw new GapiError(3, `${e.message}, or a schema name`);
|
|
145
|
+
}
|
|
146
|
+
throw e;
|
|
147
|
+
}
|
|
112
148
|
}
|
|
113
149
|
async function dispatch(argv, io) {
|
|
114
150
|
const dir = configDir(io.env);
|
|
@@ -121,6 +157,26 @@ async function dispatch(argv, io) {
|
|
|
121
157
|
stdin: io.stdin,
|
|
122
158
|
stderr: io.stderr,
|
|
123
159
|
});
|
|
160
|
+
const responses = openResponses(join(dir, 'responses'));
|
|
161
|
+
await responses.sweep();
|
|
162
|
+
// ADR 0001 eligibility, before any network call: the entry allows caching, the call is
|
|
163
|
+
// one request or pages, the method is a GET or a `reads` POST.
|
|
164
|
+
const cacheFor = async (call, model, values) => {
|
|
165
|
+
if (values.cache === undefined)
|
|
166
|
+
return undefined;
|
|
167
|
+
const maxAgeMs = parseAge(values.cache);
|
|
168
|
+
const { nocache } = catalog.entries[model.api];
|
|
169
|
+
if (nocache) {
|
|
170
|
+
throw new GapiError(3, `--cache: ${model.api} responses must not be cached: ${nocache.reason} (${nocache.source})`);
|
|
171
|
+
}
|
|
172
|
+
const method = model.byId.get(call.methodId);
|
|
173
|
+
const once = call.send === 'once' || call.send === 'pages';
|
|
174
|
+
if (!once || (method?.httpMethod !== 'GET' && !method?.read)) {
|
|
175
|
+
const command = method ? `${model.api} ${method.words.join(' ')}` : call.methodId;
|
|
176
|
+
throw new GapiError(3, `--cache: ${command} is not a cacheable read`);
|
|
177
|
+
}
|
|
178
|
+
return { key: responses.key(call, await creds.identity(call.auth)), maxAgeMs };
|
|
179
|
+
};
|
|
124
180
|
if (argv.length === 0 || (argv[0] === '--help' && argv.length === 1)) {
|
|
125
181
|
io.stdout.write(rootHelp(Object.keys(catalog.entries)));
|
|
126
182
|
return;
|
|
@@ -131,11 +187,27 @@ async function dispatch(argv, io) {
|
|
|
131
187
|
manual: { type: 'boolean' },
|
|
132
188
|
help: { type: 'boolean' },
|
|
133
189
|
});
|
|
134
|
-
return runAuth(positionals[0], values, { io, creds, catalog });
|
|
190
|
+
return runAuth(positionals[0], values, { io, creds, catalog, responses });
|
|
191
|
+
}
|
|
192
|
+
if (api === 'search') {
|
|
193
|
+
// No option parse: a word like `-list` is query text (story 11).
|
|
194
|
+
if (rest.includes('--help')) {
|
|
195
|
+
io.stdout.write(`${SEARCH_USAGE}\n`);
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
const query = rest.join(' ');
|
|
199
|
+
matchQuery(query); // a query with no token exits 3 before any document loads
|
|
200
|
+
const models = await Promise.all(Object.keys(catalog.entries).map((a) => catalog.load(a)));
|
|
201
|
+
const hits = search(models, query);
|
|
202
|
+
if (hits.length === 0)
|
|
203
|
+
io.stderr.write('gapi: no match\n');
|
|
204
|
+
io.stdout.write(searchText(hits));
|
|
205
|
+
return;
|
|
135
206
|
}
|
|
136
207
|
if (api === 'schema') {
|
|
137
208
|
const { values, positionals } = parse(rest, {
|
|
138
209
|
'resolve-refs': { type: 'boolean' },
|
|
210
|
+
find: { type: 'string' },
|
|
139
211
|
help: { type: 'boolean' },
|
|
140
212
|
});
|
|
141
213
|
if (values.help) {
|
|
@@ -145,9 +217,19 @@ async function dispatch(argv, io) {
|
|
|
145
217
|
if (positionals.length !== 2)
|
|
146
218
|
throw new GapiError(3, `usage: ${SCHEMA_USAGE}`);
|
|
147
219
|
const [name, path] = positionals;
|
|
220
|
+
if (values.find === '')
|
|
221
|
+
throw new GapiError(3, '--find: needs a word');
|
|
148
222
|
const model = await catalog.load(name);
|
|
223
|
+
const target = schemaTarget(model, path);
|
|
224
|
+
if (values.find !== undefined) {
|
|
225
|
+
const found = schemaFind(model, target, values.find);
|
|
226
|
+
if (found === '')
|
|
227
|
+
io.stderr.write('gapi: no match\n');
|
|
228
|
+
io.stdout.write(found);
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
149
231
|
const resolveRefs = values['resolve-refs'] === true;
|
|
150
|
-
io.stdout.write(schemaView(model,
|
|
232
|
+
io.stdout.write(schemaView(model, target, { resolveRefs }));
|
|
151
233
|
return;
|
|
152
234
|
}
|
|
153
235
|
if (api === 'discovery') {
|
|
@@ -205,13 +287,18 @@ async function dispatch(argv, io) {
|
|
|
205
287
|
throw new GapiError(3, `cannot read ${path}: ${reasonOf(e)}`);
|
|
206
288
|
}),
|
|
207
289
|
});
|
|
208
|
-
|
|
290
|
+
const cache = await cacheFor(call, model, values);
|
|
291
|
+
return perform(call, values['dry-run'] === true, io, creds.authorize, { responses, cache });
|
|
209
292
|
}
|
|
210
293
|
const entry = Object.hasOwn(catalog.entries, api) ? catalog.entries[api] : undefined;
|
|
211
294
|
const { values, positionals } = parse(rest, { ...GLOBAL_OPTIONS, ...headerOptions(entry) });
|
|
212
295
|
const model = await catalog.load(api);
|
|
213
296
|
const target = resolve(model, positionals);
|
|
214
297
|
if (!target.method) {
|
|
298
|
+
// A listing sends nothing: a request flag here means the method word is missing.
|
|
299
|
+
const flag = Object.keys(values).find((k) => k !== 'help');
|
|
300
|
+
if (flag)
|
|
301
|
+
throw new GapiError(3, `--${flag}: ${[api, ...positionals].join(' ')} is not a method`);
|
|
215
302
|
const helpers = positionals.length === 0 ? catalog.entries[api].helpers : undefined;
|
|
216
303
|
io.stdout.write(levelText(model, positionals, target.node, helpers));
|
|
217
304
|
return;
|
|
@@ -232,12 +319,17 @@ async function dispatch(argv, io) {
|
|
|
232
319
|
uploadProtocol: values['upload-protocol'],
|
|
233
320
|
output: values.output,
|
|
234
321
|
}, await creds.settings(values, model.headers));
|
|
235
|
-
|
|
322
|
+
const cache = await cacheFor(call, model, values);
|
|
323
|
+
return perform(call, dryRun, io, creds.authorize, { responses, cache });
|
|
236
324
|
}
|
|
237
|
-
// Prints the plan's warnings, then the redacted request (--dry-run)
|
|
238
|
-
|
|
325
|
+
// Prints the plan's warnings, then the redacted request (--dry-run), the stored response
|
|
326
|
+
// (a --cache hit) or the live response, which a --cache miss stores after a full success.
|
|
327
|
+
async function perform(call, dryRun, io, authorize, { responses, cache }) {
|
|
239
328
|
for (const w of call.warnings)
|
|
240
329
|
io.stderr.write(`gapi: warning: ${w}\n`);
|
|
330
|
+
const mtime = cache && (await responses.mtime(cache.key));
|
|
331
|
+
const fresh = cache && mtime !== undefined && Date.now() - mtime < cache.maxAgeMs;
|
|
332
|
+
const stored = fresh ? await responses.get(cache.key) : undefined;
|
|
241
333
|
if (dryRun) {
|
|
242
334
|
const headers = { ...call.headers, ...(await authorize(call.auth, { redact: true })) };
|
|
243
335
|
const request = {
|
|
@@ -255,9 +347,28 @@ async function perform(call, dryRun, io, authorize) {
|
|
|
255
347
|
body: p.body,
|
|
256
348
|
}))
|
|
257
349
|
: undefined,
|
|
350
|
+
cache: cache && (stored === undefined ? 'miss' : 'hit'),
|
|
258
351
|
};
|
|
259
352
|
io.stdout.write(`${JSON.stringify(request, null, 2)}\n`);
|
|
260
353
|
return;
|
|
261
354
|
}
|
|
262
|
-
|
|
355
|
+
if (stored !== undefined && mtime !== undefined) {
|
|
356
|
+
io.stdout.write(stored);
|
|
357
|
+
io.stderr.write(`gapi: cached response from ${new Date(mtime).toISOString()}\n`);
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
const chunks = [];
|
|
361
|
+
const out = {
|
|
362
|
+
write: (chunk) => {
|
|
363
|
+
chunks.push(chunk);
|
|
364
|
+
return io.stdout.write(chunk);
|
|
365
|
+
},
|
|
366
|
+
};
|
|
367
|
+
await send(call, { fetch: io.fetch, authorize, out, sleep: io.sleep });
|
|
368
|
+
if (!cache)
|
|
369
|
+
return;
|
|
370
|
+
// Best effort: the response is already printed, so a failed write only warns.
|
|
371
|
+
await responses.put(cache.key, chunks.join('')).catch((e) => {
|
|
372
|
+
io.stderr.write(`gapi: warning: cache write failed: ${reasonOf(e)}\n`);
|
|
373
|
+
});
|
|
263
374
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// Search: finds methods by intent across API entries with an in-memory SQLite FTS5 index (#182 §1).
|
|
2
|
+
// shortcut: DatabaseSync is a documentation-only deprecated alias of Database since Node v26.11.0
|
|
3
|
+
// (DEP0210); switch when @types/node types Database.
|
|
4
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
5
|
+
import { GapiError } from './errors.js';
|
|
6
|
+
import { SEARCH_USAGE } from './render.js';
|
|
7
|
+
// `conversionActions` → `conversion Actions`, `URLInspection` → `URL Inspection`, `a-b_c.d` → `a b c d`.
|
|
8
|
+
const split = (text) => text
|
|
9
|
+
.replace(/([\p{Ll}\p{N}])(\p{Lu})/gu, '$1 $2')
|
|
10
|
+
.replace(/(\p{Lu})(\p{Lu}\p{Ll})/gu, '$1 $2')
|
|
11
|
+
.replace(/[-_.]/g, ' ');
|
|
12
|
+
// Tokens hold only letters and digits, so no FTS5 syntax reaches the match expression.
|
|
13
|
+
export function matchQuery(query) {
|
|
14
|
+
const tokens = split(query)
|
|
15
|
+
.toLowerCase()
|
|
16
|
+
.match(/[\p{L}\p{N}]+/gu);
|
|
17
|
+
if (!tokens)
|
|
18
|
+
throw new GapiError(3, `usage: ${SEARCH_USAGE}`);
|
|
19
|
+
return tokens.map((t) => `"${t}"`).join(' OR ');
|
|
20
|
+
}
|
|
21
|
+
export function search(models, query) {
|
|
22
|
+
const match = matchQuery(query);
|
|
23
|
+
const db = new DatabaseSync(':memory:');
|
|
24
|
+
try {
|
|
25
|
+
db.exec("create virtual table m using fts5(api unindexed, id unindexed, words, extra, descr, tokenize = 'porter unicode61')");
|
|
26
|
+
const insert = db.prepare('insert into m values (?, ?, ?, ?, ?)');
|
|
27
|
+
db.exec('begin');
|
|
28
|
+
for (const model of models) {
|
|
29
|
+
for (const m of model.byId.values()) {
|
|
30
|
+
const words = split([model.api, ...m.words].join(' '));
|
|
31
|
+
const extra = [
|
|
32
|
+
split([model.title, m.request, m.response].filter(Boolean).join(' ')),
|
|
33
|
+
m.upload && 'upload',
|
|
34
|
+
m.download && 'download',
|
|
35
|
+
].filter(Boolean);
|
|
36
|
+
insert.run(model.api, m.id, words, extra.join(' '), m.description ?? '');
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
db.exec('commit');
|
|
40
|
+
const byApi = new Map(models.map((model) => [model.api, model]));
|
|
41
|
+
return db
|
|
42
|
+
.prepare('select api, id from m where m match ? order by bm25(m, 0, 0, 6, 2, 1) limit 20')
|
|
43
|
+
.all(match)
|
|
44
|
+
.flatMap(({ api, id }) => {
|
|
45
|
+
const model = byApi.get(String(api));
|
|
46
|
+
const method = model?.byId.get(String(id));
|
|
47
|
+
return model && method ? [{ model, method }] : [];
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
finally {
|
|
51
|
+
db.close();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
// Media download (epic #30 D7): the response body streams to the --output file.
|
|
2
2
|
import { createWriteStream } from 'node:fs';
|
|
3
|
-
import { rm } from 'node:fs/promises';
|
|
3
|
+
import { access, constants, rm } from 'node:fs/promises';
|
|
4
|
+
import { dirname } from 'node:path';
|
|
4
5
|
import { Readable } from 'node:stream';
|
|
5
6
|
import { pipeline } from 'node:stream/promises';
|
|
7
|
+
import { GapiError, reasonOf } from '../errors.js';
|
|
6
8
|
// The file is created only after a 2xx, so a Google error leaves no empty file behind.
|
|
7
9
|
export async function download(plan, call) {
|
|
10
|
+
// A bad local path is the caller's input (exit 3), found before Google sends any byte.
|
|
11
|
+
await access(dirname(plan.output), constants.W_OK).catch((e) => {
|
|
12
|
+
throw new GapiError(3, `--output: cannot write to ${dirname(plan.output)}: ${reasonOf(e)}`);
|
|
13
|
+
});
|
|
8
14
|
const res = await call(plan.url, { method: plan.httpMethod, headers: plan.headers });
|
|
9
15
|
const file = createWriteStream(plan.output);
|
|
10
16
|
await pipeline(res.body ?? Readable.from([]), file).catch(async (e) => {
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Retry policy shared by Transport and the resumable upload (Google client idioms, #194).
|
|
2
|
+
import { GapiError } from '../errors.js';
|
|
3
|
+
// The request never got an HTTP answer (DNS, reset, timeout): exit 1, like a Google error.
|
|
4
|
+
export class NetworkError extends GapiError {
|
|
5
|
+
}
|
|
6
|
+
// Google's retry rule: 408, 429, 5xx, and a 403 for a rate limit (google-api-python-client).
|
|
7
|
+
export const RETRIES = 3;
|
|
8
|
+
export const retryable = (status, text = '') => status === 408 ||
|
|
9
|
+
status === 429 ||
|
|
10
|
+
status >= 500 ||
|
|
11
|
+
(status === 403 && /"reason":\s*"(user)?[rR]ateLimitExceeded"/.test(text));
|
|
12
|
+
// Truncated exponential backoff with jitter: min(32 s, 1 s × 2^n + 0–1 s), Google's docs.
|
|
13
|
+
export const backoff = (attempt) => Math.min(32_000, 1000 * 2 ** attempt + Math.random() * 1000);
|
|
14
|
+
// Our choice, no Google client reads it: `Retry-After` (seconds or HTTP date) is a floor.
|
|
15
|
+
export const retryAfter = (res) => {
|
|
16
|
+
const v = res.headers.get('Retry-After');
|
|
17
|
+
if (!v)
|
|
18
|
+
return 0;
|
|
19
|
+
const ms = /^\d+$/.test(v) ? Number(v) * 1000 : Date.parse(v) - Date.now();
|
|
20
|
+
return Number.isFinite(ms) ? Math.max(0, ms) : 0;
|
|
21
|
+
};
|
|
@@ -4,19 +4,20 @@ import { openAsBlob } from 'node:fs';
|
|
|
4
4
|
import { open } from 'node:fs/promises';
|
|
5
5
|
import { GapiError, reasonOf } from '../errors.js';
|
|
6
6
|
import { parseOrText } from '../json.js';
|
|
7
|
+
import { backoff, NetworkError, RETRIES, retryable } from './retry.js';
|
|
7
8
|
// Google wants chunks in multiples of 256 KiB; 8 MiB keeps the request count low.
|
|
8
9
|
const CHUNK = 8 * 1024 * 1024;
|
|
9
10
|
// Consecutive 308s that move the upload no further before gapi gives up.
|
|
10
11
|
const MAX_STALLS = 3;
|
|
11
12
|
// Returns the final response; Transport prints it like a `once` call.
|
|
12
|
-
export async function upload(plan, call) {
|
|
13
|
+
export async function upload(plan, call, sleep) {
|
|
13
14
|
const file = await readable(plan.upload.path);
|
|
14
15
|
const { protocol, contentType } = plan.upload;
|
|
15
16
|
if (protocol === 'media') {
|
|
16
17
|
return call(plan.url, { method: plan.httpMethod, headers: plan.headers, body: file });
|
|
17
18
|
}
|
|
18
19
|
if (protocol === 'resumable')
|
|
19
|
-
return resumable(plan, file, call);
|
|
20
|
+
return resumable(plan, file, call, sleep);
|
|
20
21
|
// multipart/related: the JSON metadata part, then the media part.
|
|
21
22
|
const boundary = randomUUID();
|
|
22
23
|
const body = new Blob([
|
|
@@ -45,7 +46,9 @@ async function readable(path) {
|
|
|
45
46
|
return openAsBlob(path);
|
|
46
47
|
}
|
|
47
48
|
// Session request on the upload URL, then PUT chunks to the session URI; 308 = send more.
|
|
48
|
-
|
|
49
|
+
// A 5xx or a network error on a chunk asks the session what it kept (`bytes */<size>`) and
|
|
50
|
+
// goes on from there (google-api-python-client `next_chunk`, @google-cloud/storage).
|
|
51
|
+
async function resumable(plan, file, call, sleep) {
|
|
49
52
|
const session = await call(plan.url, {
|
|
50
53
|
method: plan.httpMethod,
|
|
51
54
|
headers: { ...plan.headers, 'X-Upload-Content-Length': String(file.size) },
|
|
@@ -58,24 +61,46 @@ async function resumable(plan, file, call) {
|
|
|
58
61
|
}
|
|
59
62
|
// undici holds the connection until a body is read or cancelled; only headers matter here.
|
|
60
63
|
await session.body?.cancel();
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
64
|
+
// A retryable answer or no answer at all is undefined: the caller asks the session again.
|
|
65
|
+
const put = (range, body) => call(location, { method: 'PUT', headers: { 'Content-Range': range }, body }, (s) => s === 308 || retryable(s)).then(async (res) => (retryable(res.status) ? void (await res.body?.cancel()) : res), (e) => {
|
|
66
|
+
if (e instanceof NetworkError)
|
|
67
|
+
return undefined;
|
|
68
|
+
throw e;
|
|
69
|
+
});
|
|
70
|
+
// Every final failure names the session, so the operator sees where the upload stopped.
|
|
71
|
+
try {
|
|
72
|
+
let start = 0;
|
|
73
|
+
let stalls = 0;
|
|
74
|
+
let failures = 0;
|
|
75
|
+
for (;;) {
|
|
76
|
+
const end = Math.min(start + CHUNK, file.size);
|
|
77
|
+
const range = file.size === 0 ? 'bytes */0' : `bytes ${start}-${end - 1}/${file.size}`;
|
|
78
|
+
let res = await put(range, file.slice(start, end));
|
|
79
|
+
while (!res) {
|
|
80
|
+
if (++failures > RETRIES)
|
|
81
|
+
throw new GapiError(1, `resumable upload: no answer at byte ${start}`);
|
|
82
|
+
await sleep(backoff(failures - 1));
|
|
83
|
+
res = await put(`bytes */${file.size}`);
|
|
84
|
+
}
|
|
85
|
+
if (res.status !== 308)
|
|
86
|
+
return res;
|
|
87
|
+
// `Range: bytes=0-<last>` names the bytes Google kept; no Range means it kept none.
|
|
88
|
+
const kept = /^bytes=0-(\d+)$/.exec(res.headers.get('Range') ?? '');
|
|
89
|
+
const next = kept ? Number(kept[1]) + 1 : 0;
|
|
90
|
+
stalls = next > start ? 0 : stalls + 1;
|
|
91
|
+
if (next > start)
|
|
92
|
+
failures = 0;
|
|
93
|
+
if (stalls === MAX_STALLS || next >= file.size) {
|
|
94
|
+
const body = parseOrText(await res.text());
|
|
95
|
+
throw new GapiError(1, `resumable upload: 308 at byte ${next} of ${file.size}, no progress`, body);
|
|
96
|
+
}
|
|
97
|
+
await res.body?.cancel();
|
|
98
|
+
start = next;
|
|
77
99
|
}
|
|
78
|
-
|
|
79
|
-
|
|
100
|
+
}
|
|
101
|
+
catch (e) {
|
|
102
|
+
if (!(e instanceof GapiError))
|
|
103
|
+
throw e;
|
|
104
|
+
throw new GapiError(e.exit, `${e.message}; session ${location}`, e.body);
|
|
80
105
|
}
|
|
81
106
|
}
|
package/dist/src/transport.js
CHANGED
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
// Transport: sends a CallPlan and prints the response (epic #30 D13).
|
|
2
2
|
// `authorize` adds the credential before every request; a non-2xx exits 1, a missing scope 2.
|
|
3
|
-
import {
|
|
3
|
+
import { setTimeout } from 'node:timers/promises';
|
|
4
|
+
import { GapiError, reasonOf } from './errors.js';
|
|
4
5
|
import { isRecord, parseOrText } from './json.js';
|
|
5
6
|
import { batch } from './transport/batch.js';
|
|
6
7
|
import { download } from './transport/download.js';
|
|
8
|
+
import { backoff, NetworkError, RETRIES, retryAfter, retryable, } from './transport/retry.js';
|
|
7
9
|
import { upload } from './transport/upload.js';
|
|
8
10
|
export async function send(plan, deps) {
|
|
9
11
|
const call = caller(plan, deps);
|
|
10
12
|
const print = (data) => deps.out.write(`${JSON.stringify(data, null, 2)}\n`);
|
|
11
13
|
if (plan.send === 'upload') {
|
|
12
|
-
print(await json(await upload(plan, call)));
|
|
14
|
+
print(await json(await upload(plan, call, deps.sleep ?? setTimeout)));
|
|
13
15
|
return;
|
|
14
16
|
}
|
|
15
17
|
if (plan.send === 'download') {
|
|
@@ -27,9 +29,14 @@ export async function send(plan, deps) {
|
|
|
27
29
|
print(plan.decode ? plan.decode(results) : results);
|
|
28
30
|
if (sent.error)
|
|
29
31
|
throw sent.error;
|
|
30
|
-
const failed = results.filter((r) => r.status < 200 || r.status > 299)
|
|
31
|
-
|
|
32
|
-
|
|
32
|
+
const failed = sent.results.filter((r) => r.status < 200 || r.status > 299);
|
|
33
|
+
// A part that lacks a scope gets the same exit 2 and hint as a single call.
|
|
34
|
+
const scope = failed.map((r) => failure(plan, r.status, r.text)).find((e) => e.exit === 2);
|
|
35
|
+
if (scope)
|
|
36
|
+
throw scope;
|
|
37
|
+
if (failed.length > 0) {
|
|
38
|
+
throw new GapiError(1, `${failed.length} of ${results.length} batch parts failed`);
|
|
39
|
+
}
|
|
33
40
|
return;
|
|
34
41
|
}
|
|
35
42
|
const init = {
|
|
@@ -60,13 +67,30 @@ export async function send(plan, deps) {
|
|
|
60
67
|
}
|
|
61
68
|
}
|
|
62
69
|
}
|
|
63
|
-
|
|
70
|
+
// Uploads and batches never retry here; a resumable upload resumes on its own.
|
|
71
|
+
function caller(plan, { fetch, authorize, sleep = setTimeout }) {
|
|
72
|
+
const retry = plan.retry && plan.send !== 'upload' && plan.send !== 'batch';
|
|
64
73
|
return async (url, init, allowed) => {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
74
|
+
for (let attempt = 0;; attempt++) {
|
|
75
|
+
const headers = { ...init.headers, ...(await authorize(plan.auth)) };
|
|
76
|
+
const res = await fetch(url, { ...init, headers }).catch((e) => {
|
|
77
|
+
const cause = e instanceof Error && e.cause !== undefined ? e.cause : e;
|
|
78
|
+
return new NetworkError(1, `network: ${reasonOf(cause)}`);
|
|
79
|
+
});
|
|
80
|
+
const last = !retry || attempt === RETRIES;
|
|
81
|
+
if (res instanceof NetworkError) {
|
|
82
|
+
if (last)
|
|
83
|
+
throw res;
|
|
84
|
+
await sleep(backoff(attempt));
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
if (res.ok || allowed?.(res.status))
|
|
88
|
+
return res;
|
|
89
|
+
const text = await res.text();
|
|
90
|
+
if (last || !retryable(res.status, text))
|
|
91
|
+
throw failure(plan, res.status, text);
|
|
92
|
+
await sleep(Math.max(backoff(attempt), retryAfter(res)));
|
|
93
|
+
}
|
|
70
94
|
};
|
|
71
95
|
}
|
|
72
96
|
// A 2xx body that is not JSON is Google's surprise, not gapi's: exit 1 with the body.
|
|
@@ -81,9 +105,11 @@ async function json(res) {
|
|
|
81
105
|
throw new GapiError(1, `HTTP ${res.status}: the response body is not JSON`, text);
|
|
82
106
|
}
|
|
83
107
|
}
|
|
84
|
-
function failure(plan, status, text) {
|
|
108
|
+
export function failure(plan, status, text) {
|
|
85
109
|
// A front-end error page can be HTML; keep it as text rather than fail on parse.
|
|
86
|
-
|
|
110
|
+
// A streaming method wraps the error in an array (Go googleapi errorReplyFromBody).
|
|
111
|
+
const parsed = parseOrText(text);
|
|
112
|
+
const body = Array.isArray(parsed) ? parsed[0] : parsed;
|
|
87
113
|
const error = isRecord(body) && isRecord(body.error) ? body.error : {};
|
|
88
114
|
// D8: the stored grant lacks this method's scope; a new consent fixes it. AIP-193: the
|
|
89
115
|
// (reason, domain) pair of the ErrorInfo detail identifies the error.
|
package/package.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "google-apis-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "One discovery-driven CLI (gapi) for 47 Google marketing and data APIs: Ads, GA4, Search Console, Tag Manager, Indexing, Merchant, Business Profile, SA360, CM360, DV360, YouTube, PageSpeed, CrUX, Sheets, Places and more. One login, read+write.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
7
|
"gapi": "dist/bin/gapi.js"
|
|
8
8
|
},
|
|
9
9
|
"engines": {
|
|
10
|
-
"node": ">=
|
|
10
|
+
"node": ">=26"
|
|
11
11
|
},
|
|
12
12
|
"scripts": {
|
|
13
13
|
"test": "node --test",
|
|
@@ -55,13 +55,13 @@
|
|
|
55
55
|
"url": "https://github.com/realroboto/google-apis-cli/issues"
|
|
56
56
|
},
|
|
57
57
|
"dependencies": {
|
|
58
|
-
"google-auth-library": "^11.3.
|
|
58
|
+
"google-auth-library": "^11.3.3"
|
|
59
59
|
},
|
|
60
60
|
"license": "MIT",
|
|
61
61
|
"devDependencies": {
|
|
62
|
-
"@biomejs/biome": "^2.5
|
|
63
|
-
"@types/node": "^
|
|
62
|
+
"@biomejs/biome": "^2.6.5",
|
|
63
|
+
"@types/node": "^26.6.6",
|
|
64
64
|
"typescript": "^7.0.2"
|
|
65
65
|
},
|
|
66
|
-
"packageManager": "pnpm@12.
|
|
66
|
+
"packageManager": "pnpm@12.16.1+sha512.091d1dee7927575661b25dab3c2ae31cafc1c54d6b71ed6a6600cf0c1cb10434a447eae14015f1b23e2e6bedeed79df94a932e2164d2832280dce3fb32f656d1"
|
|
67
67
|
}
|