google-apis-cli 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -14
- package/dist/bin/gapi.js +10 -3
- package/dist/src/apis/ads.js +43 -1
- package/dist/src/apis/bigquery.js +191 -0
- package/dist/src/apis/business-information.js +32 -0
- package/dist/src/apis/cloudquotas.js +4 -0
- package/dist/src/apis/crux.js +4 -0
- package/dist/src/apis/ga4-admin.js +47 -0
- package/dist/src/apis/ga4-data-alpha.js +1 -0
- package/dist/src/apis/ga4-data.js +8 -0
- package/dist/src/apis/gsc.js +12 -1
- package/dist/src/apis/gtm.js +35 -0
- package/dist/src/apis/merchant-accounts.js +13 -0
- package/dist/src/apis/merchant-reports.js +1 -0
- package/dist/src/apis/pagespeed.js +4 -0
- package/dist/src/apis/places.js +4 -0
- package/dist/src/apis/sa360.js +1 -0
- package/dist/src/apis/web-risk.js +4 -0
- package/dist/src/apis/youtube.js +15 -0
- package/dist/src/catalog.js +49 -7
- package/dist/src/credentials.js +42 -16
- package/dist/src/entities.js +180 -0
- package/dist/src/plan.js +55 -8
- package/dist/src/render.js +95 -4
- package/dist/src/responses.js +37 -0
- package/dist/src/run.js +301 -21
- package/dist/src/search.js +208 -0
- package/dist/src/transport/download.js +7 -1
- package/dist/src/transport/retry.js +21 -0
- package/dist/src/transport/upload.js +46 -21
- package/dist/src/transport.js +72 -39
- package/package.json +7 -7
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
export default {
|
|
2
2
|
api: 'web-risk',
|
|
3
3
|
discovery: 'https://webrisk.googleapis.com/$discovery/rest?version=v1',
|
|
4
|
+
nocache: {
|
|
5
|
+
reason: 'Lookup results expire at the response expireTime, not a fixed age',
|
|
6
|
+
source: 'https://cloud.google.com/web-risk/docs/caching',
|
|
7
|
+
},
|
|
4
8
|
paid: {
|
|
5
9
|
'webrisk.hashes.search': {
|
|
6
10
|
reason: 'Paid only',
|
package/dist/src/apis/youtube.js
CHANGED
|
@@ -1,4 +1,19 @@
|
|
|
1
|
+
import { items, text } from '../entities.js';
|
|
2
|
+
import { isRecord } from '../json.js';
|
|
1
3
|
export default {
|
|
2
4
|
api: 'youtube',
|
|
3
5
|
discovery: 'https://youtube.googleapis.com/$discovery/rest?version=v3',
|
|
6
|
+
// The operator's own channels; the custom URL (@handle) joins the title in the name.
|
|
7
|
+
entities: [
|
|
8
|
+
{
|
|
9
|
+
method: 'youtube.channels.list',
|
|
10
|
+
params: () => ({ part: 'snippet', mine: true }),
|
|
11
|
+
entities: (data) => items(data, 'items').flatMap((c) => {
|
|
12
|
+
const id = text(c.id);
|
|
13
|
+
const s = isRecord(c.snippet) ? c.snippet : {};
|
|
14
|
+
const name = [text(s.title), text(s.customUrl)].filter(Boolean).join(' ');
|
|
15
|
+
return id ? [{ kind: 'channel', id, name: name || undefined }] : [];
|
|
16
|
+
}),
|
|
17
|
+
},
|
|
18
|
+
],
|
|
4
19
|
};
|
package/dist/src/catalog.js
CHANGED
|
@@ -8,10 +8,11 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
|
|
|
8
8
|
};
|
|
9
9
|
// Catalog: the API entries found by glob, their discovery documents behind a
|
|
10
10
|
// 24 h disk cache, and the one normalized ApiModel per API (epic #30 D2, D8, D13).
|
|
11
|
+
import { randomUUID } from 'node:crypto';
|
|
11
12
|
import { mkdir, readdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
|
|
12
13
|
import { basename, dirname, extname, join } from 'node:path';
|
|
13
14
|
import { pathToFileURL } from 'node:url';
|
|
14
|
-
import { GapiError } from './errors.js';
|
|
15
|
+
import { GapiError, reasonOf } from './errors.js';
|
|
15
16
|
import { isRecord, parseOrText } from './json.js';
|
|
16
17
|
const TTL_MS = 24 * 60 * 60 * 1000;
|
|
17
18
|
const AUTH_PREFIX = 'https://www.googleapis.com/auth/';
|
|
@@ -42,14 +43,17 @@ export async function openCatalog({ dir, fetch, warn, apisDir = join(import.meta
|
|
|
42
43
|
async function document(entry, refresh) {
|
|
43
44
|
const file = join(cacheDir, `${entry.api}.json`);
|
|
44
45
|
const mtime = await stat(file).then((s) => s.mtimeMs, () => undefined);
|
|
45
|
-
|
|
46
|
+
// A future mtime (clock skew) counts as stale, so no document freezes.
|
|
47
|
+
const age = mtime === undefined ? -1 : Date.now() - Math.floor(mtime); // mtimeMs has sub-ms digits, Date.now() not: a just-written file reads as future
|
|
48
|
+
if (!refresh && age >= 0 && age < TTL_MS) {
|
|
46
49
|
const doc = await cached(file);
|
|
47
50
|
if (doc)
|
|
48
51
|
return doc;
|
|
49
52
|
}
|
|
50
53
|
const got = await download(entry.discovery).catch((e) => e instanceof Error ? e : new Error(String(e)));
|
|
51
54
|
if (!(got instanceof Error)) {
|
|
52
|
-
|
|
55
|
+
// The document is already here; a read-only cache dir only costs the next fetch.
|
|
56
|
+
await writeAtomic(file, got.text).catch((e) => warn(`cannot cache the ${entry.api} document: ${reasonOf(e)}`));
|
|
53
57
|
return got.doc;
|
|
54
58
|
}
|
|
55
59
|
const stale = mtime === undefined ? undefined : await cached(file);
|
|
@@ -72,6 +76,25 @@ export async function openCatalog({ dir, fetch, warn, apisDir = join(import.meta
|
|
|
72
76
|
}
|
|
73
77
|
return { entries, load, cacheDir };
|
|
74
78
|
}
|
|
79
|
+
// Every entity source of an entry, children included.
|
|
80
|
+
export function entitySources(entry) {
|
|
81
|
+
const sources = [...(entry.entities ?? [])];
|
|
82
|
+
for (const source of sources)
|
|
83
|
+
sources.push(...Object.values(source.children ?? {}));
|
|
84
|
+
return sources;
|
|
85
|
+
}
|
|
86
|
+
// `discovery:check` part: entity sources that call another entry's method, checked against
|
|
87
|
+
// that entry's model. Kept out of `load`, so one API never fetches another API's document.
|
|
88
|
+
export function foreignSourceProblems(entry, models) {
|
|
89
|
+
return entitySources(entry).flatMap((s) => {
|
|
90
|
+
if (s.api === undefined || s.api === entry.api)
|
|
91
|
+
return [];
|
|
92
|
+
const model = models.get(s.api);
|
|
93
|
+
if (!model)
|
|
94
|
+
return [`entities ${s.api} ${s.method}: api not loaded`];
|
|
95
|
+
return model.byId.has(s.method) ? [] : [`entities ${s.api} ${s.method}: not in the document`];
|
|
96
|
+
});
|
|
97
|
+
}
|
|
75
98
|
// `https://www.googleapis.com/auth/x` → `x`; any other scope URL stays as is.
|
|
76
99
|
export const shortScope = (scope) => scope.startsWith(AUTH_PREFIX) ? scope.slice(AUTH_PREFIX.length) : scope;
|
|
77
100
|
const isNever = (scope) => {
|
|
@@ -119,6 +142,21 @@ function toModel(doc, entry) {
|
|
|
119
142
|
overlay('paid', entry.paid, (m, v) => {
|
|
120
143
|
m.paid = v;
|
|
121
144
|
});
|
|
145
|
+
// Only a POST: a `reads` id on a write would let --cache store it and Transport resend it.
|
|
146
|
+
overlay('reads', Object.fromEntries((entry.reads ?? []).map((id) => [id, true])), (m) => {
|
|
147
|
+
if (m.httpMethod === 'POST')
|
|
148
|
+
m.read = true;
|
|
149
|
+
else
|
|
150
|
+
problems.push(`reads ${m.id}: ${m.httpMethod}, not a POST`);
|
|
151
|
+
});
|
|
152
|
+
// An entity source on a method the document lacks would fail every sync of the API. A
|
|
153
|
+
// source of another entry's method is checked by `foreignSourceProblems`.
|
|
154
|
+
for (const source of entitySources(entry)) {
|
|
155
|
+
if (source.api !== undefined && source.api !== entry.api)
|
|
156
|
+
continue;
|
|
157
|
+
if (!byId.has(source.method))
|
|
158
|
+
problems.push(`entities ${source.method}: not in the document`);
|
|
159
|
+
}
|
|
122
160
|
return {
|
|
123
161
|
...root,
|
|
124
162
|
api: entry.api,
|
|
@@ -129,6 +167,7 @@ function toModel(doc, entry) {
|
|
|
129
167
|
documentationLink: doc.documentationLink,
|
|
130
168
|
rootUrl: doc.rootUrl ?? '',
|
|
131
169
|
servicePath: doc.servicePath ?? '',
|
|
170
|
+
batchPath: doc.batchPath ?? 'batch',
|
|
132
171
|
auth,
|
|
133
172
|
scopes,
|
|
134
173
|
byId,
|
|
@@ -190,14 +229,17 @@ async function loadEntries(dir) {
|
|
|
190
229
|
const name = basename(f, extname(f));
|
|
191
230
|
if (e.api !== name)
|
|
192
231
|
throw new Error(`${file}: api '${e.api}' must equal '${name}'`);
|
|
232
|
+
if (!/^[a-z0-9-]+$/.test(e.api))
|
|
233
|
+
throw new Error(`${file}: api '${e.api}' must match [a-z0-9-]+`);
|
|
193
234
|
entries[e.api] = e;
|
|
194
235
|
}
|
|
195
236
|
return entries;
|
|
196
237
|
}
|
|
197
|
-
//
|
|
198
|
-
|
|
238
|
+
// gcloud WriteFileAtomically: a new temp file (mode 600, `wx` never reuses a looser one),
|
|
239
|
+
// flushed to disk, then renamed over the target; readers see the old or the new file.
|
|
240
|
+
export async function writeAtomic(file, text) {
|
|
199
241
|
await mkdir(dirname(file), { recursive: true, mode: 0o700 });
|
|
200
|
-
const tmp = `${file}.${
|
|
201
|
-
await writeFile(tmp, text, { mode: 0o600 });
|
|
242
|
+
const tmp = `${file}.${randomUUID()}.tmp`;
|
|
243
|
+
await writeFile(tmp, text, { mode: 0o600, flag: 'wx', flush: true });
|
|
202
244
|
await rename(tmp, file);
|
|
203
245
|
}
|
package/dist/src/credentials.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
// Credentials: the only reader and writer of config.json and credentials.json (epic #30 D8, D13).
|
|
2
2
|
// Settings precedence, Bearer vs API key, token refresh + persistence, loopback/manual login.
|
|
3
|
-
import { randomBytes } from 'node:crypto';
|
|
4
|
-
import { chmod,
|
|
3
|
+
import { hash, randomBytes } from 'node:crypto';
|
|
4
|
+
import { chmod, readFile, rm } from 'node:fs/promises';
|
|
5
5
|
import { createServer } from 'node:http';
|
|
6
6
|
import { homedir } from 'node:os';
|
|
7
7
|
import { join } from 'node:path';
|
|
8
8
|
import { createInterface } from 'node:readline/promises';
|
|
9
9
|
import { Writable } from 'node:stream';
|
|
10
10
|
import { CodeChallengeMethod, OAuth2Client } from 'google-auth-library';
|
|
11
|
-
import { shortScope } from './catalog.js';
|
|
11
|
+
import { shortScope, writeAtomic } from './catalog.js';
|
|
12
12
|
import { codeOf, GapiError, messageOf } from './errors.js';
|
|
13
13
|
import { isRecord } from './json.js';
|
|
14
14
|
const LOOPBACK_HOST = '127.0.0.1';
|
|
@@ -62,12 +62,11 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
62
62
|
throw new GapiError(2, `${configFile} is not a JSON object; fix or remove it`);
|
|
63
63
|
return cfg;
|
|
64
64
|
};
|
|
65
|
-
// Owner-only:
|
|
66
|
-
//
|
|
65
|
+
// Owner-only: file 600 by temp file + rename, so a crash or a concurrent gapi never
|
|
66
|
+
// leaves a broken file. `mkdir` mode applies only on create: narrow an existing dir too.
|
|
67
67
|
const writeSecure = async (file, data) => {
|
|
68
|
-
await
|
|
69
|
-
await
|
|
70
|
-
await chmod(file, 0o600);
|
|
68
|
+
await writeAtomic(file, JSON.stringify(data, null, 2));
|
|
69
|
+
await chmod(dir, 0o700);
|
|
71
70
|
};
|
|
72
71
|
const readCredentials = async () => {
|
|
73
72
|
let creds;
|
|
@@ -95,6 +94,9 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
95
94
|
const { tokens } = await c.getToken({ code, codeVerifier }).catch((e) => {
|
|
96
95
|
throw new GapiError(2, `token exchange failed: ${messageOf(e)}`);
|
|
97
96
|
});
|
|
97
|
+
// Google: "refresh tokens are always returned for installed applications".
|
|
98
|
+
if (!tokens.refresh_token)
|
|
99
|
+
throw new GapiError(2, `no refresh token returned; ${RELOGIN}`);
|
|
98
100
|
await writeSecure(credentialsFile, tokens);
|
|
99
101
|
return tokens;
|
|
100
102
|
};
|
|
@@ -137,8 +139,10 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
137
139
|
const c = await client();
|
|
138
140
|
c.setCredentials(await readCredentials());
|
|
139
141
|
// An event listener is not awaited: keep the write so the refresh below can await it.
|
|
142
|
+
// Copy now: google-auth-library puts the old refresh token back on this object.
|
|
140
143
|
c.on('tokens', (t) => {
|
|
141
|
-
|
|
144
|
+
const fresh = { ...t };
|
|
145
|
+
saving = readCredentials().then((stored) => writeSecure(credentialsFile, { ...stored, ...fresh }));
|
|
142
146
|
});
|
|
143
147
|
tokenClient = c;
|
|
144
148
|
}
|
|
@@ -158,6 +162,14 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
158
162
|
throw new GapiError(2, `no access token; ${RELOGIN}`);
|
|
159
163
|
return { Authorization: `Bearer ${token}` };
|
|
160
164
|
};
|
|
165
|
+
// Response cache key part (ADR 0001): one per Google account, never the token itself.
|
|
166
|
+
// No stored token → undefined; entries are stored only after an authorized success.
|
|
167
|
+
async function identity(auth) {
|
|
168
|
+
if (auth.kind === 'apiKey')
|
|
169
|
+
return 'apiKey';
|
|
170
|
+
const token = (await readCredentials().catch(() => undefined))?.refresh_token;
|
|
171
|
+
return token ? hash('sha256', token) : undefined;
|
|
172
|
+
}
|
|
161
173
|
// Both flows send `state` and a PKCE S256 challenge, and redeem one redirect URL.
|
|
162
174
|
async function login({ consent, manual = false, prompt = ask, }) {
|
|
163
175
|
const c = await client(REDIRECT_URI);
|
|
@@ -171,17 +183,24 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
171
183
|
code_challenge_method: CodeChallengeMethod.S256,
|
|
172
184
|
code_challenge: codeChallenge,
|
|
173
185
|
});
|
|
174
|
-
|
|
186
|
+
// `state` first, so only Google's own redirect can end the login (CSRF).
|
|
187
|
+
const redeem = async (redirect) => {
|
|
175
188
|
const query = redirect.searchParams;
|
|
189
|
+
if (query.get('state') !== state)
|
|
190
|
+
throw new GapiError(2, 'state mismatch');
|
|
176
191
|
const error = query.get('error');
|
|
177
192
|
if (error)
|
|
178
193
|
throw new GapiError(2, `consent failed: ${error}`);
|
|
179
|
-
if (query.get('state') !== state)
|
|
180
|
-
throw new GapiError(2, 'state mismatch');
|
|
181
194
|
const code = query.get('code');
|
|
182
195
|
if (!code)
|
|
183
196
|
throw new GapiError(2, 'no code in redirect');
|
|
184
|
-
|
|
197
|
+
const tokens = await exchange(c, code, codeVerifier);
|
|
198
|
+
// Google: "Your app must verify which scopes were actually granted".
|
|
199
|
+
const granted = new Set((tokens.scope ?? '').split(' '));
|
|
200
|
+
const missing = consent.filter((s) => !granted.has(s)).map(shortScope);
|
|
201
|
+
if (missing.length > 0)
|
|
202
|
+
warn(`scopes not granted: ${missing.join(', ')}`);
|
|
203
|
+
return tokens;
|
|
185
204
|
};
|
|
186
205
|
if (manual) {
|
|
187
206
|
stderr.write(`Open this URL and grant access:\n${url}\n` +
|
|
@@ -200,6 +219,11 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
200
219
|
res.end();
|
|
201
220
|
return;
|
|
202
221
|
}
|
|
222
|
+
if (redirect.searchParams.get('state') !== state) {
|
|
223
|
+
res.statusCode = 400; // not Google's redirect: keep waiting for it
|
|
224
|
+
res.end('gapi: state mismatch');
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
203
227
|
try {
|
|
204
228
|
const tokens = await redeem(redirect);
|
|
205
229
|
res.end('gapi: authorized. You can close this tab.');
|
|
@@ -235,7 +259,8 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
235
259
|
};
|
|
236
260
|
}
|
|
237
261
|
// Revoke, then delete (the `gcloud auth revoke` pattern, #151). Revoking the refresh
|
|
238
|
-
// token ends the whole grant. Google answers 400 for a token already
|
|
262
|
+
// token ends the whole grant. Google answers 400 `invalid_token` for a token already
|
|
263
|
+
// invalid or revoked; another 400 error code is a failed request.
|
|
239
264
|
// Any other failure keeps credentials.json, so the logout can be retried.
|
|
240
265
|
// No stored token (missing or unreadable file) leaves nothing to revoke. The revoke
|
|
241
266
|
// endpoint takes only the token, so no OAuth client id or secret is needed.
|
|
@@ -245,7 +270,8 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
245
270
|
await oauth()
|
|
246
271
|
.revokeToken(token)
|
|
247
272
|
.catch((e) => {
|
|
248
|
-
|
|
273
|
+
const data = isRecord(e) && isRecord(e.response) ? e.response.data : undefined;
|
|
274
|
+
if (!isRecord(data) || data.error !== 'invalid_token')
|
|
249
275
|
throw new GapiError(1, `token revoke failed, credentials kept: ${messageOf(e)}`);
|
|
250
276
|
warn('refresh token already invalid or revoked');
|
|
251
277
|
});
|
|
@@ -277,7 +303,7 @@ export function openCredentials({ dir, env, oauth = (opts) => new OAuth2Client(o
|
|
|
277
303
|
await writeSecure(configFile, cfg);
|
|
278
304
|
return configFile;
|
|
279
305
|
}
|
|
280
|
-
return { settings, authorize, login, status, logout, setup };
|
|
306
|
+
return { settings, authorize, identity, login, status, logout, setup };
|
|
281
307
|
}
|
|
282
308
|
// Decode a JWT payload without verifying (display only; it is our own token).
|
|
283
309
|
function accountFromIdToken(idToken) {
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
// Entities (#200 D6, ADR 0002): the offline index of the operator's own accounts, properties
|
|
2
|
+
// and sites in <config dir>/entities.db, mode 600. Rows hold the Credentials identity and live
|
|
3
|
+
// at most 7 days; login and logout delete the file. Nothing here ever picks an ID for a call.
|
|
4
|
+
// shortcut: DatabaseSync is a deprecated alias of Database since Node v26.11.0 (DEP0210), switch
|
|
5
|
+
// when @types/node types Database.
|
|
6
|
+
import { existsSync } from 'node:fs';
|
|
7
|
+
import { mkdir, rm, stat, writeFile } from 'node:fs/promises';
|
|
8
|
+
import { dirname } from 'node:path';
|
|
9
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
10
|
+
import { GapiError } from './errors.js';
|
|
11
|
+
import { isRecord } from './json.js';
|
|
12
|
+
import { MAX_AGE_MS } from './responses.js';
|
|
13
|
+
const VERSION = 1; // PRAGMA user_version; another value drops the file, sync rebuilds it
|
|
14
|
+
const NOT_A_DATABASE = 26; // SQLITE_NOTADB: the file is dropped like another version
|
|
15
|
+
const BUSY_MS = 5000; // another gapi process may hold the write lock
|
|
16
|
+
const SCHEMA = `create virtual table if not exists entity using fts5(
|
|
17
|
+
api unindexed, kind unindexed, id, name, parent, domain, identity unindexed, synced unindexed,
|
|
18
|
+
tokenize = 'trigram')`;
|
|
19
|
+
// Domain key: URL host, lower case (IDN as punycode), no leading `www.`.
|
|
20
|
+
// shortcut: no public-suffix parsing, so `shop.acme.com` does not link to `acme.com`; upgrade
|
|
21
|
+
// when a source needs subdomain linking.
|
|
22
|
+
export function domainOf(uri) {
|
|
23
|
+
if (typeof uri !== 'string' || uri === '')
|
|
24
|
+
return undefined;
|
|
25
|
+
const bare = uri.replace(/^sc-domain:/, '');
|
|
26
|
+
const url = URL.parse(/^[a-z][a-z\d+.-]*:\/\//i.test(bare) ? bare : `https://${bare}`);
|
|
27
|
+
return url?.hostname.replace(/^www\./, '') || undefined;
|
|
28
|
+
}
|
|
29
|
+
// The record items of `data[key]`, for a source's `entities` mapping.
|
|
30
|
+
export const items = (data, key) => isRecord(data) && Array.isArray(data[key]) ? data[key].filter(isRecord) : [];
|
|
31
|
+
export const text = (v) => typeof v === 'string' && v ? v : undefined;
|
|
32
|
+
// Schema version; undefined for a file that is not a database. Any other error (a lock) throws,
|
|
33
|
+
// so a busy index is never dropped.
|
|
34
|
+
function versionOf(db) {
|
|
35
|
+
try {
|
|
36
|
+
return Number(db.prepare('pragma user_version').get()?.user_version);
|
|
37
|
+
}
|
|
38
|
+
catch (e) {
|
|
39
|
+
if (e instanceof Error && 'errcode' in e && e.errcode === NOT_A_DATABASE)
|
|
40
|
+
return undefined;
|
|
41
|
+
throw e;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
export function openEntities(file) {
|
|
45
|
+
const clear = async () => {
|
|
46
|
+
await Promise.all(['', '-journal'].map((s) => rm(`${file}${s}`, { force: true })));
|
|
47
|
+
};
|
|
48
|
+
// Created mode 600 before SQLite opens it (SQLite gives its journal the same mode). `fresh`:
|
|
49
|
+
// the file is empty, so it is new (or a racing process has not written it yet).
|
|
50
|
+
const create = async () => {
|
|
51
|
+
await mkdir(dirname(file), { recursive: true, mode: 0o700 });
|
|
52
|
+
await writeFile(file, '', { mode: 0o600, flag: 'a' });
|
|
53
|
+
const fresh = (await stat(file)).size === 0;
|
|
54
|
+
return { db: new DatabaseSync(file, { timeout: BUSY_MS }), fresh };
|
|
55
|
+
};
|
|
56
|
+
// A new file gets the schema in place. A file of another schema version, or not a database
|
|
57
|
+
// at all, is dropped and rebuilt empty.
|
|
58
|
+
async function open() {
|
|
59
|
+
let { db, fresh } = await create();
|
|
60
|
+
try {
|
|
61
|
+
if (!fresh && versionOf(db) === VERSION)
|
|
62
|
+
return db;
|
|
63
|
+
if (!fresh) {
|
|
64
|
+
db.close();
|
|
65
|
+
await clear();
|
|
66
|
+
({ db } = await create());
|
|
67
|
+
}
|
|
68
|
+
db.exec(SCHEMA);
|
|
69
|
+
db.exec(`pragma user_version = ${VERSION}`);
|
|
70
|
+
return db;
|
|
71
|
+
}
|
|
72
|
+
catch (e) {
|
|
73
|
+
if (db.isOpen)
|
|
74
|
+
db.close();
|
|
75
|
+
throw e;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
async function use(fn) {
|
|
79
|
+
const db = await open();
|
|
80
|
+
try {
|
|
81
|
+
return fn(db);
|
|
82
|
+
}
|
|
83
|
+
finally {
|
|
84
|
+
db.close();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Every run: rows older than the response cache max age go (ADR 0001 terms, ADR 0002).
|
|
88
|
+
async function sweep() {
|
|
89
|
+
if (!existsSync(file))
|
|
90
|
+
return;
|
|
91
|
+
await use((db) => db.prepare('delete from entity where synced < ?').run(Date.now() - MAX_AGE_MS));
|
|
92
|
+
}
|
|
93
|
+
// Reads every source of one API, then replaces that API's rows in one transaction; a
|
|
94
|
+
// failed call throws before the write, so the old rows stay. Rows of one kind and ID merge
|
|
95
|
+
// (later defined fields win), so a child can add a domain or name. Returns the row count.
|
|
96
|
+
async function sync(api, sources, identity, request) {
|
|
97
|
+
const merged = new Map();
|
|
98
|
+
const walk = async (source, parent) => {
|
|
99
|
+
const found = [];
|
|
100
|
+
const call = {
|
|
101
|
+
api: source.api,
|
|
102
|
+
method: source.method,
|
|
103
|
+
params: source.params?.(parent) ?? {},
|
|
104
|
+
body: source.body?.(parent),
|
|
105
|
+
headers: source.headers?.(parent),
|
|
106
|
+
};
|
|
107
|
+
for await (const page of request(call))
|
|
108
|
+
found.push(...source.entities(page, parent));
|
|
109
|
+
for (const e of found) {
|
|
110
|
+
const key = `${e.kind}\0${e.id}`;
|
|
111
|
+
const was = merged.get(key);
|
|
112
|
+
merged.set(key, {
|
|
113
|
+
...e,
|
|
114
|
+
name: e.name ?? was?.name,
|
|
115
|
+
parent: e.parent ?? was?.parent,
|
|
116
|
+
domain: e.domain ?? was?.domain,
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
for (const e of found) {
|
|
120
|
+
const { children } = source;
|
|
121
|
+
const child = children && Object.hasOwn(children, e.kind) ? children[e.kind] : undefined;
|
|
122
|
+
if (child)
|
|
123
|
+
await walk(child, e);
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
for (const source of sources)
|
|
127
|
+
await walk(source);
|
|
128
|
+
const rows = [...merged.values()];
|
|
129
|
+
const now = Date.now();
|
|
130
|
+
return use((db) => {
|
|
131
|
+
db.exec('begin');
|
|
132
|
+
try {
|
|
133
|
+
db.prepare('delete from entity where api = ? and identity = ?').run(api, identity);
|
|
134
|
+
const insert = db.prepare('insert into entity values (?, ?, ?, ?, ?, ?, ?, ?)');
|
|
135
|
+
for (const r of rows) {
|
|
136
|
+
insert.run(api, r.kind, r.id, r.name ?? '', r.parent ?? '', r.domain ?? '', identity, now);
|
|
137
|
+
}
|
|
138
|
+
db.exec('commit');
|
|
139
|
+
}
|
|
140
|
+
catch (e) {
|
|
141
|
+
db.exec('rollback');
|
|
142
|
+
throw e;
|
|
143
|
+
}
|
|
144
|
+
return rows.length;
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
// Trigram match of each whitespace-separated term (any substring of 3+ characters), joined
|
|
148
|
+
// by OR, best 20 by bm25. Each term is one quoted FTS5 string, so no FTS5 syntax reaches the
|
|
149
|
+
// match. `empty`: the identity has no row (for `api`) at all.
|
|
150
|
+
async function find(query, { api, identity }) {
|
|
151
|
+
const terms = query.split(/\s+/).filter(Boolean);
|
|
152
|
+
const short = terms.find((t) => [...t].length < 3);
|
|
153
|
+
if (terms.length === 0 || short !== undefined) {
|
|
154
|
+
throw new GapiError(3, `find: each term needs 3 or more characters${short ? `: '${short}'` : ''}`);
|
|
155
|
+
}
|
|
156
|
+
if (identity === undefined || !existsSync(file))
|
|
157
|
+
return { hits: [], empty: true };
|
|
158
|
+
const match = terms.map((t) => `"${t.replaceAll('"', '""')}"`).join(' OR ');
|
|
159
|
+
return use((db) => {
|
|
160
|
+
const scope = 'identity = ? and (? is null or api = ?)';
|
|
161
|
+
const count = db.prepare(`select count(*) as n from entity where ${scope}`);
|
|
162
|
+
const empty = count.get(identity, api ?? null, api ?? null)?.n === 0;
|
|
163
|
+
const hits = db
|
|
164
|
+
.prepare(`select api, kind, id, name, parent, domain, synced from entity
|
|
165
|
+
where entity match ? and ${scope} order by rank limit 20`)
|
|
166
|
+
.all(match, identity, api ?? null, api ?? null)
|
|
167
|
+
.map((r) => ({
|
|
168
|
+
api: String(r.api),
|
|
169
|
+
kind: String(r.kind),
|
|
170
|
+
id: String(r.id),
|
|
171
|
+
name: text(r.name),
|
|
172
|
+
parent: text(r.parent),
|
|
173
|
+
domain: text(r.domain),
|
|
174
|
+
synced: Number(r.synced),
|
|
175
|
+
}));
|
|
176
|
+
return { hits, empty };
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
return { sync, find, sweep, clear };
|
|
180
|
+
}
|
package/dist/src/plan.js
CHANGED
|
@@ -15,6 +15,8 @@ const OWNED = new Set([
|
|
|
15
15
|
'upload_protocol',
|
|
16
16
|
'$.xgafv',
|
|
17
17
|
]);
|
|
18
|
+
// gaxios httpMethodsToRetry, without OPTIONS: a resend of these changes nothing more.
|
|
19
|
+
const IDEMPOTENT = new Set(['GET', 'HEAD', 'PUT', 'DELETE']);
|
|
18
20
|
// The document-level parameters a caller may pass (D5).
|
|
19
21
|
const SHARED = ['fields', 'quotaUser'];
|
|
20
22
|
const PROTOCOLS = ['media', 'multipart', 'resumable'];
|
|
@@ -78,7 +80,7 @@ export function plan(model, target, input, settings) {
|
|
|
78
80
|
if (input.body !== undefined && !method.request)
|
|
79
81
|
throw invalid(`--body: ${command} takes no request body`);
|
|
80
82
|
const mode = sendMode(method, command, input);
|
|
81
|
-
const body = mode.send === 'pages' &&
|
|
83
|
+
const body = mode.send === 'pages' && method.paging === 'body' ? pagingBody(input) : input.body;
|
|
82
84
|
const url = buildUrl(model, method, values, mode);
|
|
83
85
|
const headers = contentHeaders(mode, body !== undefined);
|
|
84
86
|
if (model.auth === 'oauth' && settings.project)
|
|
@@ -92,17 +94,18 @@ export function plan(model, target, input, settings) {
|
|
|
92
94
|
body,
|
|
93
95
|
auth: { kind: model.auth, scopes: method.scopes },
|
|
94
96
|
warnings,
|
|
97
|
+
...(IDEMPOTENT.has(method.httpMethod) || method.read ? { retry: true } : {}),
|
|
95
98
|
...mode,
|
|
96
99
|
};
|
|
97
100
|
}
|
|
98
|
-
// The outer CallPlan of a batch call: one POST to the
|
|
101
|
+
// The outer CallPlan of a batch call: one POST to the document's `batchPath` endpoint that
|
|
99
102
|
// carries `parts`, each planned by `plan` with its own headers (D7).
|
|
100
103
|
export function batchPlan(model, parts) {
|
|
101
104
|
const [first] = parts;
|
|
102
105
|
return {
|
|
103
106
|
methodId: first.methodId,
|
|
104
107
|
httpMethod: 'POST',
|
|
105
|
-
url: `${model.rootUrl}
|
|
108
|
+
url: `${model.rootUrl}${model.batchPath}`,
|
|
106
109
|
headers: {},
|
|
107
110
|
auth: first.auth,
|
|
108
111
|
warnings: first.warnings,
|
|
@@ -160,13 +163,27 @@ function sendMode(method, command, input) {
|
|
|
160
163
|
if (!/^[1-9]\d*$/.test(limit))
|
|
161
164
|
throw invalid('--page-limit must be a positive integer');
|
|
162
165
|
const pageLimit = Number(limit);
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
+
return { send: 'pages', pageLimit, follow: nextPage(method.paging) };
|
|
167
|
+
}
|
|
168
|
+
// Body paging: every page sends this object, from page 2 on with pageToken set (D6).
|
|
169
|
+
function pagingBody(input) {
|
|
166
170
|
const body = input.body ?? {};
|
|
167
171
|
if (!isRecord(body))
|
|
168
172
|
throw invalid('--page-all: --body must be a JSON object');
|
|
169
|
-
return
|
|
173
|
+
return body;
|
|
174
|
+
}
|
|
175
|
+
// AIP-158 follow rule: repeat the request with only pageToken changed, until no nextPageToken.
|
|
176
|
+
function nextPage(paging) {
|
|
177
|
+
return (data, last) => {
|
|
178
|
+
const token = isRecord(data) ? data.nextPageToken : undefined;
|
|
179
|
+
if (typeof token !== 'string' || !token)
|
|
180
|
+
return undefined;
|
|
181
|
+
if (paging === 'body')
|
|
182
|
+
return { ...last, body: { ...(isRecord(last.body) && last.body), pageToken: token } };
|
|
183
|
+
const url = new URL(last.url);
|
|
184
|
+
url.searchParams.set('pageToken', token);
|
|
185
|
+
return { ...last, url: url.href };
|
|
186
|
+
};
|
|
170
187
|
}
|
|
171
188
|
// D7: protocol defaults to multipart with --body, else media; the type must fit `accept`.
|
|
172
189
|
function upload(method, command, path, input, hasBody) {
|
|
@@ -258,10 +275,25 @@ function paramValues(model, method, params) {
|
|
|
258
275
|
return String(v);
|
|
259
276
|
throw invalid(`${name} must be a string, number or boolean`);
|
|
260
277
|
});
|
|
278
|
+
// An empty list or a required '' sends nothing; a path would lose its segment.
|
|
279
|
+
if (values.length === 0)
|
|
280
|
+
throw invalid(`${name} needs at least one value`);
|
|
281
|
+
if (spec.required && values.includes(''))
|
|
282
|
+
throw invalid(`${name} needs a value`);
|
|
261
283
|
for (const v of values) {
|
|
262
284
|
if (spec.enum && !spec.enum.includes(v)) {
|
|
263
285
|
throw invalid(`${name} must be one of: ${spec.enum.join(', ')}`);
|
|
264
286
|
}
|
|
287
|
+
const type = spec.type === 'integer' || /int64$/.test(spec.format ?? '') ? 'integer' : spec.type;
|
|
288
|
+
if (type === 'integer' && !/^-?\d+$/.test(v))
|
|
289
|
+
throw invalid(`${name} must be an integer`);
|
|
290
|
+
if (type === 'boolean' && v !== 'true' && v !== 'false') {
|
|
291
|
+
throw invalid(`${name} must be true or false`);
|
|
292
|
+
}
|
|
293
|
+
// Path patterns are checked in buildUrl, after the bare-id hint (#28).
|
|
294
|
+
if (spec.location !== 'path' && spec.pattern && !matches(spec.pattern, v)) {
|
|
295
|
+
throw invalid(`${name} must match ${spec.pattern}`);
|
|
296
|
+
}
|
|
265
297
|
}
|
|
266
298
|
out.set(name, { spec, values });
|
|
267
299
|
}
|
|
@@ -280,9 +312,14 @@ function buildUrl(model, method, params, mode) {
|
|
|
280
312
|
if (!param)
|
|
281
313
|
throw invalid(`missing required param ${name}`);
|
|
282
314
|
const value = param.values[0];
|
|
283
|
-
if (param.spec.pattern && !
|
|
315
|
+
if (param.spec.pattern && !matches(param.spec.pattern, value)) {
|
|
284
316
|
throw invalid(`${name} must match ${param.spec.pattern}`);
|
|
285
317
|
}
|
|
318
|
+
// googleapis-common validateAndEncodeParams: a `.`/`..` segment would make the URL
|
|
319
|
+
// parser leave the method path; `%2e` too, since WHATWG URL reads it as a dot.
|
|
320
|
+
const dot = (reserved ? value.split('/') : [value]).find((s) => /^(?:\.|%2e){1,2}$/i.test(s));
|
|
321
|
+
if (dot !== undefined)
|
|
322
|
+
throw invalid(`${name}: '${dot}' is a dot segment; pass a real id`);
|
|
286
323
|
if (reserved)
|
|
287
324
|
return value.split('/').map(encodeURIComponent).join('/');
|
|
288
325
|
// #28: `properties/{property}` + `properties/123` would encode to properties%2F123.
|
|
@@ -310,6 +347,16 @@ function buildUrl(model, method, params, mode) {
|
|
|
310
347
|
: model.rootUrl;
|
|
311
348
|
return `${root}${model.servicePath}${path}${qs ? `?${qs}` : ''}`;
|
|
312
349
|
}
|
|
350
|
+
// shortcut: a Java-only discovery pattern (e.g. `(?i)`) is skipped, not checked; translate
|
|
351
|
+
// Java regex to JS if Google ever relies on one for a path.
|
|
352
|
+
function matches(pattern, value) {
|
|
353
|
+
try {
|
|
354
|
+
return new RegExp(pattern).test(value);
|
|
355
|
+
}
|
|
356
|
+
catch {
|
|
357
|
+
return true;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
313
360
|
// google/api/http.proto: {x} percent-encodes every character except [-_.~0-9a-zA-Z];
|
|
314
361
|
// encodeURIComponent leaves !'()* as is.
|
|
315
362
|
const encodeSegment = (value) => encodeURIComponent(value).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
|