@rawdash/connector-google-search-console 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -2
- package/dist/index.js +202 -127
- package/dist/index.js.map +1 -1
- package/package.json +4 -3
package/dist/index.d.ts
CHANGED
|
@@ -326,8 +326,7 @@ declare class GSCConnector extends BaseConnector<GSCSettings, GSCCredentials> {
|
|
|
326
326
|
auth: "optional";
|
|
327
327
|
};
|
|
328
328
|
};
|
|
329
|
-
private
|
|
330
|
-
private fetchOAuthToken;
|
|
329
|
+
private tokenProvider?;
|
|
331
330
|
private getAccessToken;
|
|
332
331
|
private runReport;
|
|
333
332
|
sync(options: SyncOptions, storage: StorageHandle, signal?: AbortSignal): Promise<SyncResult>;
|
package/dist/index.js
CHANGED
|
@@ -1,8 +1,173 @@
|
|
|
1
|
-
//
|
|
1
|
+
// ../gcp-shared/dist/index.js
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { z as z2 } from "zod";
|
|
4
|
+
import { z as z3 } from "zod";
|
|
5
|
+
var serviceAccountKeySchema = z.object({
|
|
6
|
+
client_email: z.string().min(1),
|
|
7
|
+
private_key: z.string().min(1),
|
|
8
|
+
token_uri: z.string().url().optional(),
|
|
9
|
+
project_id: z.string().optional()
|
|
10
|
+
});
|
|
11
|
+
var tokenResponseSchema = z.object({
|
|
12
|
+
access_token: z.string().min(1),
|
|
13
|
+
expires_in: z.number().int().positive().optional()
|
|
14
|
+
});
|
|
15
|
+
function base64urlFromBytes(bytes) {
|
|
16
|
+
let binary = "";
|
|
17
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
18
|
+
binary += String.fromCharCode(bytes[i]);
|
|
19
|
+
}
|
|
20
|
+
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "");
|
|
21
|
+
}
|
|
22
|
+
function base64urlFromString(str) {
|
|
23
|
+
return base64urlFromBytes(new TextEncoder().encode(str));
|
|
24
|
+
}
|
|
25
|
+
async function signRS256JWT(payload, privateKeyPem) {
|
|
26
|
+
const header = { alg: "RS256", typ: "JWT" };
|
|
27
|
+
const headerB64 = base64urlFromString(JSON.stringify(header));
|
|
28
|
+
const payloadB64 = base64urlFromString(JSON.stringify(payload));
|
|
29
|
+
const signingInput = `${headerB64}.${payloadB64}`;
|
|
30
|
+
const pemContent = privateKeyPem.replace(/-----BEGIN PRIVATE KEY-----/g, "").replace(/-----END PRIVATE KEY-----/g, "").replace(/\s/g, "");
|
|
31
|
+
const der = Uint8Array.from(atob(pemContent), (c) => c.charCodeAt(0));
|
|
32
|
+
const key = await globalThis.crypto.subtle.importKey(
|
|
33
|
+
"pkcs8",
|
|
34
|
+
der,
|
|
35
|
+
{ name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
|
|
36
|
+
false,
|
|
37
|
+
["sign"]
|
|
38
|
+
);
|
|
39
|
+
const signature = await globalThis.crypto.subtle.sign(
|
|
40
|
+
"RSASSA-PKCS1-v1_5",
|
|
41
|
+
key,
|
|
42
|
+
new TextEncoder().encode(signingInput)
|
|
43
|
+
);
|
|
44
|
+
return `${signingInput}.${base64urlFromBytes(new Uint8Array(signature))}`;
|
|
45
|
+
}
|
|
46
|
+
function parseServiceAccountJson(value) {
|
|
47
|
+
const trimmed = value.trim();
|
|
48
|
+
if (trimmed.startsWith("{")) {
|
|
49
|
+
return serviceAccountKeySchema.parse(JSON.parse(trimmed));
|
|
50
|
+
}
|
|
51
|
+
const binary = atob(trimmed);
|
|
52
|
+
const bytes = new Uint8Array(binary.length);
|
|
53
|
+
for (let i = 0; i < binary.length; i++) {
|
|
54
|
+
bytes[i] = binary.charCodeAt(i);
|
|
55
|
+
}
|
|
56
|
+
const decoded = new TextDecoder().decode(bytes);
|
|
57
|
+
return serviceAccountKeySchema.parse(JSON.parse(decoded));
|
|
58
|
+
}
|
|
59
|
+
async function buildServiceAccountJwt(serviceAccountJson, scope) {
|
|
60
|
+
const sa = parseServiceAccountJson(serviceAccountJson);
|
|
61
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
62
|
+
const jwt = await signRS256JWT(
|
|
63
|
+
{
|
|
64
|
+
iss: sa.client_email,
|
|
65
|
+
scope,
|
|
66
|
+
aud: sa.token_uri ?? "https://oauth2.googleapis.com/token",
|
|
67
|
+
exp: now + 3600,
|
|
68
|
+
iat: now
|
|
69
|
+
},
|
|
70
|
+
sa.private_key
|
|
71
|
+
);
|
|
72
|
+
const body = new URLSearchParams({
|
|
73
|
+
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
74
|
+
assertion: jwt
|
|
75
|
+
}).toString();
|
|
76
|
+
return {
|
|
77
|
+
url: sa.token_uri ?? "https://oauth2.googleapis.com/token",
|
|
78
|
+
body
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
function buildRefreshTokenGrant(credentials) {
|
|
82
|
+
const body = new URLSearchParams({
|
|
83
|
+
grant_type: "refresh_token",
|
|
84
|
+
refresh_token: credentials.refreshToken,
|
|
85
|
+
client_id: credentials.clientId,
|
|
86
|
+
client_secret: credentials.clientSecret
|
|
87
|
+
}).toString();
|
|
88
|
+
return { url: "https://oauth2.googleapis.com/token", body };
|
|
89
|
+
}
|
|
90
|
+
var gcpAuthConfigShape = {
|
|
91
|
+
serviceAccountJson: z2.object({ $secret: z2.string().trim().min(1) }).meta({
|
|
92
|
+
label: "Service Account JSON",
|
|
93
|
+
description: "Contents of the JSON key file for a Google service account with the role required by this connector. Create one at Google Cloud -> IAM & Admin -> Service Accounts and store the JSON as a secret.",
|
|
94
|
+
secret: true
|
|
95
|
+
})
|
|
96
|
+
};
|
|
97
|
+
var HttpClientError = class extends Error {
|
|
98
|
+
response;
|
|
99
|
+
constructor(message, response) {
|
|
100
|
+
super(message);
|
|
101
|
+
this.name = new.target.name;
|
|
102
|
+
this.response = response;
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
var AuthError = class extends HttpClientError {
|
|
106
|
+
kind = "auth";
|
|
107
|
+
};
|
|
2
108
|
var HTTP_CLIENT_VERSION = "0.0.0";
|
|
3
109
|
var DEFAULT_USER_AGENT = `rawdash-connector/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;
|
|
110
|
+
var bqQueryResponseSchema = z3.object({
|
|
111
|
+
jobComplete: z3.boolean().optional(),
|
|
112
|
+
schema: z3.object({
|
|
113
|
+
fields: z3.array(z3.object({ name: z3.string(), type: z3.string() }))
|
|
114
|
+
}).optional(),
|
|
115
|
+
rows: z3.array(
|
|
116
|
+
z3.object({
|
|
117
|
+
f: z3.array(z3.object({ v: z3.string().nullable().optional() }))
|
|
118
|
+
})
|
|
119
|
+
).optional(),
|
|
120
|
+
pageToken: z3.string().optional(),
|
|
121
|
+
jobReference: z3.object({
|
|
122
|
+
projectId: z3.string(),
|
|
123
|
+
jobId: z3.string(),
|
|
124
|
+
location: z3.string().optional()
|
|
125
|
+
}).optional()
|
|
126
|
+
});
|
|
127
|
+
var GcpAccessTokenProvider = class {
|
|
128
|
+
constructor(opts) {
|
|
129
|
+
this.opts = opts;
|
|
130
|
+
}
|
|
131
|
+
opts;
|
|
132
|
+
cached = null;
|
|
133
|
+
async resolveGrant() {
|
|
134
|
+
const serviceAccountJson = this.opts.getServiceAccountJson();
|
|
135
|
+
if (serviceAccountJson) {
|
|
136
|
+
return buildServiceAccountJwt(serviceAccountJson, this.opts.scope);
|
|
137
|
+
}
|
|
138
|
+
const refreshTokenCredentials = this.opts.getRefreshTokenCredentials?.();
|
|
139
|
+
if (refreshTokenCredentials) {
|
|
140
|
+
return buildRefreshTokenGrant(refreshTokenCredentials);
|
|
141
|
+
}
|
|
142
|
+
throw new AuthError(
|
|
143
|
+
`${this.opts.connectorId}: missing serviceAccountJson or refresh-token credentials`
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
async getToken(signal) {
|
|
147
|
+
if (this.cached && Date.now() < this.cached.expiresAt) {
|
|
148
|
+
return this.cached.token;
|
|
149
|
+
}
|
|
150
|
+
const { url, body } = await this.resolveGrant();
|
|
151
|
+
const res = await this.opts.post(url, {
|
|
152
|
+
resource: "oauth_token",
|
|
153
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
154
|
+
body,
|
|
155
|
+
signal
|
|
156
|
+
});
|
|
157
|
+
const expiresIn = res.body.expires_in ?? 3600;
|
|
158
|
+
this.cached = {
|
|
159
|
+
token: res.body.access_token,
|
|
160
|
+
expiresAt: Date.now() + (expiresIn - 60) * 1e3
|
|
161
|
+
};
|
|
162
|
+
return this.cached.token;
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
// ../../connector-shared/dist/index.js
|
|
167
|
+
var HTTP_CLIENT_VERSION2 = "0.0.0";
|
|
168
|
+
var DEFAULT_USER_AGENT2 = `rawdash-connector/${HTTP_CLIENT_VERSION2} (+https://rawdash.dev)`;
|
|
4
169
|
function connectorUserAgent(connectorId) {
|
|
5
|
-
return `rawdash-connector-${connectorId}/${
|
|
170
|
+
return `rawdash-connector-${connectorId}/${HTTP_CLIENT_VERSION2} (+https://rawdash.dev)`;
|
|
6
171
|
}
|
|
7
172
|
|
|
8
173
|
// src/google-search-console.ts
|
|
@@ -13,35 +178,35 @@ import {
|
|
|
13
178
|
defineResources,
|
|
14
179
|
schemasFromResources
|
|
15
180
|
} from "@rawdash/core";
|
|
16
|
-
import { z } from "zod";
|
|
181
|
+
import { z as z4 } from "zod";
|
|
17
182
|
var configFields = defineConfigFields(
|
|
18
|
-
|
|
19
|
-
siteUrl:
|
|
183
|
+
z4.object({
|
|
184
|
+
siteUrl: z4.string().trim().min(1).meta({
|
|
20
185
|
label: "Site URL",
|
|
21
186
|
description: 'Verified Search Console property. URL-prefix properties look like "https://example.com/"; Domain properties look like "sc-domain:example.com".',
|
|
22
187
|
placeholder: "https://example.com/"
|
|
23
188
|
}),
|
|
24
|
-
serviceAccountJson:
|
|
189
|
+
serviceAccountJson: z4.object({ $secret: z4.string().trim().min(1) }).optional().meta({
|
|
25
190
|
label: "Service Account JSON (recommended)",
|
|
26
191
|
description: "Contents of the JSON key file for a Google service account that has been added as a Search Console user (Owner or Full user) on the property. Create one at Google Cloud -> IAM & Admin -> Service Accounts.",
|
|
27
192
|
secret: true
|
|
28
193
|
}),
|
|
29
|
-
refreshToken:
|
|
194
|
+
refreshToken: z4.object({ $secret: z4.string().trim().min(1) }).optional().meta({
|
|
30
195
|
label: "OAuth Refresh Token",
|
|
31
196
|
description: "Google OAuth 2.0 refresh token with webmasters.readonly scope. Required if not using serviceAccountJson.",
|
|
32
197
|
secret: true
|
|
33
198
|
}),
|
|
34
|
-
clientId:
|
|
199
|
+
clientId: z4.string().trim().min(1).optional().meta({
|
|
35
200
|
label: "OAuth Client ID",
|
|
36
201
|
description: "OAuth 2.0 client ID from Google Cloud Console. Required when using refreshToken auth.",
|
|
37
202
|
placeholder: "...apps.googleusercontent.com"
|
|
38
203
|
}),
|
|
39
|
-
clientSecret:
|
|
204
|
+
clientSecret: z4.object({ $secret: z4.string().trim().min(1) }).optional().meta({
|
|
40
205
|
label: "OAuth Client Secret",
|
|
41
206
|
description: "OAuth 2.0 client secret from Google Cloud Console. Required when using refreshToken auth.",
|
|
42
207
|
secret: true
|
|
43
208
|
}),
|
|
44
|
-
lookbackDays:
|
|
209
|
+
lookbackDays: z4.number().int().positive().optional().meta({
|
|
45
210
|
label: "Lookback days (full sync)",
|
|
46
211
|
description: "How many calendar days to fetch on a full sync. Defaults to 90.",
|
|
47
212
|
placeholder: "90"
|
|
@@ -60,6 +225,7 @@ var doc = defineConnectorDoc({
|
|
|
60
225
|
tagline: "Sync daily Search Console SEO metrics - clicks, impressions, CTR, and average position - by date, query, page, and country.",
|
|
61
226
|
vendor: {
|
|
62
227
|
name: "Google Search Console",
|
|
228
|
+
domain: "search.google.com",
|
|
63
229
|
apiDocs: "https://developers.google.com/webmaster-tools/v1/api_reference_index",
|
|
64
230
|
website: "https://search.google.com/search-console"
|
|
65
231
|
},
|
|
@@ -145,72 +311,6 @@ var PHASE_CONFIGS = {
|
|
|
145
311
|
};
|
|
146
312
|
var ROWS_PER_PAGE = 25e3;
|
|
147
313
|
var METRIC_FIELDS = ["clicks", "impressions", "ctr", "position"];
|
|
148
|
-
function base64urlFromBytes(bytes) {
|
|
149
|
-
let binary = "";
|
|
150
|
-
for (let i = 0; i < bytes.length; i++) {
|
|
151
|
-
binary += String.fromCharCode(bytes[i]);
|
|
152
|
-
}
|
|
153
|
-
return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=/g, "");
|
|
154
|
-
}
|
|
155
|
-
function base64urlFromString(str) {
|
|
156
|
-
return base64urlFromBytes(new TextEncoder().encode(str));
|
|
157
|
-
}
|
|
158
|
-
async function signRS256JWT(payload, privateKeyPem) {
|
|
159
|
-
const header = { alg: "RS256", typ: "JWT" };
|
|
160
|
-
const headerB64 = base64urlFromString(JSON.stringify(header));
|
|
161
|
-
const payloadB64 = base64urlFromString(JSON.stringify(payload));
|
|
162
|
-
const signingInput = `${headerB64}.${payloadB64}`;
|
|
163
|
-
const pemContent = privateKeyPem.replace(/-----BEGIN PRIVATE KEY-----/g, "").replace(/-----END PRIVATE KEY-----/g, "").replace(/\s/g, "");
|
|
164
|
-
const der = Uint8Array.from(atob(pemContent), (c) => c.charCodeAt(0));
|
|
165
|
-
const key = await globalThis.crypto.subtle.importKey(
|
|
166
|
-
"pkcs8",
|
|
167
|
-
der,
|
|
168
|
-
{ name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
|
|
169
|
-
false,
|
|
170
|
-
["sign"]
|
|
171
|
-
);
|
|
172
|
-
const signature = await globalThis.crypto.subtle.sign(
|
|
173
|
-
"RSASSA-PKCS1-v1_5",
|
|
174
|
-
key,
|
|
175
|
-
new TextEncoder().encode(signingInput)
|
|
176
|
-
);
|
|
177
|
-
return `${signingInput}.${base64urlFromBytes(new Uint8Array(signature))}`;
|
|
178
|
-
}
|
|
179
|
-
function parseServiceAccountJson(value) {
|
|
180
|
-
const trimmed = value.trim();
|
|
181
|
-
if (trimmed.startsWith("{")) {
|
|
182
|
-
return JSON.parse(trimmed);
|
|
183
|
-
}
|
|
184
|
-
const binary = atob(trimmed);
|
|
185
|
-
const bytes = new Uint8Array(binary.length);
|
|
186
|
-
for (let i = 0; i < binary.length; i++) {
|
|
187
|
-
bytes[i] = binary.charCodeAt(i);
|
|
188
|
-
}
|
|
189
|
-
const decoded = new TextDecoder().decode(bytes);
|
|
190
|
-
return JSON.parse(decoded);
|
|
191
|
-
}
|
|
192
|
-
async function buildServiceAccountJwt(serviceAccountJson) {
|
|
193
|
-
const sa = parseServiceAccountJson(serviceAccountJson);
|
|
194
|
-
const now = Math.floor(Date.now() / 1e3);
|
|
195
|
-
const jwt = await signRS256JWT(
|
|
196
|
-
{
|
|
197
|
-
iss: sa.client_email,
|
|
198
|
-
scope: "https://www.googleapis.com/auth/webmasters.readonly",
|
|
199
|
-
aud: sa.token_uri ?? "https://oauth2.googleapis.com/token",
|
|
200
|
-
exp: now + 3600,
|
|
201
|
-
iat: now
|
|
202
|
-
},
|
|
203
|
-
sa.private_key
|
|
204
|
-
);
|
|
205
|
-
const body = new URLSearchParams({
|
|
206
|
-
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
207
|
-
assertion: jwt
|
|
208
|
-
}).toString();
|
|
209
|
-
return {
|
|
210
|
-
url: sa.token_uri ?? "https://oauth2.googleapis.com/token",
|
|
211
|
-
body
|
|
212
|
-
};
|
|
213
|
-
}
|
|
214
314
|
function toGSCDate(date) {
|
|
215
315
|
const y = date.getUTCFullYear();
|
|
216
316
|
const m = String(date.getUTCMonth() + 1).padStart(2, "0");
|
|
@@ -264,14 +364,14 @@ function rowToMetricSample(row, dimensions, metricName) {
|
|
|
264
364
|
attributes
|
|
265
365
|
};
|
|
266
366
|
}
|
|
267
|
-
var dateKey =
|
|
268
|
-
var stringKey =
|
|
269
|
-
var numericField =
|
|
367
|
+
var dateKey = z4.string().regex(/^(19|20)\d{2}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$/);
|
|
368
|
+
var stringKey = z4.string();
|
|
369
|
+
var numericField = z4.number();
|
|
270
370
|
function reportSchema(dimensionCount) {
|
|
271
|
-
const keys = dimensionCount === 1 ?
|
|
272
|
-
return
|
|
273
|
-
rows:
|
|
274
|
-
|
|
371
|
+
const keys = dimensionCount === 1 ? z4.tuple([dateKey]) : z4.tuple([dateKey, ...Array(dimensionCount - 1).fill(stringKey)]);
|
|
372
|
+
return z4.object({
|
|
373
|
+
rows: z4.array(
|
|
374
|
+
z4.object({
|
|
275
375
|
keys,
|
|
276
376
|
clicks: numericField.optional(),
|
|
277
377
|
impressions: numericField.optional(),
|
|
@@ -281,9 +381,9 @@ function reportSchema(dimensionCount) {
|
|
|
281
381
|
).optional()
|
|
282
382
|
});
|
|
283
383
|
}
|
|
284
|
-
var
|
|
285
|
-
access_token:
|
|
286
|
-
expires_in:
|
|
384
|
+
var tokenResponseSchema2 = z4.object({
|
|
385
|
+
access_token: z4.string().min(1),
|
|
386
|
+
expires_in: z4.number().int().positive().optional()
|
|
287
387
|
});
|
|
288
388
|
var googleSearchConsoleResources = defineResources({
|
|
289
389
|
gsc_search_analytics_by_day: {
|
|
@@ -296,7 +396,7 @@ var googleSearchConsoleResources = defineResources({
|
|
|
296
396
|
{ name: "date", description: "Calendar day of the metric sample." }
|
|
297
397
|
],
|
|
298
398
|
responses: {
|
|
299
|
-
oauth_token:
|
|
399
|
+
oauth_token: tokenResponseSchema2,
|
|
300
400
|
search_analytics_by_day: reportSchema(1)
|
|
301
401
|
}
|
|
302
402
|
},
|
|
@@ -369,47 +469,22 @@ var GSCConnector = class _GSCConnector extends BaseConnector {
|
|
|
369
469
|
}
|
|
370
470
|
id = id;
|
|
371
471
|
credentials = gscCredentials;
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
472
|
+
tokenProvider;
|
|
473
|
+
getAccessToken(signal) {
|
|
474
|
+
this.tokenProvider ??= new GcpAccessTokenProvider({
|
|
475
|
+
connectorId: this.id,
|
|
476
|
+
scope: "https://www.googleapis.com/auth/webmasters.readonly",
|
|
477
|
+
getServiceAccountJson: () => this.creds.serviceAccountJson,
|
|
478
|
+
getRefreshTokenCredentials: () => {
|
|
479
|
+
const { refreshToken, clientId, clientSecret } = this.creds;
|
|
480
|
+
if (refreshToken && clientId && clientSecret) {
|
|
481
|
+
return { refreshToken, clientId, clientSecret };
|
|
482
|
+
}
|
|
483
|
+
return void 0;
|
|
484
|
+
},
|
|
485
|
+
post: (url, opts) => this.post(url, opts)
|
|
379
486
|
});
|
|
380
|
-
|
|
381
|
-
return {
|
|
382
|
-
token: res.body.access_token,
|
|
383
|
-
expiresAt: Date.now() + (expiresIn - 60) * 1e3
|
|
384
|
-
};
|
|
385
|
-
}
|
|
386
|
-
async getAccessToken(signal) {
|
|
387
|
-
if (this.cachedToken && Date.now() < this.cachedToken.expiresAt) {
|
|
388
|
-
return this.cachedToken.token;
|
|
389
|
-
}
|
|
390
|
-
const { serviceAccountJson, refreshToken, clientId, clientSecret } = this.creds;
|
|
391
|
-
if (serviceAccountJson) {
|
|
392
|
-
const { url, body } = await buildServiceAccountJwt(serviceAccountJson);
|
|
393
|
-
this.cachedToken = await this.fetchOAuthToken(url, body, signal);
|
|
394
|
-
return this.cachedToken.token;
|
|
395
|
-
}
|
|
396
|
-
if (refreshToken && clientId && clientSecret) {
|
|
397
|
-
const body = new URLSearchParams({
|
|
398
|
-
grant_type: "refresh_token",
|
|
399
|
-
refresh_token: refreshToken,
|
|
400
|
-
client_id: clientId,
|
|
401
|
-
client_secret: clientSecret
|
|
402
|
-
}).toString();
|
|
403
|
-
this.cachedToken = await this.fetchOAuthToken(
|
|
404
|
-
"https://oauth2.googleapis.com/token",
|
|
405
|
-
body,
|
|
406
|
-
signal
|
|
407
|
-
);
|
|
408
|
-
return this.cachedToken.token;
|
|
409
|
-
}
|
|
410
|
-
throw new Error(
|
|
411
|
-
"GSC connector: provide either serviceAccountJson or (refreshToken + clientId + clientSecret)"
|
|
412
|
-
);
|
|
487
|
+
return this.tokenProvider.getToken(signal);
|
|
413
488
|
}
|
|
414
489
|
async runReport(accessToken, phase, dateRange, startRow, signal) {
|
|
415
490
|
const { dimensions } = PHASE_CONFIGS[phase];
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../connector-shared/src/errors.ts","../../../connector-shared/src/retry.ts","../../../connector-shared/src/version.ts","../../../connector-shared/src/request.ts","../../../connector-shared/src/rate-limit.ts","../../../connector-shared/src/map-concurrent.ts","../../../connector-shared/src/sanitize.ts","../../../connector-shared/src/epoch.ts","../../../connector-shared/src/pagination.ts","../../../connector-shared/src/logger.ts","../src/google-search-console.ts","../src/index.ts"],"sourcesContent":["import type { HttpResponse } from './types';\n\nexport type HttpErrorKind =\n | 'transient'\n | 'rate_limit'\n | 'auth'\n | 'upstream_bug'\n | 'client_bug';\n\nexport abstract class HttpClientError extends Error {\n abstract readonly kind: HttpErrorKind;\n readonly response?: HttpResponse;\n\n constructor(message: string, response?: HttpResponse) {\n super(message);\n this.name = new.target.name;\n this.response = response;\n }\n}\n\nexport class TransientError extends HttpClientError {\n readonly kind = 'transient' as const;\n}\n\nexport class RateLimitError extends HttpClientError {\n readonly kind = 'rate_limit' as const;\n readonly retryAfter?: Date;\n\n constructor(message: string, response?: HttpResponse, retryAfter?: Date) {\n super(message, response);\n this.retryAfter = retryAfter;\n }\n}\n\nexport class AuthError extends HttpClientError {\n readonly kind = 'auth' as const;\n}\n\nexport class UpstreamBugError extends HttpClientError {\n readonly kind = 'upstream_bug' as const;\n}\n\nexport class ClientBugError extends HttpClientError {\n readonly kind = 'client_bug' as const;\n}\n\nexport function classifyStatus(status: number): HttpErrorKind {\n if (status === 429) {\n return 'rate_limit';\n }\n if (status === 401 || status === 403) {\n return 'auth';\n }\n if (status === 408) {\n return 'transient';\n }\n if (status >= 500) {\n return 'upstream_bug';\n }\n if (status >= 400) {\n return 'client_bug';\n }\n return 'client_bug';\n}\n\nexport function errorForStatus(\n message: string,\n response: HttpResponse,\n retryAfter?: Date,\n): HttpClientError {\n const kind = classifyStatus(response.status);\n switch (kind) {\n case 'rate_limit':\n return new RateLimitError(message, response, retryAfter);\n case 'auth':\n return new AuthError(message, response);\n case 'transient':\n return new TransientError(message, response);\n case 'upstream_bug':\n return new UpstreamBugError(message, response);\n case 'client_bug':\n return new ClientBugError(message, response);\n }\n}\n","import { HttpClientError, RateLimitError, TransientError } from './errors';\n\nexport interface RetryPolicy {\n maxAttempts?: number;\n initialDelayMs?: number;\n maxDelayMs?: number;\n retryOn?: (status: number | null, err?: Error) => boolean;\n}\n\nexport const defaultRetryOn = (status: number | null, err?: Error): boolean => {\n if (err instanceof RateLimitError) {\n return true;\n }\n if (err instanceof TransientError) {\n return true;\n }\n if (status === null) {\n return err instanceof Error && !(err instanceof HttpClientError);\n }\n if (status === 408 || status === 429) {\n return true;\n }\n if (status >= 500) {\n return true;\n }\n return false;\n};\n\nexport function backoffDelayMs(\n attempt: number,\n policy: Required<Pick<RetryPolicy, 'initialDelayMs' | 'maxDelayMs'>>,\n): number {\n const base = policy.initialDelayMs * 2 ** attempt;\n const jitter = base * 0.25 * Math.random();\n return Math.min(base + jitter, policy.maxDelayMs);\n}\n\nexport function parseRetryAfter(\n headerValue: string | null,\n now: Date = new Date(),\n): Date | undefined {\n if (!headerValue) {\n return undefined;\n }\n const trimmed = headerValue.trim();\n if (/^\\d+$/.test(trimmed)) {\n return new Date(now.getTime() + Number(trimmed) * 1000);\n }\n const parsed = Date.parse(trimmed);\n if (Number.isNaN(parsed)) {\n return undefined;\n }\n return new Date(parsed);\n}\n\nexport function sleep(ms: number, signal?: AbortSignal): Promise<void> {\n if (signal?.aborted) {\n return Promise.reject(signal.reason ?? new Error('Aborted'));\n }\n return new Promise<void>((resolve, reject) => {\n const onAbort = () => {\n clearTimeout(timer);\n reject(signal!.reason ?? new Error('Aborted'));\n };\n const timer = setTimeout(() => {\n signal?.removeEventListener('abort', onAbort);\n resolve();\n }, ms);\n signal?.addEventListener('abort', onAbort, { once: true });\n });\n}\n","export const HTTP_CLIENT_VERSION = '0.0.0';\n\nexport const DEFAULT_USER_AGENT = `rawdash-connector/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;\n\nexport function connectorUserAgent(connectorId: string): string {\n return `rawdash-connector-${connectorId}/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;\n}\n","import {\n AuthError,\n ClientBugError,\n HttpClientError,\n RateLimitError,\n TransientError,\n UpstreamBugError,\n errorForStatus,\n} from './errors';\nimport { defaultRetryOn, parseRetryAfter, sleep } from './retry';\nimport type { FetchLike, HttpMethod, HttpRequest, HttpResponse } from './types';\nimport { DEFAULT_USER_AGENT } from './version';\n\nconst DEFAULT_TIMEOUT_MS = 10_000;\nconst DEFAULT_MAX_ATTEMPTS = 3;\nconst DEFAULT_INITIAL_DELAY_MS = 1000;\nconst DEFAULT_MAX_DELAY_MS = 60_000;\nconst OBSERVER_TIMEOUT_MS = 250;\n\nexport interface RequestObservation {\n url: string;\n method: HttpMethod;\n status: number;\n resource: string;\n requestId: string;\n body: unknown;\n}\n\nexport type RequestObserver = (\n event: RequestObservation,\n) => void | Promise<void>;\n\nexport interface RequestOptions {\n fetch?: FetchLike;\n observer?: RequestObserver;\n resource: string;\n requestId?: string;\n}\n\nasync function notifyObserver(\n observer: RequestObserver,\n event: RequestObservation,\n): Promise<void> {\n let result: void | Promise<void>;\n try {\n result = observer(event);\n } catch (err) {\n console.warn('[connector-shared] request observer threw:', err);\n return;\n }\n if (!(result instanceof Promise)) {\n return;\n }\n const guarded = result.catch((err) => {\n console.warn('[connector-shared] request observer rejected:', err);\n });\n let timer: ReturnType<typeof setTimeout> | undefined;\n const timeout = new Promise<void>((resolve) => {\n timer = setTimeout(resolve, OBSERVER_TIMEOUT_MS);\n });\n try {\n await Promise.race([guarded, timeout]);\n } finally {\n if (timer) {\n clearTimeout(timer);\n }\n }\n}\n\nfunction newRequestId(): string {\n const c = (globalThis as { crypto?: { randomUUID?: () => string } }).crypto;\n if (c?.randomUUID) {\n return c.randomUUID();\n }\n return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;\n}\n\nfunction mergeHeaders(\n defaults: Record<string, string>,\n overrides: Record<string, string> | undefined,\n): Record<string, string> {\n const merged: Record<string, string> = {};\n for (const [k, v] of Object.entries(defaults)) {\n merged[k.toLowerCase()] = v;\n }\n if (overrides) {\n for (const [k, v] of Object.entries(overrides)) {\n merged[k.toLowerCase()] = v;\n }\n }\n return merged;\n}\n\nfunction linkTimeoutSignal(\n parent: AbortSignal | undefined,\n timeoutMs: number,\n): { signal: AbortSignal; cancel: () => void } {\n const controller = new AbortController();\n const onParentAbort = () => {\n controller.abort(parent?.reason);\n };\n if (parent) {\n if (parent.aborted) {\n controller.abort(parent.reason);\n } else {\n parent.addEventListener('abort', onParentAbort, { once: true });\n }\n }\n const timer = setTimeout(() => {\n controller.abort(new Error(`Request timed out after ${timeoutMs}ms`));\n }, timeoutMs);\n return {\n signal: controller.signal,\n cancel: () => {\n clearTimeout(timer);\n if (parent) {\n parent.removeEventListener('abort', onParentAbort);\n }\n },\n };\n}\n\nasync function readBody(res: Response, parseJson: boolean): Promise<unknown> {\n if (res.status === 204 || res.status === 205) {\n return null;\n }\n const contentType = res.headers.get('content-type') ?? '';\n if (parseJson && contentType.includes('application/json')) {\n const text = await res.text();\n if (text.length === 0) {\n return null;\n }\n return JSON.parse(text);\n }\n return res.text();\n}\n\nexport async function request<T = unknown>(\n req: HttpRequest,\n options: RequestOptions,\n): Promise<HttpResponse<T>> {\n const fetchImpl: FetchLike = options.fetch ?? (globalThis.fetch as FetchLike);\n const retry = req.retry ?? {};\n const maxAttempts = retry.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;\n const initialDelayMs = retry.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;\n const maxDelayMs = retry.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;\n const retryOn = retry.retryOn ?? defaultRetryOn;\n const timeoutMs = req.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n const parseJson = req.parseJson ?? true;\n\n const headers = mergeHeaders(\n {\n 'User-Agent': DEFAULT_USER_AGENT,\n Accept: 'application/json',\n },\n req.headers,\n );\n\n let lastErr: Error | undefined;\n\n for (let attempt = 0; attempt < maxAttempts; attempt++) {\n req.signal?.throwIfAborted();\n\n const { signal, cancel } = linkTimeoutSignal(req.signal, timeoutMs);\n let res: Response;\n try {\n res = await fetchImpl(req.url, {\n method: req.method ?? 'GET',\n headers,\n body: req.body as RequestInit['body'],\n signal,\n });\n } catch (err) {\n cancel();\n if (req.signal?.aborted) {\n throw req.signal.reason ?? err;\n }\n const error = err instanceof Error ? err : new Error(String(err));\n lastErr = error;\n if (attempt < maxAttempts - 1 && retryOn(null, error)) {\n const delay = computeDelay(attempt, initialDelayMs, maxDelayMs);\n await sleep(delay, req.signal);\n continue;\n }\n throw new TransientError(error.message);\n }\n cancel();\n\n const body = await readBody(res, parseJson);\n const httpResponse: HttpResponse<T> = {\n status: res.status,\n headers: res.headers,\n body: body as T,\n };\n if (req.rateLimit) {\n const state = req.rateLimit.parse(res.headers);\n if (state) {\n httpResponse.rateLimitState = state;\n }\n }\n\n if (options.observer) {\n await notifyObserver(options.observer, {\n url: req.url,\n method: req.method ?? 'GET',\n status: res.status,\n resource: options.resource,\n requestId: options.requestId ?? newRequestId(),\n body,\n });\n }\n\n if (res.ok) {\n return httpResponse;\n }\n\n const retryAfter = parseRetryAfter(res.headers.get('retry-after'));\n const message = `HTTP ${res.status} ${res.statusText} for ${req.method ?? 'GET'} ${req.url}`;\n const err = errorForStatus(message, httpResponse, retryAfter);\n\n if (\n attempt < maxAttempts - 1 &&\n retryOn(res.status, err) &&\n !(err instanceof AuthError) &&\n !(err instanceof ClientBugError)\n ) {\n lastErr = err;\n let delay = computeDelay(attempt, initialDelayMs, maxDelayMs);\n if (err instanceof RateLimitError && retryAfter) {\n const wait = retryAfter.getTime() - Date.now();\n if (wait > 0) {\n delay = Math.min(wait, maxDelayMs);\n }\n }\n await sleep(delay, req.signal);\n continue;\n }\n\n throw err;\n }\n\n throw lastErr ?? new UpstreamBugError('Exhausted retry attempts');\n}\n\nfunction computeDelay(\n attempt: number,\n initialDelayMs: number,\n maxDelayMs: number,\n): number {\n const base = initialDelayMs * 2 ** attempt;\n const jitter = base * 0.25 * Math.random();\n return Math.min(base + jitter, maxDelayMs);\n}\n\nexport { HttpClientError };\n","export interface RateLimitState {\n remaining: number;\n resetAt: Date;\n}\n\nexport interface RateLimitPolicy {\n parse(headers: Headers): RateLimitState | null;\n}\n\nexport interface StandardRateLimitPolicyConfig {\n remainingHeader: string;\n resetHeader: string;\n resetUnit: 's' | 'ms';\n resetFallbackMs?: number;\n}\n\nexport function standardRateLimitPolicy(\n config: StandardRateLimitPolicyConfig,\n): RateLimitPolicy {\n const { remainingHeader, resetHeader, resetUnit, resetFallbackMs } = config;\n const multiplier = resetUnit === 's' ? 1000 : 1;\n return {\n parse(h) {\n const remainingRaw = h.get(remainingHeader);\n if (remainingRaw === null || remainingRaw.trim() === '') {\n return null;\n }\n const remaining = Number(remainingRaw);\n if (!Number.isFinite(remaining)) {\n return null;\n }\n const resetRaw = h.get(resetHeader);\n if (resetRaw === null) {\n if (resetFallbackMs === undefined) {\n return null;\n }\n return {\n remaining,\n resetAt: new Date(Date.now() + resetFallbackMs),\n };\n }\n if (resetRaw.trim() === '') {\n return null;\n }\n const reset = Number(resetRaw);\n if (!Number.isFinite(reset) || reset < 0) {\n return null;\n }\n const resetMs = reset * multiplier;\n if (!Number.isFinite(resetMs)) {\n return null;\n }\n return { remaining, resetAt: new Date(resetMs) };\n },\n };\n}\n","export async function mapWithConcurrency<T, R>(\n items: readonly T[],\n concurrency: number,\n fn: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n const results = new Array<R>(items.length);\n if (items.length === 0) {\n return results;\n }\n const normalized = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;\n const limit = Math.max(1, Math.min(normalized, items.length));\n let next = 0;\n let failed = false;\n\n async function worker(): Promise<void> {\n while (!failed) {\n const i = next++;\n if (i >= items.length) {\n return;\n }\n try {\n results[i] = await fn(items[i]!, i);\n } catch (err) {\n failed = true;\n throw err;\n }\n }\n }\n\n const workers: Promise<void>[] = [];\n for (let w = 0; w < limit; w++) {\n workers.push(worker());\n }\n await Promise.all(workers);\n return results;\n}\n","export interface SanitizeAllowedUrlOptions {\n url: string | null;\n host: string;\n pathname: string;\n protocol?: 'https:' | 'http:';\n}\n\nexport function sanitizeAllowedUrl(\n options: SanitizeAllowedUrlOptions,\n): string | null {\n const { url, host, pathname, protocol = 'https:' } = options;\n if (url === null) {\n return null;\n }\n try {\n const u = new URL(url);\n if (u.protocol !== protocol || u.host !== host || u.pathname !== pathname) {\n return null;\n }\n return u.toString();\n } catch {\n return null;\n }\n}\n","export type EpochUnit = 'ms' | 's' | 'iso';\n\nexport function parseEpoch(\n value: number | string | null | undefined,\n unit: EpochUnit,\n): number | null {\n if (value === null || value === undefined) {\n return null;\n }\n if (unit === 'iso') {\n if (typeof value !== 'string') {\n return null;\n }\n const ms = new Date(value).getTime();\n return Number.isFinite(ms) ? ms : null;\n }\n if (typeof value === 'string' && value.trim() === '') {\n return null;\n }\n const n = typeof value === 'number' ? value : Number(value);\n if (!Number.isFinite(n)) {\n return null;\n }\n const result = unit === 's' ? n * 1000 : n;\n return Number.isFinite(result) ? result : null;\n}\n","import { request } from './request';\nimport type { HttpRequest } from './types';\n\nexport function parseLinkHeader(header: string | null): Record<string, string> {\n if (!header) {\n return {};\n }\n const result: Record<string, string> = {};\n for (const part of header.split(',')) {\n const match = part.match(/<([^>]+)>\\s*;\\s*rel=\"([^\"]+)\"/);\n if (match) {\n result[match[2]!] = match[1]!;\n }\n }\n return result;\n}\n\nexport async function* paginateLink<T>(\n initial: HttpRequest,\n parse: (body: unknown) => T[],\n options: { resource: string },\n): AsyncIterable<T> {\n let next: string | null = initial.url;\n while (next) {\n const res: Awaited<ReturnType<typeof request>> = await request(\n {\n ...initial,\n url: next,\n },\n { resource: options.resource },\n );\n for (const item of parse(res.body)) {\n yield item;\n }\n const links = parseLinkHeader(res.headers.get('link'));\n next = links['next'] ?? null;\n }\n}\n\nexport async function* paginateCursor<T>(\n initial: HttpRequest,\n parse: (body: unknown) => { items: T[]; nextCursor: string | null },\n buildNext: (req: HttpRequest, cursor: string) => HttpRequest,\n options: { resource: string },\n): AsyncIterable<T> {\n let req: HttpRequest = initial;\n while (true) {\n const res = await request(req, { resource: options.resource });\n const { items, nextCursor } = parse(res.body);\n for (const item of items) {\n yield item;\n }\n if (!nextCursor) {\n return;\n }\n req = buildNext(req, nextCursor);\n }\n}\n\nexport async function* paginatePage<T>(\n initial: HttpRequest,\n parse: (body: unknown) => { items: T[]; hasMore: boolean },\n buildPage: (req: HttpRequest, page: number) => HttpRequest,\n options: { resource: string },\n): AsyncIterable<T> {\n let page = 1;\n while (true) {\n const req = page === 1 ? initial : buildPage(initial, page);\n const res = await request(req, { resource: options.resource });\n const { items, hasMore } = parse(res.body);\n for (const item of items) {\n yield item;\n }\n if (!hasMore || items.length === 0) {\n return;\n }\n page++;\n }\n}\n","export type LogFields = Record<string, unknown>;\n\nexport interface ConnectorLogger {\n info(event: string, fields?: LogFields): void;\n warn(event: string, fields?: LogFields): void;\n}\n\nexport interface ConnectorLoggerOptions {\n scope: string;\n}\n\nconst MAX_VALUE_LEN = 120;\n\nfunction truncate(s: string, max = MAX_VALUE_LEN): string {\n if (s.length <= max) {\n return s;\n }\n return `${s.slice(0, max - 1)}…`;\n}\n\nfunction formatValue(value: unknown): string {\n if (value === null) {\n return 'null';\n }\n if (value === undefined) {\n return '';\n }\n if (typeof value === 'number' || typeof value === 'boolean') {\n return String(value);\n }\n if (typeof value === 'string') {\n const t = truncate(value);\n if (/[\\s\"=]/.test(t)) {\n return JSON.stringify(t);\n }\n return t;\n }\n if (typeof value === 'bigint') {\n return value.toString();\n }\n let json: string | undefined;\n try {\n json = JSON.stringify(value);\n } catch {\n json = undefined;\n }\n return truncate(json ?? String(value));\n}\n\nexport function formatLogFields(fields?: LogFields): string {\n if (!fields) {\n return '';\n }\n const parts: string[] = [];\n for (const [k, v] of Object.entries(fields)) {\n if (v === undefined) {\n continue;\n }\n parts.push(`${k}=${formatValue(v)}`);\n }\n return parts.length > 0 ? ` ${parts.join(' ')}` : '';\n}\n\nexport function formatLogLine(\n scope: string,\n event: string,\n fields?: LogFields,\n): string {\n return `[${scope}] ${event}${formatLogFields(fields)}`;\n}\n\nexport function createDefaultConnectorLogger(\n opts: ConnectorLoggerOptions,\n): ConnectorLogger {\n return {\n info(event, fields) {\n console.info(formatLogLine(opts.scope, event, fields));\n },\n warn(event, fields) {\n console.warn(formatLogLine(opts.scope, event, fields));\n },\n };\n}\n\nconst NOOP_LOGGER: ConnectorLogger = {\n info() {},\n warn() {},\n};\n\nexport function noopConnectorLogger(): ConnectorLogger {\n return NOOP_LOGGER;\n}\n","import { connectorUserAgent } from '@rawdash/connector-shared';\nimport {\n BaseConnector,\n type ConnectorContext,\n type ConnectorDoc,\n type CredentialsSchema,\n type StorageHandle,\n type SyncOptions,\n type SyncResult,\n defineConfigFields,\n defineConnectorDoc,\n defineResources,\n schemasFromResources,\n} from '@rawdash/core';\nimport { z } from 'zod';\n\nexport const configFields = defineConfigFields(\n z\n .object({\n siteUrl: z.string().trim().min(1).meta({\n label: 'Site URL',\n description:\n 'Verified Search Console property. URL-prefix properties look like \"https://example.com/\"; Domain properties look like \"sc-domain:example.com\".',\n placeholder: 'https://example.com/',\n }),\n serviceAccountJson: z\n .object({ $secret: z.string().trim().min(1) })\n .optional()\n .meta({\n label: 'Service Account JSON (recommended)',\n description:\n 'Contents of the JSON key file for a Google service account that has been added as a Search Console user (Owner or Full user) on the property. Create one at Google Cloud -> IAM & Admin -> Service Accounts.',\n secret: true,\n }),\n refreshToken: z\n .object({ $secret: z.string().trim().min(1) })\n .optional()\n .meta({\n label: 'OAuth Refresh Token',\n description:\n 'Google OAuth 2.0 refresh token with webmasters.readonly scope. Required if not using serviceAccountJson.',\n secret: true,\n }),\n clientId: z.string().trim().min(1).optional().meta({\n label: 'OAuth Client ID',\n description:\n 'OAuth 2.0 client ID from Google Cloud Console. Required when using refreshToken auth.',\n placeholder: '...apps.googleusercontent.com',\n }),\n clientSecret: z\n .object({ $secret: z.string().trim().min(1) })\n .optional()\n .meta({\n label: 'OAuth Client Secret',\n description:\n 'OAuth 2.0 client secret from Google Cloud Console. Required when using refreshToken auth.',\n secret: true,\n }),\n lookbackDays: z.number().int().positive().optional().meta({\n label: 'Lookback days (full sync)',\n description:\n 'How many calendar days to fetch on a full sync. Defaults to 90.',\n placeholder: '90',\n }),\n })\n .refine(\n (val) =>\n val.serviceAccountJson !== undefined ||\n (val.refreshToken !== undefined &&\n val.clientId !== undefined &&\n val.clientSecret !== undefined),\n {\n message:\n 'Provide either serviceAccountJson or the full OAuth tuple (refreshToken + clientId + clientSecret)',\n },\n ),\n);\n\nexport const doc: ConnectorDoc = defineConnectorDoc({\n displayName: 'Google Search Console',\n category: 'marketing',\n brandColor: '#458CF5',\n tagline:\n 'Sync daily Search Console SEO metrics - clicks, impressions, CTR, and average position - by date, query, page, and country.',\n vendor: {\n name: 'Google Search Console',\n apiDocs:\n 'https://developers.google.com/webmaster-tools/v1/api_reference_index',\n website: 'https://search.google.com/search-console',\n },\n auth: {\n summary:\n 'Authenticate against the Search Console API with either a Google service account JSON key (recommended) or an OAuth 2.0 refresh-token tuple. The identity must be added as a user on the Search Console property (Owner or Full user).',\n setup: [\n 'Identify the property to sync. URL-prefix properties use the full origin (e.g. https://example.com/); Domain properties use the sc-domain:example.com format.',\n 'Recommended: create a service account at Google Cloud -> IAM & Admin -> Service Accounts, generate a JSON key, then in Search Console add the service account email as a user on the property. Store the JSON as a secret and reference it as serviceAccountJson: secret(\"GSC_SERVICE_ACCOUNT_JSON\").',\n 'Alternative: provide an OAuth 2.0 refresh token with the webmasters.readonly scope together with its clientId and clientSecret from the Google Cloud Console.',\n ],\n },\n rateLimit:\n 'Search Console API quota is 1,200 queries per minute per project (default); 429 responses are retried automatically with exponential backoff.',\n limitations: [\n 'Search Console aggregates data with a 2-3 day lag, so incremental syncs refetch the trailing 3 days.',\n 'Each query is paginated 25,000 rows per page; a phase that yields more than that paginates by startRow.',\n ],\n});\n\nexport interface GSCSettings {\n siteUrl: string;\n lookbackDays?: number;\n}\n\nconst gscCredentials = {\n serviceAccountJson: {\n description: 'Google service account JSON key (base64 or raw JSON)',\n auth: 'optional' as const,\n },\n refreshToken: {\n description: 'Google OAuth 2.0 refresh token',\n auth: 'optional' as const,\n },\n clientId: {\n description: 'Google OAuth 2.0 client ID',\n auth: 'optional' as const,\n },\n clientSecret: {\n description: 'Google OAuth 2.0 client secret',\n auth: 'optional' as const,\n },\n} satisfies CredentialsSchema;\n\ntype GSCCredentials = typeof gscCredentials;\n\nconst PHASE_ORDER = [\n 'search_analytics_by_day',\n 'top_queries',\n 'top_pages',\n 'top_countries',\n] as const;\n\ntype GSCPhase = (typeof PHASE_ORDER)[number];\n\ninterface GSCDateRange {\n startDate: string;\n endDate: string;\n}\n\ninterface GSCSyncCursor {\n phase: GSCPhase;\n dateRange: GSCDateRange;\n}\n\nconst GSC_DATE_RE = /^\\d{4}-\\d{2}-\\d{2}$/;\n\nfunction isGSCDateString(value: unknown): value is string {\n return typeof value === 'string' && GSC_DATE_RE.test(value);\n}\n\nfunction isGSCDateRange(value: unknown): value is GSCDateRange {\n if (typeof value !== 'object' || value === null) {\n return false;\n }\n const v = value as { startDate?: unknown; endDate?: unknown };\n return isGSCDateString(v.startDate) && isGSCDateString(v.endDate);\n}\n\nfunction isGSCSyncCursor(value: unknown): value is GSCSyncCursor {\n if (typeof value !== 'object' || value === null) {\n return false;\n }\n const v = value as { phase?: unknown; dateRange?: unknown };\n if (typeof v.phase !== 'string') {\n return false;\n }\n if (!(PHASE_ORDER as readonly string[]).includes(v.phase)) {\n return false;\n }\n return isGSCDateRange(v.dateRange);\n}\n\ninterface PhaseConfig {\n dimensions: string[];\n metricName: string;\n}\n\nconst PHASE_CONFIGS: Record<GSCPhase, PhaseConfig> = {\n search_analytics_by_day: {\n dimensions: ['date'],\n metricName: 'gsc_search_analytics_by_day',\n },\n top_queries: {\n dimensions: ['date', 'query'],\n metricName: 'gsc_top_queries',\n },\n top_pages: {\n dimensions: ['date', 'page'],\n metricName: 'gsc_top_pages',\n },\n top_countries: {\n dimensions: ['date', 'country'],\n metricName: 'gsc_top_countries',\n },\n};\n\nconst ROWS_PER_PAGE = 25_000;\nconst METRIC_FIELDS = ['clicks', 'impressions', 'ctr', 'position'] as const;\n\nexport interface GSCReportRow {\n keys?: string[];\n clicks?: number;\n impressions?: number;\n ctr?: number;\n position?: number;\n}\n\ninterface GSCReportResponse {\n rows?: GSCReportRow[];\n responseAggregationType?: string;\n}\n\ninterface ServiceAccountKey {\n client_email: string;\n private_key: string;\n token_uri?: string;\n}\n\ninterface TokenResponse {\n access_token: string;\n expires_in?: number;\n}\n\nfunction base64urlFromBytes(bytes: Uint8Array): string {\n let binary = '';\n for (let i = 0; i < bytes.length; i++) {\n binary += String.fromCharCode(bytes[i]!);\n }\n return btoa(binary).replace(/\\+/g, '-').replace(/\\//g, '_').replace(/=/g, '');\n}\n\nfunction base64urlFromString(str: string): string {\n return base64urlFromBytes(new TextEncoder().encode(str));\n}\n\nasync function signRS256JWT(\n payload: Record<string, unknown>,\n privateKeyPem: string,\n): Promise<string> {\n const header = { alg: 'RS256', typ: 'JWT' };\n const headerB64 = base64urlFromString(JSON.stringify(header));\n const payloadB64 = base64urlFromString(JSON.stringify(payload));\n const signingInput = `${headerB64}.${payloadB64}`;\n\n const pemContent = privateKeyPem\n .replace(/-----BEGIN PRIVATE KEY-----/g, '')\n .replace(/-----END PRIVATE KEY-----/g, '')\n .replace(/\\s/g, '');\n const der = Uint8Array.from(atob(pemContent), (c) => c.charCodeAt(0));\n\n const key = await globalThis.crypto.subtle.importKey(\n 'pkcs8',\n der,\n { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' },\n false,\n ['sign'],\n );\n\n const signature = await globalThis.crypto.subtle.sign(\n 'RSASSA-PKCS1-v1_5',\n key,\n new TextEncoder().encode(signingInput),\n );\n\n return `${signingInput}.${base64urlFromBytes(new Uint8Array(signature))}`;\n}\n\nfunction parseServiceAccountJson(value: string): ServiceAccountKey {\n const trimmed = value.trim();\n if (trimmed.startsWith('{')) {\n return JSON.parse(trimmed) as ServiceAccountKey;\n }\n const binary = atob(trimmed);\n const bytes = new Uint8Array(binary.length);\n for (let i = 0; i < binary.length; i++) {\n bytes[i] = binary.charCodeAt(i);\n }\n const decoded = new TextDecoder().decode(bytes);\n return JSON.parse(decoded) as ServiceAccountKey;\n}\n\nasync function buildServiceAccountJwt(\n serviceAccountJson: string,\n): Promise<{ url: string; body: string }> {\n const sa = parseServiceAccountJson(serviceAccountJson);\n const now = Math.floor(Date.now() / 1000);\n const jwt = await signRS256JWT(\n {\n iss: sa.client_email,\n scope: 'https://www.googleapis.com/auth/webmasters.readonly',\n aud: sa.token_uri ?? 'https://oauth2.googleapis.com/token',\n exp: now + 3600,\n iat: now,\n },\n sa.private_key,\n );\n\n const body = new URLSearchParams({\n grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',\n assertion: jwt,\n }).toString();\n\n return {\n url: sa.token_uri ?? 'https://oauth2.googleapis.com/token',\n body,\n };\n}\n\nfunction toGSCDate(date: Date): string {\n const y = date.getUTCFullYear();\n const m = String(date.getUTCMonth() + 1).padStart(2, '0');\n const d = String(date.getUTCDate()).padStart(2, '0');\n return `${y}-${m}-${d}`;\n}\n\nfunction gscDateToMs(gscDate: string): number {\n const y = gscDate.slice(0, 4);\n const m = gscDate.slice(5, 7);\n const d = gscDate.slice(8, 10);\n return Date.UTC(Number(y), Number(m) - 1, Number(d));\n}\n\nconst MS_PER_DAY = 24 * 60 * 60 * 1000;\nconst INCREMENTAL_LOOKBACK_DAYS = 3;\n\nfunction getDateRange(\n options: SyncOptions,\n lookbackDays: number,\n): GSCDateRange {\n const now = Date.now();\n const endDate = toGSCDate(new Date(now));\n if (options.mode === 'latest') {\n const startMs = now - (INCREMENTAL_LOOKBACK_DAYS - 1) * MS_PER_DAY;\n return { startDate: toGSCDate(new Date(startMs)), endDate };\n }\n if (options.since) {\n const sinceMs = new Date(options.since).getTime();\n if (Number.isFinite(sinceMs)) {\n const days = Math.max(1, Math.ceil((now - sinceMs) / MS_PER_DAY));\n const cappedDays = Math.min(days, lookbackDays);\n const startMs = now - (cappedDays - 1) * MS_PER_DAY;\n return { startDate: toGSCDate(new Date(startMs)), endDate };\n }\n }\n const startMs = now - (lookbackDays - 1) * MS_PER_DAY;\n return { startDate: toGSCDate(new Date(startMs)), endDate };\n}\n\nexport function rowToMetricSample(\n row: GSCReportRow,\n dimensions: string[],\n metricName: string,\n): {\n name: string;\n ts: number;\n value: number;\n attributes: Record<string, string | number>;\n} {\n const attributes: Record<string, string | number> = {};\n const keys = row.keys ?? [];\n for (let i = 0; i < dimensions.length; i++) {\n attributes[dimensions[i]!] = keys[i] ?? '';\n }\n\n for (const field of METRIC_FIELDS) {\n const raw = row[field];\n attributes[field] =\n typeof raw === 'number' && Number.isFinite(raw) ? raw : 0;\n }\n\n const dateStr =\n typeof attributes['date'] === 'string' ? attributes['date'] : '1970-01-01';\n const ts = gscDateToMs(dateStr);\n const primaryValue = attributes['clicks'] as number;\n\n return {\n name: metricName,\n ts,\n value: primaryValue,\n attributes,\n };\n}\n\nconst dateKey = z\n .string()\n .regex(/^(19|20)\\d{2}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$/);\nconst stringKey = z.string();\nconst numericField = z.number();\n\nfunction reportSchema(dimensionCount: number) {\n const keys =\n dimensionCount === 1\n ? z.tuple([dateKey])\n : z.tuple([dateKey, ...Array(dimensionCount - 1).fill(stringKey)] as [\n typeof dateKey,\n ...z.ZodType[],\n ]);\n return z.object({\n rows: z\n .array(\n z.object({\n keys,\n clicks: numericField.optional(),\n impressions: numericField.optional(),\n ctr: numericField.optional(),\n position: numericField.optional(),\n }),\n )\n .optional(),\n });\n}\n\nconst tokenResponseSchema = z.object({\n access_token: z.string().min(1),\n expires_in: z.number().int().positive().optional(),\n});\n\nexport const googleSearchConsoleResources = defineResources({\n gsc_search_analytics_by_day: {\n shape: 'metric',\n description:\n 'Daily site totals - clicks, impressions, CTR, and average position across all queries and pages.',\n unit: 'clicks',\n granularity: 'day',\n endpoint: 'POST /webmasters/v3/sites/{siteUrl}/searchAnalytics/query',\n dimensions: [\n { name: 'date', description: 'Calendar day of the metric sample.' },\n ],\n responses: {\n oauth_token: tokenResponseSchema,\n search_analytics_by_day: reportSchema(1),\n },\n },\n gsc_top_queries: {\n shape: 'metric',\n description:\n 'Daily clicks, impressions, CTR, and average position broken down by search query.',\n unit: 'clicks',\n granularity: 'day',\n endpoint: 'POST /webmasters/v3/sites/{siteUrl}/searchAnalytics/query',\n dimensions: [\n { name: 'date', description: 'Calendar day of the metric sample.' },\n {\n name: 'query',\n description: 'Search term that surfaced a result for the property.',\n },\n ],\n responses: { top_queries: reportSchema(2) },\n },\n gsc_top_pages: {\n shape: 'metric',\n description:\n 'Daily clicks, impressions, CTR, and average position broken down by landing page URL.',\n unit: 'clicks',\n granularity: 'day',\n endpoint: 'POST /webmasters/v3/sites/{siteUrl}/searchAnalytics/query',\n dimensions: [\n { name: 'date', description: 'Calendar day of the metric sample.' },\n {\n name: 'page',\n description: 'Canonical URL of the page that surfaced in search.',\n },\n ],\n responses: { top_pages: reportSchema(2) },\n },\n gsc_top_countries: {\n shape: 'metric',\n description:\n 'Daily clicks, impressions, CTR, and average position broken down by visitor country.',\n unit: 'clicks',\n granularity: 'day',\n endpoint: 'POST /webmasters/v3/sites/{siteUrl}/searchAnalytics/query',\n dimensions: [\n { name: 'date', description: 'Calendar day of the metric sample.' },\n {\n name: 'country',\n description: 'ISO 3166-1 alpha-3 country code of the searcher.',\n },\n ],\n responses: { top_countries: reportSchema(2) },\n },\n});\n\nexport const id = 'google-search-console';\n\nexport class GSCConnector extends BaseConnector<GSCSettings, GSCCredentials> {\n static readonly id = id;\n\n static readonly resources = googleSearchConsoleResources;\n\n static readonly schemas = schemasFromResources(googleSearchConsoleResources);\n\n static create(input: unknown, ctx?: ConnectorContext): GSCConnector {\n const parsed = configFields.parse(input);\n return new GSCConnector(\n {\n siteUrl: parsed.siteUrl,\n lookbackDays: parsed.lookbackDays,\n },\n {\n serviceAccountJson: parsed.serviceAccountJson,\n refreshToken: parsed.refreshToken,\n clientId: parsed.clientId,\n clientSecret: parsed.clientSecret,\n },\n ctx,\n );\n }\n\n readonly id = id;\n override readonly credentials = gscCredentials;\n\n private cachedToken: { token: string; expiresAt: number } | null = null;\n\n private async fetchOAuthToken(\n url: string,\n body: string,\n signal: AbortSignal | undefined,\n ): Promise<{ token: string; expiresAt: number }> {\n const res = await this.post<TokenResponse>(url, {\n resource: 'oauth_token',\n headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n body,\n signal,\n });\n const expiresIn = res.body.expires_in ?? 3600;\n return {\n token: res.body.access_token,\n expiresAt: Date.now() + (expiresIn - 60) * 1000,\n };\n }\n\n private async getAccessToken(signal?: AbortSignal): Promise<string> {\n if (this.cachedToken && Date.now() < this.cachedToken.expiresAt) {\n return this.cachedToken.token;\n }\n\n const { serviceAccountJson, refreshToken, clientId, clientSecret } =\n this.creds;\n\n if (serviceAccountJson) {\n const { url, body } = await buildServiceAccountJwt(serviceAccountJson);\n this.cachedToken = await this.fetchOAuthToken(url, body, signal);\n return this.cachedToken.token;\n }\n\n if (refreshToken && clientId && clientSecret) {\n const body = new URLSearchParams({\n grant_type: 'refresh_token',\n refresh_token: refreshToken,\n client_id: clientId,\n client_secret: clientSecret,\n }).toString();\n this.cachedToken = await this.fetchOAuthToken(\n 'https://oauth2.googleapis.com/token',\n body,\n signal,\n );\n return this.cachedToken.token;\n }\n\n throw new Error(\n 'GSC connector: provide either serviceAccountJson or (refreshToken + clientId + clientSecret)',\n );\n }\n\n private async runReport(\n accessToken: string,\n phase: GSCPhase,\n dateRange: GSCDateRange,\n startRow: number,\n signal?: AbortSignal,\n ): Promise<GSCReportResponse> {\n const { dimensions } = PHASE_CONFIGS[phase];\n const url = `https://searchconsole.googleapis.com/webmasters/v3/sites/${encodeURIComponent(this.settings.siteUrl)}/searchAnalytics/query`;\n\n const body: Record<string, unknown> = {\n startDate: dateRange.startDate,\n endDate: dateRange.endDate,\n dimensions,\n rowLimit: ROWS_PER_PAGE,\n startRow,\n };\n\n const res = await this.post<GSCReportResponse>(url, {\n resource: phase,\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n 'User-Agent': connectorUserAgent('google-search-console'),\n },\n body: JSON.stringify(body),\n signal,\n });\n return res.body;\n }\n\n async sync(\n options: SyncOptions,\n storage: StorageHandle,\n signal?: AbortSignal,\n ): Promise<SyncResult> {\n const lookbackDays = this.settings.lookbackDays ?? 90;\n\n const cursor = isGSCSyncCursor(options.cursor) ? options.cursor : undefined;\n const dateRange = cursor?.dateRange ?? getDateRange(options, lookbackDays);\n\n let accessToken: string | null = null;\n const getToken = async (sig?: AbortSignal): Promise<string> => {\n if (!accessToken) {\n accessToken = await this.getAccessToken(sig);\n }\n return accessToken;\n };\n\n const runReportWithRetry = async (\n phase: GSCPhase,\n startRow: number,\n sig: AbortSignal | undefined,\n ): Promise<GSCReportResponse> => {\n const token = await getToken(sig);\n try {\n return await this.runReport(token, phase, dateRange, startRow, sig);\n } catch (err) {\n console.warn(\n `[gsc] runReport failed, refreshing token and retrying once`,\n err,\n );\n accessToken = null;\n const freshToken = await getToken(sig);\n return this.runReport(freshToken, phase, dateRange, startRow, sig);\n }\n };\n\n const drainPhase = async (phase: GSCPhase): Promise<GSCReportRow[]> => {\n const allRows: GSCReportRow[] = [];\n let startRow = 0;\n for (;;) {\n const response = await runReportWithRetry(phase, startRow, signal);\n const rows = response.rows ?? [];\n allRows.push(...rows);\n startRow += rows.length;\n if (rows.length < ROWS_PER_PAGE) {\n break;\n }\n }\n return allRows;\n };\n\n const resumeIdx = cursor ? PHASE_ORDER.indexOf(cursor.phase) : -1;\n const startIdx = resumeIdx >= 0 ? resumeIdx : 0;\n\n for (let i = startIdx; i < PHASE_ORDER.length; i++) {\n const phase = PHASE_ORDER[i]!;\n if (signal?.aborted) {\n return { done: false, cursor: { phase, dateRange } };\n }\n\n let rows: GSCReportRow[];\n try {\n rows = await drainPhase(phase);\n } catch (err) {\n if (signal?.aborted) {\n return { done: false, cursor: { phase, dateRange } };\n }\n throw err;\n }\n const cfg = PHASE_CONFIGS[phase];\n const samples = rows.map((row) =>\n rowToMetricSample(row, cfg.dimensions, cfg.metricName),\n );\n await storage.metrics(samples, { names: [cfg.metricName] });\n }\n\n return { done: true };\n }\n}\n","import { GSCConnector } from './google-search-console';\n\nexport {\n configFields,\n doc,\n GSCConnector,\n googleSearchConsoleResources as resources,\n id,\n rowToMetricSample,\n} from './google-search-console';\nexport type { GSCSettings } from './google-search-console';\nexport default GSCConnector;\n"],"mappings":";AEAO,IAAM,sBAAsB;AAE5B,IAAM,qBAAqB,qBAAqB,mBAAmB;AAEnE,SAAS,mBAAmB,aAA6B;AAC9D,SAAO,qBAAqB,WAAW,IAAI,mBAAmB;AAChE;;;AQLA;AAAA,EACE;AAAA,EAOA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,SAAS;AAEX,IAAM,eAAe;AAAA,EAC1B,EACG,OAAO;AAAA,IACN,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,KAAK;AAAA,MACrC,OAAO;AAAA,MACP,aACE;AAAA,MACF,aAAa;AAAA,IACf,CAAC;AAAA,IACD,oBAAoB,EACjB,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,EAC5C,SAAS,EACT,KAAK;AAAA,MACJ,OAAO;AAAA,MACP,aACE;AAAA,MACF,QAAQ;AAAA,IACV,CAAC;AAAA,IACH,cAAc,EACX,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,EAC5C,SAAS,EACT,KAAK;AAAA,MACJ,OAAO;AAAA,MACP,aACE;AAAA,MACF,QAAQ;AAAA,IACV,CAAC;AAAA,IACH,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK;AAAA,MACjD,OAAO;AAAA,MACP,aACE;AAAA,MACF,aAAa;AAAA,IACf,CAAC;AAAA,IACD,cAAc,EACX,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,EAC5C,SAAS,EACT,KAAK;AAAA,MACJ,OAAO;AAAA,MACP,aACE;AAAA,MACF,QAAQ;AAAA,IACV,CAAC;AAAA,IACH,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK;AAAA,MACxD,OAAO;AAAA,MACP,aACE;AAAA,MACF,aAAa;AAAA,IACf,CAAC;AAAA,EACH,CAAC,EACA;AAAA,IACC,CAAC,QACC,IAAI,uBAAuB,UAC1B,IAAI,iBAAiB,UACpB,IAAI,aAAa,UACjB,IAAI,iBAAiB;AAAA,IACzB;AAAA,MACE,SACE;AAAA,IACJ;AAAA,EACF;AACJ;AAEO,IAAM,MAAoB,mBAAmB;AAAA,EAClD,aAAa;AAAA,EACb,UAAU;AAAA,EACV,YAAY;AAAA,EACZ,SACE;AAAA,EACF,QAAQ;AAAA,IACN,MAAM;AAAA,IACN,SACE;AAAA,IACF,SAAS;AAAA,EACX;AAAA,EACA,MAAM;AAAA,IACJ,SACE;AAAA,IACF,OAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EACA,WACE;AAAA,EACF,aAAa;AAAA,IACX;AAAA,IACA;AAAA,EACF;AACF,CAAC;AAOD,IAAM,iBAAiB;AAAA,EACrB,oBAAoB;AAAA,IAClB,aAAa;AAAA,IACb,MAAM;AAAA,EACR;AAAA,EACA,cAAc;AAAA,IACZ,aAAa;AAAA,IACb,MAAM;AAAA,EACR;AAAA,EACA,UAAU;AAAA,IACR,aAAa;AAAA,IACb,MAAM;AAAA,EACR;AAAA,EACA,cAAc;AAAA,IACZ,aAAa;AAAA,IACb,MAAM;AAAA,EACR;AACF;AAIA,IAAM,cAAc;AAAA,EAClB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAcA,IAAM,cAAc;AAEpB,SAAS,gBAAgB,OAAiC;AACxD,SAAO,OAAO,UAAU,YAAY,YAAY,KAAK,KAAK;AAC5D;AAEA,SAAS,eAAe,OAAuC;AAC7D,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AACA,QAAM,IAAI;AACV,SAAO,gBAAgB,EAAE,SAAS,KAAK,gBAAgB,EAAE,OAAO;AAClE;AAEA,SAAS,gBAAgB,OAAwC;AAC/D,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AACA,QAAM,IAAI;AACV,MAAI,OAAO,EAAE,UAAU,UAAU;AAC/B,WAAO;AAAA,EACT;AACA,MAAI,CAAE,YAAkC,SAAS,EAAE,KAAK,GAAG;AACzD,WAAO;AAAA,EACT;AACA,SAAO,eAAe,EAAE,SAAS;AACnC;AAOA,IAAM,gBAA+C;AAAA,EACnD,yBAAyB;AAAA,IACvB,YAAY,CAAC,MAAM;AAAA,IACnB,YAAY;AAAA,EACd;AAAA,EACA,aAAa;AAAA,IACX,YAAY,CAAC,QAAQ,OAAO;AAAA,IAC5B,YAAY;AAAA,EACd;AAAA,EACA,WAAW;AAAA,IACT,YAAY,CAAC,QAAQ,MAAM;AAAA,IAC3B,YAAY;AAAA,EACd;AAAA,EACA,eAAe;AAAA,IACb,YAAY,CAAC,QAAQ,SAAS;AAAA,IAC9B,YAAY;AAAA,EACd;AACF;AAEA,IAAM,gBAAgB;AACtB,IAAM,gBAAgB,CAAC,UAAU,eAAe,OAAO,UAAU;AA0BjE,SAAS,mBAAmB,OAA2B;AACrD,MAAI,SAAS;AACb,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,cAAU,OAAO,aAAa,MAAM,CAAC,CAAE;AAAA,EACzC;AACA,SAAO,KAAK,MAAM,EAAE,QAAQ,OAAO,GAAG,EAAE,QAAQ,OAAO,GAAG,EAAE,QAAQ,MAAM,EAAE;AAC9E;AAEA,SAAS,oBAAoB,KAAqB;AAChD,SAAO,mBAAmB,IAAI,YAAY,EAAE,OAAO,GAAG,CAAC;AACzD;AAEA,eAAe,aACb,SACA,eACiB;AACjB,QAAM,SAAS,EAAE,KAAK,SAAS,KAAK,MAAM;AAC1C,QAAM,YAAY,oBAAoB,KAAK,UAAU,MAAM,CAAC;AAC5D,QAAM,aAAa,oBAAoB,KAAK,UAAU,OAAO,CAAC;AAC9D,QAAM,eAAe,GAAG,SAAS,IAAI,UAAU;AAE/C,QAAM,aAAa,cAChB,QAAQ,gCAAgC,EAAE,EAC1C,QAAQ,8BAA8B,EAAE,EACxC,QAAQ,OAAO,EAAE;AACpB,QAAM,MAAM,WAAW,KAAK,KAAK,UAAU,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;AAEpE,QAAM,MAAM,MAAM,WAAW,OAAO,OAAO;AAAA,IACzC;AAAA,IACA;AAAA,IACA,EAAE,MAAM,qBAAqB,MAAM,UAAU;AAAA,IAC7C;AAAA,IACA,CAAC,MAAM;AAAA,EACT;AAEA,QAAM,YAAY,MAAM,WAAW,OAAO,OAAO;AAAA,IAC/C;AAAA,IACA;AAAA,IACA,IAAI,YAAY,EAAE,OAAO,YAAY;AAAA,EACvC;AAEA,SAAO,GAAG,YAAY,IAAI,mBAAmB,IAAI,WAAW,SAAS,CAAC,CAAC;AACzE;AAEA,SAAS,wBAAwB,OAAkC;AACjE,QAAM,UAAU,MAAM,KAAK;AAC3B,MAAI,QAAQ,WAAW,GAAG,GAAG;AAC3B,WAAO,KAAK,MAAM,OAAO;AAAA,EAC3B;AACA,QAAM,SAAS,KAAK,OAAO;AAC3B,QAAM,QAAQ,IAAI,WAAW,OAAO,MAAM;AAC1C,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,UAAM,CAAC,IAAI,OAAO,WAAW,CAAC;AAAA,EAChC;AACA,QAAM,UAAU,IAAI,YAAY,EAAE,OAAO,KAAK;AAC9C,SAAO,KAAK,MAAM,OAAO;AAC3B;AAEA,eAAe,uBACb,oBACwC;AACxC,QAAM,KAAK,wBAAwB,kBAAkB;AACrD,QAAM,MAAM,KAAK,MAAM,KAAK,IAAI,IAAI,GAAI;AACxC,QAAM,MAAM,MAAM;AAAA,IAChB;AAAA,MACE,KAAK,GAAG;AAAA,MACR,OAAO;AAAA,MACP,KAAK,GAAG,aAAa;AAAA,MACrB,KAAK,MAAM;AAAA,MACX,KAAK;AAAA,IACP;AAAA,IACA,GAAG;AAAA,EACL;AAEA,QAAM,OAAO,IAAI,gBAAgB;AAAA,IAC/B,YAAY;AAAA,IACZ,WAAW;AAAA,EACb,CAAC,EAAE,SAAS;AAEZ,SAAO;AAAA,IACL,KAAK,GAAG,aAAa;AAAA,IACrB;AAAA,EACF;AACF;AAEA,SAAS,UAAU,MAAoB;AACrC,QAAM,IAAI,KAAK,eAAe;AAC9B,QAAM,IAAI,OAAO,KAAK,YAAY,IAAI,CAAC,EAAE,SAAS,GAAG,GAAG;AACxD,QAAM,IAAI,OAAO,KAAK,WAAW,CAAC,EAAE,SAAS,GAAG,GAAG;AACnD,SAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;AACvB;AAEA,SAAS,YAAY,SAAyB;AAC5C,QAAM,IAAI,QAAQ,MAAM,GAAG,CAAC;AAC5B,QAAM,IAAI,QAAQ,MAAM,GAAG,CAAC;AAC5B,QAAM,IAAI,QAAQ,MAAM,GAAG,EAAE;AAC7B,SAAO,KAAK,IAAI,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC;AACrD;AAEA,IAAM,aAAa,KAAK,KAAK,KAAK;AAClC,IAAM,4BAA4B;AAElC,SAAS,aACP,SACA,cACc;AACd,QAAM,MAAM,KAAK,IAAI;AACrB,QAAM,UAAU,UAAU,IAAI,KAAK,GAAG,CAAC;AACvC,MAAI,QAAQ,SAAS,UAAU;AAC7B,UAAMA,WAAU,OAAO,4BAA4B,KAAK;AACxD,WAAO,EAAE,WAAW,UAAU,IAAI,KAAKA,QAAO,CAAC,GAAG,QAAQ;AAAA,EAC5D;AACA,MAAI,QAAQ,OAAO;AACjB,UAAM,UAAU,IAAI,KAAK,QAAQ,KAAK,EAAE,QAAQ;AAChD,QAAI,OAAO,SAAS,OAAO,GAAG;AAC5B,YAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,MAAM,WAAW,UAAU,CAAC;AAChE,YAAM,aAAa,KAAK,IAAI,MAAM,YAAY;AAC9C,YAAMA,WAAU,OAAO,aAAa,KAAK;AACzC,aAAO,EAAE,WAAW,UAAU,IAAI,KAAKA,QAAO,CAAC,GAAG,QAAQ;AAAA,IAC5D;AAAA,EACF;AACA,QAAM,UAAU,OAAO,eAAe,KAAK;AAC3C,SAAO,EAAE,WAAW,UAAU,IAAI,KAAK,OAAO,CAAC,GAAG,QAAQ;AAC5D;AAEO,SAAS,kBACd,KACA,YACA,YAMA;AACA,QAAM,aAA8C,CAAC;AACrD,QAAM,OAAO,IAAI,QAAQ,CAAC;AAC1B,WAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,eAAW,WAAW,CAAC,CAAE,IAAI,KAAK,CAAC,KAAK;AAAA,EAC1C;AAEA,aAAW,SAAS,eAAe;AACjC,UAAM,MAAM,IAAI,KAAK;AACrB,eAAW,KAAK,IACd,OAAO,QAAQ,YAAY,OAAO,SAAS,GAAG,IAAI,MAAM;AAAA,EAC5D;AAEA,QAAM,UACJ,OAAO,WAAW,MAAM,MAAM,WAAW,WAAW,MAAM,IAAI;AAChE,QAAM,KAAK,YAAY,OAAO;AAC9B,QAAM,eAAe,WAAW,QAAQ;AAExC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA,OAAO;AAAA,IACP;AAAA,EACF;AACF;AAEA,IAAM,UAAU,EACb,OAAO,EACP,MAAM,sDAAsD;AAC/D,IAAM,YAAY,EAAE,OAAO;AAC3B,IAAM,eAAe,EAAE,OAAO;AAE9B,SAAS,aAAa,gBAAwB;AAC5C,QAAM,OACJ,mBAAmB,IACf,EAAE,MAAM,CAAC,OAAO,CAAC,IACjB,EAAE,MAAM,CAAC,SAAS,GAAG,MAAM,iBAAiB,CAAC,EAAE,KAAK,SAAS,CAAC,CAG7D;AACP,SAAO,EAAE,OAAO;AAAA,IACd,MAAM,EACH;AAAA,MACC,EAAE,OAAO;AAAA,QACP;AAAA,QACA,QAAQ,aAAa,SAAS;AAAA,QAC9B,aAAa,aAAa,SAAS;AAAA,QACnC,KAAK,aAAa,SAAS;AAAA,QAC3B,UAAU,aAAa,SAAS;AAAA,MAClC,CAAC;AAAA,IACH,EACC,SAAS;AAAA,EACd,CAAC;AACH;AAEA,IAAM,sBAAsB,EAAE,OAAO;AAAA,EACnC,cAAc,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC9B,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS;AACnD,CAAC;AAEM,IAAM,+BAA+B,gBAAgB;AAAA,EAC1D,6BAA6B;AAAA,IAC3B,OAAO;AAAA,IACP,aACE;AAAA,IACF,MAAM;AAAA,IACN,aAAa;AAAA,IACb,UAAU;AAAA,IACV,YAAY;AAAA,MACV,EAAE,MAAM,QAAQ,aAAa,qCAAqC;AAAA,IACpE;AAAA,IACA,WAAW;AAAA,MACT,aAAa;AAAA,MACb,yBAAyB,aAAa,CAAC;AAAA,IACzC;AAAA,EACF;AAAA,EACA,iBAAiB;AAAA,IACf,OAAO;AAAA,IACP,aACE;AAAA,IACF,MAAM;AAAA,IACN,aAAa;AAAA,IACb,UAAU;AAAA,IACV,YAAY;AAAA,MACV,EAAE,MAAM,QAAQ,aAAa,qCAAqC;AAAA,MAClE;AAAA,QACE,MAAM;AAAA,QACN,aAAa;AAAA,MACf;AAAA,IACF;AAAA,IACA,WAAW,EAAE,aAAa,aAAa,CAAC,EAAE;AAAA,EAC5C;AAAA,EACA,eAAe;AAAA,IACb,OAAO;AAAA,IACP,aACE;AAAA,IACF,MAAM;AAAA,IACN,aAAa;AAAA,IACb,UAAU;AAAA,IACV,YAAY;AAAA,MACV,EAAE,MAAM,QAAQ,aAAa,qCAAqC;AAAA,MAClE;AAAA,QACE,MAAM;AAAA,QACN,aAAa;AAAA,MACf;AAAA,IACF;AAAA,IACA,WAAW,EAAE,WAAW,aAAa,CAAC,EAAE;AAAA,EAC1C;AAAA,EACA,mBAAmB;AAAA,IACjB,OAAO;AAAA,IACP,aACE;AAAA,IACF,MAAM;AAAA,IACN,aAAa;AAAA,IACb,UAAU;AAAA,IACV,YAAY;AAAA,MACV,EAAE,MAAM,QAAQ,aAAa,qCAAqC;AAAA,MAClE;AAAA,QACE,MAAM;AAAA,QACN,aAAa;AAAA,MACf;AAAA,IACF;AAAA,IACA,WAAW,EAAE,eAAe,aAAa,CAAC,EAAE;AAAA,EAC9C;AACF,CAAC;AAEM,IAAM,KAAK;AAEX,IAAM,eAAN,MAAM,sBAAqB,cAA2C;AAAA,EAC3E,OAAgB,KAAK;AAAA,EAErB,OAAgB,YAAY;AAAA,EAE5B,OAAgB,UAAU,qBAAqB,4BAA4B;AAAA,EAE3E,OAAO,OAAO,OAAgB,KAAsC;AAClE,UAAM,SAAS,aAAa,MAAM,KAAK;AACvC,WAAO,IAAI;AAAA,MACT;AAAA,QACE,SAAS,OAAO;AAAA,QAChB,cAAc,OAAO;AAAA,MACvB;AAAA,MACA;AAAA,QACE,oBAAoB,OAAO;AAAA,QAC3B,cAAc,OAAO;AAAA,QACrB,UAAU,OAAO;AAAA,QACjB,cAAc,OAAO;AAAA,MACvB;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAES,KAAK;AAAA,EACI,cAAc;AAAA,EAExB,cAA2D;AAAA,EAEnE,MAAc,gBACZ,KACA,MACA,QAC+C;AAC/C,UAAM,MAAM,MAAM,KAAK,KAAoB,KAAK;AAAA,MAC9C,UAAU;AAAA,MACV,SAAS,EAAE,gBAAgB,oCAAoC;AAAA,MAC/D;AAAA,MACA;AAAA,IACF,CAAC;AACD,UAAM,YAAY,IAAI,KAAK,cAAc;AACzC,WAAO;AAAA,MACL,OAAO,IAAI,KAAK;AAAA,MAChB,WAAW,KAAK,IAAI,KAAK,YAAY,MAAM;AAAA,IAC7C;AAAA,EACF;AAAA,EAEA,MAAc,eAAe,QAAuC;AAClE,QAAI,KAAK,eAAe,KAAK,IAAI,IAAI,KAAK,YAAY,WAAW;AAC/D,aAAO,KAAK,YAAY;AAAA,IAC1B;AAEA,UAAM,EAAE,oBAAoB,cAAc,UAAU,aAAa,IAC/D,KAAK;AAEP,QAAI,oBAAoB;AACtB,YAAM,EAAE,KAAK,KAAK,IAAI,MAAM,uBAAuB,kBAAkB;AACrE,WAAK,cAAc,MAAM,KAAK,gBAAgB,KAAK,MAAM,MAAM;AAC/D,aAAO,KAAK,YAAY;AAAA,IAC1B;AAEA,QAAI,gBAAgB,YAAY,cAAc;AAC5C,YAAM,OAAO,IAAI,gBAAgB;AAAA,QAC/B,YAAY;AAAA,QACZ,eAAe;AAAA,QACf,WAAW;AAAA,QACX,eAAe;AAAA,MACjB,CAAC,EAAE,SAAS;AACZ,WAAK,cAAc,MAAM,KAAK;AAAA,QAC5B;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,aAAO,KAAK,YAAY;AAAA,IAC1B;AAEA,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,UACZ,aACA,OACA,WACA,UACA,QAC4B;AAC5B,UAAM,EAAE,WAAW,IAAI,cAAc,KAAK;AAC1C,UAAM,MAAM,4DAA4D,mBAAmB,KAAK,SAAS,OAAO,CAAC;AAEjH,UAAM,OAAgC;AAAA,MACpC,WAAW,UAAU;AAAA,MACrB,SAAS,UAAU;AAAA,MACnB;AAAA,MACA,UAAU;AAAA,MACV;AAAA,IACF;AAEA,UAAM,MAAM,MAAM,KAAK,KAAwB,KAAK;AAAA,MAClD,UAAU;AAAA,MACV,SAAS;AAAA,QACP,eAAe,UAAU,WAAW;AAAA,QACpC,gBAAgB;AAAA,QAChB,cAAc,mBAAmB,uBAAuB;AAAA,MAC1D;AAAA,MACA,MAAM,KAAK,UAAU,IAAI;AAAA,MACzB;AAAA,IACF,CAAC;AACD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,MAAM,KACJ,SACA,SACA,QACqB;AACrB,UAAM,eAAe,KAAK,SAAS,gBAAgB;AAEnD,UAAM,SAAS,gBAAgB,QAAQ,MAAM,IAAI,QAAQ,SAAS;AAClE,UAAM,YAAY,QAAQ,aAAa,aAAa,SAAS,YAAY;AAEzE,QAAI,cAA6B;AACjC,UAAM,WAAW,OAAO,QAAuC;AAC7D,UAAI,CAAC,aAAa;AAChB,sBAAc,MAAM,KAAK,eAAe,GAAG;AAAA,MAC7C;AACA,aAAO;AAAA,IACT;AAEA,UAAM,qBAAqB,OACzB,OACA,UACA,QAC+B;AAC/B,YAAM,QAAQ,MAAM,SAAS,GAAG;AAChC,UAAI;AACF,eAAO,MAAM,KAAK,UAAU,OAAO,OAAO,WAAW,UAAU,GAAG;AAAA,MACpE,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN;AAAA,UACA;AAAA,QACF;AACA,sBAAc;AACd,cAAM,aAAa,MAAM,SAAS,GAAG;AACrC,eAAO,KAAK,UAAU,YAAY,OAAO,WAAW,UAAU,GAAG;AAAA,MACnE;AAAA,IACF;AAEA,UAAM,aAAa,OAAO,UAA6C;AACrE,YAAM,UAA0B,CAAC;AACjC,UAAI,WAAW;AACf,iBAAS;AACP,cAAM,WAAW,MAAM,mBAAmB,OAAO,UAAU,MAAM;AACjE,cAAM,OAAO,SAAS,QAAQ,CAAC;AAC/B,gBAAQ,KAAK,GAAG,IAAI;AACpB,oBAAY,KAAK;AACjB,YAAI,KAAK,SAAS,eAAe;AAC/B;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,SAAS,YAAY,QAAQ,OAAO,KAAK,IAAI;AAC/D,UAAM,WAAW,aAAa,IAAI,YAAY;AAE9C,aAAS,IAAI,UAAU,IAAI,YAAY,QAAQ,KAAK;AAClD,YAAM,QAAQ,YAAY,CAAC;AAC3B,UAAI,QAAQ,SAAS;AACnB,eAAO,EAAE,MAAM,OAAO,QAAQ,EAAE,OAAO,UAAU,EAAE;AAAA,MACrD;AAEA,UAAI;AACJ,UAAI;AACF,eAAO,MAAM,WAAW,KAAK;AAAA,MAC/B,SAAS,KAAK;AACZ,YAAI,QAAQ,SAAS;AACnB,iBAAO,EAAE,MAAM,OAAO,QAAQ,EAAE,OAAO,UAAU,EAAE;AAAA,QACrD;AACA,cAAM;AAAA,MACR;AACA,YAAM,MAAM,cAAc,KAAK;AAC/B,YAAM,UAAU,KAAK;AAAA,QAAI,CAAC,QACxB,kBAAkB,KAAK,IAAI,YAAY,IAAI,UAAU;AAAA,MACvD;AACA,YAAM,QAAQ,QAAQ,SAAS,EAAE,OAAO,CAAC,IAAI,UAAU,EAAE,CAAC;AAAA,IAC5D;AAEA,WAAO,EAAE,MAAM,KAAK;AAAA,EACtB;AACF;;;ACjqBA,IAAO,gBAAQ;","names":["startMs"]}
|
|
1
|
+
{"version":3,"sources":["../../gcp-shared/src/auth.ts","../../gcp-shared/src/config.ts","../../../connector-shared/src/errors.ts","../../../connector-shared/src/retry.ts","../../../connector-shared/src/version.ts","../../../connector-shared/src/request.ts","../../../connector-shared/src/rate-limit.ts","../../../connector-shared/src/map-concurrent.ts","../../../connector-shared/src/sanitize.ts","../../../connector-shared/src/epoch.ts","../../../connector-shared/src/pagination.ts","../../../connector-shared/src/logger.ts","../../gcp-shared/src/bigquery.ts","../../gcp-shared/src/access-token.ts","../../gcp-shared/src/dates.ts","../../../connector-shared/src/errors.ts","../../../connector-shared/src/retry.ts","../../../connector-shared/src/version.ts","../../../connector-shared/src/request.ts","../../../connector-shared/src/rate-limit.ts","../../../connector-shared/src/map-concurrent.ts","../../../connector-shared/src/sanitize.ts","../../../connector-shared/src/epoch.ts","../../../connector-shared/src/pagination.ts","../../../connector-shared/src/logger.ts","../src/google-search-console.ts","../src/index.ts"],"sourcesContent":["import { z } from 'zod';\n\nexport interface ServiceAccountKey {\n client_email: string;\n private_key: string;\n token_uri?: string;\n project_id?: string;\n}\n\nconst serviceAccountKeySchema = z.object({\n client_email: z.string().min(1),\n private_key: z.string().min(1),\n token_uri: z.string().url().optional(),\n project_id: z.string().optional(),\n});\n\nexport interface TokenResponse {\n access_token: string;\n expires_in?: number;\n}\n\nexport const tokenResponseSchema = z.object({\n access_token: z.string().min(1),\n expires_in: z.number().int().positive().optional(),\n});\n\nfunction base64urlFromBytes(bytes: Uint8Array): string {\n let binary = '';\n for (let i = 0; i < bytes.length; i++) {\n binary += String.fromCharCode(bytes[i]!);\n }\n return btoa(binary).replace(/\\+/g, '-').replace(/\\//g, '_').replace(/=/g, '');\n}\n\nfunction base64urlFromString(str: string): string {\n return base64urlFromBytes(new TextEncoder().encode(str));\n}\n\nasync function signRS256JWT(\n payload: Record<string, unknown>,\n privateKeyPem: string,\n): Promise<string> {\n const header = { alg: 'RS256', typ: 'JWT' };\n const headerB64 = base64urlFromString(JSON.stringify(header));\n const payloadB64 = base64urlFromString(JSON.stringify(payload));\n const signingInput = `${headerB64}.${payloadB64}`;\n\n const pemContent = privateKeyPem\n .replace(/-----BEGIN PRIVATE KEY-----/g, '')\n .replace(/-----END PRIVATE KEY-----/g, '')\n .replace(/\\s/g, '');\n const der = Uint8Array.from(atob(pemContent), (c) => c.charCodeAt(0));\n\n const key = await globalThis.crypto.subtle.importKey(\n 'pkcs8',\n der,\n { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' },\n false,\n ['sign'],\n );\n\n const signature = await globalThis.crypto.subtle.sign(\n 'RSASSA-PKCS1-v1_5',\n key,\n new TextEncoder().encode(signingInput),\n );\n\n return `${signingInput}.${base64urlFromBytes(new Uint8Array(signature))}`;\n}\n\nexport function parseServiceAccountJson(value: string): ServiceAccountKey {\n const trimmed = value.trim();\n if (trimmed.startsWith('{')) {\n return serviceAccountKeySchema.parse(JSON.parse(trimmed));\n }\n const binary = atob(trimmed);\n const bytes = new Uint8Array(binary.length);\n for (let i = 0; i < binary.length; i++) {\n bytes[i] = binary.charCodeAt(i);\n }\n const decoded = new TextDecoder().decode(bytes);\n return serviceAccountKeySchema.parse(JSON.parse(decoded));\n}\n\nexport async function buildServiceAccountJwt(\n serviceAccountJson: string,\n scope: string,\n): Promise<{ url: string; body: string }> {\n const sa = parseServiceAccountJson(serviceAccountJson);\n const now = Math.floor(Date.now() / 1000);\n const jwt = await signRS256JWT(\n {\n iss: sa.client_email,\n scope,\n aud: sa.token_uri ?? 'https://oauth2.googleapis.com/token',\n exp: now + 3600,\n iat: now,\n },\n sa.private_key,\n );\n\n const body = new URLSearchParams({\n grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',\n assertion: jwt,\n }).toString();\n\n return {\n url: sa.token_uri ?? 'https://oauth2.googleapis.com/token',\n body,\n };\n}\n\nexport interface RefreshTokenCredentials {\n refreshToken: string;\n clientId: string;\n clientSecret: string;\n}\n\nexport function buildRefreshTokenGrant(credentials: RefreshTokenCredentials): {\n url: string;\n body: string;\n} {\n const body = new URLSearchParams({\n grant_type: 'refresh_token',\n refresh_token: credentials.refreshToken,\n client_id: credentials.clientId,\n client_secret: credentials.clientSecret,\n }).toString();\n\n return { url: 'https://oauth2.googleapis.com/token', body };\n}\n","import { z } from 'zod';\n\nexport const gcpAuthConfigShape = {\n serviceAccountJson: z.object({ $secret: z.string().trim().min(1) }).meta({\n label: 'Service Account JSON',\n description:\n 'Contents of the JSON key file for a Google service account with the role required by this connector. Create one at Google Cloud -> IAM & Admin -> Service Accounts and store the JSON as a secret.',\n secret: true,\n }),\n} as const;\n\nexport interface GcpAuthConfig {\n serviceAccountJson: { $secret: string };\n}\n","import type { HttpResponse } from './types';\n\nexport type HttpErrorKind =\n | 'transient'\n | 'rate_limit'\n | 'auth'\n | 'upstream_bug'\n | 'client_bug';\n\nexport abstract class HttpClientError extends Error {\n abstract readonly kind: HttpErrorKind;\n readonly response?: HttpResponse;\n\n constructor(message: string, response?: HttpResponse) {\n super(message);\n this.name = new.target.name;\n this.response = response;\n }\n}\n\nexport class TransientError extends HttpClientError {\n readonly kind = 'transient' as const;\n}\n\nexport class RateLimitError extends HttpClientError {\n readonly kind = 'rate_limit' as const;\n readonly retryAfter?: Date;\n\n constructor(message: string, response?: HttpResponse, retryAfter?: Date) {\n super(message, response);\n this.retryAfter = retryAfter;\n }\n}\n\nexport class AuthError extends HttpClientError {\n readonly kind = 'auth' as const;\n}\n\nexport class UpstreamBugError extends HttpClientError {\n readonly kind = 'upstream_bug' as const;\n}\n\nexport class ClientBugError extends HttpClientError {\n readonly kind = 'client_bug' as const;\n}\n\nexport function classifyStatus(status: number): HttpErrorKind {\n if (status === 429) {\n return 'rate_limit';\n }\n if (status === 401 || status === 403) {\n return 'auth';\n }\n if (status === 408) {\n return 'transient';\n }\n if (status >= 500) {\n return 'upstream_bug';\n }\n if (status >= 400) {\n return 'client_bug';\n }\n return 'client_bug';\n}\n\nexport function errorForStatus(\n message: string,\n response: HttpResponse,\n retryAfter?: Date,\n): HttpClientError {\n const kind = classifyStatus(response.status);\n switch (kind) {\n case 'rate_limit':\n return new RateLimitError(message, response, retryAfter);\n case 'auth':\n return new AuthError(message, response);\n case 'transient':\n return new TransientError(message, response);\n case 'upstream_bug':\n return new UpstreamBugError(message, response);\n case 'client_bug':\n return new ClientBugError(message, response);\n }\n}\n","import { HttpClientError, RateLimitError, TransientError } from './errors';\n\nexport interface RetryPolicy {\n maxAttempts?: number;\n initialDelayMs?: number;\n maxDelayMs?: number;\n retryOn?: (status: number | null, err?: Error) => boolean;\n}\n\nexport const defaultRetryOn = (status: number | null, err?: Error): boolean => {\n if (err instanceof RateLimitError) {\n return true;\n }\n if (err instanceof TransientError) {\n return true;\n }\n if (status === null) {\n return err instanceof Error && !(err instanceof HttpClientError);\n }\n if (status === 408 || status === 429) {\n return true;\n }\n if (status >= 500) {\n return true;\n }\n return false;\n};\n\nexport function backoffDelayMs(\n attempt: number,\n policy: Required<Pick<RetryPolicy, 'initialDelayMs' | 'maxDelayMs'>>,\n): number {\n const base = policy.initialDelayMs * 2 ** attempt;\n const jitter = base * 0.25 * Math.random();\n return Math.min(base + jitter, policy.maxDelayMs);\n}\n\nexport function parseRetryAfter(\n headerValue: string | null,\n now: Date = new Date(),\n): Date | undefined {\n if (!headerValue) {\n return undefined;\n }\n const trimmed = headerValue.trim();\n if (/^\\d+$/.test(trimmed)) {\n return new Date(now.getTime() + Number(trimmed) * 1000);\n }\n const parsed = Date.parse(trimmed);\n if (Number.isNaN(parsed)) {\n return undefined;\n }\n return new Date(parsed);\n}\n\nexport function sleep(ms: number, signal?: AbortSignal): Promise<void> {\n if (signal?.aborted) {\n return Promise.reject(signal.reason ?? new Error('Aborted'));\n }\n return new Promise<void>((resolve, reject) => {\n const onAbort = () => {\n clearTimeout(timer);\n reject(signal!.reason ?? new Error('Aborted'));\n };\n const timer = setTimeout(() => {\n signal?.removeEventListener('abort', onAbort);\n resolve();\n }, ms);\n signal?.addEventListener('abort', onAbort, { once: true });\n });\n}\n","export const HTTP_CLIENT_VERSION = '0.0.0';\n\nexport const DEFAULT_USER_AGENT = `rawdash-connector/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;\n\nexport function connectorUserAgent(connectorId: string): string {\n return `rawdash-connector-${connectorId}/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;\n}\n","import {\n AuthError,\n ClientBugError,\n HttpClientError,\n RateLimitError,\n TransientError,\n UpstreamBugError,\n errorForStatus,\n} from './errors';\nimport { defaultRetryOn, parseRetryAfter, sleep } from './retry';\nimport type { FetchLike, HttpMethod, HttpRequest, HttpResponse } from './types';\nimport { DEFAULT_USER_AGENT } from './version';\n\nconst DEFAULT_TIMEOUT_MS = 10_000;\nconst DEFAULT_MAX_ATTEMPTS = 3;\nconst DEFAULT_INITIAL_DELAY_MS = 1000;\nconst DEFAULT_MAX_DELAY_MS = 60_000;\nconst OBSERVER_TIMEOUT_MS = 250;\n\nexport interface RequestObservation {\n url: string;\n method: HttpMethod;\n status: number;\n resource: string;\n requestId: string;\n body: unknown;\n}\n\nexport type RequestObserver = (\n event: RequestObservation,\n) => void | Promise<void>;\n\nexport interface RequestOptions {\n fetch?: FetchLike;\n observer?: RequestObserver;\n resource: string;\n requestId?: string;\n}\n\nasync function notifyObserver(\n observer: RequestObserver,\n event: RequestObservation,\n): Promise<void> {\n let result: void | Promise<void>;\n try {\n result = observer(event);\n } catch (err) {\n console.warn('[connector-shared] request observer threw:', err);\n return;\n }\n if (!(result instanceof Promise)) {\n return;\n }\n const guarded = result.catch((err) => {\n console.warn('[connector-shared] request observer rejected:', err);\n });\n let timer: ReturnType<typeof setTimeout> | undefined;\n const timeout = new Promise<void>((resolve) => {\n timer = setTimeout(resolve, OBSERVER_TIMEOUT_MS);\n });\n try {\n await Promise.race([guarded, timeout]);\n } finally {\n if (timer) {\n clearTimeout(timer);\n }\n }\n}\n\nfunction newRequestId(): string {\n const c = (globalThis as { crypto?: { randomUUID?: () => string } }).crypto;\n if (c?.randomUUID) {\n return c.randomUUID();\n }\n return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;\n}\n\nfunction mergeHeaders(\n defaults: Record<string, string>,\n overrides: Record<string, string> | undefined,\n): Record<string, string> {\n const merged: Record<string, string> = {};\n for (const [k, v] of Object.entries(defaults)) {\n merged[k.toLowerCase()] = v;\n }\n if (overrides) {\n for (const [k, v] of Object.entries(overrides)) {\n merged[k.toLowerCase()] = v;\n }\n }\n return merged;\n}\n\nfunction linkTimeoutSignal(\n parent: AbortSignal | undefined,\n timeoutMs: number,\n): { signal: AbortSignal; cancel: () => void } {\n const controller = new AbortController();\n const onParentAbort = () => {\n controller.abort(parent?.reason);\n };\n if (parent) {\n if (parent.aborted) {\n controller.abort(parent.reason);\n } else {\n parent.addEventListener('abort', onParentAbort, { once: true });\n }\n }\n const timer = setTimeout(() => {\n controller.abort(new Error(`Request timed out after ${timeoutMs}ms`));\n }, timeoutMs);\n return {\n signal: controller.signal,\n cancel: () => {\n clearTimeout(timer);\n if (parent) {\n parent.removeEventListener('abort', onParentAbort);\n }\n },\n };\n}\n\nasync function readBody(res: Response, parseJson: boolean): Promise<unknown> {\n if (res.status === 204 || res.status === 205) {\n return null;\n }\n const contentType = res.headers.get('content-type') ?? '';\n if (parseJson && contentType.includes('application/json')) {\n const text = await res.text();\n if (text.length === 0) {\n return null;\n }\n return JSON.parse(text);\n }\n return res.text();\n}\n\nexport async function request<T = unknown>(\n req: HttpRequest,\n options: RequestOptions,\n): Promise<HttpResponse<T>> {\n const fetchImpl: FetchLike = options.fetch ?? (globalThis.fetch as FetchLike);\n const retry = req.retry ?? {};\n const maxAttempts = retry.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;\n const initialDelayMs = retry.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;\n const maxDelayMs = retry.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;\n const retryOn = retry.retryOn ?? defaultRetryOn;\n const timeoutMs = req.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n const parseJson = req.parseJson ?? true;\n\n const headers = mergeHeaders(\n {\n 'User-Agent': DEFAULT_USER_AGENT,\n Accept: 'application/json',\n },\n req.headers,\n );\n\n let lastErr: Error | undefined;\n\n for (let attempt = 0; attempt < maxAttempts; attempt++) {\n req.signal?.throwIfAborted();\n\n const { signal, cancel } = linkTimeoutSignal(req.signal, timeoutMs);\n let res: Response;\n try {\n res = await fetchImpl(req.url, {\n method: req.method ?? 'GET',\n headers,\n body: req.body as RequestInit['body'],\n signal,\n });\n } catch (err) {\n cancel();\n if (req.signal?.aborted) {\n throw req.signal.reason ?? err;\n }\n const error = err instanceof Error ? err : new Error(String(err));\n lastErr = error;\n if (attempt < maxAttempts - 1 && retryOn(null, error)) {\n const delay = computeDelay(attempt, initialDelayMs, maxDelayMs);\n await sleep(delay, req.signal);\n continue;\n }\n throw new TransientError(error.message);\n }\n cancel();\n\n const body = await readBody(res, parseJson);\n const httpResponse: HttpResponse<T> = {\n status: res.status,\n headers: res.headers,\n body: body as T,\n };\n if (req.rateLimit) {\n const state = req.rateLimit.parse(res.headers);\n if (state) {\n httpResponse.rateLimitState = state;\n }\n }\n\n if (options.observer) {\n await notifyObserver(options.observer, {\n url: req.url,\n method: req.method ?? 'GET',\n status: res.status,\n resource: options.resource,\n requestId: options.requestId ?? newRequestId(),\n body,\n });\n }\n\n if (res.ok) {\n return httpResponse;\n }\n\n const retryAfter = parseRetryAfter(res.headers.get('retry-after'));\n const message = `HTTP ${res.status} ${res.statusText} for ${req.method ?? 'GET'} ${req.url}`;\n const err = errorForStatus(message, httpResponse, retryAfter);\n\n if (\n attempt < maxAttempts - 1 &&\n retryOn(res.status, err) &&\n !(err instanceof AuthError) &&\n !(err instanceof ClientBugError)\n ) {\n lastErr = err;\n let delay = computeDelay(attempt, initialDelayMs, maxDelayMs);\n if (err instanceof RateLimitError && retryAfter) {\n const wait = retryAfter.getTime() - Date.now();\n if (wait > 0) {\n delay = Math.min(wait, maxDelayMs);\n }\n }\n await sleep(delay, req.signal);\n continue;\n }\n\n throw err;\n }\n\n throw lastErr ?? new UpstreamBugError('Exhausted retry attempts');\n}\n\nfunction computeDelay(\n attempt: number,\n initialDelayMs: number,\n maxDelayMs: number,\n): number {\n const base = initialDelayMs * 2 ** attempt;\n const jitter = base * 0.25 * Math.random();\n return Math.min(base + jitter, maxDelayMs);\n}\n\nexport { HttpClientError };\n","export interface RateLimitState {\n remaining: number;\n resetAt: Date;\n}\n\nexport interface RateLimitPolicy {\n parse(headers: Headers): RateLimitState | null;\n}\n\nexport interface StandardRateLimitPolicyConfig {\n remainingHeader: string;\n resetHeader: string;\n resetUnit: 's' | 'ms';\n resetFallbackMs?: number;\n}\n\nexport function standardRateLimitPolicy(\n config: StandardRateLimitPolicyConfig,\n): RateLimitPolicy {\n const { remainingHeader, resetHeader, resetUnit, resetFallbackMs } = config;\n const multiplier = resetUnit === 's' ? 1000 : 1;\n return {\n parse(h) {\n const remainingRaw = h.get(remainingHeader);\n if (remainingRaw === null || remainingRaw.trim() === '') {\n return null;\n }\n const remaining = Number(remainingRaw);\n if (!Number.isFinite(remaining)) {\n return null;\n }\n const resetRaw = h.get(resetHeader);\n if (resetRaw === null) {\n if (resetFallbackMs === undefined) {\n return null;\n }\n return {\n remaining,\n resetAt: new Date(Date.now() + resetFallbackMs),\n };\n }\n if (resetRaw.trim() === '') {\n return null;\n }\n const reset = Number(resetRaw);\n if (!Number.isFinite(reset) || reset < 0) {\n return null;\n }\n const resetMs = reset * multiplier;\n if (!Number.isFinite(resetMs)) {\n return null;\n }\n return { remaining, resetAt: new Date(resetMs) };\n },\n };\n}\n","export async function mapWithConcurrency<T, R>(\n items: readonly T[],\n concurrency: number,\n fn: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n const results = new Array<R>(items.length);\n if (items.length === 0) {\n return results;\n }\n const normalized = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;\n const limit = Math.max(1, Math.min(normalized, items.length));\n let next = 0;\n let failed = false;\n\n async function worker(): Promise<void> {\n while (!failed) {\n const i = next++;\n if (i >= items.length) {\n return;\n }\n try {\n results[i] = await fn(items[i]!, i);\n } catch (err) {\n failed = true;\n throw err;\n }\n }\n }\n\n const workers: Promise<void>[] = [];\n for (let w = 0; w < limit; w++) {\n workers.push(worker());\n }\n await Promise.all(workers);\n return results;\n}\n","export interface SanitizeAllowedUrlOptions {\n url: string | null;\n host: string;\n pathname: string;\n protocol?: 'https:' | 'http:';\n}\n\nexport function sanitizeAllowedUrl(\n options: SanitizeAllowedUrlOptions,\n): string | null {\n const { url, host, pathname, protocol = 'https:' } = options;\n if (url === null) {\n return null;\n }\n try {\n const u = new URL(url);\n if (u.protocol !== protocol || u.host !== host || u.pathname !== pathname) {\n return null;\n }\n return u.toString();\n } catch {\n return null;\n }\n}\n","export type EpochUnit = 'ms' | 's' | 'iso';\n\nexport function parseEpoch(\n value: number | string | null | undefined,\n unit: EpochUnit,\n): number | null {\n if (value === null || value === undefined) {\n return null;\n }\n if (unit === 'iso') {\n if (typeof value !== 'string') {\n return null;\n }\n const ms = new Date(value).getTime();\n return Number.isFinite(ms) ? ms : null;\n }\n if (typeof value === 'string' && value.trim() === '') {\n return null;\n }\n const n = typeof value === 'number' ? value : Number(value);\n if (!Number.isFinite(n)) {\n return null;\n }\n const result = unit === 's' ? n * 1000 : n;\n return Number.isFinite(result) ? result : null;\n}\n","import { request } from './request';\nimport type { HttpRequest } from './types';\n\nexport function parseLinkHeader(header: string | null): Record<string, string> {\n if (!header) {\n return {};\n }\n const result: Record<string, string> = {};\n for (const part of header.split(',')) {\n const match = part.match(/<([^>]+)>\\s*;\\s*rel=\"([^\"]+)\"/);\n if (match) {\n result[match[2]!] = match[1]!;\n }\n }\n return result;\n}\n\nexport async function* paginateLink<T>(\n initial: HttpRequest,\n parse: (body: unknown) => T[],\n options: { resource: string },\n): AsyncIterable<T> {\n let next: string | null = initial.url;\n while (next) {\n const res: Awaited<ReturnType<typeof request>> = await request(\n {\n ...initial,\n url: next,\n },\n { resource: options.resource },\n );\n for (const item of parse(res.body)) {\n yield item;\n }\n const links = parseLinkHeader(res.headers.get('link'));\n next = links['next'] ?? null;\n }\n}\n\nexport async function* paginateCursor<T>(\n initial: HttpRequest,\n parse: (body: unknown) => { items: T[]; nextCursor: string | null },\n buildNext: (req: HttpRequest, cursor: string) => HttpRequest,\n options: { resource: string },\n): AsyncIterable<T> {\n let req: HttpRequest = initial;\n while (true) {\n const res = await request(req, { resource: options.resource });\n const { items, nextCursor } = parse(res.body);\n for (const item of items) {\n yield item;\n }\n if (!nextCursor) {\n return;\n }\n req = buildNext(req, nextCursor);\n }\n}\n\nexport async function* paginatePage<T>(\n initial: HttpRequest,\n parse: (body: unknown) => { items: T[]; hasMore: boolean },\n buildPage: (req: HttpRequest, page: number) => HttpRequest,\n options: { resource: string },\n): AsyncIterable<T> {\n let page = 1;\n while (true) {\n const req = page === 1 ? initial : buildPage(initial, page);\n const res = await request(req, { resource: options.resource });\n const { items, hasMore } = parse(res.body);\n for (const item of items) {\n yield item;\n }\n if (!hasMore || items.length === 0) {\n return;\n }\n page++;\n }\n}\n","export type LogFields = Record<string, unknown>;\n\nexport interface ConnectorLogger {\n info(event: string, fields?: LogFields): void;\n warn(event: string, fields?: LogFields): void;\n}\n\nexport interface ConnectorLoggerOptions {\n scope: string;\n}\n\nconst MAX_VALUE_LEN = 120;\n\nfunction truncate(s: string, max = MAX_VALUE_LEN): string {\n if (s.length <= max) {\n return s;\n }\n return `${s.slice(0, max - 1)}…`;\n}\n\nfunction formatValue(value: unknown): string {\n if (value === null) {\n return 'null';\n }\n if (value === undefined) {\n return '';\n }\n if (typeof value === 'number' || typeof value === 'boolean') {\n return String(value);\n }\n if (typeof value === 'string') {\n const t = truncate(value);\n if (/[\\s\"=]/.test(t)) {\n return JSON.stringify(t);\n }\n return t;\n }\n if (typeof value === 'bigint') {\n return value.toString();\n }\n let json: string | undefined;\n try {\n json = JSON.stringify(value);\n } catch {\n json = undefined;\n }\n return truncate(json ?? String(value));\n}\n\nexport function formatLogFields(fields?: LogFields): string {\n if (!fields) {\n return '';\n }\n const parts: string[] = [];\n for (const [k, v] of Object.entries(fields)) {\n if (v === undefined) {\n continue;\n }\n parts.push(`${k}=${formatValue(v)}`);\n }\n return parts.length > 0 ? ` ${parts.join(' ')}` : '';\n}\n\nexport function formatLogLine(\n scope: string,\n event: string,\n fields?: LogFields,\n): string {\n return `[${scope}] ${event}${formatLogFields(fields)}`;\n}\n\nexport function createDefaultConnectorLogger(\n opts: ConnectorLoggerOptions,\n): ConnectorLogger {\n return {\n info(event, fields) {\n console.info(formatLogLine(opts.scope, event, fields));\n },\n warn(event, fields) {\n console.warn(formatLogLine(opts.scope, event, fields));\n },\n };\n}\n\nconst NOOP_LOGGER: ConnectorLogger = {\n info() {},\n warn() {},\n};\n\nexport function noopConnectorLogger(): ConnectorLogger {\n return NOOP_LOGGER;\n}\n","import { parseEpoch } from '@rawdash/connector-shared';\nimport { z } from 'zod';\n\nexport const BQ_IDENT_RE = /^[A-Za-z_][A-Za-z0-9_-]*$/;\nexport const BQ_DATASET_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;\n\nexport const BQ_API_BASE = 'https://bigquery.googleapis.com/bigquery/v2';\nexport const BQ_READONLY_SCOPE =\n 'https://www.googleapis.com/auth/bigquery.readonly';\nexport const BQ_PAGE_SIZE = 10_000;\nexport const BQ_QUERY_TIMEOUT_MS = 30_000;\n\nexport const bqQueryResponseSchema = z.object({\n jobComplete: z.boolean().optional(),\n schema: z\n .object({\n fields: z.array(z.object({ name: z.string(), type: z.string() })),\n })\n .optional(),\n rows: z\n .array(\n z.object({\n f: z.array(z.object({ v: z.string().nullable().optional() })),\n }),\n )\n .optional(),\n pageToken: z.string().optional(),\n jobReference: z\n .object({\n projectId: z.string(),\n jobId: z.string(),\n location: z.string().optional(),\n })\n .optional(),\n});\n\nexport type BqQueryResponse = z.infer<typeof bqQueryResponseSchema>;\nexport type BqJobReference = NonNullable<BqQueryResponse['jobReference']>;\n\nexport type BqPageRequest =\n | { method: 'POST'; url: string; body: string }\n | { method: 'GET'; url: string };\n\nexport interface BqPageLogger {\n info(message: string, meta?: Record<string, unknown>): void;\n warn(message: string, meta?: Record<string, unknown>): void;\n}\n\nexport function buildBigQueryPageRequest(opts: {\n projectId: string;\n sql: string;\n pageToken: string | undefined;\n jobReference: BqJobReference | undefined;\n location?: string;\n pageSize?: number;\n timeoutMs?: number;\n}): BqPageRequest {\n const pageSize = opts.pageSize ?? BQ_PAGE_SIZE;\n const timeoutMs = opts.timeoutMs ?? BQ_QUERY_TIMEOUT_MS;\n\n if (opts.pageToken === undefined) {\n const url = `${BQ_API_BASE}/projects/${encodeURIComponent(\n opts.projectId,\n )}/queries`;\n const body: Record<string, unknown> = {\n query: opts.sql,\n useLegacySql: false,\n maxResults: pageSize,\n timeoutMs,\n };\n if (opts.location !== undefined) {\n body['location'] = opts.location;\n }\n return { method: 'POST', url, body: JSON.stringify(body) };\n }\n\n if (opts.jobReference === undefined) {\n throw new Error(\n 'cannot fetch the next page of BigQuery results without a jobReference',\n );\n }\n\n const params = new URLSearchParams({\n pageToken: opts.pageToken,\n maxResults: String(pageSize),\n timeoutMs: String(timeoutMs),\n });\n const location = opts.jobReference.location ?? opts.location;\n if (location !== undefined) {\n params.set('location', location);\n }\n const url = `${BQ_API_BASE}/projects/${encodeURIComponent(\n opts.jobReference.projectId,\n )}/queries/${encodeURIComponent(opts.jobReference.jobId)}?${params.toString()}`;\n return { method: 'GET', url };\n}\n\nexport async function collectBigQueryPages<T>(opts: {\n projectId: string;\n sql: string;\n resource: string;\n fetchPage: (\n request: BqPageRequest,\n signal: AbortSignal | undefined,\n ) => Promise<BqQueryResponse>;\n mapRows: (response: BqQueryResponse) => T[];\n jobIncompleteMessage: string;\n location?: string;\n pageSize?: number;\n signal?: AbortSignal;\n logger?: BqPageLogger;\n}): Promise<{ rows: T[]; aborted: boolean }> {\n const rows: T[] = [];\n let pageToken: string | undefined;\n let jobReference: BqJobReference | undefined;\n let page = 0;\n const phaseStart = Date.now();\n\n do {\n if (opts.signal?.aborted) {\n return { rows, aborted: true };\n }\n const request = buildBigQueryPageRequest({\n projectId: opts.projectId,\n sql: opts.sql,\n pageToken,\n jobReference,\n location: opts.location,\n pageSize: opts.pageSize,\n });\n let response: BqQueryResponse;\n try {\n response = await opts.fetchPage(request, opts.signal);\n } catch (err) {\n opts.logger?.warn('fetch page failed', {\n resource: opts.resource,\n page: page + 1,\n error: err instanceof Error ? err.message : String(err),\n });\n throw err;\n }\n if (response.jobComplete === false) {\n throw new Error(opts.jobIncompleteMessage);\n }\n if (response.jobReference !== undefined) {\n jobReference = response.jobReference;\n }\n const pageRows = opts.mapRows(response);\n rows.push(...pageRows);\n pageToken =\n typeof response.pageToken === 'string' && response.pageToken.length > 0\n ? response.pageToken\n : undefined;\n page += 1;\n opts.logger?.info('fetched page', {\n resource: opts.resource,\n page,\n items: pageRows.length,\n next: pageToken ?? null,\n });\n } while (pageToken !== undefined);\n\n opts.logger?.info('resource done', {\n resource: opts.resource,\n pages: page,\n items: rows.length,\n duration_ms: Date.now() - phaseStart,\n });\n return { rows, aborted: false };\n}\n\nexport function indexBqFields(\n response: BqQueryResponse,\n): Record<string, number> {\n const fieldIndex: Record<string, number> = {};\n (response.schema?.fields ?? []).forEach((field, idx) => {\n fieldIndex[field.name] = idx;\n });\n return fieldIndex;\n}\n\nexport function readBqCell(\n cells: ReadonlyArray<{ v?: string | null }>,\n fieldIndex: Record<string, number>,\n name: string,\n): string | null {\n const idx = fieldIndex[name];\n if (idx === undefined) {\n return null;\n }\n const raw = cells[idx]?.v;\n if (raw === undefined || raw === null) {\n return null;\n }\n return raw;\n}\n\nexport function parseBqDateOrEpoch(value: string): number | null {\n const dateMatch = /^(\\d{4})-(\\d{2})-(\\d{2})$/.exec(value);\n if (dateMatch) {\n return Date.UTC(\n Number(dateMatch[1]),\n Number(dateMatch[2]) - 1,\n Number(dateMatch[3]),\n );\n }\n return parseEpoch(value, 'iso');\n}\n","import { AuthError } from '@rawdash/connector-shared';\n\nimport {\n type RefreshTokenCredentials,\n buildRefreshTokenGrant,\n buildServiceAccountJwt,\n} from './auth';\n\ninterface GcpTokenResponse {\n access_token: string;\n expires_in?: number;\n}\n\nexport type GcpTokenPoster = (\n url: string,\n opts: {\n resource: string;\n headers: Record<string, string>;\n body: string;\n signal?: AbortSignal;\n },\n) => Promise<{ body: GcpTokenResponse }>;\n\nexport class GcpAccessTokenProvider {\n private cached: { token: string; expiresAt: number } | null = null;\n\n constructor(\n private readonly opts: {\n connectorId: string;\n scope: string;\n getServiceAccountJson: () => string | undefined;\n getRefreshTokenCredentials?: () => RefreshTokenCredentials | undefined;\n post: GcpTokenPoster;\n },\n ) {}\n\n private async resolveGrant(): Promise<{ url: string; body: string }> {\n const serviceAccountJson = this.opts.getServiceAccountJson();\n if (serviceAccountJson) {\n return buildServiceAccountJwt(serviceAccountJson, this.opts.scope);\n }\n const refreshTokenCredentials = this.opts.getRefreshTokenCredentials?.();\n if (refreshTokenCredentials) {\n return buildRefreshTokenGrant(refreshTokenCredentials);\n }\n throw new AuthError(\n `${this.opts.connectorId}: missing serviceAccountJson or refresh-token credentials`,\n );\n }\n\n async getToken(signal?: AbortSignal): Promise<string> {\n if (this.cached && Date.now() < this.cached.expiresAt) {\n return this.cached.token;\n }\n const { url, body } = await this.resolveGrant();\n const res = await this.opts.post(url, {\n resource: 'oauth_token',\n headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n body,\n signal,\n });\n const expiresIn = res.body.expires_in ?? 3600;\n this.cached = {\n token: res.body.access_token,\n expiresAt: Date.now() + (expiresIn - 60) * 1000,\n };\n return this.cached.token;\n }\n}\n","export const MS_PER_DAY = 86_400_000;\n\nfunction pad2(n: number): string {\n return String(n).padStart(2, '0');\n}\n\nexport function toDateStr(ms: number): string {\n const d = new Date(ms);\n return `${d.getUTCFullYear()}-${pad2(d.getUTCMonth() + 1)}-${pad2(d.getUTCDate())}`;\n}\n\nexport function startOfUtcDay(ms: number): number {\n return Math.floor(ms / MS_PER_DAY) * MS_PER_DAY;\n}\n","import type { HttpResponse } from './types';\n\nexport type HttpErrorKind =\n | 'transient'\n | 'rate_limit'\n | 'auth'\n | 'upstream_bug'\n | 'client_bug';\n\nexport abstract class HttpClientError extends Error {\n abstract readonly kind: HttpErrorKind;\n readonly response?: HttpResponse;\n\n constructor(message: string, response?: HttpResponse) {\n super(message);\n this.name = new.target.name;\n this.response = response;\n }\n}\n\nexport class TransientError extends HttpClientError {\n readonly kind = 'transient' as const;\n}\n\nexport class RateLimitError extends HttpClientError {\n readonly kind = 'rate_limit' as const;\n readonly retryAfter?: Date;\n\n constructor(message: string, response?: HttpResponse, retryAfter?: Date) {\n super(message, response);\n this.retryAfter = retryAfter;\n }\n}\n\nexport class AuthError extends HttpClientError {\n readonly kind = 'auth' as const;\n}\n\nexport class UpstreamBugError extends HttpClientError {\n readonly kind = 'upstream_bug' as const;\n}\n\nexport class ClientBugError extends HttpClientError {\n readonly kind = 'client_bug' as const;\n}\n\nexport function classifyStatus(status: number): HttpErrorKind {\n if (status === 429) {\n return 'rate_limit';\n }\n if (status === 401 || status === 403) {\n return 'auth';\n }\n if (status === 408) {\n return 'transient';\n }\n if (status >= 500) {\n return 'upstream_bug';\n }\n if (status >= 400) {\n return 'client_bug';\n }\n return 'client_bug';\n}\n\nexport function errorForStatus(\n message: string,\n response: HttpResponse,\n retryAfter?: Date,\n): HttpClientError {\n const kind = classifyStatus(response.status);\n switch (kind) {\n case 'rate_limit':\n return new RateLimitError(message, response, retryAfter);\n case 'auth':\n return new AuthError(message, response);\n case 'transient':\n return new TransientError(message, response);\n case 'upstream_bug':\n return new UpstreamBugError(message, response);\n case 'client_bug':\n return new ClientBugError(message, response);\n }\n}\n","import { HttpClientError, RateLimitError, TransientError } from './errors';\n\nexport interface RetryPolicy {\n maxAttempts?: number;\n initialDelayMs?: number;\n maxDelayMs?: number;\n retryOn?: (status: number | null, err?: Error) => boolean;\n}\n\nexport const defaultRetryOn = (status: number | null, err?: Error): boolean => {\n if (err instanceof RateLimitError) {\n return true;\n }\n if (err instanceof TransientError) {\n return true;\n }\n if (status === null) {\n return err instanceof Error && !(err instanceof HttpClientError);\n }\n if (status === 408 || status === 429) {\n return true;\n }\n if (status >= 500) {\n return true;\n }\n return false;\n};\n\nexport function backoffDelayMs(\n attempt: number,\n policy: Required<Pick<RetryPolicy, 'initialDelayMs' | 'maxDelayMs'>>,\n): number {\n const base = policy.initialDelayMs * 2 ** attempt;\n const jitter = base * 0.25 * Math.random();\n return Math.min(base + jitter, policy.maxDelayMs);\n}\n\nexport function parseRetryAfter(\n headerValue: string | null,\n now: Date = new Date(),\n): Date | undefined {\n if (!headerValue) {\n return undefined;\n }\n const trimmed = headerValue.trim();\n if (/^\\d+$/.test(trimmed)) {\n return new Date(now.getTime() + Number(trimmed) * 1000);\n }\n const parsed = Date.parse(trimmed);\n if (Number.isNaN(parsed)) {\n return undefined;\n }\n return new Date(parsed);\n}\n\nexport function sleep(ms: number, signal?: AbortSignal): Promise<void> {\n if (signal?.aborted) {\n return Promise.reject(signal.reason ?? new Error('Aborted'));\n }\n return new Promise<void>((resolve, reject) => {\n const onAbort = () => {\n clearTimeout(timer);\n reject(signal!.reason ?? new Error('Aborted'));\n };\n const timer = setTimeout(() => {\n signal?.removeEventListener('abort', onAbort);\n resolve();\n }, ms);\n signal?.addEventListener('abort', onAbort, { once: true });\n });\n}\n","export const HTTP_CLIENT_VERSION = '0.0.0';\n\nexport const DEFAULT_USER_AGENT = `rawdash-connector/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;\n\nexport function connectorUserAgent(connectorId: string): string {\n return `rawdash-connector-${connectorId}/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;\n}\n","import {\n AuthError,\n ClientBugError,\n HttpClientError,\n RateLimitError,\n TransientError,\n UpstreamBugError,\n errorForStatus,\n} from './errors';\nimport { defaultRetryOn, parseRetryAfter, sleep } from './retry';\nimport type { FetchLike, HttpMethod, HttpRequest, HttpResponse } from './types';\nimport { DEFAULT_USER_AGENT } from './version';\n\nconst DEFAULT_TIMEOUT_MS = 10_000;\nconst DEFAULT_MAX_ATTEMPTS = 3;\nconst DEFAULT_INITIAL_DELAY_MS = 1000;\nconst DEFAULT_MAX_DELAY_MS = 60_000;\nconst OBSERVER_TIMEOUT_MS = 250;\n\nexport interface RequestObservation {\n url: string;\n method: HttpMethod;\n status: number;\n resource: string;\n requestId: string;\n body: unknown;\n}\n\nexport type RequestObserver = (\n event: RequestObservation,\n) => void | Promise<void>;\n\nexport interface RequestOptions {\n fetch?: FetchLike;\n observer?: RequestObserver;\n resource: string;\n requestId?: string;\n}\n\nasync function notifyObserver(\n observer: RequestObserver,\n event: RequestObservation,\n): Promise<void> {\n let result: void | Promise<void>;\n try {\n result = observer(event);\n } catch (err) {\n console.warn('[connector-shared] request observer threw:', err);\n return;\n }\n if (!(result instanceof Promise)) {\n return;\n }\n const guarded = result.catch((err) => {\n console.warn('[connector-shared] request observer rejected:', err);\n });\n let timer: ReturnType<typeof setTimeout> | undefined;\n const timeout = new Promise<void>((resolve) => {\n timer = setTimeout(resolve, OBSERVER_TIMEOUT_MS);\n });\n try {\n await Promise.race([guarded, timeout]);\n } finally {\n if (timer) {\n clearTimeout(timer);\n }\n }\n}\n\nfunction newRequestId(): string {\n const c = (globalThis as { crypto?: { randomUUID?: () => string } }).crypto;\n if (c?.randomUUID) {\n return c.randomUUID();\n }\n return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;\n}\n\nfunction mergeHeaders(\n defaults: Record<string, string>,\n overrides: Record<string, string> | undefined,\n): Record<string, string> {\n const merged: Record<string, string> = {};\n for (const [k, v] of Object.entries(defaults)) {\n merged[k.toLowerCase()] = v;\n }\n if (overrides) {\n for (const [k, v] of Object.entries(overrides)) {\n merged[k.toLowerCase()] = v;\n }\n }\n return merged;\n}\n\nfunction linkTimeoutSignal(\n parent: AbortSignal | undefined,\n timeoutMs: number,\n): { signal: AbortSignal; cancel: () => void } {\n const controller = new AbortController();\n const onParentAbort = () => {\n controller.abort(parent?.reason);\n };\n if (parent) {\n if (parent.aborted) {\n controller.abort(parent.reason);\n } else {\n parent.addEventListener('abort', onParentAbort, { once: true });\n }\n }\n const timer = setTimeout(() => {\n controller.abort(new Error(`Request timed out after ${timeoutMs}ms`));\n }, timeoutMs);\n return {\n signal: controller.signal,\n cancel: () => {\n clearTimeout(timer);\n if (parent) {\n parent.removeEventListener('abort', onParentAbort);\n }\n },\n };\n}\n\nasync function readBody(res: Response, parseJson: boolean): Promise<unknown> {\n if (res.status === 204 || res.status === 205) {\n return null;\n }\n const contentType = res.headers.get('content-type') ?? '';\n if (parseJson && contentType.includes('application/json')) {\n const text = await res.text();\n if (text.length === 0) {\n return null;\n }\n return JSON.parse(text);\n }\n return res.text();\n}\n\nexport async function request<T = unknown>(\n req: HttpRequest,\n options: RequestOptions,\n): Promise<HttpResponse<T>> {\n const fetchImpl: FetchLike = options.fetch ?? (globalThis.fetch as FetchLike);\n const retry = req.retry ?? {};\n const maxAttempts = retry.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;\n const initialDelayMs = retry.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;\n const maxDelayMs = retry.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;\n const retryOn = retry.retryOn ?? defaultRetryOn;\n const timeoutMs = req.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n const parseJson = req.parseJson ?? true;\n\n const headers = mergeHeaders(\n {\n 'User-Agent': DEFAULT_USER_AGENT,\n Accept: 'application/json',\n },\n req.headers,\n );\n\n let lastErr: Error | undefined;\n\n for (let attempt = 0; attempt < maxAttempts; attempt++) {\n req.signal?.throwIfAborted();\n\n const { signal, cancel } = linkTimeoutSignal(req.signal, timeoutMs);\n let res: Response;\n try {\n res = await fetchImpl(req.url, {\n method: req.method ?? 'GET',\n headers,\n body: req.body as RequestInit['body'],\n signal,\n });\n } catch (err) {\n cancel();\n if (req.signal?.aborted) {\n throw req.signal.reason ?? err;\n }\n const error = err instanceof Error ? err : new Error(String(err));\n lastErr = error;\n if (attempt < maxAttempts - 1 && retryOn(null, error)) {\n const delay = computeDelay(attempt, initialDelayMs, maxDelayMs);\n await sleep(delay, req.signal);\n continue;\n }\n throw new TransientError(error.message);\n }\n cancel();\n\n const body = await readBody(res, parseJson);\n const httpResponse: HttpResponse<T> = {\n status: res.status,\n headers: res.headers,\n body: body as T,\n };\n if (req.rateLimit) {\n const state = req.rateLimit.parse(res.headers);\n if (state) {\n httpResponse.rateLimitState = state;\n }\n }\n\n if (options.observer) {\n await notifyObserver(options.observer, {\n url: req.url,\n method: req.method ?? 'GET',\n status: res.status,\n resource: options.resource,\n requestId: options.requestId ?? newRequestId(),\n body,\n });\n }\n\n if (res.ok) {\n return httpResponse;\n }\n\n const retryAfter = parseRetryAfter(res.headers.get('retry-after'));\n const message = `HTTP ${res.status} ${res.statusText} for ${req.method ?? 'GET'} ${req.url}`;\n const err = errorForStatus(message, httpResponse, retryAfter);\n\n if (\n attempt < maxAttempts - 1 &&\n retryOn(res.status, err) &&\n !(err instanceof AuthError) &&\n !(err instanceof ClientBugError)\n ) {\n lastErr = err;\n let delay = computeDelay(attempt, initialDelayMs, maxDelayMs);\n if (err instanceof RateLimitError && retryAfter) {\n const wait = retryAfter.getTime() - Date.now();\n if (wait > 0) {\n delay = Math.min(wait, maxDelayMs);\n }\n }\n await sleep(delay, req.signal);\n continue;\n }\n\n throw err;\n }\n\n throw lastErr ?? new UpstreamBugError('Exhausted retry attempts');\n}\n\nfunction computeDelay(\n attempt: number,\n initialDelayMs: number,\n maxDelayMs: number,\n): number {\n const base = initialDelayMs * 2 ** attempt;\n const jitter = base * 0.25 * Math.random();\n return Math.min(base + jitter, maxDelayMs);\n}\n\nexport { HttpClientError };\n","export interface RateLimitState {\n remaining: number;\n resetAt: Date;\n}\n\nexport interface RateLimitPolicy {\n parse(headers: Headers): RateLimitState | null;\n}\n\nexport interface StandardRateLimitPolicyConfig {\n remainingHeader: string;\n resetHeader: string;\n resetUnit: 's' | 'ms';\n resetFallbackMs?: number;\n}\n\nexport function standardRateLimitPolicy(\n config: StandardRateLimitPolicyConfig,\n): RateLimitPolicy {\n const { remainingHeader, resetHeader, resetUnit, resetFallbackMs } = config;\n const multiplier = resetUnit === 's' ? 1000 : 1;\n return {\n parse(h) {\n const remainingRaw = h.get(remainingHeader);\n if (remainingRaw === null || remainingRaw.trim() === '') {\n return null;\n }\n const remaining = Number(remainingRaw);\n if (!Number.isFinite(remaining)) {\n return null;\n }\n const resetRaw = h.get(resetHeader);\n if (resetRaw === null) {\n if (resetFallbackMs === undefined) {\n return null;\n }\n return {\n remaining,\n resetAt: new Date(Date.now() + resetFallbackMs),\n };\n }\n if (resetRaw.trim() === '') {\n return null;\n }\n const reset = Number(resetRaw);\n if (!Number.isFinite(reset) || reset < 0) {\n return null;\n }\n const resetMs = reset * multiplier;\n if (!Number.isFinite(resetMs)) {\n return null;\n }\n return { remaining, resetAt: new Date(resetMs) };\n },\n };\n}\n","export async function mapWithConcurrency<T, R>(\n items: readonly T[],\n concurrency: number,\n fn: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n const results = new Array<R>(items.length);\n if (items.length === 0) {\n return results;\n }\n const normalized = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;\n const limit = Math.max(1, Math.min(normalized, items.length));\n let next = 0;\n let failed = false;\n\n async function worker(): Promise<void> {\n while (!failed) {\n const i = next++;\n if (i >= items.length) {\n return;\n }\n try {\n results[i] = await fn(items[i]!, i);\n } catch (err) {\n failed = true;\n throw err;\n }\n }\n }\n\n const workers: Promise<void>[] = [];\n for (let w = 0; w < limit; w++) {\n workers.push(worker());\n }\n await Promise.all(workers);\n return results;\n}\n","export interface SanitizeAllowedUrlOptions {\n url: string | null;\n host: string;\n pathname: string;\n protocol?: 'https:' | 'http:';\n}\n\nexport function sanitizeAllowedUrl(\n options: SanitizeAllowedUrlOptions,\n): string | null {\n const { url, host, pathname, protocol = 'https:' } = options;\n if (url === null) {\n return null;\n }\n try {\n const u = new URL(url);\n if (u.protocol !== protocol || u.host !== host || u.pathname !== pathname) {\n return null;\n }\n return u.toString();\n } catch {\n return null;\n }\n}\n","export type EpochUnit = 'ms' | 's' | 'iso';\n\nexport function parseEpoch(\n value: number | string | null | undefined,\n unit: EpochUnit,\n): number | null {\n if (value === null || value === undefined) {\n return null;\n }\n if (unit === 'iso') {\n if (typeof value !== 'string') {\n return null;\n }\n const ms = new Date(value).getTime();\n return Number.isFinite(ms) ? ms : null;\n }\n if (typeof value === 'string' && value.trim() === '') {\n return null;\n }\n const n = typeof value === 'number' ? value : Number(value);\n if (!Number.isFinite(n)) {\n return null;\n }\n const result = unit === 's' ? n * 1000 : n;\n return Number.isFinite(result) ? result : null;\n}\n","import { request } from './request';\nimport type { HttpRequest } from './types';\n\nexport function parseLinkHeader(header: string | null): Record<string, string> {\n if (!header) {\n return {};\n }\n const result: Record<string, string> = {};\n for (const part of header.split(',')) {\n const match = part.match(/<([^>]+)>\\s*;\\s*rel=\"([^\"]+)\"/);\n if (match) {\n result[match[2]!] = match[1]!;\n }\n }\n return result;\n}\n\nexport async function* paginateLink<T>(\n initial: HttpRequest,\n parse: (body: unknown) => T[],\n options: { resource: string },\n): AsyncIterable<T> {\n let next: string | null = initial.url;\n while (next) {\n const res: Awaited<ReturnType<typeof request>> = await request(\n {\n ...initial,\n url: next,\n },\n { resource: options.resource },\n );\n for (const item of parse(res.body)) {\n yield item;\n }\n const links = parseLinkHeader(res.headers.get('link'));\n next = links['next'] ?? null;\n }\n}\n\nexport async function* paginateCursor<T>(\n initial: HttpRequest,\n parse: (body: unknown) => { items: T[]; nextCursor: string | null },\n buildNext: (req: HttpRequest, cursor: string) => HttpRequest,\n options: { resource: string },\n): AsyncIterable<T> {\n let req: HttpRequest = initial;\n while (true) {\n const res = await request(req, { resource: options.resource });\n const { items, nextCursor } = parse(res.body);\n for (const item of items) {\n yield item;\n }\n if (!nextCursor) {\n return;\n }\n req = buildNext(req, nextCursor);\n }\n}\n\nexport async function* paginatePage<T>(\n initial: HttpRequest,\n parse: (body: unknown) => { items: T[]; hasMore: boolean },\n buildPage: (req: HttpRequest, page: number) => HttpRequest,\n options: { resource: string },\n): AsyncIterable<T> {\n let page = 1;\n while (true) {\n const req = page === 1 ? initial : buildPage(initial, page);\n const res = await request(req, { resource: options.resource });\n const { items, hasMore } = parse(res.body);\n for (const item of items) {\n yield item;\n }\n if (!hasMore || items.length === 0) {\n return;\n }\n page++;\n }\n}\n","export type LogFields = Record<string, unknown>;\n\nexport interface ConnectorLogger {\n info(event: string, fields?: LogFields): void;\n warn(event: string, fields?: LogFields): void;\n}\n\nexport interface ConnectorLoggerOptions {\n scope: string;\n}\n\nconst MAX_VALUE_LEN = 120;\n\nfunction truncate(s: string, max = MAX_VALUE_LEN): string {\n if (s.length <= max) {\n return s;\n }\n return `${s.slice(0, max - 1)}…`;\n}\n\nfunction formatValue(value: unknown): string {\n if (value === null) {\n return 'null';\n }\n if (value === undefined) {\n return '';\n }\n if (typeof value === 'number' || typeof value === 'boolean') {\n return String(value);\n }\n if (typeof value === 'string') {\n const t = truncate(value);\n if (/[\\s\"=]/.test(t)) {\n return JSON.stringify(t);\n }\n return t;\n }\n if (typeof value === 'bigint') {\n return value.toString();\n }\n let json: string | undefined;\n try {\n json = JSON.stringify(value);\n } catch {\n json = undefined;\n }\n return truncate(json ?? String(value));\n}\n\nexport function formatLogFields(fields?: LogFields): string {\n if (!fields) {\n return '';\n }\n const parts: string[] = [];\n for (const [k, v] of Object.entries(fields)) {\n if (v === undefined) {\n continue;\n }\n parts.push(`${k}=${formatValue(v)}`);\n }\n return parts.length > 0 ? ` ${parts.join(' ')}` : '';\n}\n\nexport function formatLogLine(\n scope: string,\n event: string,\n fields?: LogFields,\n): string {\n return `[${scope}] ${event}${formatLogFields(fields)}`;\n}\n\nexport function createDefaultConnectorLogger(\n opts: ConnectorLoggerOptions,\n): ConnectorLogger {\n return {\n info(event, fields) {\n console.info(formatLogLine(opts.scope, event, fields));\n },\n warn(event, fields) {\n console.warn(formatLogLine(opts.scope, event, fields));\n },\n };\n}\n\nconst NOOP_LOGGER: ConnectorLogger = {\n info() {},\n warn() {},\n};\n\nexport function noopConnectorLogger(): ConnectorLogger {\n return NOOP_LOGGER;\n}\n","import { GcpAccessTokenProvider } from '@rawdash/connector-gcp-shared';\nimport { connectorUserAgent } from '@rawdash/connector-shared';\nimport {\n BaseConnector,\n type ConnectorContext,\n type ConnectorDoc,\n type CredentialsSchema,\n type StorageHandle,\n type SyncOptions,\n type SyncResult,\n defineConfigFields,\n defineConnectorDoc,\n defineResources,\n schemasFromResources,\n} from '@rawdash/core';\nimport { z } from 'zod';\n\nexport const configFields = defineConfigFields(\n z\n .object({\n siteUrl: z.string().trim().min(1).meta({\n label: 'Site URL',\n description:\n 'Verified Search Console property. URL-prefix properties look like \"https://example.com/\"; Domain properties look like \"sc-domain:example.com\".',\n placeholder: 'https://example.com/',\n }),\n serviceAccountJson: z\n .object({ $secret: z.string().trim().min(1) })\n .optional()\n .meta({\n label: 'Service Account JSON (recommended)',\n description:\n 'Contents of the JSON key file for a Google service account that has been added as a Search Console user (Owner or Full user) on the property. Create one at Google Cloud -> IAM & Admin -> Service Accounts.',\n secret: true,\n }),\n refreshToken: z\n .object({ $secret: z.string().trim().min(1) })\n .optional()\n .meta({\n label: 'OAuth Refresh Token',\n description:\n 'Google OAuth 2.0 refresh token with webmasters.readonly scope. Required if not using serviceAccountJson.',\n secret: true,\n }),\n clientId: z.string().trim().min(1).optional().meta({\n label: 'OAuth Client ID',\n description:\n 'OAuth 2.0 client ID from Google Cloud Console. Required when using refreshToken auth.',\n placeholder: '...apps.googleusercontent.com',\n }),\n clientSecret: z\n .object({ $secret: z.string().trim().min(1) })\n .optional()\n .meta({\n label: 'OAuth Client Secret',\n description:\n 'OAuth 2.0 client secret from Google Cloud Console. Required when using refreshToken auth.',\n secret: true,\n }),\n lookbackDays: z.number().int().positive().optional().meta({\n label: 'Lookback days (full sync)',\n description:\n 'How many calendar days to fetch on a full sync. Defaults to 90.',\n placeholder: '90',\n }),\n })\n .refine(\n (val) =>\n val.serviceAccountJson !== undefined ||\n (val.refreshToken !== undefined &&\n val.clientId !== undefined &&\n val.clientSecret !== undefined),\n {\n message:\n 'Provide either serviceAccountJson or the full OAuth tuple (refreshToken + clientId + clientSecret)',\n },\n ),\n);\n\nexport const doc: ConnectorDoc = defineConnectorDoc({\n displayName: 'Google Search Console',\n category: 'marketing',\n brandColor: '#458CF5',\n tagline:\n 'Sync daily Search Console SEO metrics - clicks, impressions, CTR, and average position - by date, query, page, and country.',\n vendor: {\n name: 'Google Search Console',\n domain: 'search.google.com',\n apiDocs:\n 'https://developers.google.com/webmaster-tools/v1/api_reference_index',\n website: 'https://search.google.com/search-console',\n },\n auth: {\n summary:\n 'Authenticate against the Search Console API with either a Google service account JSON key (recommended) or an OAuth 2.0 refresh-token tuple. The identity must be added as a user on the Search Console property (Owner or Full user).',\n setup: [\n 'Identify the property to sync. URL-prefix properties use the full origin (e.g. https://example.com/); Domain properties use the sc-domain:example.com format.',\n 'Recommended: create a service account at Google Cloud -> IAM & Admin -> Service Accounts, generate a JSON key, then in Search Console add the service account email as a user on the property. Store the JSON as a secret and reference it as serviceAccountJson: secret(\"GSC_SERVICE_ACCOUNT_JSON\").',\n 'Alternative: provide an OAuth 2.0 refresh token with the webmasters.readonly scope together with its clientId and clientSecret from the Google Cloud Console.',\n ],\n },\n rateLimit:\n 'Search Console API quota is 1,200 queries per minute per project (default); 429 responses are retried automatically with exponential backoff.',\n limitations: [\n 'Search Console aggregates data with a 2-3 day lag, so incremental syncs refetch the trailing 3 days.',\n 'Each query is paginated 25,000 rows per page; a phase that yields more than that paginates by startRow.',\n ],\n});\n\nexport interface GSCSettings {\n siteUrl: string;\n lookbackDays?: number;\n}\n\nconst gscCredentials = {\n serviceAccountJson: {\n description: 'Google service account JSON key (base64 or raw JSON)',\n auth: 'optional' as const,\n },\n refreshToken: {\n description: 'Google OAuth 2.0 refresh token',\n auth: 'optional' as const,\n },\n clientId: {\n description: 'Google OAuth 2.0 client ID',\n auth: 'optional' as const,\n },\n clientSecret: {\n description: 'Google OAuth 2.0 client secret',\n auth: 'optional' as const,\n },\n} satisfies CredentialsSchema;\n\ntype GSCCredentials = typeof gscCredentials;\n\nconst PHASE_ORDER = [\n 'search_analytics_by_day',\n 'top_queries',\n 'top_pages',\n 'top_countries',\n] as const;\n\ntype GSCPhase = (typeof PHASE_ORDER)[number];\n\ninterface GSCDateRange {\n startDate: string;\n endDate: string;\n}\n\ninterface GSCSyncCursor {\n phase: GSCPhase;\n dateRange: GSCDateRange;\n}\n\nconst GSC_DATE_RE = /^\\d{4}-\\d{2}-\\d{2}$/;\n\nfunction isGSCDateString(value: unknown): value is string {\n return typeof value === 'string' && GSC_DATE_RE.test(value);\n}\n\nfunction isGSCDateRange(value: unknown): value is GSCDateRange {\n if (typeof value !== 'object' || value === null) {\n return false;\n }\n const v = value as { startDate?: unknown; endDate?: unknown };\n return isGSCDateString(v.startDate) && isGSCDateString(v.endDate);\n}\n\nfunction isGSCSyncCursor(value: unknown): value is GSCSyncCursor {\n if (typeof value !== 'object' || value === null) {\n return false;\n }\n const v = value as { phase?: unknown; dateRange?: unknown };\n if (typeof v.phase !== 'string') {\n return false;\n }\n if (!(PHASE_ORDER as readonly string[]).includes(v.phase)) {\n return false;\n }\n return isGSCDateRange(v.dateRange);\n}\n\ninterface PhaseConfig {\n dimensions: string[];\n metricName: string;\n}\n\nconst PHASE_CONFIGS: Record<GSCPhase, PhaseConfig> = {\n search_analytics_by_day: {\n dimensions: ['date'],\n metricName: 'gsc_search_analytics_by_day',\n },\n top_queries: {\n dimensions: ['date', 'query'],\n metricName: 'gsc_top_queries',\n },\n top_pages: {\n dimensions: ['date', 'page'],\n metricName: 'gsc_top_pages',\n },\n top_countries: {\n dimensions: ['date', 'country'],\n metricName: 'gsc_top_countries',\n },\n};\n\nconst ROWS_PER_PAGE = 25_000;\nconst METRIC_FIELDS = ['clicks', 'impressions', 'ctr', 'position'] as const;\n\nexport interface GSCReportRow {\n keys?: string[];\n clicks?: number;\n impressions?: number;\n ctr?: number;\n position?: number;\n}\n\ninterface GSCReportResponse {\n rows?: GSCReportRow[];\n responseAggregationType?: string;\n}\n\nfunction toGSCDate(date: Date): string {\n const y = date.getUTCFullYear();\n const m = String(date.getUTCMonth() + 1).padStart(2, '0');\n const d = String(date.getUTCDate()).padStart(2, '0');\n return `${y}-${m}-${d}`;\n}\n\nfunction gscDateToMs(gscDate: string): number {\n const y = gscDate.slice(0, 4);\n const m = gscDate.slice(5, 7);\n const d = gscDate.slice(8, 10);\n return Date.UTC(Number(y), Number(m) - 1, Number(d));\n}\n\nconst MS_PER_DAY = 24 * 60 * 60 * 1000;\nconst INCREMENTAL_LOOKBACK_DAYS = 3;\n\nfunction getDateRange(\n options: SyncOptions,\n lookbackDays: number,\n): GSCDateRange {\n const now = Date.now();\n const endDate = toGSCDate(new Date(now));\n if (options.mode === 'latest') {\n const startMs = now - (INCREMENTAL_LOOKBACK_DAYS - 1) * MS_PER_DAY;\n return { startDate: toGSCDate(new Date(startMs)), endDate };\n }\n if (options.since) {\n const sinceMs = new Date(options.since).getTime();\n if (Number.isFinite(sinceMs)) {\n const days = Math.max(1, Math.ceil((now - sinceMs) / MS_PER_DAY));\n const cappedDays = Math.min(days, lookbackDays);\n const startMs = now - (cappedDays - 1) * MS_PER_DAY;\n return { startDate: toGSCDate(new Date(startMs)), endDate };\n }\n }\n const startMs = now - (lookbackDays - 1) * MS_PER_DAY;\n return { startDate: toGSCDate(new Date(startMs)), endDate };\n}\n\nexport function rowToMetricSample(\n row: GSCReportRow,\n dimensions: string[],\n metricName: string,\n): {\n name: string;\n ts: number;\n value: number;\n attributes: Record<string, string | number>;\n} {\n const attributes: Record<string, string | number> = {};\n const keys = row.keys ?? [];\n for (let i = 0; i < dimensions.length; i++) {\n attributes[dimensions[i]!] = keys[i] ?? '';\n }\n\n for (const field of METRIC_FIELDS) {\n const raw = row[field];\n attributes[field] =\n typeof raw === 'number' && Number.isFinite(raw) ? raw : 0;\n }\n\n const dateStr =\n typeof attributes['date'] === 'string' ? attributes['date'] : '1970-01-01';\n const ts = gscDateToMs(dateStr);\n const primaryValue = attributes['clicks'] as number;\n\n return {\n name: metricName,\n ts,\n value: primaryValue,\n attributes,\n };\n}\n\nconst dateKey = z\n .string()\n .regex(/^(19|20)\\d{2}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$/);\nconst stringKey = z.string();\nconst numericField = z.number();\n\nfunction reportSchema(dimensionCount: number) {\n const keys =\n dimensionCount === 1\n ? z.tuple([dateKey])\n : z.tuple([dateKey, ...Array(dimensionCount - 1).fill(stringKey)] as [\n typeof dateKey,\n ...z.ZodType[],\n ]);\n return z.object({\n rows: z\n .array(\n z.object({\n keys,\n clicks: numericField.optional(),\n impressions: numericField.optional(),\n ctr: numericField.optional(),\n position: numericField.optional(),\n }),\n )\n .optional(),\n });\n}\n\nconst tokenResponseSchema = z.object({\n access_token: z.string().min(1),\n expires_in: z.number().int().positive().optional(),\n});\n\nexport const googleSearchConsoleResources = defineResources({\n gsc_search_analytics_by_day: {\n shape: 'metric',\n description:\n 'Daily site totals - clicks, impressions, CTR, and average position across all queries and pages.',\n unit: 'clicks',\n granularity: 'day',\n endpoint: 'POST /webmasters/v3/sites/{siteUrl}/searchAnalytics/query',\n dimensions: [\n { name: 'date', description: 'Calendar day of the metric sample.' },\n ],\n responses: {\n oauth_token: tokenResponseSchema,\n search_analytics_by_day: reportSchema(1),\n },\n },\n gsc_top_queries: {\n shape: 'metric',\n description:\n 'Daily clicks, impressions, CTR, and average position broken down by search query.',\n unit: 'clicks',\n granularity: 'day',\n endpoint: 'POST /webmasters/v3/sites/{siteUrl}/searchAnalytics/query',\n dimensions: [\n { name: 'date', description: 'Calendar day of the metric sample.' },\n {\n name: 'query',\n description: 'Search term that surfaced a result for the property.',\n },\n ],\n responses: { top_queries: reportSchema(2) },\n },\n gsc_top_pages: {\n shape: 'metric',\n description:\n 'Daily clicks, impressions, CTR, and average position broken down by landing page URL.',\n unit: 'clicks',\n granularity: 'day',\n endpoint: 'POST /webmasters/v3/sites/{siteUrl}/searchAnalytics/query',\n dimensions: [\n { name: 'date', description: 'Calendar day of the metric sample.' },\n {\n name: 'page',\n description: 'Canonical URL of the page that surfaced in search.',\n },\n ],\n responses: { top_pages: reportSchema(2) },\n },\n gsc_top_countries: {\n shape: 'metric',\n description:\n 'Daily clicks, impressions, CTR, and average position broken down by visitor country.',\n unit: 'clicks',\n granularity: 'day',\n endpoint: 'POST /webmasters/v3/sites/{siteUrl}/searchAnalytics/query',\n dimensions: [\n { name: 'date', description: 'Calendar day of the metric sample.' },\n {\n name: 'country',\n description: 'ISO 3166-1 alpha-3 country code of the searcher.',\n },\n ],\n responses: { top_countries: reportSchema(2) },\n },\n});\n\nexport const id = 'google-search-console';\n\nexport class GSCConnector extends BaseConnector<GSCSettings, GSCCredentials> {\n static readonly id = id;\n\n static readonly resources = googleSearchConsoleResources;\n\n static readonly schemas = schemasFromResources(googleSearchConsoleResources);\n\n static create(input: unknown, ctx?: ConnectorContext): GSCConnector {\n const parsed = configFields.parse(input);\n return new GSCConnector(\n {\n siteUrl: parsed.siteUrl,\n lookbackDays: parsed.lookbackDays,\n },\n {\n serviceAccountJson: parsed.serviceAccountJson,\n refreshToken: parsed.refreshToken,\n clientId: parsed.clientId,\n clientSecret: parsed.clientSecret,\n },\n ctx,\n );\n }\n\n readonly id = id;\n override readonly credentials = gscCredentials;\n\n private tokenProvider?: GcpAccessTokenProvider;\n\n private getAccessToken(signal?: AbortSignal): Promise<string> {\n this.tokenProvider ??= new GcpAccessTokenProvider({\n connectorId: this.id,\n scope: 'https://www.googleapis.com/auth/webmasters.readonly',\n getServiceAccountJson: () => this.creds.serviceAccountJson,\n getRefreshTokenCredentials: () => {\n const { refreshToken, clientId, clientSecret } = this.creds;\n if (refreshToken && clientId && clientSecret) {\n return { refreshToken, clientId, clientSecret };\n }\n return undefined;\n },\n post: (url, opts) =>\n this.post<{ access_token: string; expires_in?: number }>(url, opts),\n });\n return this.tokenProvider.getToken(signal);\n }\n\n private async runReport(\n accessToken: string,\n phase: GSCPhase,\n dateRange: GSCDateRange,\n startRow: number,\n signal?: AbortSignal,\n ): Promise<GSCReportResponse> {\n const { dimensions } = PHASE_CONFIGS[phase];\n const url = `https://searchconsole.googleapis.com/webmasters/v3/sites/${encodeURIComponent(this.settings.siteUrl)}/searchAnalytics/query`;\n\n const body: Record<string, unknown> = {\n startDate: dateRange.startDate,\n endDate: dateRange.endDate,\n dimensions,\n rowLimit: ROWS_PER_PAGE,\n startRow,\n };\n\n const res = await this.post<GSCReportResponse>(url, {\n resource: phase,\n headers: {\n Authorization: `Bearer ${accessToken}`,\n 'Content-Type': 'application/json',\n 'User-Agent': connectorUserAgent('google-search-console'),\n },\n body: JSON.stringify(body),\n signal,\n });\n return res.body;\n }\n\n async sync(\n options: SyncOptions,\n storage: StorageHandle,\n signal?: AbortSignal,\n ): Promise<SyncResult> {\n const lookbackDays = this.settings.lookbackDays ?? 90;\n\n const cursor = isGSCSyncCursor(options.cursor) ? options.cursor : undefined;\n const dateRange = cursor?.dateRange ?? getDateRange(options, lookbackDays);\n\n let accessToken: string | null = null;\n const getToken = async (sig?: AbortSignal): Promise<string> => {\n if (!accessToken) {\n accessToken = await this.getAccessToken(sig);\n }\n return accessToken;\n };\n\n const runReportWithRetry = async (\n phase: GSCPhase,\n startRow: number,\n sig: AbortSignal | undefined,\n ): Promise<GSCReportResponse> => {\n const token = await getToken(sig);\n try {\n return await this.runReport(token, phase, dateRange, startRow, sig);\n } catch (err) {\n console.warn(\n `[gsc] runReport failed, refreshing token and retrying once`,\n err,\n );\n accessToken = null;\n const freshToken = await getToken(sig);\n return this.runReport(freshToken, phase, dateRange, startRow, sig);\n }\n };\n\n const drainPhase = async (phase: GSCPhase): Promise<GSCReportRow[]> => {\n const allRows: GSCReportRow[] = [];\n let startRow = 0;\n for (;;) {\n const response = await runReportWithRetry(phase, startRow, signal);\n const rows = response.rows ?? [];\n allRows.push(...rows);\n startRow += rows.length;\n if (rows.length < ROWS_PER_PAGE) {\n break;\n }\n }\n return allRows;\n };\n\n const resumeIdx = cursor ? PHASE_ORDER.indexOf(cursor.phase) : -1;\n const startIdx = resumeIdx >= 0 ? resumeIdx : 0;\n\n for (let i = startIdx; i < PHASE_ORDER.length; i++) {\n const phase = PHASE_ORDER[i]!;\n if (signal?.aborted) {\n return { done: false, cursor: { phase, dateRange } };\n }\n\n let rows: GSCReportRow[];\n try {\n rows = await drainPhase(phase);\n } catch (err) {\n if (signal?.aborted) {\n return { done: false, cursor: { phase, dateRange } };\n }\n throw err;\n }\n const cfg = PHASE_CONFIGS[phase];\n const samples = rows.map((row) =>\n rowToMetricSample(row, cfg.dimensions, cfg.metricName),\n );\n await storage.metrics(samples, { names: [cfg.metricName] });\n }\n\n return { done: true };\n }\n}\n","import { GSCConnector } from './google-search-console';\n\nexport {\n configFields,\n doc,\n GSCConnector,\n googleSearchConsoleResources as resources,\n id,\n rowToMetricSample,\n} from './google-search-console';\nexport type { GSCSettings } from './google-search-console';\nexport default GSCConnector;\n"],"mappings":";AAAA,SAAS,SAAS;ACAlB,SAAS,KAAAA,UAAS;AWClB,SAAS,KAAAA,UAAS;AZQlB,IAAM,0BAA0B,EAAE,OAAO;EACvC,cAAc,EAAE,OAAO,EAAE,IAAI,CAAC;EAC9B,aAAa,EAAE,OAAO,EAAE,IAAI,CAAC;EAC7B,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS;EACrC,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAOM,IAAM,sBAAsB,EAAE,OAAO;EAC1C,cAAc,EAAE,OAAO,EAAE,IAAI,CAAC;EAC9B,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS;AACnD,CAAC;AAED,SAAS,mBAAmB,OAA2B;AACrD,MAAI,SAAS;AACb,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,cAAU,OAAO,aAAa,MAAM,CAAC,CAAE;EACzC;AACA,SAAO,KAAK,MAAM,EAAE,QAAQ,OAAO,GAAG,EAAE,QAAQ,OAAO,GAAG,EAAE,QAAQ,MAAM,EAAE;AAC9E;AAEA,SAAS,oBAAoB,KAAqB;AAChD,SAAO,mBAAmB,IAAI,YAAY,EAAE,OAAO,GAAG,CAAC;AACzD;AAEA,eAAe,aACb,SACA,eACiB;AACjB,QAAM,SAAS,EAAE,KAAK,SAAS,KAAK,MAAM;AAC1C,QAAM,YAAY,oBAAoB,KAAK,UAAU,MAAM,CAAC;AAC5D,QAAM,aAAa,oBAAoB,KAAK,UAAU,OAAO,CAAC;AAC9D,QAAM,eAAe,GAAG,SAAS,IAAI,UAAU;AAE/C,QAAM,aAAa,cAChB,QAAQ,gCAAgC,EAAE,EAC1C,QAAQ,8BAA8B,EAAE,EACxC,QAAQ,OAAO,EAAE;AACpB,QAAM,MAAM,WAAW,KAAK,KAAK,UAAU,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;AAEpE,QAAM,MAAM,MAAM,WAAW,OAAO,OAAO;IACzC;IACA;IACA,EAAE,MAAM,qBAAqB,MAAM,UAAU;IAC7C;IACA,CAAC,MAAM;EACT;AAEA,QAAM,YAAY,MAAM,WAAW,OAAO,OAAO;IAC/C;IACA;IACA,IAAI,YAAY,EAAE,OAAO,YAAY;EACvC;AAEA,SAAO,GAAG,YAAY,IAAI,mBAAmB,IAAI,WAAW,SAAS,CAAC,CAAC;AACzE;AAEO,SAAS,wBAAwB,OAAkC;AACxE,QAAM,UAAU,MAAM,KAAK;AAC3B,MAAI,QAAQ,WAAW,GAAG,GAAG;AAC3B,WAAO,wBAAwB,MAAM,KAAK,MAAM,OAAO,CAAC;EAC1D;AACA,QAAM,SAAS,KAAK,OAAO;AAC3B,QAAM,QAAQ,IAAI,WAAW,OAAO,MAAM;AAC1C,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,UAAM,CAAC,IAAI,OAAO,WAAW,CAAC;EAChC;AACA,QAAM,UAAU,IAAI,YAAY,EAAE,OAAO,KAAK;AAC9C,SAAO,wBAAwB,MAAM,KAAK,MAAM,OAAO,CAAC;AAC1D;AAEA,eAAsB,uBACpB,oBACA,OACwC;AACxC,QAAM,KAAK,wBAAwB,kBAAkB;AACrD,QAAM,MAAM,KAAK,MAAM,KAAK,IAAI,IAAI,GAAI;AACxC,QAAM,MAAM,MAAM;IAChB;MACE,KAAK,GAAG;MACR;MACA,KAAK,GAAG,aAAa;MACrB,KAAK,MAAM;MACX,KAAK;IACP;IACA,GAAG;EACL;AAEA,QAAM,OAAO,IAAI,gBAAgB;IAC/B,YAAY;IACZ,WAAW;EACb,CAAC,EAAE,SAAS;AAEZ,SAAO;IACL,KAAK,GAAG,aAAa;IACrB;EACF;AACF;AAQO,SAAS,uBAAuB,aAGrC;AACA,QAAM,OAAO,IAAI,gBAAgB;IAC/B,YAAY;IACZ,eAAe,YAAY;IAC3B,WAAW,YAAY;IACvB,eAAe,YAAY;EAC7B,CAAC,EAAE,SAAS;AAEZ,SAAO,EAAE,KAAK,uCAAuC,KAAK;AAC5D;AChIO,IAAM,qBAAqB;EAChC,oBAAoBA,GAAE,OAAO,EAAE,SAASA,GAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,EAAE,KAAK;IACvE,OAAO;IACP,aACE;IACF,QAAQ;EACV,CAAC;AACH;ACAO,IAAe,kBAAf,cAAuC,MAAM;EAEzC;EAET,YAAY,SAAiB,UAAyB;AACpD,UAAM,OAAO;AACb,SAAK,OAAO,WAAW;AACvB,SAAK,WAAW;EAClB;AACF;AAgBO,IAAM,YAAN,cAAwB,gBAAgB;EACpC,OAAO;AAClB;AEpCO,IAAM,sBAAsB;AAE5B,IAAM,qBAAqB,qBAAqB,mBAAmB;AQUnE,IAAM,wBAAwBC,GAAE,OAAO;EAC5C,aAAaA,GAAE,QAAQ,EAAE,SAAS;EAClC,QAAQA,GACL,OAAO;IACN,QAAQA,GAAE,MAAMA,GAAE,OAAO,EAAE,MAAMA,GAAE,OAAO,GAAG,MAAMA,GAAE,OAAO,EAAE,CAAC,CAAC;EAClE,CAAC,EACA,SAAS;EACZ,MAAMA,GACH;IACCA,GAAE,OAAO;MACP,GAAGA,GAAE,MAAMA,GAAE,OAAO,EAAE,GAAGA,GAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IAC9D,CAAC;EACH,EACC,SAAS;EACZ,WAAWA,GAAE,OAAO,EAAE,SAAS;EAC/B,cAAcA,GACX,OAAO;IACN,WAAWA,GAAE,OAAO;IACpB,OAAOA,GAAE,OAAO;IAChB,UAAUA,GAAE,OAAO,EAAE,SAAS;EAChC,CAAC,EACA,SAAS;AACd,CAAC;ACXM,IAAM,yBAAN,MAA6B;EAGlC,YACmB,MAOjB;AAPiB,SAAA,OAAA;EAOhB;EAPgB;EAHX,SAAsD;EAY9D,MAAc,eAAuD;AACnE,UAAM,qBAAqB,KAAK,KAAK,sBAAsB;AAC3D,QAAI,oBAAoB;AACtB,aAAO,uBAAuB,oBAAoB,KAAK,KAAK,KAAK;IACnE;AACA,UAAM,0BAA0B,KAAK,KAAK,6BAA6B;AACvE,QAAI,yBAAyB;AAC3B,aAAO,uBAAuB,uBAAuB;IACvD;AACA,UAAM,IAAI;MACR,GAAG,KAAK,KAAK,WAAW;IAC1B;EACF;EAEA,MAAM,SAAS,QAAuC;AACpD,QAAI,KAAK,UAAU,KAAK,IAAI,IAAI,KAAK,OAAO,WAAW;AACrD,aAAO,KAAK,OAAO;IACrB;AACA,UAAM,EAAE,KAAK,KAAK,IAAI,MAAM,KAAK,aAAa;AAC9C,UAAM,MAAM,MAAM,KAAK,KAAK,KAAK,KAAK;MACpC,UAAU;MACV,SAAS,EAAE,gBAAgB,oCAAoC;MAC/D;MACA;IACF,CAAC;AACD,UAAM,YAAY,IAAI,KAAK,cAAc;AACzC,SAAK,SAAS;MACZ,OAAO,IAAI,KAAK;MAChB,WAAW,KAAK,IAAI,KAAK,YAAY,MAAM;IAC7C;AACA,WAAO,KAAK,OAAO;EACrB;AACF;;;AIpEO,IAAMC,uBAAsB;AAE5B,IAAMC,sBAAqB,qBAAqBD,oBAAmB;AAEnE,SAAS,mBAAmB,aAA6B;AAC9D,SAAO,qBAAqB,WAAW,IAAIA,oBAAmB;AAChE;;;AQJA;AAAA,EACE;AAAA,EAOA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,KAAAE,UAAS;AAEX,IAAM,eAAe;AAAA,EAC1BA,GACG,OAAO;AAAA,IACN,SAASA,GAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,KAAK;AAAA,MACrC,OAAO;AAAA,MACP,aACE;AAAA,MACF,aAAa;AAAA,IACf,CAAC;AAAA,IACD,oBAAoBA,GACjB,OAAO,EAAE,SAASA,GAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,EAC5C,SAAS,EACT,KAAK;AAAA,MACJ,OAAO;AAAA,MACP,aACE;AAAA,MACF,QAAQ;AAAA,IACV,CAAC;AAAA,IACH,cAAcA,GACX,OAAO,EAAE,SAASA,GAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,EAC5C,SAAS,EACT,KAAK;AAAA,MACJ,OAAO;AAAA,MACP,aACE;AAAA,MACF,QAAQ;AAAA,IACV,CAAC;AAAA,IACH,UAAUA,GAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK;AAAA,MACjD,OAAO;AAAA,MACP,aACE;AAAA,MACF,aAAa;AAAA,IACf,CAAC;AAAA,IACD,cAAcA,GACX,OAAO,EAAE,SAASA,GAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,EAC5C,SAAS,EACT,KAAK;AAAA,MACJ,OAAO;AAAA,MACP,aACE;AAAA,MACF,QAAQ;AAAA,IACV,CAAC;AAAA,IACH,cAAcA,GAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK;AAAA,MACxD,OAAO;AAAA,MACP,aACE;AAAA,MACF,aAAa;AAAA,IACf,CAAC;AAAA,EACH,CAAC,EACA;AAAA,IACC,CAAC,QACC,IAAI,uBAAuB,UAC1B,IAAI,iBAAiB,UACpB,IAAI,aAAa,UACjB,IAAI,iBAAiB;AAAA,IACzB;AAAA,MACE,SACE;AAAA,IACJ;AAAA,EACF;AACJ;AAEO,IAAM,MAAoB,mBAAmB;AAAA,EAClD,aAAa;AAAA,EACb,UAAU;AAAA,EACV,YAAY;AAAA,EACZ,SACE;AAAA,EACF,QAAQ;AAAA,IACN,MAAM;AAAA,IACN,QAAQ;AAAA,IACR,SACE;AAAA,IACF,SAAS;AAAA,EACX;AAAA,EACA,MAAM;AAAA,IACJ,SACE;AAAA,IACF,OAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EACA,WACE;AAAA,EACF,aAAa;AAAA,IACX;AAAA,IACA;AAAA,EACF;AACF,CAAC;AAOD,IAAM,iBAAiB;AAAA,EACrB,oBAAoB;AAAA,IAClB,aAAa;AAAA,IACb,MAAM;AAAA,EACR;AAAA,EACA,cAAc;AAAA,IACZ,aAAa;AAAA,IACb,MAAM;AAAA,EACR;AAAA,EACA,UAAU;AAAA,IACR,aAAa;AAAA,IACb,MAAM;AAAA,EACR;AAAA,EACA,cAAc;AAAA,IACZ,aAAa;AAAA,IACb,MAAM;AAAA,EACR;AACF;AAIA,IAAM,cAAc;AAAA,EAClB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAcA,IAAM,cAAc;AAEpB,SAAS,gBAAgB,OAAiC;AACxD,SAAO,OAAO,UAAU,YAAY,YAAY,KAAK,KAAK;AAC5D;AAEA,SAAS,eAAe,OAAuC;AAC7D,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AACA,QAAM,IAAI;AACV,SAAO,gBAAgB,EAAE,SAAS,KAAK,gBAAgB,EAAE,OAAO;AAClE;AAEA,SAAS,gBAAgB,OAAwC;AAC/D,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AACA,QAAM,IAAI;AACV,MAAI,OAAO,EAAE,UAAU,UAAU;AAC/B,WAAO;AAAA,EACT;AACA,MAAI,CAAE,YAAkC,SAAS,EAAE,KAAK,GAAG;AACzD,WAAO;AAAA,EACT;AACA,SAAO,eAAe,EAAE,SAAS;AACnC;AAOA,IAAM,gBAA+C;AAAA,EACnD,yBAAyB;AAAA,IACvB,YAAY,CAAC,MAAM;AAAA,IACnB,YAAY;AAAA,EACd;AAAA,EACA,aAAa;AAAA,IACX,YAAY,CAAC,QAAQ,OAAO;AAAA,IAC5B,YAAY;AAAA,EACd;AAAA,EACA,WAAW;AAAA,IACT,YAAY,CAAC,QAAQ,MAAM;AAAA,IAC3B,YAAY;AAAA,EACd;AAAA,EACA,eAAe;AAAA,IACb,YAAY,CAAC,QAAQ,SAAS;AAAA,IAC9B,YAAY;AAAA,EACd;AACF;AAEA,IAAM,gBAAgB;AACtB,IAAM,gBAAgB,CAAC,UAAU,eAAe,OAAO,UAAU;AAejE,SAAS,UAAU,MAAoB;AACrC,QAAM,IAAI,KAAK,eAAe;AAC9B,QAAM,IAAI,OAAO,KAAK,YAAY,IAAI,CAAC,EAAE,SAAS,GAAG,GAAG;AACxD,QAAM,IAAI,OAAO,KAAK,WAAW,CAAC,EAAE,SAAS,GAAG,GAAG;AACnD,SAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;AACvB;AAEA,SAAS,YAAY,SAAyB;AAC5C,QAAM,IAAI,QAAQ,MAAM,GAAG,CAAC;AAC5B,QAAM,IAAI,QAAQ,MAAM,GAAG,CAAC;AAC5B,QAAM,IAAI,QAAQ,MAAM,GAAG,EAAE;AAC7B,SAAO,KAAK,IAAI,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC;AACrD;AAEA,IAAM,aAAa,KAAK,KAAK,KAAK;AAClC,IAAM,4BAA4B;AAElC,SAAS,aACP,SACA,cACc;AACd,QAAM,MAAM,KAAK,IAAI;AACrB,QAAM,UAAU,UAAU,IAAI,KAAK,GAAG,CAAC;AACvC,MAAI,QAAQ,SAAS,UAAU;AAC7B,UAAMC,WAAU,OAAO,4BAA4B,KAAK;AACxD,WAAO,EAAE,WAAW,UAAU,IAAI,KAAKA,QAAO,CAAC,GAAG,QAAQ;AAAA,EAC5D;AACA,MAAI,QAAQ,OAAO;AACjB,UAAM,UAAU,IAAI,KAAK,QAAQ,KAAK,EAAE,QAAQ;AAChD,QAAI,OAAO,SAAS,OAAO,GAAG;AAC5B,YAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,MAAM,WAAW,UAAU,CAAC;AAChE,YAAM,aAAa,KAAK,IAAI,MAAM,YAAY;AAC9C,YAAMA,WAAU,OAAO,aAAa,KAAK;AACzC,aAAO,EAAE,WAAW,UAAU,IAAI,KAAKA,QAAO,CAAC,GAAG,QAAQ;AAAA,IAC5D;AAAA,EACF;AACA,QAAM,UAAU,OAAO,eAAe,KAAK;AAC3C,SAAO,EAAE,WAAW,UAAU,IAAI,KAAK,OAAO,CAAC,GAAG,QAAQ;AAC5D;AAEO,SAAS,kBACd,KACA,YACA,YAMA;AACA,QAAM,aAA8C,CAAC;AACrD,QAAM,OAAO,IAAI,QAAQ,CAAC;AAC1B,WAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,eAAW,WAAW,CAAC,CAAE,IAAI,KAAK,CAAC,KAAK;AAAA,EAC1C;AAEA,aAAW,SAAS,eAAe;AACjC,UAAM,MAAM,IAAI,KAAK;AACrB,eAAW,KAAK,IACd,OAAO,QAAQ,YAAY,OAAO,SAAS,GAAG,IAAI,MAAM;AAAA,EAC5D;AAEA,QAAM,UACJ,OAAO,WAAW,MAAM,MAAM,WAAW,WAAW,MAAM,IAAI;AAChE,QAAM,KAAK,YAAY,OAAO;AAC9B,QAAM,eAAe,WAAW,QAAQ;AAExC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA,OAAO;AAAA,IACP;AAAA,EACF;AACF;AAEA,IAAM,UAAUD,GACb,OAAO,EACP,MAAM,sDAAsD;AAC/D,IAAM,YAAYA,GAAE,OAAO;AAC3B,IAAM,eAAeA,GAAE,OAAO;AAE9B,SAAS,aAAa,gBAAwB;AAC5C,QAAM,OACJ,mBAAmB,IACfA,GAAE,MAAM,CAAC,OAAO,CAAC,IACjBA,GAAE,MAAM,CAAC,SAAS,GAAG,MAAM,iBAAiB,CAAC,EAAE,KAAK,SAAS,CAAC,CAG7D;AACP,SAAOA,GAAE,OAAO;AAAA,IACd,MAAMA,GACH;AAAA,MACCA,GAAE,OAAO;AAAA,QACP;AAAA,QACA,QAAQ,aAAa,SAAS;AAAA,QAC9B,aAAa,aAAa,SAAS;AAAA,QACnC,KAAK,aAAa,SAAS;AAAA,QAC3B,UAAU,aAAa,SAAS;AAAA,MAClC,CAAC;AAAA,IACH,EACC,SAAS;AAAA,EACd,CAAC;AACH;AAEA,IAAME,uBAAsBF,GAAE,OAAO;AAAA,EACnC,cAAcA,GAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC9B,YAAYA,GAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS;AACnD,CAAC;AAEM,IAAM,+BAA+B,gBAAgB;AAAA,EAC1D,6BAA6B;AAAA,IAC3B,OAAO;AAAA,IACP,aACE;AAAA,IACF,MAAM;AAAA,IACN,aAAa;AAAA,IACb,UAAU;AAAA,IACV,YAAY;AAAA,MACV,EAAE,MAAM,QAAQ,aAAa,qCAAqC;AAAA,IACpE;AAAA,IACA,WAAW;AAAA,MACT,aAAaE;AAAA,MACb,yBAAyB,aAAa,CAAC;AAAA,IACzC;AAAA,EACF;AAAA,EACA,iBAAiB;AAAA,IACf,OAAO;AAAA,IACP,aACE;AAAA,IACF,MAAM;AAAA,IACN,aAAa;AAAA,IACb,UAAU;AAAA,IACV,YAAY;AAAA,MACV,EAAE,MAAM,QAAQ,aAAa,qCAAqC;AAAA,MAClE;AAAA,QACE,MAAM;AAAA,QACN,aAAa;AAAA,MACf;AAAA,IACF;AAAA,IACA,WAAW,EAAE,aAAa,aAAa,CAAC,EAAE;AAAA,EAC5C;AAAA,EACA,eAAe;AAAA,IACb,OAAO;AAAA,IACP,aACE;AAAA,IACF,MAAM;AAAA,IACN,aAAa;AAAA,IACb,UAAU;AAAA,IACV,YAAY;AAAA,MACV,EAAE,MAAM,QAAQ,aAAa,qCAAqC;AAAA,MAClE;AAAA,QACE,MAAM;AAAA,QACN,aAAa;AAAA,MACf;AAAA,IACF;AAAA,IACA,WAAW,EAAE,WAAW,aAAa,CAAC,EAAE;AAAA,EAC1C;AAAA,EACA,mBAAmB;AAAA,IACjB,OAAO;AAAA,IACP,aACE;AAAA,IACF,MAAM;AAAA,IACN,aAAa;AAAA,IACb,UAAU;AAAA,IACV,YAAY;AAAA,MACV,EAAE,MAAM,QAAQ,aAAa,qCAAqC;AAAA,MAClE;AAAA,QACE,MAAM;AAAA,QACN,aAAa;AAAA,MACf;AAAA,IACF;AAAA,IACA,WAAW,EAAE,eAAe,aAAa,CAAC,EAAE;AAAA,EAC9C;AACF,CAAC;AAEM,IAAM,KAAK;AAEX,IAAM,eAAN,MAAM,sBAAqB,cAA2C;AAAA,EAC3E,OAAgB,KAAK;AAAA,EAErB,OAAgB,YAAY;AAAA,EAE5B,OAAgB,UAAU,qBAAqB,4BAA4B;AAAA,EAE3E,OAAO,OAAO,OAAgB,KAAsC;AAClE,UAAM,SAAS,aAAa,MAAM,KAAK;AACvC,WAAO,IAAI;AAAA,MACT;AAAA,QACE,SAAS,OAAO;AAAA,QAChB,cAAc,OAAO;AAAA,MACvB;AAAA,MACA;AAAA,QACE,oBAAoB,OAAO;AAAA,QAC3B,cAAc,OAAO;AAAA,QACrB,UAAU,OAAO;AAAA,QACjB,cAAc,OAAO;AAAA,MACvB;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAES,KAAK;AAAA,EACI,cAAc;AAAA,EAExB;AAAA,EAEA,eAAe,QAAuC;AAC5D,SAAK,kBAAkB,IAAI,uBAAuB;AAAA,MAChD,aAAa,KAAK;AAAA,MAClB,OAAO;AAAA,MACP,uBAAuB,MAAM,KAAK,MAAM;AAAA,MACxC,4BAA4B,MAAM;AAChC,cAAM,EAAE,cAAc,UAAU,aAAa,IAAI,KAAK;AACtD,YAAI,gBAAgB,YAAY,cAAc;AAC5C,iBAAO,EAAE,cAAc,UAAU,aAAa;AAAA,QAChD;AACA,eAAO;AAAA,MACT;AAAA,MACA,MAAM,CAAC,KAAK,SACV,KAAK,KAAoD,KAAK,IAAI;AAAA,IACtE,CAAC;AACD,WAAO,KAAK,cAAc,SAAS,MAAM;AAAA,EAC3C;AAAA,EAEA,MAAc,UACZ,aACA,OACA,WACA,UACA,QAC4B;AAC5B,UAAM,EAAE,WAAW,IAAI,cAAc,KAAK;AAC1C,UAAM,MAAM,4DAA4D,mBAAmB,KAAK,SAAS,OAAO,CAAC;AAEjH,UAAM,OAAgC;AAAA,MACpC,WAAW,UAAU;AAAA,MACrB,SAAS,UAAU;AAAA,MACnB;AAAA,MACA,UAAU;AAAA,MACV;AAAA,IACF;AAEA,UAAM,MAAM,MAAM,KAAK,KAAwB,KAAK;AAAA,MAClD,UAAU;AAAA,MACV,SAAS;AAAA,QACP,eAAe,UAAU,WAAW;AAAA,QACpC,gBAAgB;AAAA,QAChB,cAAc,mBAAmB,uBAAuB;AAAA,MAC1D;AAAA,MACA,MAAM,KAAK,UAAU,IAAI;AAAA,MACzB;AAAA,IACF,CAAC;AACD,WAAO,IAAI;AAAA,EACb;AAAA,EAEA,MAAM,KACJ,SACA,SACA,QACqB;AACrB,UAAM,eAAe,KAAK,SAAS,gBAAgB;AAEnD,UAAM,SAAS,gBAAgB,QAAQ,MAAM,IAAI,QAAQ,SAAS;AAClE,UAAM,YAAY,QAAQ,aAAa,aAAa,SAAS,YAAY;AAEzE,QAAI,cAA6B;AACjC,UAAM,WAAW,OAAO,QAAuC;AAC7D,UAAI,CAAC,aAAa;AAChB,sBAAc,MAAM,KAAK,eAAe,GAAG;AAAA,MAC7C;AACA,aAAO;AAAA,IACT;AAEA,UAAM,qBAAqB,OACzB,OACA,UACA,QAC+B;AAC/B,YAAM,QAAQ,MAAM,SAAS,GAAG;AAChC,UAAI;AACF,eAAO,MAAM,KAAK,UAAU,OAAO,OAAO,WAAW,UAAU,GAAG;AAAA,MACpE,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN;AAAA,UACA;AAAA,QACF;AACA,sBAAc;AACd,cAAM,aAAa,MAAM,SAAS,GAAG;AACrC,eAAO,KAAK,UAAU,YAAY,OAAO,WAAW,UAAU,GAAG;AAAA,MACnE;AAAA,IACF;AAEA,UAAM,aAAa,OAAO,UAA6C;AACrE,YAAM,UAA0B,CAAC;AACjC,UAAI,WAAW;AACf,iBAAS;AACP,cAAM,WAAW,MAAM,mBAAmB,OAAO,UAAU,MAAM;AACjE,cAAM,OAAO,SAAS,QAAQ,CAAC;AAC/B,gBAAQ,KAAK,GAAG,IAAI;AACpB,oBAAY,KAAK;AACjB,YAAI,KAAK,SAAS,eAAe;AAC/B;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAEA,UAAM,YAAY,SAAS,YAAY,QAAQ,OAAO,KAAK,IAAI;AAC/D,UAAM,WAAW,aAAa,IAAI,YAAY;AAE9C,aAAS,IAAI,UAAU,IAAI,YAAY,QAAQ,KAAK;AAClD,YAAM,QAAQ,YAAY,CAAC;AAC3B,UAAI,QAAQ,SAAS;AACnB,eAAO,EAAE,MAAM,OAAO,QAAQ,EAAE,OAAO,UAAU,EAAE;AAAA,MACrD;AAEA,UAAI;AACJ,UAAI;AACF,eAAO,MAAM,WAAW,KAAK;AAAA,MAC/B,SAAS,KAAK;AACZ,YAAI,QAAQ,SAAS;AACnB,iBAAO,EAAE,MAAM,OAAO,QAAQ,EAAE,OAAO,UAAU,EAAE;AAAA,QACrD;AACA,cAAM;AAAA,MACR;AACA,YAAM,MAAM,cAAc,KAAK;AAC/B,YAAM,UAAU,KAAK;AAAA,QAAI,CAAC,QACxB,kBAAkB,KAAK,IAAI,YAAY,IAAI,UAAU;AAAA,MACvD;AACA,YAAM,QAAQ,QAAQ,SAAS,EAAE,OAAO,CAAC,IAAI,UAAU,EAAE,CAAC;AAAA,IAC5D;AAEA,WAAO,EAAE,MAAM,KAAK;AAAA,EACtB;AACF;;;ACjiBA,IAAO,gBAAQ;","names":["z","z","HTTP_CLIENT_VERSION","DEFAULT_USER_AGENT","z","startMs","tokenResponseSchema"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rawdash/connector-google-search-console",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.22.0",
|
|
4
4
|
"description": "Rawdash connector for Google Search Console - syncs SEO impressions, clicks, CTR, and position by day, query, page, and country into the six-shape storage model",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -24,15 +24,16 @@
|
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
26
|
"zod": "^4.4.3",
|
|
27
|
-
"@rawdash/core": "0.
|
|
27
|
+
"@rawdash/core": "0.22.0"
|
|
28
28
|
},
|
|
29
29
|
"devDependencies": {
|
|
30
30
|
"fast-check": "^4.8.0",
|
|
31
31
|
"tsup": "^8.0.0",
|
|
32
32
|
"typescript": "^5.7.2",
|
|
33
33
|
"vitest": "^4.1.4",
|
|
34
|
+
"@rawdash/connector-gcp-shared": "0.1.0",
|
|
34
35
|
"@rawdash/connector-shared": "0.3.1",
|
|
35
|
-
"@rawdash/connector-test-utils": "0.0.
|
|
36
|
+
"@rawdash/connector-test-utils": "0.0.10"
|
|
36
37
|
},
|
|
37
38
|
"scripts": {
|
|
38
39
|
"build": "tsup",
|